mirror of https://github.com/apache/lucene.git
SOLR-13710: Persist package jars locally & expose them over http
This commit is contained in:
Noble Paul
GitHub
parent
7e5c071c08
commit
e23c50b2a8
|
|
@ -16,17 +16,20 @@
|
|||
*/
|
||||
package org.apache.solr.core;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.math.BigInteger;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
|
|
@ -34,10 +37,14 @@ import java.util.concurrent.Callable;
|
|||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.solr.api.Api;
|
||||
import org.apache.solr.api.V2HttpCall;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.cloud.ClusterState;
|
||||
import org.apache.solr.common.cloud.DocCollection;
|
||||
|
|
@ -45,16 +52,26 @@ import org.apache.solr.common.cloud.Replica;
|
|||
import org.apache.solr.common.cloud.Slice;
|
||||
import org.apache.solr.common.cloud.ZkStateReader;
|
||||
import org.apache.solr.common.params.CollectionAdminParams;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.handler.RequestHandlerBase;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.security.AuthorizationContext;
|
||||
import org.apache.solr.security.PermissionNameProvider;
|
||||
import org.apache.solr.util.SimplePostTool;
|
||||
import org.apache.zookeeper.server.ByteBufferInputStream;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
|
||||
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
|
||||
import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE;
|
||||
import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
|
||||
import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
|
||||
|
||||
/**
|
||||
* The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the Jar in a single node.
|
||||
|
|
@ -88,6 +105,14 @@ public class BlobRepository {
|
|||
this.coreContainer = coreContainer;
|
||||
}
|
||||
|
||||
public Collection<String> getFiles() {
|
||||
return Arrays.asList(getBlobsPath().toFile().list());
|
||||
}
|
||||
|
||||
public Path getBlobsPath() {
|
||||
return SolrResourceLoader.getBlobsDirPath(this.coreContainer.getResourceLoader().getInstancePath());
|
||||
}
|
||||
|
||||
// I wanted to {@link SolrCore#loadDecodeAndCacheBlob(String, Decoder)} below but precommit complains
|
||||
|
||||
/**
|
||||
|
|
@ -116,12 +141,12 @@ public class BlobRepository {
|
|||
return getBlobIncRef(key.concat(decoder.getName()), () -> addBlob(key, decoder));
|
||||
}
|
||||
|
||||
BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha512) {
|
||||
BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha256) {
|
||||
StringBuffer keyBuilder = new StringBuffer(key);
|
||||
if (decoder != null) keyBuilder.append(decoder.getName());
|
||||
keyBuilder.append("/").append(sha512);
|
||||
keyBuilder.append("/").append(sha256);
|
||||
|
||||
return getBlobIncRef(keyBuilder.toString(), () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha512), decoder));
|
||||
return getBlobIncRef(keyBuilder.toString(), () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha256), decoder));
|
||||
}
|
||||
|
||||
// do the actual work returning the appropriate type...
|
||||
|
|
@ -166,35 +191,79 @@ public class BlobRepository {
|
|||
return aBlob;
|
||||
}
|
||||
|
||||
static String INVALID_JAR_MSG = "Invalid jar from {0} , expected sha512 hash : {1} , actual : {2}";
|
||||
static String INVALID_JAR_MSG = "Invalid jar from {0} , expected sha256 hash : {1} , actual : {2}";
|
||||
|
||||
private ByteBuffer fetchBlobAndVerify(String key, String url, String sha512) {
|
||||
private ByteBuffer fetchBlobAndVerify(String key, String url, String sha256) throws IOException {
|
||||
ByteBuffer byteBuffer = null;
|
||||
if (sha256 != null) {
|
||||
byteBuffer = getFromLocalFs(sha256);
|
||||
}
|
||||
if (byteBuffer == null) byteBuffer = getAndValidate(key, url, sha256);
|
||||
return byteBuffer;
|
||||
}
|
||||
|
||||
private ByteBuffer getAndValidate(String key, String url, String sha256) throws IOException {
|
||||
ByteBuffer byteBuffer = fetchFromUrl(key, url);
|
||||
String computedDigest = sha512Digest(byteBuffer);
|
||||
if (!computedDigest.equals(sha512)) {
|
||||
throw new SolrException(SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha512, computedDigest));
|
||||
|
||||
String computedDigest = sha256Digest(byteBuffer);
|
||||
if (!computedDigest.equals(sha256)) {
|
||||
throw new SolrException(SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha256, computedDigest));
|
||||
}
|
||||
File file = new File(getBlobsPath().toFile(), sha256);
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) {
|
||||
fos.write(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.limit());
|
||||
IOUtils.fsync(file.toPath(), false);
|
||||
}
|
||||
return byteBuffer;
|
||||
}
|
||||
|
||||
public static String sha512Digest(ByteBuffer byteBuffer) {
|
||||
MessageDigest digest = null;
|
||||
public String putBlob(InputStream is) throws SolrException {
|
||||
byte[] b = new byte[(int) MAX_JAR_SIZE + 1];
|
||||
String sha256 = null;
|
||||
try {
|
||||
digest = MessageDigest.getInstance("SHA-512");
|
||||
} catch (NoSuchAlgorithmException e) {
|
||||
//unlikely
|
||||
throw new SolrException(SERVER_ERROR, e);
|
||||
}
|
||||
byteBuffer = ByteBuffer.wrap(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.limit());
|
||||
digest.update(byteBuffer);
|
||||
int sz = is.read(b);
|
||||
|
||||
if (sz > MAX_JAR_SIZE)
|
||||
throw new SolrException(BAD_REQUEST, "size is more than permitted , use system property runtime.lib.size to change it");
|
||||
sha256 = sha256Digest(ByteBuffer.wrap(b, 0, sz));
|
||||
File file = new File(getBlobsPath().toFile(), sha256);
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) {
|
||||
fos.write(b, 0, sz);
|
||||
}
|
||||
IOUtils.fsync(file.toPath(), false);
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(BAD_REQUEST, e);
|
||||
}
|
||||
return sha256;
|
||||
|
||||
return String.format(
|
||||
Locale.ROOT,
|
||||
"%0128x",
|
||||
new BigInteger(1, digest.digest()));
|
||||
}
|
||||
|
||||
private ByteBuffer getFromLocalFs(String sha256) throws IOException {
|
||||
Path p = getBlobsPath();
|
||||
File f = new File(p.toFile(), sha256);
|
||||
if (!f.exists()) return null;
|
||||
byte[] b = new byte[(int) f.length()];
|
||||
try (FileInputStream fis = new FileInputStream(f)) {
|
||||
fis.read(b);
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(b);
|
||||
if (sha256.equals(sha256Digest(byteBuffer))) {
|
||||
return byteBuffer;
|
||||
} else {
|
||||
return null;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static String sha256Digest(ByteBuffer buf) {
|
||||
try {
|
||||
return DigestUtils.sha256Hex(new ByteBufferInputStream(ByteBuffer.wrap( buf.array(), buf.arrayOffset(), buf.limit())));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Unable to compute sha256", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Package local for unit tests only please do not use elsewhere
|
||||
|
|
@ -216,14 +285,14 @@ public class BlobRepository {
|
|||
entity = response.getEntity();
|
||||
int statusCode = response.getStatusLine().getStatusCode();
|
||||
if (statusCode != 200) {
|
||||
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such resource available: " + key + ", url : "+ url);
|
||||
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such resource available: " + key + ", url : " + url);
|
||||
}
|
||||
|
||||
try (InputStream is = entity.getContent()) {
|
||||
b = SimplePostTool.inputStreamToByteArray(is, MAX_JAR_SIZE);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error loading resource "+ url, e);
|
||||
log.error("Error loading resource " + url, e);
|
||||
if (e instanceof SolrException) {
|
||||
throw (SolrException) e;
|
||||
} else {
|
||||
|
|
@ -284,6 +353,68 @@ public class BlobRepository {
|
|||
}
|
||||
}
|
||||
|
||||
BlobRead blobRead = new BlobRead();
|
||||
|
||||
|
||||
class BlobRead extends RequestHandlerBase implements PermissionNameProvider {
|
||||
|
||||
|
||||
@Override
|
||||
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDescription() {
|
||||
return "List fetch blobs";
|
||||
}
|
||||
|
||||
@Override
|
||||
public Name getPermissionName(AuthorizationContext request) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<Api> getApis() {
|
||||
return Collections.singleton(new Api(Utils.getSpec("node.blob.GET")) {
|
||||
@Override
|
||||
public void call(SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
String sha256 = ((V2HttpCall) req.getHttpSolrCall()).getUrlParts().get("sha256");
|
||||
if (sha256 == null) {
|
||||
rsp.add("blob", getFiles());
|
||||
} else {
|
||||
try {
|
||||
ByteBuffer buf = getFromLocalFs(sha256);
|
||||
if(buf == null){
|
||||
throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such blob");
|
||||
} else {
|
||||
ModifiableSolrParams solrParams = new ModifiableSolrParams();
|
||||
solrParams.add(CommonParams.WT, FILE_STREAM);
|
||||
req.setParams( SolrParams.wrapDefaults(solrParams, req.getParams()));
|
||||
rsp.add(FILE_STREAM, (SolrCore.RawWriter) os -> os.write(buf.array(), buf.arrayOffset(), buf.limit()));
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(SERVER_ERROR,e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean registerV1() {
|
||||
return Boolean.FALSE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean registerV2() {
|
||||
return Boolean.TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static class BlobContent<T> {
|
||||
public final String key;
|
||||
private final T content; // holds byte buffer or cached object, holding both is a waste of memory
|
||||
|
|
@ -337,7 +468,7 @@ public class BlobRepository {
|
|||
public static class BlobContentRef<T> {
|
||||
public final BlobContent<T> blob;
|
||||
|
||||
private BlobContentRef(BlobContent<T> blob) {
|
||||
public BlobContentRef(BlobContent<T> blob) {
|
||||
this.blob = blob;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -642,6 +642,7 @@ public class CoreContainer {
|
|||
this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());
|
||||
|
||||
containerHandlers.put("/ext", clusterPropertiesListener.extHandler);
|
||||
containerHandlers.put("/blob-get", blobRepository.blobRead);
|
||||
createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
|
||||
createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);
|
||||
collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
|
||||
|
|
@ -1542,7 +1543,7 @@ public class CoreContainer {
|
|||
} catch (SolrCoreState.CoreIsClosedException e) {
|
||||
throw e;
|
||||
} catch (Exception e) {
|
||||
coreInitFailures.put(cd.getName(), new CoreLoadFailure(cd, (Exception) e));
|
||||
coreInitFailures.put(cd.getName(), new CoreLoadFailure(cd, e));
|
||||
throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to reload core [" + cd.getName() + "]", e);
|
||||
} finally {
|
||||
if (!success && newCore != null && newCore.getOpenCount() > 0) {
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory;
|
|||
import static org.apache.solr.common.params.CommonParams.NAME;
|
||||
import static org.apache.solr.common.params.CommonParams.PACKAGE;
|
||||
import static org.apache.solr.common.params.CommonParams.VERSION;
|
||||
import static org.apache.solr.core.RuntimeLib.SHA512;
|
||||
import static org.apache.solr.core.RuntimeLib.SHA256;
|
||||
|
||||
public class PackageManager implements ClusterPropertiesListener {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
|
@ -94,8 +94,8 @@ public class PackageManager implements ClusterPropertiesListener {
|
|||
|
||||
|
||||
public boolean isModified(Map map) {
|
||||
return (!Objects.equals(lib.getSha512(), (map).get(SHA512)) ||
|
||||
!Objects.equals(lib.getSig(), (map).get(SHA512)));
|
||||
return (!Objects.equals(lib.getSha256(), (map).get(SHA256)) ||
|
||||
!Objects.equals(lib.getSig(), (map).get(SHA256)));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -167,7 +167,7 @@ public class PackageManager implements ClusterPropertiesListener {
|
|||
needsReload[0] = true;
|
||||
}
|
||||
if (needsReload[0]) {
|
||||
createNewClassLoader(m, ver);
|
||||
createNewClassLoaders(m, ver);
|
||||
}
|
||||
return needsReload[0];
|
||||
}
|
||||
|
|
@ -177,7 +177,7 @@ public class PackageManager implements ClusterPropertiesListener {
|
|||
return p == null ? coreContainer.getResourceLoader() : p.loader;
|
||||
}
|
||||
|
||||
void createNewClassLoader(Map m, int ver) {
|
||||
void createNewClassLoaders(Map m, int ver) {
|
||||
boolean[] loadedAll = new boolean[1];
|
||||
loadedAll[0] = true;
|
||||
Map<String, Package> newPkgs = new LinkedHashMap<>();
|
||||
|
|
|
|||
|
|
@ -44,10 +44,10 @@ import static org.apache.solr.common.params.CommonParams.NAME;
|
|||
*/
|
||||
public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWriter {
|
||||
public static final String TYPE = "runtimeLib";
|
||||
public static final String SHA512 = "sha512";
|
||||
public static final String SHA256 = "sha256";
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
private final CoreContainer coreContainer;
|
||||
private String name, version, sig, sha512, url;
|
||||
private String name, version, sig, sha256, url;
|
||||
private BlobRepository.BlobContentRef<ByteBuffer> jarContent;
|
||||
private boolean verified = false;
|
||||
int znodeVersion = -1;
|
||||
|
|
@ -57,7 +57,7 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit
|
|||
ew.putIfNotNull(NAME, name);
|
||||
ew.putIfNotNull("url", url);
|
||||
ew.putIfNotNull(version, version);
|
||||
ew.putIfNotNull(sha512, sha512);
|
||||
ew.putIfNotNull("sha256", sha256);
|
||||
ew.putIfNotNull("sig", sig);
|
||||
if (znodeVersion > -1) {
|
||||
ew.put(ConfigOverlay.ZNODEVER, znodeVersion);
|
||||
|
|
@ -101,19 +101,19 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit
|
|||
}
|
||||
version = String.valueOf(v);
|
||||
} else {
|
||||
sha512 = info.attributes.get(SHA512);
|
||||
if (sha512 == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute");
|
||||
sha256 = info.attributes.get(SHA256);
|
||||
if (sha256 == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha256' attribute");
|
||||
}
|
||||
ByteBuffer buf = coreContainer.getBlobRepository().fetchFromUrl(name, url);
|
||||
|
||||
String digest = BlobRepository.sha512Digest(buf);
|
||||
if (!sha512.equals(digest)) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest));
|
||||
String digest = BlobRepository.sha256Digest(buf);
|
||||
if (!sha256.equals(digest)) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha256, digest));
|
||||
}
|
||||
verifyJarSignature(buf);
|
||||
|
||||
log.debug("dynamic library verified {}, sha512: {}", url, sha512);
|
||||
log.debug("dynamic library verified {}, sha256: {}", url, sha256);
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -130,7 +130,7 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit
|
|||
|
||||
jarContent = url == null ?
|
||||
coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version) :
|
||||
coreContainer.getBlobRepository().getBlobIncRef(name, null, url, sha512);
|
||||
coreContainer.getBlobRepository().getBlobIncRef(name, null, url, sha256);
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -148,8 +148,8 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit
|
|||
|
||||
}
|
||||
|
||||
public String getSha512() {
|
||||
return sha512;
|
||||
public String getSha256() {
|
||||
return sha256;
|
||||
}
|
||||
|
||||
public ByteBuffer getFileContent(String entryName) throws IOException {
|
||||
|
|
|
|||
|
|
@ -576,8 +576,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
static final String empty[] = new String[0];
|
||||
|
||||
static final String[] empty = new String[0];
|
||||
|
||||
@Override
|
||||
public <T> T newInstance(String name, Class<T> expectedType) {
|
||||
|
|
@ -808,6 +808,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
|
|||
* manipulated using select Solr features (e.g. streaming expressions).
|
||||
*/
|
||||
public static final String USER_FILES_DIRECTORY = "userfiles";
|
||||
public static final String BLOBS_DIRECTORY = "blobs";
|
||||
public static void ensureUserFilesDataDir(Path solrHome) {
|
||||
final Path userFilesPath = getUserFilesPath(solrHome);
|
||||
final File userFilesDirectory = new File(userFilesPath.toString());
|
||||
|
|
@ -823,10 +824,28 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
|
|||
}
|
||||
}
|
||||
|
||||
public static void ensureBlobsDir(Path solrHome) {
|
||||
final Path blobsDir = getBlobsDirPath(solrHome);
|
||||
final File blobsFilesDirectory = new File(blobsDir.toString());
|
||||
if (! blobsFilesDirectory.exists()) {
|
||||
try {
|
||||
final boolean created = blobsFilesDirectory.mkdir();
|
||||
if (! created) {
|
||||
log.warn("Unable to create [{}] directory in SOLR_HOME [{}]. Features requiring this directory may fail.", BLOBS_DIRECTORY, solrHome);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Unable to create [" + BLOBS_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static Path getBlobsDirPath(Path solrHome) {
|
||||
return Paths.get(solrHome.toAbsolutePath().toString(), BLOBS_DIRECTORY).toAbsolutePath();
|
||||
}
|
||||
|
||||
public static Path getUserFilesPath(Path solrHome) {
|
||||
return Paths.get(solrHome.toAbsolutePath().toString(), USER_FILES_DIRECTORY).toAbsolutePath();
|
||||
}
|
||||
|
||||
// Logs a message only once per startup
|
||||
private static void logOnceInfo(String key, String msg) {
|
||||
if (!loggedOnce.contains(key)) {
|
||||
|
|
@ -923,7 +942,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable
|
|||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg);
|
||||
}
|
||||
}
|
||||
try (OutputStream out = new FileOutputStream(confFile);) {
|
||||
try (OutputStream out = new FileOutputStream(confFile)) {
|
||||
out.write(content);
|
||||
}
|
||||
log.info("Written confile " + resourceName);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,9 @@
|
|||
|
||||
package org.apache.solr.handler.admin;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
|
@ -29,6 +31,7 @@ import java.util.Objects;
|
|||
|
||||
import org.apache.solr.api.ApiBag;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.request.CollectionApiMapping;
|
||||
import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta;
|
||||
import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta;
|
||||
|
|
@ -60,7 +63,7 @@ import static java.util.Arrays.asList;
|
|||
import static java.util.Collections.singletonList;
|
||||
import static org.apache.solr.common.util.CommandOperation.captureErrors;
|
||||
import static org.apache.solr.common.util.StrUtils.formatString;
|
||||
import static org.apache.solr.core.RuntimeLib.SHA512;
|
||||
import static org.apache.solr.core.RuntimeLib.SHA256;
|
||||
|
||||
public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
|
@ -218,8 +221,8 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
|||
op.addError(StrUtils.formatString("The jar with a name ''{0}'' does not exist", name));
|
||||
return false;
|
||||
}
|
||||
if (Objects.equals(existing.get(SHA512), op.getDataMap().get(SHA512))) {
|
||||
op.addError("Trying to update a jar with the same sha512");
|
||||
if (Objects.equals(existing.get(SHA256), op.getDataMap().get(SHA256))) {
|
||||
op.addError("Trying to update a jar with the same sha256");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -365,4 +368,10 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport {
|
|||
}
|
||||
}
|
||||
|
||||
public static void postBlob(String baseUrl, ByteBuffer buf) throws IOException {
|
||||
try(HttpSolrClient client = new HttpSolrClient.Builder(baseUrl+"/____v2/node/blob" ).build()){
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,6 +52,8 @@ public interface PermissionNameProvider {
|
|||
AUTOSCALING_WRITE_PERM("autoscaling-write", null),
|
||||
AUTOSCALING_HISTORY_READ_PERM("autoscaling-history-read", null),
|
||||
METRICS_HISTORY_READ_PERM("metrics-history-read", null),
|
||||
BLOB_READ("blob-read", null),
|
||||
BLOB_WRITE("blob-write", null),
|
||||
ALL("all", unmodifiableSet(new HashSet<>(asList("*", null))))
|
||||
;
|
||||
final String name;
|
||||
|
|
|
|||
|
|
@ -82,9 +82,9 @@ import org.apache.solr.security.AuthenticationPlugin;
|
|||
import org.apache.solr.security.PKIAuthenticationPlugin;
|
||||
import org.apache.solr.security.PublicKeyHandler;
|
||||
import org.apache.solr.util.SolrFileCleaningTracker;
|
||||
import org.apache.solr.util.tracing.GlobalTracer;
|
||||
import org.apache.solr.util.StartupLoggingUtils;
|
||||
import org.apache.solr.util.configuration.SSLConfigurationsFactory;
|
||||
import org.apache.solr.util.tracing.GlobalTracer;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
|
@ -180,6 +180,7 @@ public class SolrDispatchFilter extends BaseSolrFilter {
|
|||
final Path solrHomePath = solrHome == null ? SolrResourceLoader.locateSolrHome() : Paths.get(solrHome);
|
||||
coresInit = createCoreContainer(solrHomePath, extraProperties);
|
||||
SolrResourceLoader.ensureUserFilesDataDir(solrHomePath);
|
||||
SolrResourceLoader.ensureBlobsDir(solrHomePath);
|
||||
this.httpClient = coresInit.getUpdateShardHandler().getDefaultHttpClient();
|
||||
setupJvmMetrics(coresInit);
|
||||
log.debug("user.dir=" + System.getProperty("user.dir"));
|
||||
|
|
@ -626,8 +627,8 @@ public class SolrDispatchFilter extends BaseSolrFilter {
|
|||
public void close() {
|
||||
// even though we skip closes, we let local tests know not to close so that a full understanding can take
|
||||
// place
|
||||
assert Thread.currentThread().getStackTrace()[2].getClassName().matches(
|
||||
"org\\.apache\\.(?:solr|lucene).*") ? false : true : CLOSE_STREAM_MSG;
|
||||
assert !Thread.currentThread().getStackTrace()[2].getClassName().matches(
|
||||
"org\\.apache\\.(?:solr|lucene).*") : CLOSE_STREAM_MSG;
|
||||
this.stream = ClosedServletInputStream.CLOSED_SERVLET_INPUT_STREAM;
|
||||
}
|
||||
};
|
||||
|
|
@ -661,9 +662,8 @@ public class SolrDispatchFilter extends BaseSolrFilter {
|
|||
public void close() {
|
||||
// even though we skip closes, we let local tests know not to close so that a full understanding can take
|
||||
// place
|
||||
assert Thread.currentThread().getStackTrace()[2].getClassName().matches(
|
||||
"org\\.apache\\.(?:solr|lucene).*") ? false
|
||||
: true : CLOSE_STREAM_MSG;
|
||||
assert !Thread.currentThread().getStackTrace()[2].getClassName().matches(
|
||||
"org\\.apache\\.(?:solr|lucene).*") : CLOSE_STREAM_MSG;
|
||||
stream = ClosedServletOutputStream.CLOSED_SERVLET_OUTPUT_STREAM;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -71,3 +71,27 @@ openssl dgst -sha512 cache.jar.bin
|
|||
openssl dgst -sha512 cache_v2.jar.bin
|
||||
|
||||
aa3f42fb640636dd8126beca36ac389486d0fcb1c3a2e2c387d043d57637535ce8db3b17983853322f78bb8f447ed75fe7b405675debe652ed826ee95e8ce328
|
||||
|
||||
=============sha256============================
|
||||
|
||||
openssl dgst -sha256 runtimelibs.jar.bin
|
||||
|
||||
e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc
|
||||
|
||||
openssl dgst -sha512 runtimelibs_v2.jar.bin
|
||||
|
||||
79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4
|
||||
|
||||
openssl dgst -sha256 runtimelibs_v3.jar.bin
|
||||
|
||||
20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3
|
||||
|
||||
openssl dgst -sha256 cache.jar.bin
|
||||
|
||||
380c2a61759f01b4d5d2570496c3d2737e3cc6968347faa94d93e906e03e077f
|
||||
|
||||
openssl dgst -sha256 cache_v2.jar.bin
|
||||
|
||||
22551e42e6fd9646a641ebc1380472ec66fba62f35febad46c8165376b41161d
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,6 @@ import static org.junit.Assert.assertEquals;
|
|||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.fail;
|
||||
import static org.mockito.Mockito.any;
|
||||
import static org.mockito.Mockito.eq;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
|
@ -63,6 +62,7 @@ public class BlobRepositoryMockingTest {
|
|||
boolean blobFetched = false;
|
||||
String blobKey = "";
|
||||
String url = null;
|
||||
String sha256 = null;
|
||||
ByteBuffer filecontent = null;
|
||||
|
||||
@BeforeClass
|
||||
|
|
@ -91,6 +91,14 @@ public class BlobRepositoryMockingTest {
|
|||
return filecontent;
|
||||
}
|
||||
|
||||
@Override
|
||||
BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha256) {
|
||||
if(!Objects.equals(sha256, BlobRepositoryMockingTest.this.sha256)) return null;
|
||||
blobKey = key;
|
||||
blobFetched = true;
|
||||
return new BlobContentRef(new BlobContent(key, filecontent)) ;
|
||||
}
|
||||
|
||||
@Override
|
||||
ConcurrentHashMap<String, BlobContent> createMap() {
|
||||
return mapMock;
|
||||
|
|
@ -130,21 +138,13 @@ public class BlobRepositoryMockingTest {
|
|||
when(mockContainer.isZooKeeperAware()).thenReturn(true);
|
||||
filecontent = TestDynamicLoading.getFileContent("runtimecode/runtimelibs_v2.jar.bin");
|
||||
url = "http://localhost:8080/myjar/location.jar";
|
||||
sha256 = "79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4";
|
||||
BlobRepository.BlobContentRef ref = repository.getBlobIncRef( "filefoo",null,url,
|
||||
"bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417");
|
||||
"79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4");
|
||||
assertTrue("filefoo".equals(blobKey));
|
||||
assertTrue(blobFetched);
|
||||
assertNotNull(ref.blob);
|
||||
assertEquals(filecontent, ref.blob.get());
|
||||
verify(mockContainer).isZooKeeperAware();
|
||||
try {
|
||||
repository.getBlobIncRef( "filefoo",null,url,
|
||||
"WRONG-SHA512-KEY");
|
||||
fail("expected exception");
|
||||
} catch (Exception e) {
|
||||
assertTrue(e.getMessage().contains(" expected sha512 hash : WRONG-SHA512-KEY , actual :"));
|
||||
}
|
||||
|
||||
url = null;
|
||||
filecontent = null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase {
|
|||
try {
|
||||
String payload = "{\n" +
|
||||
"'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" +
|
||||
" 'sha512':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
|
||||
" 'sha256':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
|
||||
"}";
|
||||
RestTestHarness client = randomRestTestHarness();
|
||||
TestSolrConfigHandler.runConfigCommandExpectFailure(client, "/config", payload, "Invalid jar");
|
||||
|
|
@ -85,7 +85,7 @@ public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase {
|
|||
|
||||
payload = "{\n" +
|
||||
"'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" +
|
||||
" 'sha512':'d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" +
|
||||
" 'sha256':'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}" +
|
||||
"}";
|
||||
client = randomRestTestHarness();
|
||||
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
|
||||
|
|
@ -93,8 +93,8 @@ public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase {
|
|||
null,
|
||||
"/config/overlay",
|
||||
null,
|
||||
Arrays.asList("overlay", "runtimeLib", "urljar", "sha512"),
|
||||
"d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", 120);
|
||||
Arrays.asList("overlay", "runtimeLib", "urljar", "sha256"),
|
||||
"e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc", 120);
|
||||
|
||||
payload = "{\n" +
|
||||
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}" +
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ import java.io.IOException;
|
|||
import java.io.InputStream;
|
||||
import java.io.Reader;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.URL;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
|
@ -37,12 +38,14 @@ import org.apache.solr.client.solrj.SolrRequest;
|
|||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
|
||||
import org.apache.solr.client.solrj.request.GenericSolrRequest;
|
||||
import org.apache.solr.client.solrj.request.QueryRequest;
|
||||
import org.apache.solr.client.solrj.request.UpdateRequest;
|
||||
import org.apache.solr.client.solrj.request.V2Request;
|
||||
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
|
||||
import org.apache.solr.client.solrj.response.V2Response;
|
||||
import org.apache.solr.cloud.ConfigRequest;
|
||||
import org.apache.solr.cloud.MiniSolrCloudCluster;
|
||||
|
|
@ -50,6 +53,7 @@ import org.apache.solr.cloud.SolrCloudTestCase;
|
|||
import org.apache.solr.common.cloud.ClusterProperties;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.common.params.MapSolrParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.Pair;
|
||||
|
|
@ -150,19 +154,19 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
String payload = null;
|
||||
try {
|
||||
payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sha512 : 'wrong-sha512'}}";
|
||||
"sha256 : 'wrong-sha256'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
fail("Expected error");
|
||||
} catch (BaseHttpSolrClient.RemoteExecutionException e) {
|
||||
assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("expected sha512 hash :"));
|
||||
assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("expected sha256 hash :"));
|
||||
}
|
||||
|
||||
try {
|
||||
payload = "{add-package:{name : 'foo', url: 'http://localhost:" + port + "/jar0.jar', " +
|
||||
"sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
|
||||
"sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
|
|
@ -173,13 +177,13 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
}
|
||||
|
||||
payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
|
||||
"sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
|
|
@ -203,13 +207,13 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
|
||||
payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar3.jar', " +
|
||||
"sha512 : '60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922'}}";
|
||||
"sha256 : '20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
|
||||
request = new V2Request.Builder("/node/ext/bar")
|
||||
|
|
@ -239,6 +243,13 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
(Predicate<Object>) o -> o instanceof List && ((List) o).isEmpty()));
|
||||
|
||||
|
||||
URL baseUrl = cluster.getRandomJetty(random()).getBaseUrl();
|
||||
try(HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()){
|
||||
SimpleSolrResponse rsp = new GenericSolrRequest(SolrRequest.METHOD.GET, "/____v2/node/blob", new ModifiableSolrParams()).process(client);
|
||||
List l = (List) rsp.nl.get("blob");
|
||||
assertTrue(l.contains("e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc"));
|
||||
assertTrue(l.contains("20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3"));
|
||||
}
|
||||
} finally {
|
||||
cluster.shutdown();
|
||||
server.first().stop();
|
||||
|
|
@ -266,7 +277,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
String payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sig : 'EdYkvRpMZbvElN93/xUmyKXcj6xHP16AVk71TlTascEwCb5cFQ2AeKhPIlwYpkLWXEOcLZKfeXoWwOLaV5ZNhg==' ," +
|
||||
"sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
|
||||
"sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}";
|
||||
try {
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
|
|
@ -280,14 +291,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sig : '" + signature + "'," +
|
||||
"sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
|
||||
"sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}";
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload("{add-requesthandler:{name : 'bar', class : 'org.apache.solr.core.RuntimeLibReqHandler' package : global}}")
|
||||
|
|
@ -310,14 +321,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar3.jar', " +
|
||||
"sig : 'YxFr6SpYrDwG85miDfRWHTjU9UltjtIWQZEhcV55C2rczRUVowCYBxmsDv5mAM8j0CTv854xpI1DtBT86wpoTdbF95LQuP9FJId4TS1j8bZ9cxHP5Cqyz1uBHFfUUNUrnpzTHQkVTp02O9NAjh3c2W41bL4U7j6jQ32+4CW2M+x00TDG0y0H75rQDR8zbLt31oWCz+sBOdZ3rGKJgAvdoGm/wVCTmsabZN+xoz4JaDeBXF16O9Uk9SSq4G0dz5YXFuLxHK7ciB5t0+q6pXlF/tdlDqF76Abze0R3d2/0MhXBzyNp3UxJmj6DiprgysfB0TbQtJG0XGfdSmx0VChvcA==' ," +
|
||||
"sha512 : '60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922'}}";
|
||||
"sha256 : '20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3'}}";
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
|
||||
request = new V2Request.Builder("/node/ext/bar")
|
||||
|
|
@ -357,14 +368,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
String payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sig : '" + signature + "'," +
|
||||
"sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
|
||||
"sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}";
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload("{add-requesthandler:{name : 'bar', class : 'org.apache.solr.core.RuntimeLibReqHandler' package : global }}")
|
||||
|
|
@ -387,14 +398,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar3.jar', " +
|
||||
"sig : 'a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==' ," +
|
||||
"sha512 : '60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922'}}";
|
||||
"sha256 : '20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3'}}";
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
|
||||
request = new V2Request.Builder("/node/ext/bar")
|
||||
|
|
@ -468,16 +479,16 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
cluster.waitForActiveCollection(COLLECTION_NAME, 2, 2);
|
||||
String payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}";
|
||||
"sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
String sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512");
|
||||
String sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256");
|
||||
String url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/url");
|
||||
|
||||
assertEquals(sha512,
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(sha256,
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
|
||||
payload = "{\n" +
|
||||
|
|
@ -501,7 +512,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/queryResponseWriter/json1", params),
|
||||
Utils.makeMap(
|
||||
"/config/queryResponseWriter/json1/_packageinfo_/url", url,
|
||||
"/config/queryResponseWriter/json1/_meta_/sha512", sha512
|
||||
"/config/queryResponseWriter/json1/_meta_/sha256", sha256
|
||||
));
|
||||
|
||||
params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
|
||||
|
|
@ -513,7 +524,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/searchComponent/get", params),
|
||||
Utils.makeMap(
|
||||
"config/searchComponent/get/_packageinfo_/url", url,
|
||||
"config/searchComponent/get/_packageinfo_/sha512", sha512
|
||||
"config/searchComponent/get/_packageinfo_/sha256", sha256
|
||||
));
|
||||
|
||||
params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
|
||||
|
|
@ -525,7 +536,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/requestHandler/runtime", params),
|
||||
Utils.makeMap(
|
||||
":config:requestHandler:/runtime:_packageinfo_:url", url,
|
||||
":config:requestHandler:/runtime:_packageinfo_:sha512", sha512
|
||||
":config:requestHandler:/runtime:_packageinfo_:sha256", sha256
|
||||
));
|
||||
|
||||
|
||||
|
|
@ -578,16 +589,16 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
|
||||
payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar2.jar', " +
|
||||
"sha512 : 'bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417'}}";
|
||||
"sha256 : '79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512");
|
||||
sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256");
|
||||
url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/url");
|
||||
|
||||
assertEquals(sha512,
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512"));
|
||||
assertEquals(sha256,
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256"));
|
||||
|
||||
params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
|
||||
WT, JAVABIN,
|
||||
|
|
@ -598,7 +609,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/queryResponseWriter/json1", params),
|
||||
Utils.makeMap(
|
||||
"/config/queryResponseWriter/json1/_packageinfo_/url", url,
|
||||
"/config/queryResponseWriter/json1/_packageinfo_/sha512", sha512
|
||||
"/config/queryResponseWriter/json1/_packageinfo_/sha256", sha256
|
||||
));
|
||||
|
||||
params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
|
||||
|
|
@ -610,7 +621,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/searchComponent/get", params),
|
||||
Utils.makeMap(
|
||||
"/config/searchComponent/get/_packageinfo_/url", url,
|
||||
"/config/searchComponent/get/_packageinfo_/sha512", sha512
|
||||
"/config/searchComponent/get/_packageinfo_/sha256", sha256
|
||||
));
|
||||
|
||||
params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
|
||||
|
|
@ -622,7 +633,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/requestHandler/runtime", params),
|
||||
Utils.makeMap(
|
||||
":config:requestHandler:/runtime:_packageinfo_:url", url,
|
||||
":config:requestHandler:/runtime:_packageinfo_:sha512", sha512
|
||||
":config:requestHandler:/runtime:_packageinfo_:sha256", sha256
|
||||
));
|
||||
|
||||
|
||||
|
|
@ -634,7 +645,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
.build().process(cluster.getSolrClient());
|
||||
fail("should have failed");
|
||||
} catch (BaseHttpSolrClient.RemoteExecutionException e) {
|
||||
assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("Trying to update a jar with the same sha512"));
|
||||
assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("Trying to update a jar with the same sha256"));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -652,6 +663,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
}
|
||||
|
||||
@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13650")
|
||||
public void testCacheLoadFromPackage() throws Exception {
|
||||
String COLLECTION_NAME = "globalCacheColl";
|
||||
Map<String, Object> jars = Utils.makeMap(
|
||||
|
|
@ -672,14 +684,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
.configure();
|
||||
try {
|
||||
String payload = "{add-package:{name : 'cache_pkg', url: 'http://localhost:" + port + "/jar1.jar', " +
|
||||
"sha512 : '1a3739b629ce85895c9b2a8c12dd7d98161ff47634b0693f1e1c5b444fb38343f95c6ee955cd99103bd24cfde6c205234b63823818660ac08392cdc626caf585'}}";
|
||||
"sha256 : '380c2a61759f01b4d5d2570496c3d2737e3cc6968347faa94d93e906e03e077f'}}";
|
||||
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha256"));
|
||||
|
||||
CollectionAdminRequest
|
||||
.createCollection(COLLECTION_NAME, "conf", 2, 1)
|
||||
|
|
@ -693,7 +705,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
NamedList<Object> rsp = cluster.getSolrClient().request(new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/overlay", params));
|
||||
assertEquals("org.apache.solr.core.MyDocCache", rsp._getStr("overlay/props/query/documentCache/class", null));
|
||||
|
||||
String sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512");
|
||||
String sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256");
|
||||
String url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/url");
|
||||
|
||||
|
||||
|
|
@ -706,7 +718,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/query/documentCache", params),
|
||||
Utils.makeMap(
|
||||
"/config/query/documentCache/_packageinfo_/url", url,
|
||||
"/config/query/documentCache/_packageinfo_/sha512", sha512
|
||||
"/config/query/documentCache/_packageinfo_/sha256", sha256
|
||||
));
|
||||
|
||||
|
||||
|
|
@ -725,15 +737,15 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
|
||||
|
||||
payload = "{update-package:{name : 'cache_pkg', url: 'http://localhost:" + port + "/jar2.jar', " +
|
||||
"sha512 : 'aa3f42fb640636dd8126beca36ac389486d0fcb1c3a2e2c387d043d57637535ce8db3b17983853322f78bb8f447ed75fe7b405675debe652ed826ee95e8ce328'}}";
|
||||
"sha256 : '22551e42e6fd9646a641ebc1380472ec66fba62f35febad46c8165376b41161d'}}";
|
||||
new V2Request.Builder("/cluster")
|
||||
.withPayload(payload)
|
||||
.withMethod(SolrRequest.METHOD.POST)
|
||||
.build().process(cluster.getSolrClient());
|
||||
sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512");
|
||||
sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256");
|
||||
url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/url");
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha512"));
|
||||
assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"),
|
||||
getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha256"));
|
||||
|
||||
params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME,
|
||||
WT, JAVABIN,
|
||||
|
|
@ -744,7 +756,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/query/documentCache", params),
|
||||
Utils.makeMap(
|
||||
"/config/query/documentCache/_packageinfo_/url", url,
|
||||
"/config/query/documentCache/_packageinfo_/sha512", sha512
|
||||
"/config/query/documentCache/_packageinfo_/sha256", sha256
|
||||
));
|
||||
req = new UpdateRequest();
|
||||
req.add("id", "2", "desc_s", "document 1")
|
||||
|
|
@ -764,8 +776,6 @@ public class TestContainerReqHandler extends SolrCloudTestCase {
|
|||
cluster.shutdown();
|
||||
server.first().stop();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -128,11 +128,11 @@ Step 1: Download a jar from github to the current directory
|
|||
----
|
||||
curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true
|
||||
----
|
||||
Step 2: Get the `sha512` hash of the jar
|
||||
Step 2: Get the `sha256` hash of the jar
|
||||
|
||||
[source,bash]
|
||||
----
|
||||
openssl dgst -sha512 runtimelibs.jar
|
||||
openssl dgst -sha256 runtimelibs.jar
|
||||
----
|
||||
|
||||
Step 3 : Start solr with runtime lib enabled
|
||||
|
|
@ -156,7 +156,7 @@ Step 5: Add the jar to your collection `gettingstarted`
|
|||
curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
|
||||
"add-package": { "name" : "my-pkg",
|
||||
"url":"http://localhost:8000/runtimelibs.jar" ,
|
||||
"sha512" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"}
|
||||
"sha256" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"}
|
||||
}'
|
||||
----
|
||||
|
||||
|
|
@ -198,7 +198,7 @@ output:
|
|||
Example:
|
||||
|
||||
* Host the new jar to a new url. eg: http://localhost:8000/runtimelibs_v2.jar
|
||||
* get the `sha512` hash of the new jar
|
||||
* get the `sha256` hash of the new jar
|
||||
* run the update-runtime lib command
|
||||
|
||||
[source,bash]
|
||||
|
|
@ -206,7 +206,7 @@ Example:
|
|||
curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{
|
||||
"update-package": { "name" : "my-pkg",
|
||||
"url":"http://localhost:8000/runtimelibs_v2.jar" ,
|
||||
"sha512" : "<replace-the-new-sha512-digest-here>"}
|
||||
"sha256" : "<replace-the-new-sha256-digest-here>"}
|
||||
}'
|
||||
----
|
||||
NOTE: Always upload your jar to a new url as the Solr cluster is still referring to the old jar. If the existing jar is modified it can cause errors as the hash may not match
|
||||
|
|
|
|||
|
|
@ -9,15 +9,15 @@
|
|||
"type": "string",
|
||||
"description": "The remote url"
|
||||
},
|
||||
"sha512": {
|
||||
"sha256": {
|
||||
"type": "string",
|
||||
"description": "The sha512 hash of the jar"
|
||||
"description": "The sha256 hash of the jar"
|
||||
},
|
||||
"sig": {
|
||||
"type": "string",
|
||||
"description": "the signature of the jar"
|
||||
}
|
||||
},
|
||||
"required" : ["name","url","sha512"]
|
||||
"required" : ["name","url","sha256"]
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"methods": [
|
||||
"GET"
|
||||
],
|
||||
"url": {
|
||||
"paths": [
|
||||
"/node/blob",
|
||||
"/node/blob/{sha256}"
|
||||
]
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue