diff --git a/solr/core/src/java/org/apache/solr/core/BlobRepository.java b/solr/core/src/java/org/apache/solr/core/BlobRepository.java index 02199e23045..ea2f6d79126 100644 --- a/solr/core/src/java/org/apache/solr/core/BlobRepository.java +++ b/solr/core/src/java/org/apache/solr/core/BlobRepository.java @@ -16,17 +16,20 @@ */ package org.apache.solr.core; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; -import java.math.BigInteger; import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Random; import java.util.Set; @@ -34,10 +37,14 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.api.Api; +import org.apache.solr.api.V2HttpCall; import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -45,16 +52,26 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionAdminParams; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.security.AuthorizationContext; +import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.util.SimplePostTool; import org.apache.zookeeper.server.ByteBufferInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; import static org.apache.solr.common.SolrException.ErrorCode.SERVICE_UNAVAILABLE; import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP; +import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; /** * The purpose of this class is to store the Jars loaded in memory and to keep only one copy of the Jar in a single node. @@ -88,6 +105,14 @@ public class BlobRepository { this.coreContainer = coreContainer; } + public Collection getFiles() { + return Arrays.asList(getBlobsPath().toFile().list()); + } + + public Path getBlobsPath() { + return SolrResourceLoader.getBlobsDirPath(this.coreContainer.getResourceLoader().getInstancePath()); + } + // I wanted to {@link SolrCore#loadDecodeAndCacheBlob(String, Decoder)} below but precommit complains /** @@ -116,12 +141,12 @@ public class BlobRepository { return getBlobIncRef(key.concat(decoder.getName()), () -> addBlob(key, decoder)); } - BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha512) { + BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha256) { StringBuffer keyBuilder = new StringBuffer(key); if (decoder != null) keyBuilder.append(decoder.getName()); - keyBuilder.append("/").append(sha512); + keyBuilder.append("/").append(sha256); - return getBlobIncRef(keyBuilder.toString(), () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha512), decoder)); + return getBlobIncRef(keyBuilder.toString(), () -> new BlobContent<>(key, fetchBlobAndVerify(key, url, sha256), decoder)); } // do the actual work returning the appropriate type... @@ -166,35 +191,79 @@ public class BlobRepository { return aBlob; } - static String INVALID_JAR_MSG = "Invalid jar from {0} , expected sha512 hash : {1} , actual : {2}"; + static String INVALID_JAR_MSG = "Invalid jar from {0} , expected sha256 hash : {1} , actual : {2}"; - private ByteBuffer fetchBlobAndVerify(String key, String url, String sha512) { + private ByteBuffer fetchBlobAndVerify(String key, String url, String sha256) throws IOException { + ByteBuffer byteBuffer = null; + if (sha256 != null) { + byteBuffer = getFromLocalFs(sha256); + } + if (byteBuffer == null) byteBuffer = getAndValidate(key, url, sha256); + return byteBuffer; + } + + private ByteBuffer getAndValidate(String key, String url, String sha256) throws IOException { ByteBuffer byteBuffer = fetchFromUrl(key, url); - String computedDigest = sha512Digest(byteBuffer); - if (!computedDigest.equals(sha512)) { - throw new SolrException(SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha512, computedDigest)); - + String computedDigest = sha256Digest(byteBuffer); + if (!computedDigest.equals(sha256)) { + throw new SolrException(SERVER_ERROR, StrUtils.formatString(INVALID_JAR_MSG, url, sha256, computedDigest)); + } + File file = new File(getBlobsPath().toFile(), sha256); + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.limit()); + IOUtils.fsync(file.toPath(), false); } return byteBuffer; } - public static String sha512Digest(ByteBuffer byteBuffer) { - MessageDigest digest = null; + public String putBlob(InputStream is) throws SolrException { + byte[] b = new byte[(int) MAX_JAR_SIZE + 1]; + String sha256 = null; try { - digest = MessageDigest.getInstance("SHA-512"); - } catch (NoSuchAlgorithmException e) { - //unlikely - throw new SolrException(SERVER_ERROR, e); - } - byteBuffer = ByteBuffer.wrap(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.limit()); - digest.update(byteBuffer); + int sz = is.read(b); + + if (sz > MAX_JAR_SIZE) + throw new SolrException(BAD_REQUEST, "size is more than permitted , use system property runtime.lib.size to change it"); + sha256 = sha256Digest(ByteBuffer.wrap(b, 0, sz)); + File file = new File(getBlobsPath().toFile(), sha256); + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(b, 0, sz); + } + IOUtils.fsync(file.toPath(), false); + } catch (IOException e) { + throw new SolrException(BAD_REQUEST, e); + } + return sha256; - return String.format( - Locale.ROOT, - "%0128x", - new BigInteger(1, digest.digest())); } + private ByteBuffer getFromLocalFs(String sha256) throws IOException { + Path p = getBlobsPath(); + File f = new File(p.toFile(), sha256); + if (!f.exists()) return null; + byte[] b = new byte[(int) f.length()]; + try (FileInputStream fis = new FileInputStream(f)) { + fis.read(b); + ByteBuffer byteBuffer = ByteBuffer.wrap(b); + if (sha256.equals(sha256Digest(byteBuffer))) { + return byteBuffer; + } else { + return null; + + } + } + } + + public static String sha256Digest(ByteBuffer buf) { + try { + return DigestUtils.sha256Hex(new ByteBufferInputStream(ByteBuffer.wrap( buf.array(), buf.arrayOffset(), buf.limit()))); + } catch (IOException e) { + throw new RuntimeException("Unable to compute sha256", e); + } + } + + + /** * Package local for unit tests only please do not use elsewhere @@ -216,14 +285,14 @@ public class BlobRepository { entity = response.getEntity(); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != 200) { - throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such resource available: " + key + ", url : "+ url); + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "no such resource available: " + key + ", url : " + url); } try (InputStream is = entity.getContent()) { b = SimplePostTool.inputStreamToByteArray(is, MAX_JAR_SIZE); } } catch (Exception e) { - log.error("Error loading resource "+ url, e); + log.error("Error loading resource " + url, e); if (e instanceof SolrException) { throw (SolrException) e; } else { @@ -284,6 +353,68 @@ public class BlobRepository { } } + BlobRead blobRead = new BlobRead(); + + + class BlobRead extends RequestHandlerBase implements PermissionNameProvider { + + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) { + + } + + @Override + public String getDescription() { + return "List fetch blobs"; + } + + @Override + public Name getPermissionName(AuthorizationContext request) { + return null; + } + + @Override + public Collection getApis() { + return Collections.singleton(new Api(Utils.getSpec("node.blob.GET")) { + @Override + public void call(SolrQueryRequest req, SolrQueryResponse rsp) { + String sha256 = ((V2HttpCall) req.getHttpSolrCall()).getUrlParts().get("sha256"); + if (sha256 == null) { + rsp.add("blob", getFiles()); + } else { + try { + ByteBuffer buf = getFromLocalFs(sha256); + if(buf == null){ + throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No such blob"); + } else { + ModifiableSolrParams solrParams = new ModifiableSolrParams(); + solrParams.add(CommonParams.WT, FILE_STREAM); + req.setParams( SolrParams.wrapDefaults(solrParams, req.getParams())); + rsp.add(FILE_STREAM, (SolrCore.RawWriter) os -> os.write(buf.array(), buf.arrayOffset(), buf.limit())); + } + + } catch (IOException e) { + throw new SolrException(SERVER_ERROR,e); + } + } + + } + }); + } + + @Override + public Boolean registerV1() { + return Boolean.FALSE; + } + + @Override + public Boolean registerV2() { + return Boolean.TRUE; + } + } + + public static class BlobContent { public final String key; private final T content; // holds byte buffer or cached object, holding both is a waste of memory @@ -337,7 +468,7 @@ public class BlobRepository { public static class BlobContentRef { public final BlobContent blob; - private BlobContentRef(BlobContent blob) { + public BlobContentRef(BlobContent blob) { this.blob = blob; } } diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 2c2f4080115..b229ca5f1f1 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -642,6 +642,7 @@ public class CoreContainer { this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins()); containerHandlers.put("/ext", clusterPropertiesListener.extHandler); + containerHandlers.put("/blob-get", blobRepository.blobRead); createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class); createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class); collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class); @@ -1542,7 +1543,7 @@ public class CoreContainer { } catch (SolrCoreState.CoreIsClosedException e) { throw e; } catch (Exception e) { - coreInitFailures.put(cd.getName(), new CoreLoadFailure(cd, (Exception) e)); + coreInitFailures.put(cd.getName(), new CoreLoadFailure(cd, e)); throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to reload core [" + cd.getName() + "]", e); } finally { if (!success && newCore != null && newCore.getOpenCount() > 0) { diff --git a/solr/core/src/java/org/apache/solr/core/PackageManager.java b/solr/core/src/java/org/apache/solr/core/PackageManager.java index ecc6950d257..7eb00a53735 100644 --- a/solr/core/src/java/org/apache/solr/core/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/core/PackageManager.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.common.params.CommonParams.PACKAGE; import static org.apache.solr.common.params.CommonParams.VERSION; -import static org.apache.solr.core.RuntimeLib.SHA512; +import static org.apache.solr.core.RuntimeLib.SHA256; public class PackageManager implements ClusterPropertiesListener { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -94,8 +94,8 @@ public class PackageManager implements ClusterPropertiesListener { public boolean isModified(Map map) { - return (!Objects.equals(lib.getSha512(), (map).get(SHA512)) || - !Objects.equals(lib.getSig(), (map).get(SHA512))); + return (!Objects.equals(lib.getSha256(), (map).get(SHA256)) || + !Objects.equals(lib.getSig(), (map).get(SHA256))); } } @@ -167,7 +167,7 @@ public class PackageManager implements ClusterPropertiesListener { needsReload[0] = true; } if (needsReload[0]) { - createNewClassLoader(m, ver); + createNewClassLoaders(m, ver); } return needsReload[0]; } @@ -177,7 +177,7 @@ public class PackageManager implements ClusterPropertiesListener { return p == null ? coreContainer.getResourceLoader() : p.loader; } - void createNewClassLoader(Map m, int ver) { + void createNewClassLoaders(Map m, int ver) { boolean[] loadedAll = new boolean[1]; loadedAll[0] = true; Map newPkgs = new LinkedHashMap<>(); diff --git a/solr/core/src/java/org/apache/solr/core/RuntimeLib.java b/solr/core/src/java/org/apache/solr/core/RuntimeLib.java index 507fe02fdaa..1e1f5f708f8 100644 --- a/solr/core/src/java/org/apache/solr/core/RuntimeLib.java +++ b/solr/core/src/java/org/apache/solr/core/RuntimeLib.java @@ -44,10 +44,10 @@ import static org.apache.solr.common.params.CommonParams.NAME; */ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWriter { public static final String TYPE = "runtimeLib"; - public static final String SHA512 = "sha512"; + public static final String SHA256 = "sha256"; private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final CoreContainer coreContainer; - private String name, version, sig, sha512, url; + private String name, version, sig, sha256, url; private BlobRepository.BlobContentRef jarContent; private boolean verified = false; int znodeVersion = -1; @@ -57,7 +57,7 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit ew.putIfNotNull(NAME, name); ew.putIfNotNull("url", url); ew.putIfNotNull(version, version); - ew.putIfNotNull(sha512, sha512); + ew.putIfNotNull("sha256", sha256); ew.putIfNotNull("sig", sig); if (znodeVersion > -1) { ew.put(ConfigOverlay.ZNODEVER, znodeVersion); @@ -101,19 +101,19 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit } version = String.valueOf(v); } else { - sha512 = info.attributes.get(SHA512); - if (sha512 == null) { - throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha512' attribute"); + sha256 = info.attributes.get(SHA256); + if (sha256 == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "runtimeLib with url must have a 'sha256' attribute"); } ByteBuffer buf = coreContainer.getBlobRepository().fetchFromUrl(name, url); - String digest = BlobRepository.sha512Digest(buf); - if (!sha512.equals(digest)) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha512, digest)); + String digest = BlobRepository.sha256Digest(buf); + if (!sha256.equals(digest)) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, StrUtils.formatString(BlobRepository.INVALID_JAR_MSG, url, sha256, digest)); } verifyJarSignature(buf); - log.debug("dynamic library verified {}, sha512: {}", url, sha512); + log.debug("dynamic library verified {}, sha256: {}", url, sha256); } @@ -130,7 +130,7 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit jarContent = url == null ? coreContainer.getBlobRepository().getBlobIncRef(name + "/" + version) : - coreContainer.getBlobRepository().getBlobIncRef(name, null, url, sha512); + coreContainer.getBlobRepository().getBlobIncRef(name, null, url, sha256); } } @@ -148,8 +148,8 @@ public class RuntimeLib implements PluginInfoInitialized, AutoCloseable, MapWrit } - public String getSha512() { - return sha512; + public String getSha256() { + return sha256; } public ByteBuffer getFileContent(String entryName) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java index f27edbc5149..b3dc5e450d9 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java +++ b/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java @@ -576,8 +576,8 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } } } - - static final String empty[] = new String[0]; + + static final String[] empty = new String[0]; @Override public T newInstance(String name, Class expectedType) { @@ -808,6 +808,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable * manipulated using select Solr features (e.g. streaming expressions). */ public static final String USER_FILES_DIRECTORY = "userfiles"; + public static final String BLOBS_DIRECTORY = "blobs"; public static void ensureUserFilesDataDir(Path solrHome) { final Path userFilesPath = getUserFilesPath(solrHome); final File userFilesDirectory = new File(userFilesPath.toString()); @@ -823,10 +824,28 @@ public class SolrResourceLoader implements ResourceLoader,Closeable } } + public static void ensureBlobsDir(Path solrHome) { + final Path blobsDir = getBlobsDirPath(solrHome); + final File blobsFilesDirectory = new File(blobsDir.toString()); + if (! blobsFilesDirectory.exists()) { + try { + final boolean created = blobsFilesDirectory.mkdir(); + if (! created) { + log.warn("Unable to create [{}] directory in SOLR_HOME [{}]. Features requiring this directory may fail.", BLOBS_DIRECTORY, solrHome); + } + } catch (Exception e) { + log.warn("Unable to create [" + BLOBS_DIRECTORY + "] directory in SOLR_HOME [" + solrHome + "]. Features requiring this directory may fail.", e); + } + } + } + + public static Path getBlobsDirPath(Path solrHome) { + return Paths.get(solrHome.toAbsolutePath().toString(), BLOBS_DIRECTORY).toAbsolutePath(); + } + public static Path getUserFilesPath(Path solrHome) { return Paths.get(solrHome.toAbsolutePath().toString(), USER_FILES_DIRECTORY).toAbsolutePath(); } - // Logs a message only once per startup private static void logOnceInfo(String key, String msg) { if (!loggedOnce.contains(key)) { @@ -923,7 +942,7 @@ public class SolrResourceLoader implements ResourceLoader,Closeable throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg); } } - try (OutputStream out = new FileOutputStream(confFile);) { + try (OutputStream out = new FileOutputStream(confFile)) { out.write(content); } log.info("Written confile " + resourceName); diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java index 1a1e2b6fa3a..2259a0e4867 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionHandlerApi.java @@ -17,7 +17,9 @@ package org.apache.solr.handler.admin; +import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -29,6 +31,7 @@ import java.util.Objects; import org.apache.solr.api.ApiBag; import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionApiMapping; import org.apache.solr.client.solrj.request.CollectionApiMapping.CommandMeta; import org.apache.solr.client.solrj.request.CollectionApiMapping.Meta; @@ -60,7 +63,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.apache.solr.common.util.CommandOperation.captureErrors; import static org.apache.solr.common.util.StrUtils.formatString; -import static org.apache.solr.core.RuntimeLib.SHA512; +import static org.apache.solr.core.RuntimeLib.SHA256; public class CollectionHandlerApi extends BaseHandlerApiSupport { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -218,8 +221,8 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport { op.addError(StrUtils.formatString("The jar with a name ''{0}'' does not exist", name)); return false; } - if (Objects.equals(existing.get(SHA512), op.getDataMap().get(SHA512))) { - op.addError("Trying to update a jar with the same sha512"); + if (Objects.equals(existing.get(SHA256), op.getDataMap().get(SHA256))) { + op.addError("Trying to update a jar with the same sha256"); return false; } } @@ -365,4 +368,10 @@ public class CollectionHandlerApi extends BaseHandlerApiSupport { } } + public static void postBlob(String baseUrl, ByteBuffer buf) throws IOException { + try(HttpSolrClient client = new HttpSolrClient.Builder(baseUrl+"/____v2/node/blob" ).build()){ + + } + } + } diff --git a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java index f1d216a189d..16b39a41435 100644 --- a/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java +++ b/solr/core/src/java/org/apache/solr/security/PermissionNameProvider.java @@ -52,6 +52,8 @@ public interface PermissionNameProvider { AUTOSCALING_WRITE_PERM("autoscaling-write", null), AUTOSCALING_HISTORY_READ_PERM("autoscaling-history-read", null), METRICS_HISTORY_READ_PERM("metrics-history-read", null), + BLOB_READ("blob-read", null), + BLOB_WRITE("blob-write", null), ALL("all", unmodifiableSet(new HashSet<>(asList("*", null)))) ; final String name; diff --git a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java index 90d6b17539f..a385479534b 100644 --- a/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java +++ b/solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java @@ -82,9 +82,9 @@ import org.apache.solr.security.AuthenticationPlugin; import org.apache.solr.security.PKIAuthenticationPlugin; import org.apache.solr.security.PublicKeyHandler; import org.apache.solr.util.SolrFileCleaningTracker; -import org.apache.solr.util.tracing.GlobalTracer; import org.apache.solr.util.StartupLoggingUtils; import org.apache.solr.util.configuration.SSLConfigurationsFactory; +import org.apache.solr.util.tracing.GlobalTracer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -180,6 +180,7 @@ public class SolrDispatchFilter extends BaseSolrFilter { final Path solrHomePath = solrHome == null ? SolrResourceLoader.locateSolrHome() : Paths.get(solrHome); coresInit = createCoreContainer(solrHomePath, extraProperties); SolrResourceLoader.ensureUserFilesDataDir(solrHomePath); + SolrResourceLoader.ensureBlobsDir(solrHomePath); this.httpClient = coresInit.getUpdateShardHandler().getDefaultHttpClient(); setupJvmMetrics(coresInit); log.debug("user.dir=" + System.getProperty("user.dir")); @@ -626,8 +627,8 @@ public class SolrDispatchFilter extends BaseSolrFilter { public void close() { // even though we skip closes, we let local tests know not to close so that a full understanding can take // place - assert Thread.currentThread().getStackTrace()[2].getClassName().matches( - "org\\.apache\\.(?:solr|lucene).*") ? false : true : CLOSE_STREAM_MSG; + assert !Thread.currentThread().getStackTrace()[2].getClassName().matches( + "org\\.apache\\.(?:solr|lucene).*") : CLOSE_STREAM_MSG; this.stream = ClosedServletInputStream.CLOSED_SERVLET_INPUT_STREAM; } }; @@ -661,9 +662,8 @@ public class SolrDispatchFilter extends BaseSolrFilter { public void close() { // even though we skip closes, we let local tests know not to close so that a full understanding can take // place - assert Thread.currentThread().getStackTrace()[2].getClassName().matches( - "org\\.apache\\.(?:solr|lucene).*") ? false - : true : CLOSE_STREAM_MSG; + assert !Thread.currentThread().getStackTrace()[2].getClassName().matches( + "org\\.apache\\.(?:solr|lucene).*") : CLOSE_STREAM_MSG; stream = ClosedServletOutputStream.CLOSED_SERVLET_OUTPUT_STREAM; } }; diff --git a/solr/core/src/test-files/runtimecode/sig.txt b/solr/core/src/test-files/runtimecode/sig.txt index 8e1ba98e447..44ead80a96b 100644 --- a/solr/core/src/test-files/runtimecode/sig.txt +++ b/solr/core/src/test-files/runtimecode/sig.txt @@ -71,3 +71,27 @@ openssl dgst -sha512 cache.jar.bin openssl dgst -sha512 cache_v2.jar.bin aa3f42fb640636dd8126beca36ac389486d0fcb1c3a2e2c387d043d57637535ce8db3b17983853322f78bb8f447ed75fe7b405675debe652ed826ee95e8ce328 + +=============sha256============================ + +openssl dgst -sha256 runtimelibs.jar.bin + +e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc + +openssl dgst -sha512 runtimelibs_v2.jar.bin + +79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4 + +openssl dgst -sha256 runtimelibs_v3.jar.bin + +20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3 + +openssl dgst -sha256 cache.jar.bin + +380c2a61759f01b4d5d2570496c3d2737e3cc6968347faa94d93e906e03e077f + +openssl dgst -sha256 cache_v2.jar.bin + +22551e42e6fd9646a641ebc1380472ec66fba62f35febad46c8165376b41161d + + diff --git a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java index 4a0f1ba7eb6..ce1f68ef89a 100644 --- a/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java +++ b/solr/core/src/test/org/apache/solr/core/BlobRepositoryMockingTest.java @@ -36,7 +36,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; import static org.mockito.Mockito.any; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; @@ -63,6 +62,7 @@ public class BlobRepositoryMockingTest { boolean blobFetched = false; String blobKey = ""; String url = null; + String sha256 = null; ByteBuffer filecontent = null; @BeforeClass @@ -91,6 +91,14 @@ public class BlobRepositoryMockingTest { return filecontent; } + @Override + BlobContentRef getBlobIncRef(String key, Decoder decoder, String url, String sha256) { + if(!Objects.equals(sha256, BlobRepositoryMockingTest.this.sha256)) return null; + blobKey = key; + blobFetched = true; + return new BlobContentRef(new BlobContent(key, filecontent)) ; + } + @Override ConcurrentHashMap createMap() { return mapMock; @@ -130,21 +138,13 @@ public class BlobRepositoryMockingTest { when(mockContainer.isZooKeeperAware()).thenReturn(true); filecontent = TestDynamicLoading.getFileContent("runtimecode/runtimelibs_v2.jar.bin"); url = "http://localhost:8080/myjar/location.jar"; + sha256 = "79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4"; BlobRepository.BlobContentRef ref = repository.getBlobIncRef( "filefoo",null,url, - "bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417"); + "79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4"); assertTrue("filefoo".equals(blobKey)); assertTrue(blobFetched); assertNotNull(ref.blob); assertEquals(filecontent, ref.blob.get()); - verify(mockContainer).isZooKeeperAware(); - try { - repository.getBlobIncRef( "filefoo",null,url, - "WRONG-SHA512-KEY"); - fail("expected exception"); - } catch (Exception e) { - assertTrue(e.getMessage().contains(" expected sha512 hash : WRONG-SHA512-KEY , actual :")); - } - url = null; filecontent = null; } diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java index 575cf9e2da8..8fec3a433a7 100644 --- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java +++ b/solr/core/src/test/org/apache/solr/core/TestDynamicLoadingUrl.java @@ -77,7 +77,7 @@ public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase { try { String payload = "{\n" + "'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" + - " 'sha512':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" + + " 'sha256':'e01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" + "}"; RestTestHarness client = randomRestTestHarness(); TestSolrConfigHandler.runConfigCommandExpectFailure(client, "/config", payload, "Invalid jar"); @@ -85,7 +85,7 @@ public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase { payload = "{\n" + "'add-runtimelib' : { 'name' : 'urljar', url : 'http://localhost:" + port + "/jar1.jar'" + - " 'sha512':'d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}" + + " 'sha256':'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}" + "}"; client = randomRestTestHarness(); TestSolrConfigHandler.runConfigCommand(client, "/config", payload); @@ -93,8 +93,8 @@ public class TestDynamicLoadingUrl extends AbstractFullDistribZkTestBase { null, "/config/overlay", null, - Arrays.asList("overlay", "runtimeLib", "urljar", "sha512"), - "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", 120); + Arrays.asList("overlay", "runtimeLib", "urljar", "sha256"), + "e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc", 120); payload = "{\n" + "'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler', 'runtimeLib' : true}" + diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java b/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java index a393086821b..a09f215bca1 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerReqHandler.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; import java.lang.invoke.MethodHandles; +import java.net.URL; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -37,12 +38,14 @@ import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.BaseHttpSolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; +import org.apache.solr.client.solrj.response.SimpleSolrResponse; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.ConfigRequest; import org.apache.solr.cloud.MiniSolrCloudCluster; @@ -50,6 +53,7 @@ import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.ClusterProperties; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Pair; @@ -150,19 +154,19 @@ public class TestContainerReqHandler extends SolrCloudTestCase { String payload = null; try { payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " + - "sha512 : 'wrong-sha512'}}"; + "sha256 : 'wrong-sha256'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); fail("Expected error"); } catch (BaseHttpSolrClient.RemoteExecutionException e) { - assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("expected sha512 hash :")); + assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("expected sha256 hash :")); } try { payload = "{add-package:{name : 'foo', url: 'http://localhost:" + port + "/jar0.jar', " + - "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}"; + "sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) @@ -173,13 +177,13 @@ public class TestContainerReqHandler extends SolrCloudTestCase { } payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " + - "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}"; + "sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); new V2Request.Builder("/cluster") @@ -203,13 +207,13 @@ public class TestContainerReqHandler extends SolrCloudTestCase { payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar3.jar', " + - "sha512 : '60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922'}}"; + "sha256 : '20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); request = new V2Request.Builder("/node/ext/bar") @@ -239,6 +243,13 @@ public class TestContainerReqHandler extends SolrCloudTestCase { (Predicate) o -> o instanceof List && ((List) o).isEmpty())); + URL baseUrl = cluster.getRandomJetty(random()).getBaseUrl(); + try(HttpSolrClient client = new HttpSolrClient.Builder(baseUrl.toString()).build()){ + SimpleSolrResponse rsp = new GenericSolrRequest(SolrRequest.METHOD.GET, "/____v2/node/blob", new ModifiableSolrParams()).process(client); + List l = (List) rsp.nl.get("blob"); + assertTrue(l.contains("e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc")); + assertTrue(l.contains("20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3")); + } } finally { cluster.shutdown(); server.first().stop(); @@ -266,7 +277,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { String payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " + "sig : 'EdYkvRpMZbvElN93/xUmyKXcj6xHP16AVk71TlTascEwCb5cFQ2AeKhPIlwYpkLWXEOcLZKfeXoWwOLaV5ZNhg==' ," + - "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}"; + "sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}"; try { new V2Request.Builder("/cluster") .withPayload(payload) @@ -280,14 +291,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase { payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " + "sig : '" + signature + "'," + - "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}"; + "sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); new V2Request.Builder("/cluster") .withPayload("{add-requesthandler:{name : 'bar', class : 'org.apache.solr.core.RuntimeLibReqHandler' package : global}}") @@ -310,14 +321,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase { payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar3.jar', " + "sig : 'YxFr6SpYrDwG85miDfRWHTjU9UltjtIWQZEhcV55C2rczRUVowCYBxmsDv5mAM8j0CTv854xpI1DtBT86wpoTdbF95LQuP9FJId4TS1j8bZ9cxHP5Cqyz1uBHFfUUNUrnpzTHQkVTp02O9NAjh3c2W41bL4U7j6jQ32+4CW2M+x00TDG0y0H75rQDR8zbLt31oWCz+sBOdZ3rGKJgAvdoGm/wVCTmsabZN+xoz4JaDeBXF16O9Uk9SSq4G0dz5YXFuLxHK7ciB5t0+q6pXlF/tdlDqF76Abze0R3d2/0MhXBzyNp3UxJmj6DiprgysfB0TbQtJG0XGfdSmx0VChvcA==' ," + - "sha512 : '60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922'}}"; + "sha256 : '20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); request = new V2Request.Builder("/node/ext/bar") @@ -357,14 +368,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase { String payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " + "sig : '" + signature + "'," + - "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}"; + "sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); new V2Request.Builder("/cluster") .withPayload("{add-requesthandler:{name : 'bar', class : 'org.apache.solr.core.RuntimeLibReqHandler' package : global }}") @@ -387,14 +398,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase { payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar3.jar', " + "sig : 'a400n4T7FT+2gM0SC6+MfSOExjud8MkhTSFylhvwNjtWwUgKdPFn434Wv7Qc4QEqDVLhQoL3WqYtQmLPti0G4Q==' ," + - "sha512 : '60ec88c2a2e9b409f7afc309273383810a0d07a078b482434eda9674f7e25b8adafa8a67c9913c996cbfb78a7f6ad2b9db26dbd4fe0ca4068f248d5db563f922'}}"; + "sha256 : '20e0bfaec71b2e93c4da9f2ed3745dda04dc3fc915b66cc0275863982e73b2a3'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); request = new V2Request.Builder("/node/ext/bar") @@ -468,16 +479,16 @@ public class TestContainerReqHandler extends SolrCloudTestCase { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 2); String payload = "{add-package:{name : 'global', url: 'http://localhost:" + port + "/jar1.jar', " + - "sha512 : 'd01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420'}}"; + "sha256 : 'e1f9e23988c19619402f1040c9251556dcd6e02b9d3e3b966a129ea1be5c70fc'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - String sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"); + String sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"); String url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/url"); - assertEquals(sha512, - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(sha256, + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); payload = "{\n" + @@ -501,7 +512,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/queryResponseWriter/json1", params), Utils.makeMap( "/config/queryResponseWriter/json1/_packageinfo_/url", url, - "/config/queryResponseWriter/json1/_meta_/sha512", sha512 + "/config/queryResponseWriter/json1/_meta_/sha256", sha256 )); params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, @@ -513,7 +524,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/searchComponent/get", params), Utils.makeMap( "config/searchComponent/get/_packageinfo_/url", url, - "config/searchComponent/get/_packageinfo_/sha512", sha512 + "config/searchComponent/get/_packageinfo_/sha256", sha256 )); params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, @@ -525,7 +536,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/requestHandler/runtime", params), Utils.makeMap( ":config:requestHandler:/runtime:_packageinfo_:url", url, - ":config:requestHandler:/runtime:_packageinfo_:sha512", sha512 + ":config:requestHandler:/runtime:_packageinfo_:sha256", sha256 )); @@ -578,16 +589,16 @@ public class TestContainerReqHandler extends SolrCloudTestCase { payload = "{update-package:{name : 'global', url: 'http://localhost:" + port + "/jar2.jar', " + - "sha512 : 'bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417'}}"; + "sha256 : '79298d7d5c3e60d91154efe7d72f4536eac46698edfa22ab894b85492d562ed4'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"); + sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"); url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/url"); - assertEquals(sha512, - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha512")); + assertEquals(sha256, + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/global/sha256")); params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, WT, JAVABIN, @@ -598,7 +609,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/queryResponseWriter/json1", params), Utils.makeMap( "/config/queryResponseWriter/json1/_packageinfo_/url", url, - "/config/queryResponseWriter/json1/_packageinfo_/sha512", sha512 + "/config/queryResponseWriter/json1/_packageinfo_/sha256", sha256 )); params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, @@ -610,7 +621,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/searchComponent/get", params), Utils.makeMap( "/config/searchComponent/get/_packageinfo_/url", url, - "/config/searchComponent/get/_packageinfo_/sha512", sha512 + "/config/searchComponent/get/_packageinfo_/sha256", sha256 )); params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, @@ -622,7 +633,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/requestHandler/runtime", params), Utils.makeMap( ":config:requestHandler:/runtime:_packageinfo_:url", url, - ":config:requestHandler:/runtime:_packageinfo_:sha512", sha512 + ":config:requestHandler:/runtime:_packageinfo_:sha256", sha256 )); @@ -634,7 +645,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { .build().process(cluster.getSolrClient()); fail("should have failed"); } catch (BaseHttpSolrClient.RemoteExecutionException e) { - assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("Trying to update a jar with the same sha512")); + assertTrue("actual output : " + Utils.toJSONString(e.getMetaData()), e.getMetaData()._getStr("error/details[0]/errorMessages[0]", "").contains("Trying to update a jar with the same sha256")); } @@ -652,6 +663,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { } + @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13650") public void testCacheLoadFromPackage() throws Exception { String COLLECTION_NAME = "globalCacheColl"; Map jars = Utils.makeMap( @@ -672,14 +684,14 @@ public class TestContainerReqHandler extends SolrCloudTestCase { .configure(); try { String payload = "{add-package:{name : 'cache_pkg', url: 'http://localhost:" + port + "/jar1.jar', " + - "sha512 : '1a3739b629ce85895c9b2a8c12dd7d98161ff47634b0693f1e1c5b444fb38343f95c6ee955cd99103bd24cfde6c205234b63823818660ac08392cdc626caf585'}}"; + "sha256 : '380c2a61759f01b4d5d2570496c3d2737e3cc6968347faa94d93e906e03e077f'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha256")); CollectionAdminRequest .createCollection(COLLECTION_NAME, "conf", 2, 1) @@ -693,7 +705,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { NamedList rsp = cluster.getSolrClient().request(new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/overlay", params)); assertEquals("org.apache.solr.core.MyDocCache", rsp._getStr("overlay/props/query/documentCache/class", null)); - String sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha512"); + String sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/sha256"); String url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "add-package/url"); @@ -706,7 +718,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/query/documentCache", params), Utils.makeMap( "/config/query/documentCache/_packageinfo_/url", url, - "/config/query/documentCache/_packageinfo_/sha512", sha512 + "/config/query/documentCache/_packageinfo_/sha256", sha256 )); @@ -725,15 +737,15 @@ public class TestContainerReqHandler extends SolrCloudTestCase { payload = "{update-package:{name : 'cache_pkg', url: 'http://localhost:" + port + "/jar2.jar', " + - "sha512 : 'aa3f42fb640636dd8126beca36ac389486d0fcb1c3a2e2c387d043d57637535ce8db3b17983853322f78bb8f447ed75fe7b405675debe652ed826ee95e8ce328'}}"; + "sha256 : '22551e42e6fd9646a641ebc1380472ec66fba62f35febad46c8165376b41161d'}}"; new V2Request.Builder("/cluster") .withPayload(payload) .withMethod(SolrRequest.METHOD.POST) .build().process(cluster.getSolrClient()); - sha512 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"); + sha256 = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"); url = (String) getObjectByPath(Utils.fromJSONString(payload), true, "update-package/url"); - assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha512"), - getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha512")); + assertEquals(getObjectByPath(Utils.fromJSONString(payload), true, "update-package/sha256"), + getObjectByPath(new ClusterProperties(cluster.getZkClient()).getClusterProperties(), true, "package/cache_pkg/sha256")); params = new MapSolrParams((Map) Utils.makeMap("collection", COLLECTION_NAME, WT, JAVABIN, @@ -744,7 +756,7 @@ public class TestContainerReqHandler extends SolrCloudTestCase { new GenericSolrRequest(SolrRequest.METHOD.GET, "/config/query/documentCache", params), Utils.makeMap( "/config/query/documentCache/_packageinfo_/url", url, - "/config/query/documentCache/_packageinfo_/sha512", sha512 + "/config/query/documentCache/_packageinfo_/sha256", sha256 )); req = new UpdateRequest(); req.add("id", "2", "desc_s", "document 1") @@ -764,8 +776,6 @@ public class TestContainerReqHandler extends SolrCloudTestCase { cluster.shutdown(); server.first().stop(); } - - } } diff --git a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc index 88ec077e75b..5738c6f26d4 100644 --- a/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc +++ b/solr/solr-ref-guide/src/adding-custom-plugins-in-solrcloud-mode.adoc @@ -128,11 +128,11 @@ Step 1: Download a jar from github to the current directory ---- curl -o runtimelibs.jar -LO https://github.com/apache/lucene-solr/blob/master/solr/core/src/test-files/runtimecode/runtimelibs.jar.bin?raw=true ---- -Step 2: Get the `sha512` hash of the jar +Step 2: Get the `sha256` hash of the jar [source,bash] ---- - openssl dgst -sha512 runtimelibs.jar + openssl dgst -sha256 runtimelibs.jar ---- Step 3 : Start solr with runtime lib enabled @@ -156,7 +156,7 @@ Step 5: Add the jar to your collection `gettingstarted` curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{ "add-package": { "name" : "my-pkg", "url":"http://localhost:8000/runtimelibs.jar" , - "sha512" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"} + "sha256" : "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420"} }' ---- @@ -198,7 +198,7 @@ output: Example: * Host the new jar to a new url. eg: http://localhost:8000/runtimelibs_v2.jar -* get the `sha512` hash of the new jar +* get the `sha256` hash of the new jar * run the update-runtime lib command [source,bash] @@ -206,7 +206,7 @@ Example: curl http://localhost:8983/solr/gettingstarted/config -H 'Content-type:application/json' -d '{ "update-package": { "name" : "my-pkg", "url":"http://localhost:8000/runtimelibs_v2.jar" , - "sha512" : ""} + "sha256" : ""} }' ---- NOTE: Always upload your jar to a new url as the Solr cluster is still referring to the old jar. If the existing jar is modified it can cause errors as the hash may not match diff --git a/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json b/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json index 5a8a66f595c..ab334b56c54 100644 --- a/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json +++ b/solr/solrj/src/resources/apispec/cluster.Commands.runtimelib.properties.json @@ -9,15 +9,15 @@ "type": "string", "description": "The remote url" }, - "sha512": { + "sha256": { "type": "string", - "description": "The sha512 hash of the jar" + "description": "The sha256 hash of the jar" }, "sig": { "type": "string", "description": "the signature of the jar" } }, - "required" : ["name","url","sha512"] + "required" : ["name","url","sha256"] } \ No newline at end of file diff --git a/solr/solrj/src/resources/apispec/node.blob.GET.json b/solr/solrj/src/resources/apispec/node.blob.GET.json new file mode 100644 index 00000000000..273333e8414 --- /dev/null +++ b/solr/solrj/src/resources/apispec/node.blob.GET.json @@ -0,0 +1,11 @@ +{ + "methods": [ + "GET" + ], + "url": { + "paths": [ + "/node/blob", + "/node/blob/{sha256}" + ] + } +}