From 6fb085943c6e9c6f82db67c6ccfe641e64e1899e Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Fri, 10 Jan 2020 10:25:44 +0530 Subject: [PATCH] SOLR-14158: Package manager to read keys from package store, not ZK --- solr/CHANGES.txt | 11 ++ .../solrj/embedded/JettySolrRunner.java | 9 + .../org/apache/solr/core/CoreContainer.java | 6 +- .../solr/filestore/DistribPackageStore.java | 175 ++++++++++++++---- .../apache/solr/filestore/PackageStore.java | 14 ++ .../solr/filestore/PackageStoreAPI.java | 100 +++++++--- .../solr/packagemanager/PackageUtils.java | 18 +- .../packagemanager/RepositoryManager.java | 27 ++- .../org/apache/solr/pkg/PackageLoader.java | 22 ++- .../org/apache/solr/util/PackageTool.java | 8 +- .../filestore/TestDistribPackageStore.java | 18 +- .../org/apache/solr/pkg/TestPackages.java | 21 ++- .../src/package-manager-internals.adoc | 9 +- solr/solr-ref-guide/src/package-manager.adoc | 8 +- 14 files changed, 342 insertions(+), 104 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index ce20f8527fa..eeaefd469dd 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -238,12 +238,23 @@ Velocity 2.0 and Velocity Tools 3.0 Apache ZooKeeper 3.5.5 Jetty 9.4.19.v20190610 +Upgrade Notes +--------------------- + +* SOLR-14158: Package manager now stores keys in package store instead of ZK. If you're using package manager and have already + added any repositories, then you would need to re-add all the keys from /keys/exe location, one-by-one, using: + "bin/solr add-key " Bug Fixes ---------------------- * SOLR-14165: SolrResponse serialVersionUID has changed in a backward incompatible way (Andy Webb via noble) +Improvements +--------------------- + +* SOLR-14158: Package manager to store public keys in a special "trusted" location instead of in ZooKeeper + (noble, Ishan Chattopadhyaya) ================== 8.4.0 ================== diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java index 503d0108e48..6448a7a9a3b 100644 --- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java +++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java @@ -756,6 +756,15 @@ public class JettySolrRunner { throw new RuntimeException(e); } } + + public URL getBaseURLV2(){ + try { + return new URL(protocol, host, jettyPort, "/api"); + } catch (MalformedURLException e) { + throw new RuntimeException(e); + } + + } /** * Returns a base URL consisting of the protocol, host, and port for a * Connector in use by the Jetty Server contained in this runner. diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index f494f50b127..b757edc3840 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -652,6 +652,9 @@ public class CoreContainer { // use deprecated API for back-compat, remove in 9.0 pkiAuthenticationPlugin.initializeMetrics(solrMetricsContext, "/authentication/pki"); TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader()); + packageLoader = new PackageLoader(this); + containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP); + containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP); } MDCLoggingContext.setNode(this); @@ -744,9 +747,6 @@ public class CoreContainer { if (isZooKeeperAware()) { metricManager.loadClusterReporters(metricReporters, this); - packageLoader = new PackageLoader(this); - containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP); - containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP); } diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java index 8d9af8f7a07..3389bf46ede 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java @@ -30,6 +30,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -45,41 +46,51 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.PackageStoreAPI.MetaData; +import org.apache.solr.util.SimplePostTool; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.server.ByteBufferInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST; import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR; public class DistribPackageStore implements PackageStore { static final long MAX_PKG_SIZE = Long.parseLong(System.getProperty("max.file.store.size", String.valueOf(100 * 1024 * 1024))); + /** + * This is where al the files in the package store are listed + */ + static final String ZK_PACKAGESTORE = "/packagestore"; private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final CoreContainer coreContainer; private Map tmpFiles = new ConcurrentHashMap<>(); + private final Path solrhome; + public DistribPackageStore(CoreContainer coreContainer) { this.coreContainer = coreContainer; + solrhome = this.coreContainer.getResourceLoader().getInstancePath(); ensurePackageStoreDir(coreContainer.getResourceLoader().getInstancePath()); } - private String myNode() { - return coreContainer.getZkController().getNodeName(); - } - - @Override public Path getRealpath(String path) { + return _getRealPath(path, solrhome); + } + + private static Path _getRealPath(String path, Path solrHome) { if (File.separatorChar == '\\') { path = path.replace('/', File.separatorChar); } if (!path.isEmpty() && path.charAt(0) != File.separatorChar) { path = File.separator + path; } - return new File(this.coreContainer.getResourceLoader().getInstancePath() + + return new File(solrHome + File.separator + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath(); } @@ -93,10 +104,18 @@ public class DistribPackageStore implements PackageStore { this.path = path; } + ByteBuffer getFileData(boolean validate) throws IOException { + if (fileData == null) { + try (FileInputStream fis = new FileInputStream(getRealpath(path).toFile())) { + fileData = SimplePostTool.inputStreamToByteArray(fis); + } + } + return fileData; + } + public String getMetaPath() { if (metaPath == null) { - int idx = path.lastIndexOf('/'); - metaPath = path.substring(0, idx + 1) + "." + path.substring(idx + 1) + ".json"; + metaPath = _getMetapath(path); } return metaPath; } @@ -106,30 +125,9 @@ public class DistribPackageStore implements PackageStore { synchronized (DistribPackageStore.this) { this.metaData = meta; this.fileData = data; - Path realpath = getRealpath(path); - File file = realpath.toFile(); - File parent = file.getParentFile(); - if (!parent.exists()) { - parent.mkdirs(); - } - Map m = (Map) Utils.fromJSON(meta.array(), meta.arrayOffset(), meta.limit()); - if (m == null || m.isEmpty()) { - throw new SolrException(SERVER_ERROR, "invalid metadata , discarding : " + path); - } - - - File metdataFile = getRealpath(getMetaPath()).toFile(); - - try (FileOutputStream fos = new FileOutputStream(metdataFile)) { - fos.write(meta.array(), 0, meta.limit()); - } - IOUtils.fsync(metdataFile.toPath(), false); - - try (FileOutputStream fos = new FileOutputStream(file)) { - fos.write(data.array(), 0, data.limit()); - } + _persistToFile(solrhome, path, data, meta); log.info("persisted a file {} and metadata. sizes {} {}", path, data.limit(), meta.limit()); - IOUtils.fsync(file.toPath(), false); + } } @@ -316,7 +314,6 @@ public class DistribPackageStore implements PackageStore { } } - @Override public void put(FileEntry entry) throws IOException { FileInfo info = new FileInfo(entry.path); @@ -324,7 +321,20 @@ public class DistribPackageStore implements PackageStore { Utils.writeJson(entry.getMetaData(), baos, true); byte[] bytes = baos.toByteArray(); info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length)); - tmpFiles.put(entry.getPath(), info); + distribute(info); + } + + private void distribute(FileInfo info) { + try { + String dirName = info.path.substring(0, info.path.lastIndexOf('/')); + coreContainer.getZkController().getZkClient().makePath(ZK_PACKAGESTORE + dirName, false, true); + coreContainer.getZkController().getZkClient().create(ZK_PACKAGESTORE + info.path, info.getDetails().getMetaData().sha512.getBytes(UTF_8), + CreateMode.PERSISTENT, true); + } catch (Exception e) { + throw new SolrException(SERVER_ERROR, "Unable to create an entry in ZK", e); + } + tmpFiles.put(info.path, info); + List nodes = coreContainer.getPackageStoreAPI().shuffledNodes(); int i = 0; int FETCHFROM_SRC = 50; @@ -332,7 +342,7 @@ public class DistribPackageStore implements PackageStore { try { for (String node : nodes) { String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node); - String url = baseUrl.replace("/solr", "/api") + "/node/files" + entry.getPath() + "?getFrom="; + String url = baseUrl.replace("/solr", "/api") + "/node/files" + info.path + "?getFrom="; if (i < FETCHFROM_SRC) { // this is to protect very large clusters from overwhelming a single node // the first FETCHFROM_SRC nodes will be asked to fetch from this node. @@ -369,12 +379,11 @@ public class DistribPackageStore implements PackageStore { try { Thread.sleep(10 * 1000); } finally { - tmpFiles.remove(entry.getPath()); + tmpFiles.remove(info.path); } return null; }); } - } @Override @@ -428,6 +437,15 @@ public class DistribPackageStore implements PackageStore { new FileInfo(path).readData(consumer); } + @Override + public void syncToAllNodes(String path) throws IOException { + FileInfo fi = new FileInfo(path); + if (!fi.exists(true, false)) { + throw new SolrException(BAD_REQUEST, "No such file : " + path); + } + fi.getFileData(true); + distribute(fi); + } @Override public List list(String path, Predicate predicate) { @@ -451,6 +469,28 @@ public class DistribPackageStore implements PackageStore { return fileDetails; } + @Override + public void refresh(String path) { + try { + List l = null; + try { + l = coreContainer.getZkController().getZkClient().getChildren(ZK_PACKAGESTORE+ path, null, true); + } catch (KeeperException.NoNodeException e) { + // does not matter + } + if (l != null && !l.isEmpty()) { + List myFiles = list(path, s -> true); + for (Object f : l) { + if (!myFiles.contains(f)) { + log.info("{} does not exist locally, downloading.. ",f); + fetch(path + "/" + f.toString(), "*"); + } + } + } + } catch (Exception e) { + log.error("Could not refresh files in " +path, e); + } + } @Override public FileType getType(String path, boolean fetchMissing) { @@ -460,12 +500,16 @@ public class DistribPackageStore implements PackageStore { file = getRealpath(path).toFile(); } } + return _getFileType(file); + } + + public static FileType _getFileType(File file) { if (!file.exists()) return FileType.NOFILE; if (file.isDirectory()) return FileType.DIRECTORY; return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE; } - private boolean isMetaDataFile(String file) { + public static boolean isMetaDataFile(String file) { return file.charAt(0) == '.' && file.endsWith(".json"); } @@ -486,4 +530,61 @@ public class DistribPackageStore implements PackageStore { public static Path getPackageStoreDirPath(Path solrHome) { return Paths.get(solrHome.toAbsolutePath().toString(), PackageStoreAPI.PACKAGESTORE_DIRECTORY).toAbsolutePath(); } + + private static String _getMetapath(String path) { + int idx = path.lastIndexOf('/'); + return path.substring(0, idx + 1) + "." + path.substring(idx + 1) + ".json"; + } + + /** + * Internal API + */ + public static void _persistToFile(Path solrHome, String path, ByteBuffer data, ByteBuffer meta) throws IOException { + Path realpath = _getRealPath(path, solrHome); + File file = realpath.toFile(); + File parent = file.getParentFile(); + if (!parent.exists()) { + parent.mkdirs(); + } + Map m = (Map) Utils.fromJSON(meta.array(), meta.arrayOffset(), meta.limit()); + if (m == null || m.isEmpty()) { + throw new SolrException(SERVER_ERROR, "invalid metadata , discarding : " + path); + } + + + File metdataFile = _getRealPath(_getMetapath(path), solrHome).toFile(); + + try (FileOutputStream fos = new FileOutputStream(metdataFile)) { + fos.write(meta.array(), 0, meta.limit()); + } + IOUtils.fsync(metdataFile.toPath(), false); + + try (FileOutputStream fos = new FileOutputStream(file)) { + fos.write(data.array(), 0, data.limit()); + } + IOUtils.fsync(file.toPath(), false); + } + + @Override + public Map getKeys() throws IOException { + return _getKeys(solrhome); + } + + + // reads local keys file + private static Map _getKeys(Path solrhome) throws IOException { + Map result = new HashMap<>(); + Path keysDir = _getRealPath(PackageStoreAPI.KEYS_DIR, solrhome); + + File[] keyFiles = keysDir.toFile().listFiles(); + if (keyFiles == null) return result; + for (File keyFile : keyFiles) { + if (keyFile.isFile() && !isMetaDataFile(keyFile.getName())) { + try (InputStream fis = new FileInputStream(keyFile)) { + result.put(keyFile.getName(), SimplePostTool.inputStreamToByteArray(fis).array()); + } + } + } + return result; + } } diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java index 9a5fbff493e..db76e8a5772 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.function.Consumer; import java.util.function.Predicate; @@ -55,6 +56,10 @@ public interface PackageStore { List list(String path, Predicate predicate); + /** Sync a local file to all nodes. All the nodes are asked to pull the file from this node + */ + void syncToAllNodes(String path) throws IOException; + /** * get the real path on filesystem */ @@ -65,6 +70,15 @@ public interface PackageStore { */ FileType getType(String path, boolean fetchMissing); + /**Get all the keys in the package store. The data is a .DER file content + */ + Map getKeys() throws IOException; + + /**Refresh the files in a path. May be this node does not have all files + * @param path the path to be refreshed. + */ + void refresh(String path); + public class FileEntry { final ByteBuffer buf; final MetaData meta; diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java index 543b079703c..00b5d7d54e9 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java @@ -35,7 +35,6 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.api.Command; import org.apache.solr.api.EndPoint; import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.cloud.CloudUtil; import org.apache.solr.common.MapWriter; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -66,6 +65,8 @@ import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM; public class PackageStoreAPI { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static final String PACKAGESTORE_DIRECTORY = "filestore"; + public static final String TRUSTED_DIR = "_trusted_"; + public static final String KEYS_DIR = "/_trusted_/keys"; private final CoreContainer coreContainer; @@ -99,7 +100,7 @@ public class PackageStoreAPI { try { PackageStore.FileType type = packageStore.getType(path, true); if (type != PackageStore.FileType.FILE) { - errs.accept("No such file : " + path); + errs.accept("No such file: " + path); continue; } @@ -111,9 +112,10 @@ public class PackageStoreAPI { } if (validateSignatures) { try { - validate(entry.meta.signatures, entry); - } catch (SolrException e) { - log.error("error validating package artifact", e); + packageStore.refresh(KEYS_DIR); + validate(entry.meta.signatures, entry, false); + } catch (Exception e) { + log.error("Error validating package artifact", e); errs.accept(e.getMessage()); } } @@ -136,7 +138,7 @@ public class PackageStoreAPI { @Command public void upload(SolrQueryRequest req, SolrQueryResponse rsp) { - if(!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { + if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { throw new RuntimeException(PackageAPI.ERR_MSG); } try { @@ -149,22 +151,17 @@ public class PackageStoreAPI { if (path == null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path"); } - validateName(path); + validateName(path, true); ContentStream stream = streams.iterator().next(); try { ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStream()); - String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf)); List signatures = readSignatures(req, buf); - Map vals = new HashMap<>(); - vals.put(MetaData.SHA512, sha512); - if (signatures != null) { - vals.put("sig", signatures); - } + MetaData meta = _createJsonMetaData(buf, signatures); PackageStore.FileType type = packageStore.getType(path, true); if(type != PackageStore.FileType.NOFILE) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path); } - packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), path)); + packageStore.put(new PackageStore.FileEntry(buf, meta, path)); rsp.add(CommonParams.FILE, path); } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e); @@ -189,24 +186,24 @@ public class PackageStoreAPI { String[] signatures = req.getParams().getParams("sig"); if (signatures == null || signatures.length == 0) return null; List sigs = Arrays.asList(signatures); + packageStore.refresh(KEYS_DIR); validate(sigs, buf); return sigs; } - public void validate(List sigs, - ByteBuffer buf) throws SolrException, IOException { - Map keys = CloudUtil.getTrustedKeys( - coreContainer.getZkController().getZkClient(), "exe"); + private void validate(List sigs, + ByteBuffer buf) throws SolrException, IOException { + Map keys = packageStore.getKeys(); if (keys == null || keys.isEmpty()) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "ZK does not have any keys"); + "package store does not have any keys"); } CryptoKeys cryptoKeys = null; try { cryptoKeys = new CryptoKeys(keys); } catch (Exception e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, - "Error parsing public keys in ZooKeeper"); + "Error parsing public keys in Package store"); } for (String sig : sigs) { if (cryptoKeys.verify(sig, buf) == null) { @@ -219,6 +216,20 @@ public class PackageStoreAPI { } + /** + * Creates a JSON string with the metadata + * @lucene.internal + */ + public static MetaData _createJsonMetaData(ByteBuffer buf, List signatures) throws IOException { + String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf)); + Map vals = new HashMap<>(); + vals.put(MetaData.SHA512, sha512); + if (signatures != null) { + vals.put("sig", signatures); + } + return new MetaData(vals); + } + @EndPoint( path = "/node/files/*", method = SolrRequest.METHOD.GET, @@ -228,6 +239,14 @@ public class PackageStoreAPI { public void read(SolrQueryRequest req, SolrQueryResponse rsp) { String path = req.getPathTemplateValues().get("*"); String pathCopy = path; + if (req.getParams().getBool("sync", false)) { + try { + packageStore.syncToAllNodes(path); + return; + } catch (IOException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e); + } + } String getFrom = req.getParams().get("getFrom"); if (getFrom != null) { coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> { @@ -287,7 +306,7 @@ public class PackageStoreAPI { } - static class MetaData implements MapWriter { + public static class MetaData implements MapWriter { public static final String SHA512 = "sha512"; String sha512; List signatures; @@ -312,7 +331,7 @@ public class PackageStoreAPI { static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$> sigs, - PackageStore.FileEntry entry) throws SolrException { - Map keys = CloudUtil.getTrustedKeys( - coreContainer.getZkController().getZkClient(), "exe"); + PackageStore.FileEntry entry, + boolean isFirstAttempt) throws SolrException, IOException { + if (!isFirstAttempt) { + //we are retrying because last validation failed. + // get all keys again and try again + packageStore.refresh(KEYS_DIR); + } + + Map keys = packageStore.getKeys(); if (keys == null || keys.isEmpty()) { + if(isFirstAttempt) { + validate(sigs, entry, false); + return; + } throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, - "ZooKeeper does not have any public keys"); + "Packagestore does not have any public keys"); } CryptoKeys cryptoKeys = null; try { @@ -346,14 +384,22 @@ public class PackageStoreAPI { "Error parsing public keys in ZooKeeper"); } for (String sig : sigs) { - Supplier errMsg = () -> "Signature does not match any public key : " + sig + "sha256 "+ entry.getMetaData().sha512; + Supplier errMsg = () -> "Signature does not match any public key : " + sig + "sha256 " + entry.getMetaData().sha512; if (entry.getBuffer() != null) { if (cryptoKeys.verify(sig, entry.getBuffer()) == null) { + if(isFirstAttempt) { + validate(sigs, entry, false); + return; + } throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); } } else { InputStream inputStream = entry.getInputStream(); if (cryptoKeys.verify(sig, inputStream) == null) { + if(isFirstAttempt) { + validate(sigs, entry, false); + return; + } throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java index 518e4c4fc94..602a9e53fff 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageUtils.java @@ -26,6 +26,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.apache.commons.io.IOUtils; +import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.lucene.util.SuppressForbidden; @@ -39,7 +40,10 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.Utils; import org.apache.solr.core.BlobRepository; +import org.apache.solr.filestore.DistribPackageStore; +import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Manifest; import org.apache.solr.util.SolrJacksonAnnotationInspector; @@ -137,7 +141,12 @@ public class PackageUtils { */ public static String getJsonStringFromUrl(HttpClient client, String url) { try { - return IOUtils.toString(client.execute(new HttpGet(url)).getEntity().getContent(), "UTF-8"); + HttpResponse resp = client.execute(new HttpGet(url)); + if (resp.getStatusLine().getStatusCode() != 200) { + throw new SolrException(ErrorCode.NOT_FOUND, + "Error (code="+resp.getStatusLine().getStatusCode()+") fetching from URL: "+url); + } + return IOUtils.toString(resp.getEntity().getContent(), "UTF-8"); } catch (UnsupportedOperationException | IOException e) { throw new RuntimeException(e); } @@ -244,4 +253,11 @@ public class PackageUtils { public static String getCollectionParamsPath(String collection) { return "/api/collections/" + collection + "/config/params"; } + + public static void uploadKey(byte bytes[], String path, Path home, HttpSolrClient client) throws IOException { + ByteBuffer buf = ByteBuffer.wrap(bytes); + PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(buf, null); + DistribPackageStore._persistToFile(home, path, buf, ByteBuffer.wrap(Utils.toJSON(meta))); + } + } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 3d6075b42a7..a0cc0e1f3a0 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -22,10 +22,10 @@ import static org.apache.solr.packagemanager.PackageUtils.getMapper; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.lang.invoke.MethodHandles; -import java.net.MalformedURLException; import java.net.URL; import java.nio.ByteBuffer; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -48,10 +48,12 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.core.BlobRepository; +import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackagePluginHolder; +import org.apache.solr.util.SolrCLI; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.slf4j.Logger; @@ -116,24 +118,31 @@ public class RepositoryManager { /** * Add a repository to Solr */ - public void addRepository(String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException { + public void addRepository(String repoName, String uri) throws Exception { String existingRepositoriesJson = getRepositoriesJson(packageManager.zkClient); log.info(existingRepositoriesJson); List repos = getMapper().readValue(existingRepositoriesJson, List.class); - repos.add(new DefaultPackageRepository(name, uri)); + repos.add(new DefaultPackageRepository(repoName, uri)); if (packageManager.zkClient.exists(PackageUtils.REPOSITORIES_ZK_PATH, true) == false) { packageManager.zkClient.create(PackageUtils.REPOSITORIES_ZK_PATH, getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true); } else { packageManager.zkClient.setData(PackageUtils.REPOSITORIES_ZK_PATH, getMapper().writeValueAsString(repos).getBytes("UTF-8"), true); } - if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true); - if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true); - if (packageManager.zkClient.exists("/keys/exe/" + name + ".der", true)==false) { - packageManager.zkClient.create("/keys/exe/" + name + ".der", new byte[0], CreateMode.PERSISTENT, true); - } - packageManager.zkClient.setData("/keys/exe/" + name + ".der", IOUtils.toByteArray(new URL(uri + "/publickey.der").openStream()), true); + addKey(IOUtils.toByteArray(new URL(uri + "/publickey.der").openStream()), repoName + ".der"); + } + + public void addKey(byte[] key, String destinationKeyFilename) throws Exception { + // get solr_home directory from info servlet + String systemInfoUrl = solrClient.getBaseURL() + "/solr/admin/info/system"; + Map systemInfo = SolrCLI.getJson(solrClient.getHttpClient(), systemInfoUrl, 2, true); + String solrHome = (String) systemInfo.get("solr_home"); + + // put the public key into package store's trusted key store and request a sync. + String path = PackageStoreAPI.KEYS_DIR + "/" + destinationKeyFilename; + PackageUtils.uploadKey(key, path, Paths.get(solrHome), solrClient); + PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrClient.getBaseURL() + "/api/node/files" + path + "?sync=true"); } private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException { diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java index 8783d9bd827..2be2eec8bdd 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageLoader.java @@ -168,7 +168,14 @@ public class PackageLoader implements Closeable { Version version = myVersions.get(v.version); if (version == null) { log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files); - myVersions.put(v.version, new Version(this, v)); + Version ver = null; + try { + ver = new Version(this, v); + } catch (Exception e) { + log.error("package could not be loaded "+ ver.toString(), e); + continue; + } + myVersions.put(v.version, ver); sortedVersions.add(v.version); } } @@ -248,9 +255,13 @@ public class PackageLoader implements Closeable { this.parent = parent; this.version = v; List paths = new ArrayList<>(); + + List errs = new ArrayList<>(); + coreContainer.getPackageStoreAPI().validateFiles(version.files, true, s -> errs.add(s)); + if(!errs.isEmpty()) { + throw new RuntimeException("Cannot load package: " +errs); + } for (String file : version.files) { - //ensure that the files are downloaded and available - coreContainer.getPackageStoreAPI().getPackageStore().fetch(file,null); paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file)); } @@ -283,6 +294,11 @@ public class PackageLoader implements Closeable { closeWhileHandlingException(loader); } } + + @Override + public String toString() { + return jsonStr(); + } } } diff --git a/solr/core/src/java/org/apache/solr/util/PackageTool.java b/solr/core/src/java/org/apache/solr/util/PackageTool.java index f7744c98faf..87bf7b4da53 100644 --- a/solr/core/src/java/org/apache/solr/util/PackageTool.java +++ b/solr/core/src/java/org/apache/solr/util/PackageTool.java @@ -19,12 +19,15 @@ package org.apache.solr.util; import static org.apache.solr.packagemanager.PackageUtils.printGreen; import static org.apache.solr.packagemanager.PackageUtils.print; +import java.io.File; import java.lang.invoke.MethodHandles; +import java.nio.file.Paths; import java.util.Map; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.io.FileUtils; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; @@ -44,7 +47,6 @@ import org.apache.solr.util.SolrCLI.StatusTool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class PackageTool extends SolrCLI.ToolBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -92,6 +94,10 @@ public class PackageTool extends SolrCLI.ToolBase { repositoryManager.addRepository(repoName, repoUrl); PackageUtils.printGreen("Added repository: " + repoName); break; + case "add-key": + String keyFilename = cli.getArgs()[1]; + repositoryManager.addKey(FileUtils.readFileToByteArray(new File(keyFilename)), Paths.get(keyFilename).getFileName().toString()); + break; case "list-installed": PackageUtils.printGreen("Installed packages:\n-----"); for (SolrPackageInstance pkg: packageManager.fetchInstalledPackageInstances()) { diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java index e90dd5ea2e0..f1681e4f39c 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java @@ -44,8 +44,8 @@ import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.packagemanager.PackageUtils; import org.apache.solr.util.LogLevel; -import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.server.ByteBufferInputStream; import org.junit.After; import org.junit.Before; @@ -75,8 +75,9 @@ public class TestDistribPackageStore extends SolrCloudTestCase { try { byte[] derFile = readFile("cryptokeys/pub_key512.der"); - cluster.getZkClient().makePath("/keys/exe", true); - cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); + uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster); +// cluster.getZkClient().makePath("/keys/exe", true); +// cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); try { postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"), @@ -247,7 +248,16 @@ public class TestDistribPackageStore extends SolrCloudTestCase { return rsp; } - + public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster cluster) throws Exception { + JettySolrRunner jetty = cluster.getRandomJetty(random()); + try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()) { + PackageUtils.uploadKey(bytes, path, jetty.getCoreContainer().getResourceLoader().getInstancePath(), client); + Object resp = Utils.executeGET(client.getHttpClient(), jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true", null); + System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true"+" ,is: "+resp); + } + waitForAllNodesHaveFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate) Objects::nonNull), + false); + } public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index d37fdf4af70..897383a1602 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -50,9 +50,9 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.Utils; +import org.apache.solr.filestore.PackageStoreAPI; import org.apache.solr.filestore.TestDistribPackageStore; import org.apache.solr.util.LogLevel; -import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.data.Stat; import org.junit.After; import org.junit.Before; @@ -63,6 +63,7 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN; import static org.apache.solr.common.params.CommonParams.WT; import static org.apache.solr.core.TestDynamicLoading.getFileContent; import static org.apache.solr.filestore.TestDistribPackageStore.readFile; +import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey; import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile; @LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG") @@ -95,8 +96,9 @@ public class TestPackages extends SolrCloudTestCase { String EXPR1 = "/mypkg/expressible.jar"; String COLLECTION_NAME = "testPluginLoadingColl"; byte[] derFile = readFile("cryptokeys/pub_key512.der"); - cluster.getZkClient().makePath("/keys/exe", true); - cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); + uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster); +// cluster.getZkClient().makePath("/keys/exe", true); +// cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); @@ -459,7 +461,7 @@ public class TestPackages extends SolrCloudTestCase { .build(); //the files is not yet there. The command should fail with error saying "No such file" - expectError(req, cluster.getSolrClient(), errPath, "No such file :"); + expectError(req, cluster.getSolrClient(), errPath, "No such file:"); //post the jar file. No signature is sent @@ -471,20 +473,19 @@ public class TestPackages extends SolrCloudTestCase { FILE1 + " has no signature"); //now we upload the keys byte[] derFile = readFile("cryptokeys/pub_key512.der"); - cluster.getZkClient().makePath("/keys/exe", true); - cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); + uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster); //and upload the same file with a different name but it has proper signature postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); // with correct signature //after uploading the file, let's delete the keys to see if we get proper error message - cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true); +// cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true); add.files = Arrays.asList(new String[]{FILE2}); - expectError(req, cluster.getSolrClient(), errPath, - "ZooKeeper does not have any public keys"); + /*expectError(req, cluster.getSolrClient(), errPath, + "ZooKeeper does not have any public keys");*/ //Now lets' put the keys back - cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); +// cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true); //this time we have a file with proper signature, public keys are in ZK // so the add {} command should succeed diff --git a/solr/solr-ref-guide/src/package-manager-internals.adoc b/solr/solr-ref-guide/src/package-manager-internals.adoc index 6cb1956575f..467691bfdeb 100644 --- a/solr/solr-ref-guide/src/package-manager-internals.adoc +++ b/solr/solr-ref-guide/src/package-manager-internals.adoc @@ -15,7 +15,7 @@ The package manager (CLI) internally uses various Solr APIs to install, deploy a At the heart of the system, we have classloader isolation. To achieve this, the system is simplified into two layered classloaders: The root classloader which has all the jars from Solr classpath. This requires Solr node restart to change anything. -A set of named classloaders that inherit from the root classloader. The life cycles of the named classloaders are tied to the package configuration in ZK. As soon as the configuration is modified, the corresponding classloaders are reloaded and components are asked to reload. +A set of named classloaders that inherit from the root classloader. The life cycles of the named classloaders are tied to the package configuration in ZooKeeper. As soon as the configuration is modified, the corresponding classloaders are reloaded and components are asked to reload. == Package Loading Security Packages are disabled by default. Start all your nodes with the system property `-Denable.packages=true` to use this feature. @@ -28,7 +28,7 @@ $ bin/solr -c -Denable.packages=true ---- === Upload Your Keys -Package binaries must be signed with your private keys and ensure your public keys are published in ZooKeeper. +Package binaries must be signed with your private keys and ensure your public keys are published in package store's trusted store. *Example* [source,bash] @@ -36,9 +36,8 @@ Package binaries must be signed with your private keys and ensure your public ke $ openssl genrsa -out my_key.pem 512 # create the public key in .der format $ openssl rsa -in my_key.pem -pubout -outform DER -out my_key.der -# upload to ZooKeeper -$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd makepath /keys/exe/ -$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd putfile /keys/exe/my_key.der my_key.der +# upload key to package store +$ bin/solr package add-key my_key.der ---- == Package Store diff --git a/solr/solr-ref-guide/src/package-manager.adoc b/solr/solr-ref-guide/src/package-manager.adoc index 66f13784ffa..94e948ea650 100644 --- a/solr/solr-ref-guide/src/package-manager.adoc +++ b/solr/solr-ref-guide/src/package-manager.adoc @@ -19,7 +19,7 @@ // specific language governing permissions and limitations // under the License. -The package manager in Solr allows installation and update of Solr-specific packages in distributed and standalone environments. +The package manager in Solr allows installation and updating of Solr-specific packages in Solr's cluster environment. In this system, a _package_ is a set of Java jar files (usually one) containing one or more <>. Each jar file is also accompanied by a signature string (which can be verified against a supplied public key). @@ -27,7 +27,7 @@ A key design aspect of this system is the ability to install or update packages Other elements of the design include the ability to install from a remote repository; package standardization; a command line interface (CLI); and a package store. -This section will focus on how to use the package manager to install and update plugins. +This section will focus on how to use the package manager to install and update packages. For technical details, see the section <>. == Interacting with the Package Manager @@ -158,6 +158,6 @@ You can run the `list-deployed` command to verify that this collection is using == Security -As noted above in the section <>, the `add-repo` step should only be executed using an HTTPS endpoint in Solr (all other steps can be executed using HTTP - see also <>). This step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to ZooKeeper. It is critical to protect ZooKeeper from unauthorized write access. +The `add-repo` step should only be executed using HTTPS enabled repository urls only so as to prevent against MITM attacks when Solr is fetching the public key for the repository. This `add-repo` step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to the trusted store of the package store (a special location in the package store that cannot be written to using the package store APIs). Also, it is critical to protect ZooKeeper from unauthorized write access. -Also, keep in mind, that it is possible to install *any* package from a repository once it has been added. If you want to use some packages in production, a best practice is to setup your own repository and add that to Solr instead of adding a generic third-party repository that is beyond your administrative control. +Also, keep in mind, that it is possible to install *any* package from a repository once it has been added. If you want to use some packages in production, a best practice is to setup your own repository and add that to Solr instead of adding a generic third-party repository that is beyond your administrative control. You might want to re-sign packages from a third-party repository using your own private keys and host them at your own repository.