From 88f457ee2a517f8c9656209846c16d402edfef0e Mon Sep 17 00:00:00 2001 From: noble Date: Sat, 12 Oct 2019 16:59:14 +1100 Subject: [PATCH] SOLR-13821: refactored the code to change the API to suit package loader --- .../org/apache/solr/core/CoreContainer.java | 3 + .../solr/filestore/DistribPackageStore.java | 60 +++++------- .../apache/solr/filestore/PackageStore.java | 6 +- .../solr/filestore/PackageStoreAPI.java | 86 ++++++++++++++++- .../java/org/apache/solr/util/CryptoKeys.java | 20 ++++ .../filestore/TestDistribPackageStore.java | 92 +++++++++++-------- 6 files changed, 187 insertions(+), 80 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 03592c3abaf..054bd67542d 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -580,6 +580,9 @@ public class CoreContainer { return replayUpdatesExecutor; } + public PackageStoreAPI getPackageStoreAPI() { + return packageStoreAPI; + } //------------------------------------------------------------------- // Initialization / Cleanup //------------------------------------------------------------------- diff --git a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java index 4e86926150f..ded379fddc7 100644 --- a/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/DistribPackageStore.java @@ -29,12 +29,10 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Consumer; import java.util.function.Predicate; @@ -45,7 +43,6 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.BlobRepository; import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.PackageStoreAPI.MetaData; import org.apache.zookeeper.server.ByteBufferInputStream; @@ -62,6 +59,7 @@ public class DistribPackageStore implements PackageStore { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final CoreContainer coreContainer; private Map tmpFiles = new ConcurrentHashMap<>(); + public DistribPackageStore(CoreContainer coreContainer) { this.coreContainer = coreContainer; ensurePackageStoreDir(coreContainer.getResourceLoader().getInstancePath()); @@ -73,23 +71,13 @@ public class DistribPackageStore implements PackageStore { } - /** - * get a list of nodes randomly shuffled - * * @lucene.internal - */ - public ArrayList shuffledNodes() { - Set liveNodes = coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); - ArrayList l = new ArrayList(liveNodes); - l.remove(myNode()); - Collections.shuffle(l, BlobRepository.RANDOM); - return l; - } + @Override public Path getRealpath(String path) { if (File.separatorChar == '\\') { - path = path.replace('/' , File.separatorChar); + path = path.replace('/', File.separatorChar); } if (path.charAt(0) != File.separatorChar) { path = File.separator + path; @@ -182,7 +170,7 @@ public class DistribPackageStore implements PackageStore { try { IOUtils.deleteFilesIfExist(getRealpath(path), getRealpath(getMetaPath())); } catch (IOException e) { - log.error("Unable to delete files: "+path); + log.error("Unable to delete files: " + path); } } @@ -224,8 +212,7 @@ public class DistribPackageStore implements PackageStore { } boolean fetchFromAnyNode() { - - ArrayList l = shuffledNodes(); + ArrayList l = coreContainer.getPackageStoreAPI().shuffledNodes(); ZkStateReader stateReader = coreContainer.getZkController().getZkStateReader(); for (String liveNode : l) { try { @@ -273,17 +260,15 @@ public class DistribPackageStore implements PackageStore { } - - public FileDetails getDetails() { - FileType type = getType(path); + FileType type = getType(path, false); return new FileDetails() { @Override public MetaData getMetaData() { try { return readMetaData(); - } catch (Exception e){ + } catch (Exception e) { throw new RuntimeException(e); } } @@ -307,18 +292,17 @@ public class DistribPackageStore implements PackageStore { return; } ew.put("timestamp", getTimeStamp()); - metaData.writeMap(ew); + if(metaData != null) + metaData.writeMap(ew); } }; - - } public void readData(Consumer consumer) throws IOException { MetaData meta = readMetaData(); try (InputStream is = new FileInputStream(realPath().toFile())) { - consumer.accept(new FileEntry(null, meta,path ){ + consumer.accept(new FileEntry(null, meta, path) { @Override public InputStream getInputStream() { return is; @@ -337,10 +321,10 @@ public class DistribPackageStore implements PackageStore { byte[] bytes = baos.toByteArray(); info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length)); tmpFiles.put(entry.getPath(), info); - List nodes = shuffledNodes(); + List nodes = coreContainer.getPackageStoreAPI().shuffledNodes(); int i = 0; int FETCHFROM_SRC = 50; - String myNodeName = myNode(); + String myNodeName = coreContainer.getZkController().getNodeName(); try { for (String node : nodes) { String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node); @@ -395,11 +379,11 @@ public class DistribPackageStore implements PackageStore { } @Override - public synchronized boolean fetch(String path, String from) { + public boolean fetch(String path, String from) { if (path == null || path.isEmpty()) return false; FileInfo f = new FileInfo(path); try { - if(f.exists(true, false)){ + if (f.exists(true, false)) { return true; } } catch (IOException e) { @@ -419,7 +403,7 @@ public class DistribPackageStore implements PackageStore { } @Override - public synchronized void get(String path, Consumer consumer) throws IOException { + public void get(String path, Consumer consumer, boolean fetchmissing) throws IOException { File file = getRealpath(path).toFile(); String simpleName = file.getName(); if (isMetaDataFile(simpleName)) { @@ -440,10 +424,10 @@ public class DistribPackageStore implements PackageStore { @Override - public synchronized List list(String path, Predicate predicate) { + public List list(String path, Predicate predicate) { File file = getRealpath(path).toFile(); List fileDetails = new ArrayList<>(); - FileType type = getType(path); + FileType type = getType(path, false); if (type == FileType.DIRECTORY) { file.list((dir, name) -> { if (predicate == null || predicate.test(name)) { @@ -455,7 +439,6 @@ public class DistribPackageStore implements PackageStore { }); } else if (type == FileType.FILE) { - fileDetails.add(new FileInfo(path).getDetails()); } @@ -464,9 +447,14 @@ public class DistribPackageStore implements PackageStore { @Override - public synchronized FileType getType(String path) { + public FileType getType(String path, boolean fetchMissing) { File file = getRealpath(path).toFile(); - if (!file.exists()) return FileType.NOFILE; + if (!file.exists() && fetchMissing) { + if (fetch(path, null)) { + file = getRealpath(path).toFile(); + } + if (!file.exists()) return FileType.NOFILE; + } if (file.isDirectory()) return FileType.DIRECTORY; return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE; } diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java index b9be69196dd..8ac23886aba 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStore.java @@ -45,11 +45,11 @@ public interface PackageStore { /** * read file content from a given path */ - void get(String path, Consumer filecontent) throws IOException; + void get(String path, Consumer filecontent, boolean getMissing) throws IOException; /** * Fetch a resource from another node - * internal + * internal API */ boolean fetch(String path, String from); @@ -63,7 +63,7 @@ public interface PackageStore { /** * The type of the resource */ - FileType getType(String path); + FileType getType(String path, boolean fetchMissing); public class FileEntry { final ByteBuffer buf; diff --git a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java index 71ee9d84dda..a51b3662951 100644 --- a/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java +++ b/solr/core/src/java/org/apache/solr/filestore/PackageStoreAPI.java @@ -18,13 +18,18 @@ package org.apache.solr.filestore; import java.io.IOException; +import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Supplier; import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.api.Command; @@ -39,6 +44,7 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.StrUtils; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.BlobRepository; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; @@ -75,6 +81,50 @@ public class PackageStoreAPI { return packageStore; } + /** + * get a list of nodes randomly shuffled + * * @lucene.internal + */ + public ArrayList shuffledNodes() { + Set liveNodes = coreContainer.getZkController().getZkStateReader().getClusterState().getLiveNodes(); + ArrayList l = new ArrayList(liveNodes); + l.remove(coreContainer.getZkController().getNodeName()); + Collections.shuffle(l, BlobRepository.RANDOM); + return l; + } + + public void validateFiles(List files, boolean validateSignatures, Consumer errs) { + for (String path : files) { + try { + PackageStore.FileType type = packageStore.getType(path, true); + if (type != PackageStore.FileType.FILE) { + errs.accept("No such file : " + path); + continue; + } + + packageStore.get(path, entry -> { + if (entry.getMetaData().signatures == null || + entry.getMetaData().signatures.isEmpty()) { + errs.accept(path + " has no signature"); + return; + } + if (validateSignatures) { + try { + validate(entry.meta.signatures, entry); + } catch (SolrException e) { + log.error("error validating package artifact", e); + errs.accept(e.getMessage()); + } + } + }, false); + } catch (Exception e) { + log.error("Error reading file ", e); + errs.accept("Error reading file " + path + " " + e.getMessage()); + } + } + + } + @EndPoint( path = "/cluster/files/*", method = SolrRequest.METHOD.PUT, @@ -187,7 +237,7 @@ public class PackageStoreAPI { path = ""; } - PackageStore.FileType typ = packageStore.getType(path); + PackageStore.FileType typ = packageStore.getType(path, false); if (typ == PackageStore.FileType.NOFILE) { rsp.add("files", Collections.singletonMap(path, null)); return; @@ -221,7 +271,7 @@ public class PackageStoreAPI { } catch (IOException e) { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading file" + path); } - }); + }, false); }); } @@ -270,4 +320,36 @@ public class PackageStoreAPI { } } } + + public void validate(List sigs, + PackageStore.FileEntry entry) throws SolrException { + Map keys = CloudUtil.getTrustedKeys( + coreContainer.getZkController().getZkClient(), "exe"); + if (keys == null || keys.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, + "ZooKeeper does not have any public keys"); + } + CryptoKeys cryptoKeys = null; + try { + cryptoKeys = new CryptoKeys(keys); + } catch (Exception e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, + "Error parsing public keys in ZooKeeper"); + } + for (String sig : sigs) { + Supplier errMsg = () -> "Signature does not match any public key : " + sig; + if (entry.getBuffer() != null) { + if (cryptoKeys.verify(sig, entry.getBuffer()) == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); + } + } else { + InputStream inputStream = entry.getInputStream(); + if (cryptoKeys.verify(sig, inputStream) == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get()); + } + + } + + } + } } diff --git a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java index 8bac73c34bd..d8c058f2abb 100644 --- a/solr/core/src/java/org/apache/solr/util/CryptoKeys.java +++ b/solr/core/src/java/org/apache/solr/util/CryptoKeys.java @@ -87,6 +87,26 @@ public final class CryptoKeys implements CLIO { return null; } + public String verify(String sig, InputStream is) { + exception = null; + for (Map.Entry entry : keys.entrySet()) { + boolean verified; + try { + verified = CryptoKeys.verify(entry.getValue(), Base64.base64ToByteArray(sig), is); + log.debug("verified {} ", verified); + if (verified) return entry.getKey(); + } catch (Exception e) { + exception = e; + log.debug("NOT verified "); + } + + } + + return null; + } + + + /** * Create PublicKey from a .DER file diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java index a99028a5522..9ca5e1be2a5 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribPackageStore.java @@ -51,7 +51,7 @@ import org.apache.zookeeper.server.ByteBufferInputStream; import static org.apache.solr.common.util.Utils.JAVABINCONSUMER; import static org.apache.solr.core.TestDynamicLoading.getFileContent; -@LogLevel("org.apache.solr.core.PackageStoreAPI=DEBUG;org.apache.solr.core.DistribPackageStore=DEBUG") +@LogLevel("org.apache.solr.filestore.PackageStoreAPI=DEBUG;org.apache.solr.filestore.DistribPackageStore=DEBUG") public class TestDistribPackageStore extends SolrCloudTestCase { public void testPackageStoreManagement() throws Exception { @@ -104,54 +104,26 @@ public class TestDistribPackageStore extends SolrCloudTestCase { ) ); - class Fetcher implements Callable { - String url; - JettySolrRunner jetty; - Fetcher(String s, JettySolrRunner jettySolrRunner){ - this.url = s; - this.jetty = jettySolrRunner; - } - @Override - public NavigableObject call() throws Exception { - try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) { - return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), this.url, JAVABINCONSUMER); - } - } - - @Override - public String toString() { - return url; - } - - } - Map expected = Utils.makeMap( ":files:/package/mypkg/v1.0/runtimelibs.jar:name", "runtimelibs.jar", ":files:/package/mypkg/v1.0[0]:sha512", "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420" ); - for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { - String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api"); - String url = baseUrl + "/node/files/package/mypkg/v1.0/runtimelibs.jar?wt=javabin&meta=true"; + waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs.jar", expected, true); - assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected); - - try (HttpSolrClient solrClient = (HttpSolrClient) jettySolrRunner.newClient()) { - ByteBuffer buf = Utils.executeGET(solrClient.getHttpClient(), baseUrl + "/node/files/package/mypkg/v1.0/runtimelibs.jar", - Utils.newBytesConsumer(Integer.MAX_VALUE)); - assertEquals( - "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", - DigestUtils.sha512Hex(new ByteBufferInputStream(buf)) - ); - - } - - } postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs_v2.jar.bin"), "/package/mypkg/v1.0/runtimelibs_v2.jar", null ); + expected = Utils.makeMap( + ":files:/package/mypkg/v1.0/runtimelibs_v2.jar:name", "runtimelibs_v2.jar", + ":files:/package/mypkg/v1.0[0]:sha512", + "bc5ce45ad281b6a08fb7e529b1eb475040076834816570902acb6ebdd809410e31006efdeaa7f78a6c35574f3504963f5f7e4d92247d0eb4db3fc9abdda5d417" + + ); + waitForAllNodesHaveFile(cluster,"/package/mypkg/v1.0/runtimelibs_v2.jar", expected, false); + expected = Utils.makeMap( ":files:/package/mypkg/v1.0", (Predicate) o -> { @@ -180,6 +152,48 @@ public class TestDistribPackageStore extends SolrCloudTestCase { } } + public static void waitForAllNodesHaveFile(MiniSolrCloudCluster cluster, String path, Map expected , boolean verifyContent) throws Exception { + for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) { + String baseUrl = jettySolrRunner.getBaseUrl().toString().replace("/solr", "/api"); + String url = baseUrl + "/node/files" + path + "?wt=javabin&meta=true"; + assertResponseValues(10, new Fetcher(url, jettySolrRunner), expected); + + if(verifyContent) { + try (HttpSolrClient solrClient = (HttpSolrClient) jettySolrRunner.newClient()) { + ByteBuffer buf = Utils.executeGET(solrClient.getHttpClient(), baseUrl + "/node/files" + path, + Utils.newBytesConsumer(Integer.MAX_VALUE)); + assertEquals( + "d01b51de67ae1680a84a813983b1de3b592fc32f1a22b662fc9057da5953abd1b72476388ba342cad21671cd0b805503c78ab9075ff2f3951fdf75fa16981420", + DigestUtils.sha512Hex(new ByteBufferInputStream(buf)) + ); + + } + } + + } + } + + static class Fetcher implements Callable { + String url; + JettySolrRunner jetty; + Fetcher(String s, JettySolrRunner jettySolrRunner){ + this.url = s; + this.jetty = jettySolrRunner; + } + @Override + public NavigableObject call() throws Exception { + try (HttpSolrClient solrClient = (HttpSolrClient) jetty.newClient()) { + return (NavigableObject) Utils.executeGET(solrClient.getHttpClient(), this.url, JAVABINCONSUMER); + } + } + + @Override + public String toString() { + return url; + } + + } + public static NavigableObject assertResponseValues(int repeats, SolrClient client, SolrRequest req, Map vals) throws Exception { Callable callable = () -> req.process(client); @@ -223,7 +237,7 @@ public class TestDistribPackageStore extends SolrCloudTestCase { - private void postFile(SolrClient client, ByteBuffer buffer, String name, String sig) + public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { String resource = "/cluster/files" + name; ModifiableSolrParams params = new ModifiableSolrParams();