mirror of https://github.com/apache/lucene.git
SOLR-14158: Package manager to read keys from package store, not ZK
This commit is contained in:
Ishan Chattopadhyaya
parent
239d9a6726
commit
6fb085943c
|
|
@ -238,12 +238,23 @@ Velocity 2.0 and Velocity Tools 3.0
|
|||
Apache ZooKeeper 3.5.5
|
||||
Jetty 9.4.19.v20190610
|
||||
|
||||
Upgrade Notes
|
||||
---------------------
|
||||
|
||||
* SOLR-14158: Package manager now stores keys in package store instead of ZK. If you're using package manager and have already
|
||||
added any repositories, then you would need to re-add all the keys from /keys/exe location, one-by-one, using:
|
||||
"bin/solr add-key <keyfile.der>"
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
* SOLR-14165: SolrResponse serialVersionUID has changed in a backward incompatible way (Andy Webb via noble)
|
||||
|
||||
Improvements
|
||||
---------------------
|
||||
|
||||
* SOLR-14158: Package manager to store public keys in a special "trusted" location instead of in ZooKeeper
|
||||
(noble, Ishan Chattopadhyaya)
|
||||
|
||||
================== 8.4.0 ==================
|
||||
|
||||
|
|
|
|||
|
|
@ -756,6 +756,15 @@ public class JettySolrRunner {
|
|||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public URL getBaseURLV2(){
|
||||
try {
|
||||
return new URL(protocol, host, jettyPort, "/api");
|
||||
} catch (MalformedURLException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
}
|
||||
/**
|
||||
* Returns a base URL consisting of the protocol, host, and port for a
|
||||
* Connector in use by the Jetty Server contained in this runner.
|
||||
|
|
|
|||
|
|
@ -652,6 +652,9 @@ public class CoreContainer {
|
|||
// use deprecated API for back-compat, remove in 9.0
|
||||
pkiAuthenticationPlugin.initializeMetrics(solrMetricsContext, "/authentication/pki");
|
||||
TracerConfigurator.loadTracer(loader, cfg.getTracerConfiguratorPluginInfo(), getZkController().getZkStateReader());
|
||||
packageLoader = new PackageLoader(this);
|
||||
containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP);
|
||||
containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP);
|
||||
}
|
||||
|
||||
MDCLoggingContext.setNode(this);
|
||||
|
|
@ -744,9 +747,6 @@ public class CoreContainer {
|
|||
|
||||
if (isZooKeeperAware()) {
|
||||
metricManager.loadClusterReporters(metricReporters, this);
|
||||
packageLoader = new PackageLoader(this);
|
||||
containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().editAPI), Collections.EMPTY_MAP);
|
||||
containerHandlers.getApiBag().register(new AnnotatedApi(packageLoader.getPackageAPI().readAPI), Collections.EMPTY_MAP);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ import java.nio.file.Paths;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
|
|
@ -45,41 +46,51 @@ import org.apache.solr.common.params.CommonParams;
|
|||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.core.CoreContainer;
|
||||
import org.apache.solr.filestore.PackageStoreAPI.MetaData;
|
||||
import org.apache.solr.util.SimplePostTool;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.apache.zookeeper.server.ByteBufferInputStream;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static java.nio.charset.StandardCharsets.UTF_8;
|
||||
import static org.apache.solr.common.SolrException.ErrorCode.BAD_REQUEST;
|
||||
import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
|
||||
|
||||
|
||||
public class DistribPackageStore implements PackageStore {
|
||||
static final long MAX_PKG_SIZE = Long.parseLong(System.getProperty("max.file.store.size", String.valueOf(100 * 1024 * 1024)));
|
||||
/**
|
||||
* This is where al the files in the package store are listed
|
||||
*/
|
||||
static final String ZK_PACKAGESTORE = "/packagestore";
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
private final CoreContainer coreContainer;
|
||||
private Map<String, FileInfo> tmpFiles = new ConcurrentHashMap<>();
|
||||
|
||||
private final Path solrhome;
|
||||
|
||||
public DistribPackageStore(CoreContainer coreContainer) {
|
||||
this.coreContainer = coreContainer;
|
||||
solrhome = this.coreContainer.getResourceLoader().getInstancePath();
|
||||
ensurePackageStoreDir(coreContainer.getResourceLoader().getInstancePath());
|
||||
|
||||
}
|
||||
|
||||
private String myNode() {
|
||||
return coreContainer.getZkController().getNodeName();
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Path getRealpath(String path) {
|
||||
return _getRealPath(path, solrhome);
|
||||
}
|
||||
|
||||
private static Path _getRealPath(String path, Path solrHome) {
|
||||
if (File.separatorChar == '\\') {
|
||||
path = path.replace('/', File.separatorChar);
|
||||
}
|
||||
if (!path.isEmpty() && path.charAt(0) != File.separatorChar) {
|
||||
path = File.separator + path;
|
||||
}
|
||||
return new File(this.coreContainer.getResourceLoader().getInstancePath() +
|
||||
return new File(solrHome +
|
||||
File.separator + PackageStoreAPI.PACKAGESTORE_DIRECTORY + path).toPath();
|
||||
}
|
||||
|
||||
|
|
@ -93,10 +104,18 @@ public class DistribPackageStore implements PackageStore {
|
|||
this.path = path;
|
||||
}
|
||||
|
||||
ByteBuffer getFileData(boolean validate) throws IOException {
|
||||
if (fileData == null) {
|
||||
try (FileInputStream fis = new FileInputStream(getRealpath(path).toFile())) {
|
||||
fileData = SimplePostTool.inputStreamToByteArray(fis);
|
||||
}
|
||||
}
|
||||
return fileData;
|
||||
}
|
||||
|
||||
public String getMetaPath() {
|
||||
if (metaPath == null) {
|
||||
int idx = path.lastIndexOf('/');
|
||||
metaPath = path.substring(0, idx + 1) + "." + path.substring(idx + 1) + ".json";
|
||||
metaPath = _getMetapath(path);
|
||||
}
|
||||
return metaPath;
|
||||
}
|
||||
|
|
@ -106,30 +125,9 @@ public class DistribPackageStore implements PackageStore {
|
|||
synchronized (DistribPackageStore.this) {
|
||||
this.metaData = meta;
|
||||
this.fileData = data;
|
||||
Path realpath = getRealpath(path);
|
||||
File file = realpath.toFile();
|
||||
File parent = file.getParentFile();
|
||||
if (!parent.exists()) {
|
||||
parent.mkdirs();
|
||||
}
|
||||
Map m = (Map) Utils.fromJSON(meta.array(), meta.arrayOffset(), meta.limit());
|
||||
if (m == null || m.isEmpty()) {
|
||||
throw new SolrException(SERVER_ERROR, "invalid metadata , discarding : " + path);
|
||||
}
|
||||
|
||||
|
||||
File metdataFile = getRealpath(getMetaPath()).toFile();
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(metdataFile)) {
|
||||
fos.write(meta.array(), 0, meta.limit());
|
||||
}
|
||||
IOUtils.fsync(metdataFile.toPath(), false);
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) {
|
||||
fos.write(data.array(), 0, data.limit());
|
||||
}
|
||||
_persistToFile(solrhome, path, data, meta);
|
||||
log.info("persisted a file {} and metadata. sizes {} {}", path, data.limit(), meta.limit());
|
||||
IOUtils.fsync(file.toPath(), false);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -316,7 +314,6 @@ public class DistribPackageStore implements PackageStore {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void put(FileEntry entry) throws IOException {
|
||||
FileInfo info = new FileInfo(entry.path);
|
||||
|
|
@ -324,7 +321,20 @@ public class DistribPackageStore implements PackageStore {
|
|||
Utils.writeJson(entry.getMetaData(), baos, true);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
info.persistToFile(entry.buf, ByteBuffer.wrap(bytes, 0, bytes.length));
|
||||
tmpFiles.put(entry.getPath(), info);
|
||||
distribute(info);
|
||||
}
|
||||
|
||||
private void distribute(FileInfo info) {
|
||||
try {
|
||||
String dirName = info.path.substring(0, info.path.lastIndexOf('/'));
|
||||
coreContainer.getZkController().getZkClient().makePath(ZK_PACKAGESTORE + dirName, false, true);
|
||||
coreContainer.getZkController().getZkClient().create(ZK_PACKAGESTORE + info.path, info.getDetails().getMetaData().sha512.getBytes(UTF_8),
|
||||
CreateMode.PERSISTENT, true);
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SERVER_ERROR, "Unable to create an entry in ZK", e);
|
||||
}
|
||||
tmpFiles.put(info.path, info);
|
||||
|
||||
List<String> nodes = coreContainer.getPackageStoreAPI().shuffledNodes();
|
||||
int i = 0;
|
||||
int FETCHFROM_SRC = 50;
|
||||
|
|
@ -332,7 +342,7 @@ public class DistribPackageStore implements PackageStore {
|
|||
try {
|
||||
for (String node : nodes) {
|
||||
String baseUrl = coreContainer.getZkController().getZkStateReader().getBaseUrlForNodeName(node);
|
||||
String url = baseUrl.replace("/solr", "/api") + "/node/files" + entry.getPath() + "?getFrom=";
|
||||
String url = baseUrl.replace("/solr", "/api") + "/node/files" + info.path + "?getFrom=";
|
||||
if (i < FETCHFROM_SRC) {
|
||||
// this is to protect very large clusters from overwhelming a single node
|
||||
// the first FETCHFROM_SRC nodes will be asked to fetch from this node.
|
||||
|
|
@ -369,12 +379,11 @@ public class DistribPackageStore implements PackageStore {
|
|||
try {
|
||||
Thread.sleep(10 * 1000);
|
||||
} finally {
|
||||
tmpFiles.remove(entry.getPath());
|
||||
tmpFiles.remove(info.path);
|
||||
}
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -428,6 +437,15 @@ public class DistribPackageStore implements PackageStore {
|
|||
new FileInfo(path).readData(consumer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void syncToAllNodes(String path) throws IOException {
|
||||
FileInfo fi = new FileInfo(path);
|
||||
if (!fi.exists(true, false)) {
|
||||
throw new SolrException(BAD_REQUEST, "No such file : " + path);
|
||||
}
|
||||
fi.getFileData(true);
|
||||
distribute(fi);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List list(String path, Predicate<String> predicate) {
|
||||
|
|
@ -451,6 +469,28 @@ public class DistribPackageStore implements PackageStore {
|
|||
return fileDetails;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refresh(String path) {
|
||||
try {
|
||||
List l = null;
|
||||
try {
|
||||
l = coreContainer.getZkController().getZkClient().getChildren(ZK_PACKAGESTORE+ path, null, true);
|
||||
} catch (KeeperException.NoNodeException e) {
|
||||
// does not matter
|
||||
}
|
||||
if (l != null && !l.isEmpty()) {
|
||||
List myFiles = list(path, s -> true);
|
||||
for (Object f : l) {
|
||||
if (!myFiles.contains(f)) {
|
||||
log.info("{} does not exist locally, downloading.. ",f);
|
||||
fetch(path + "/" + f.toString(), "*");
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Could not refresh files in " +path, e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileType getType(String path, boolean fetchMissing) {
|
||||
|
|
@ -460,12 +500,16 @@ public class DistribPackageStore implements PackageStore {
|
|||
file = getRealpath(path).toFile();
|
||||
}
|
||||
}
|
||||
return _getFileType(file);
|
||||
}
|
||||
|
||||
public static FileType _getFileType(File file) {
|
||||
if (!file.exists()) return FileType.NOFILE;
|
||||
if (file.isDirectory()) return FileType.DIRECTORY;
|
||||
return isMetaDataFile(file.getName()) ? FileType.METADATA : FileType.FILE;
|
||||
}
|
||||
|
||||
private boolean isMetaDataFile(String file) {
|
||||
public static boolean isMetaDataFile(String file) {
|
||||
return file.charAt(0) == '.' && file.endsWith(".json");
|
||||
}
|
||||
|
||||
|
|
@ -486,4 +530,61 @@ public class DistribPackageStore implements PackageStore {
|
|||
public static Path getPackageStoreDirPath(Path solrHome) {
|
||||
return Paths.get(solrHome.toAbsolutePath().toString(), PackageStoreAPI.PACKAGESTORE_DIRECTORY).toAbsolutePath();
|
||||
}
|
||||
|
||||
private static String _getMetapath(String path) {
|
||||
int idx = path.lastIndexOf('/');
|
||||
return path.substring(0, idx + 1) + "." + path.substring(idx + 1) + ".json";
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal API
|
||||
*/
|
||||
public static void _persistToFile(Path solrHome, String path, ByteBuffer data, ByteBuffer meta) throws IOException {
|
||||
Path realpath = _getRealPath(path, solrHome);
|
||||
File file = realpath.toFile();
|
||||
File parent = file.getParentFile();
|
||||
if (!parent.exists()) {
|
||||
parent.mkdirs();
|
||||
}
|
||||
Map m = (Map) Utils.fromJSON(meta.array(), meta.arrayOffset(), meta.limit());
|
||||
if (m == null || m.isEmpty()) {
|
||||
throw new SolrException(SERVER_ERROR, "invalid metadata , discarding : " + path);
|
||||
}
|
||||
|
||||
|
||||
File metdataFile = _getRealPath(_getMetapath(path), solrHome).toFile();
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(metdataFile)) {
|
||||
fos.write(meta.array(), 0, meta.limit());
|
||||
}
|
||||
IOUtils.fsync(metdataFile.toPath(), false);
|
||||
|
||||
try (FileOutputStream fos = new FileOutputStream(file)) {
|
||||
fos.write(data.array(), 0, data.limit());
|
||||
}
|
||||
IOUtils.fsync(file.toPath(), false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, byte[]> getKeys() throws IOException {
|
||||
return _getKeys(solrhome);
|
||||
}
|
||||
|
||||
|
||||
// reads local keys file
|
||||
private static Map<String, byte[]> _getKeys(Path solrhome) throws IOException {
|
||||
Map<String, byte[]> result = new HashMap<>();
|
||||
Path keysDir = _getRealPath(PackageStoreAPI.KEYS_DIR, solrhome);
|
||||
|
||||
File[] keyFiles = keysDir.toFile().listFiles();
|
||||
if (keyFiles == null) return result;
|
||||
for (File keyFile : keyFiles) {
|
||||
if (keyFile.isFile() && !isMetaDataFile(keyFile.getName())) {
|
||||
try (InputStream fis = new FileInputStream(keyFile)) {
|
||||
result.put(keyFile.getName(), SimplePostTool.inputStreamToByteArray(fis).array());
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
|
|||
import java.nio.file.Path;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
|
||||
|
|
@ -55,6 +56,10 @@ public interface PackageStore {
|
|||
|
||||
List<FileDetails> list(String path, Predicate<String> predicate);
|
||||
|
||||
/** Sync a local file to all nodes. All the nodes are asked to pull the file from this node
|
||||
*/
|
||||
void syncToAllNodes(String path) throws IOException;
|
||||
|
||||
/**
|
||||
* get the real path on filesystem
|
||||
*/
|
||||
|
|
@ -65,6 +70,15 @@ public interface PackageStore {
|
|||
*/
|
||||
FileType getType(String path, boolean fetchMissing);
|
||||
|
||||
/**Get all the keys in the package store. The data is a .DER file content
|
||||
*/
|
||||
Map<String,byte[]> getKeys() throws IOException;
|
||||
|
||||
/**Refresh the files in a path. May be this node does not have all files
|
||||
* @param path the path to be refreshed.
|
||||
*/
|
||||
void refresh(String path);
|
||||
|
||||
public class FileEntry {
|
||||
final ByteBuffer buf;
|
||||
final MetaData meta;
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ import org.apache.commons.codec.digest.DigestUtils;
|
|||
import org.apache.solr.api.Command;
|
||||
import org.apache.solr.api.EndPoint;
|
||||
import org.apache.solr.client.solrj.SolrRequest;
|
||||
import org.apache.solr.cloud.CloudUtil;
|
||||
import org.apache.solr.common.MapWriter;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
|
|
@ -66,6 +65,8 @@ import static org.apache.solr.handler.ReplicationHandler.FILE_STREAM;
|
|||
public class PackageStoreAPI {
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
public static final String PACKAGESTORE_DIRECTORY = "filestore";
|
||||
public static final String TRUSTED_DIR = "_trusted_";
|
||||
public static final String KEYS_DIR = "/_trusted_/keys";
|
||||
|
||||
|
||||
private final CoreContainer coreContainer;
|
||||
|
|
@ -99,7 +100,7 @@ public class PackageStoreAPI {
|
|||
try {
|
||||
PackageStore.FileType type = packageStore.getType(path, true);
|
||||
if (type != PackageStore.FileType.FILE) {
|
||||
errs.accept("No such file : " + path);
|
||||
errs.accept("No such file: " + path);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -111,9 +112,10 @@ public class PackageStoreAPI {
|
|||
}
|
||||
if (validateSignatures) {
|
||||
try {
|
||||
validate(entry.meta.signatures, entry);
|
||||
} catch (SolrException e) {
|
||||
log.error("error validating package artifact", e);
|
||||
packageStore.refresh(KEYS_DIR);
|
||||
validate(entry.meta.signatures, entry, false);
|
||||
} catch (Exception e) {
|
||||
log.error("Error validating package artifact", e);
|
||||
errs.accept(e.getMessage());
|
||||
}
|
||||
}
|
||||
|
|
@ -136,7 +138,7 @@ public class PackageStoreAPI {
|
|||
|
||||
@Command
|
||||
public void upload(SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
if(!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
|
||||
if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) {
|
||||
throw new RuntimeException(PackageAPI.ERR_MSG);
|
||||
}
|
||||
try {
|
||||
|
|
@ -149,22 +151,17 @@ public class PackageStoreAPI {
|
|||
if (path == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No path");
|
||||
}
|
||||
validateName(path);
|
||||
validateName(path, true);
|
||||
ContentStream stream = streams.iterator().next();
|
||||
try {
|
||||
ByteBuffer buf = SimplePostTool.inputStreamToByteArray(stream.getStream());
|
||||
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));
|
||||
List<String> signatures = readSignatures(req, buf);
|
||||
Map<String, Object> vals = new HashMap<>();
|
||||
vals.put(MetaData.SHA512, sha512);
|
||||
if (signatures != null) {
|
||||
vals.put("sig", signatures);
|
||||
}
|
||||
MetaData meta = _createJsonMetaData(buf, signatures);
|
||||
PackageStore.FileType type = packageStore.getType(path, true);
|
||||
if(type != PackageStore.FileType.NOFILE) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Path already exists "+ path);
|
||||
}
|
||||
packageStore.put(new PackageStore.FileEntry(buf, new MetaData(vals), path));
|
||||
packageStore.put(new PackageStore.FileEntry(buf, meta, path));
|
||||
rsp.add(CommonParams.FILE, path);
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
|
||||
|
|
@ -189,24 +186,24 @@ public class PackageStoreAPI {
|
|||
String[] signatures = req.getParams().getParams("sig");
|
||||
if (signatures == null || signatures.length == 0) return null;
|
||||
List<String> sigs = Arrays.asList(signatures);
|
||||
packageStore.refresh(KEYS_DIR);
|
||||
validate(sigs, buf);
|
||||
return sigs;
|
||||
}
|
||||
|
||||
public void validate(List<String> sigs,
|
||||
ByteBuffer buf) throws SolrException, IOException {
|
||||
Map<String, byte[]> keys = CloudUtil.getTrustedKeys(
|
||||
coreContainer.getZkController().getZkClient(), "exe");
|
||||
private void validate(List<String> sigs,
|
||||
ByteBuffer buf) throws SolrException, IOException {
|
||||
Map<String, byte[]> keys = packageStore.getKeys();
|
||||
if (keys == null || keys.isEmpty()) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"ZK does not have any keys");
|
||||
"package store does not have any keys");
|
||||
}
|
||||
CryptoKeys cryptoKeys = null;
|
||||
try {
|
||||
cryptoKeys = new CryptoKeys(keys);
|
||||
} catch (Exception e) {
|
||||
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
|
||||
"Error parsing public keys in ZooKeeper");
|
||||
"Error parsing public keys in Package store");
|
||||
}
|
||||
for (String sig : sigs) {
|
||||
if (cryptoKeys.verify(sig, buf) == null) {
|
||||
|
|
@ -219,6 +216,20 @@ public class PackageStoreAPI {
|
|||
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a JSON string with the metadata
|
||||
* @lucene.internal
|
||||
*/
|
||||
public static MetaData _createJsonMetaData(ByteBuffer buf, List<String> signatures) throws IOException {
|
||||
String sha512 = DigestUtils.sha512Hex(new ByteBufferInputStream(buf));
|
||||
Map<String, Object> vals = new HashMap<>();
|
||||
vals.put(MetaData.SHA512, sha512);
|
||||
if (signatures != null) {
|
||||
vals.put("sig", signatures);
|
||||
}
|
||||
return new MetaData(vals);
|
||||
}
|
||||
|
||||
@EndPoint(
|
||||
path = "/node/files/*",
|
||||
method = SolrRequest.METHOD.GET,
|
||||
|
|
@ -228,6 +239,14 @@ public class PackageStoreAPI {
|
|||
public void read(SolrQueryRequest req, SolrQueryResponse rsp) {
|
||||
String path = req.getPathTemplateValues().get("*");
|
||||
String pathCopy = path;
|
||||
if (req.getParams().getBool("sync", false)) {
|
||||
try {
|
||||
packageStore.syncToAllNodes(path);
|
||||
return;
|
||||
} catch (IOException e) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error getting file ", e);
|
||||
}
|
||||
}
|
||||
String getFrom = req.getParams().get("getFrom");
|
||||
if (getFrom != null) {
|
||||
coreContainer.getUpdateShardHandler().getUpdateExecutor().submit(() -> {
|
||||
|
|
@ -287,7 +306,7 @@ public class PackageStoreAPI {
|
|||
|
||||
}
|
||||
|
||||
static class MetaData implements MapWriter {
|
||||
public static class MetaData implements MapWriter {
|
||||
public static final String SHA512 = "sha512";
|
||||
String sha512;
|
||||
List<String> signatures;
|
||||
|
|
@ -312,7 +331,7 @@ public class PackageStoreAPI {
|
|||
|
||||
static final String INVALIDCHARS = " /\\#&*\n\t%@~`=+^$><?{}[]|:;!";
|
||||
|
||||
public static void validateName(String path) {
|
||||
public static void validateName(String path, boolean failForTrusted) {
|
||||
if (path == null) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "empty path");
|
||||
}
|
||||
|
|
@ -328,15 +347,34 @@ public class PackageStoreAPI {
|
|||
}
|
||||
}
|
||||
}
|
||||
if (failForTrusted && TRUSTED_DIR.equals(parts.get(0))) {
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "trying to write into /_trusted_/ directory");
|
||||
}
|
||||
}
|
||||
|
||||
/**Validate a file for signature
|
||||
*
|
||||
* @param sigs the signatures. atleast one should succeed
|
||||
* @param entry The file details
|
||||
* @param isFirstAttempt If there is a failure
|
||||
*/
|
||||
public void validate(List<String> sigs,
|
||||
PackageStore.FileEntry entry) throws SolrException {
|
||||
Map<String, byte[]> keys = CloudUtil.getTrustedKeys(
|
||||
coreContainer.getZkController().getZkClient(), "exe");
|
||||
PackageStore.FileEntry entry,
|
||||
boolean isFirstAttempt) throws SolrException, IOException {
|
||||
if (!isFirstAttempt) {
|
||||
//we are retrying because last validation failed.
|
||||
// get all keys again and try again
|
||||
packageStore.refresh(KEYS_DIR);
|
||||
}
|
||||
|
||||
Map<String, byte[]> keys = packageStore.getKeys();
|
||||
if (keys == null || keys.isEmpty()) {
|
||||
if(isFirstAttempt) {
|
||||
validate(sigs, entry, false);
|
||||
return;
|
||||
}
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
|
||||
"ZooKeeper does not have any public keys");
|
||||
"Packagestore does not have any public keys");
|
||||
}
|
||||
CryptoKeys cryptoKeys = null;
|
||||
try {
|
||||
|
|
@ -346,14 +384,22 @@ public class PackageStoreAPI {
|
|||
"Error parsing public keys in ZooKeeper");
|
||||
}
|
||||
for (String sig : sigs) {
|
||||
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig + "sha256 "+ entry.getMetaData().sha512;
|
||||
Supplier<String> errMsg = () -> "Signature does not match any public key : " + sig + "sha256 " + entry.getMetaData().sha512;
|
||||
if (entry.getBuffer() != null) {
|
||||
if (cryptoKeys.verify(sig, entry.getBuffer()) == null) {
|
||||
if(isFirstAttempt) {
|
||||
validate(sigs, entry, false);
|
||||
return;
|
||||
}
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
|
||||
}
|
||||
} else {
|
||||
InputStream inputStream = entry.getInputStream();
|
||||
if (cryptoKeys.verify(sig, inputStream) == null) {
|
||||
if(isFirstAttempt) {
|
||||
validate(sigs, entry, false);
|
||||
return;
|
||||
}
|
||||
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, errMsg.get());
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import java.util.zip.ZipEntry;
|
|||
import java.util.zip.ZipFile;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.lucene.util.SuppressForbidden;
|
||||
|
|
@ -39,7 +40,10 @@ import org.apache.solr.common.SolrException;
|
|||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.core.BlobRepository;
|
||||
import org.apache.solr.filestore.DistribPackageStore;
|
||||
import org.apache.solr.filestore.PackageStoreAPI;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Manifest;
|
||||
import org.apache.solr.util.SolrJacksonAnnotationInspector;
|
||||
|
||||
|
|
@ -137,7 +141,12 @@ public class PackageUtils {
|
|||
*/
|
||||
public static String getJsonStringFromUrl(HttpClient client, String url) {
|
||||
try {
|
||||
return IOUtils.toString(client.execute(new HttpGet(url)).getEntity().getContent(), "UTF-8");
|
||||
HttpResponse resp = client.execute(new HttpGet(url));
|
||||
if (resp.getStatusLine().getStatusCode() != 200) {
|
||||
throw new SolrException(ErrorCode.NOT_FOUND,
|
||||
"Error (code="+resp.getStatusLine().getStatusCode()+") fetching from URL: "+url);
|
||||
}
|
||||
return IOUtils.toString(resp.getEntity().getContent(), "UTF-8");
|
||||
} catch (UnsupportedOperationException | IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
|
@ -244,4 +253,11 @@ public class PackageUtils {
|
|||
public static String getCollectionParamsPath(String collection) {
|
||||
return "/api/collections/" + collection + "/config/params";
|
||||
}
|
||||
|
||||
public static void uploadKey(byte bytes[], String path, Path home, HttpSolrClient client) throws IOException {
|
||||
ByteBuffer buf = ByteBuffer.wrap(bytes);
|
||||
PackageStoreAPI.MetaData meta = PackageStoreAPI._createJsonMetaData(buf, null);
|
||||
DistribPackageStore._persistToFile(home, path, buf, ByteBuffer.wrap(Utils.toJSON(meta)));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,10 +22,10 @@ import static org.apache.solr.packagemanager.PackageUtils.getMapper;
|
|||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
|
|
@ -48,10 +48,12 @@ import org.apache.solr.common.SolrException;
|
|||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.cloud.SolrZkClient;
|
||||
import org.apache.solr.core.BlobRepository;
|
||||
import org.apache.solr.filestore.PackageStoreAPI;
|
||||
import org.apache.solr.packagemanager.SolrPackage.Artifact;
|
||||
import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease;
|
||||
import org.apache.solr.pkg.PackageAPI;
|
||||
import org.apache.solr.pkg.PackagePluginHolder;
|
||||
import org.apache.solr.util.SolrCLI;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -116,24 +118,31 @@ public class RepositoryManager {
|
|||
/**
|
||||
* Add a repository to Solr
|
||||
*/
|
||||
public void addRepository(String name, String uri) throws KeeperException, InterruptedException, MalformedURLException, IOException {
|
||||
public void addRepository(String repoName, String uri) throws Exception {
|
||||
String existingRepositoriesJson = getRepositoriesJson(packageManager.zkClient);
|
||||
log.info(existingRepositoriesJson);
|
||||
|
||||
List<PackageRepository> repos = getMapper().readValue(existingRepositoriesJson, List.class);
|
||||
repos.add(new DefaultPackageRepository(name, uri));
|
||||
repos.add(new DefaultPackageRepository(repoName, uri));
|
||||
if (packageManager.zkClient.exists(PackageUtils.REPOSITORIES_ZK_PATH, true) == false) {
|
||||
packageManager.zkClient.create(PackageUtils.REPOSITORIES_ZK_PATH, getMapper().writeValueAsString(repos).getBytes("UTF-8"), CreateMode.PERSISTENT, true);
|
||||
} else {
|
||||
packageManager.zkClient.setData(PackageUtils.REPOSITORIES_ZK_PATH, getMapper().writeValueAsString(repos).getBytes("UTF-8"), true);
|
||||
}
|
||||
|
||||
if (packageManager.zkClient.exists("/keys", true)==false) packageManager.zkClient.create("/keys", new byte[0], CreateMode.PERSISTENT, true);
|
||||
if (packageManager.zkClient.exists("/keys/exe", true)==false) packageManager.zkClient.create("/keys/exe", new byte[0], CreateMode.PERSISTENT, true);
|
||||
if (packageManager.zkClient.exists("/keys/exe/" + name + ".der", true)==false) {
|
||||
packageManager.zkClient.create("/keys/exe/" + name + ".der", new byte[0], CreateMode.PERSISTENT, true);
|
||||
}
|
||||
packageManager.zkClient.setData("/keys/exe/" + name + ".der", IOUtils.toByteArray(new URL(uri + "/publickey.der").openStream()), true);
|
||||
addKey(IOUtils.toByteArray(new URL(uri + "/publickey.der").openStream()), repoName + ".der");
|
||||
}
|
||||
|
||||
public void addKey(byte[] key, String destinationKeyFilename) throws Exception {
|
||||
// get solr_home directory from info servlet
|
||||
String systemInfoUrl = solrClient.getBaseURL() + "/solr/admin/info/system";
|
||||
Map<String,Object> systemInfo = SolrCLI.getJson(solrClient.getHttpClient(), systemInfoUrl, 2, true);
|
||||
String solrHome = (String) systemInfo.get("solr_home");
|
||||
|
||||
// put the public key into package store's trusted key store and request a sync.
|
||||
String path = PackageStoreAPI.KEYS_DIR + "/" + destinationKeyFilename;
|
||||
PackageUtils.uploadKey(key, path, Paths.get(solrHome), solrClient);
|
||||
PackageUtils.getJsonStringFromUrl(solrClient.getHttpClient(), solrClient.getBaseURL() + "/api/node/files" + path + "?sync=true");
|
||||
}
|
||||
|
||||
private String getRepositoriesJson(SolrZkClient zkClient) throws UnsupportedEncodingException, KeeperException, InterruptedException {
|
||||
|
|
|
|||
|
|
@ -168,7 +168,14 @@ public class PackageLoader implements Closeable {
|
|||
Version version = myVersions.get(v.version);
|
||||
if (version == null) {
|
||||
log.info("A new version: {} added for package: {} with artifacts {}", v.version, this.name, v.files);
|
||||
myVersions.put(v.version, new Version(this, v));
|
||||
Version ver = null;
|
||||
try {
|
||||
ver = new Version(this, v);
|
||||
} catch (Exception e) {
|
||||
log.error("package could not be loaded "+ ver.toString(), e);
|
||||
continue;
|
||||
}
|
||||
myVersions.put(v.version, ver);
|
||||
sortedVersions.add(v.version);
|
||||
}
|
||||
}
|
||||
|
|
@ -248,9 +255,13 @@ public class PackageLoader implements Closeable {
|
|||
this.parent = parent;
|
||||
this.version = v;
|
||||
List<Path> paths = new ArrayList<>();
|
||||
|
||||
List<String> errs = new ArrayList<>();
|
||||
coreContainer.getPackageStoreAPI().validateFiles(version.files, true, s -> errs.add(s));
|
||||
if(!errs.isEmpty()) {
|
||||
throw new RuntimeException("Cannot load package: " +errs);
|
||||
}
|
||||
for (String file : version.files) {
|
||||
//ensure that the files are downloaded and available
|
||||
coreContainer.getPackageStoreAPI().getPackageStore().fetch(file,null);
|
||||
paths.add(coreContainer.getPackageStoreAPI().getPackageStore().getRealpath(file));
|
||||
}
|
||||
|
||||
|
|
@ -283,6 +294,11 @@ public class PackageLoader implements Closeable {
|
|||
closeWhileHandlingException(loader);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return jsonStr();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -19,12 +19,15 @@ package org.apache.solr.util;
|
|||
import static org.apache.solr.packagemanager.PackageUtils.printGreen;
|
||||
import static org.apache.solr.packagemanager.PackageUtils.print;
|
||||
|
||||
import java.io.File;
|
||||
import java.lang.invoke.MethodHandles;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.cli.OptionBuilder;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.logging.log4j.Level;
|
||||
import org.apache.logging.log4j.core.config.Configurator;
|
||||
|
|
@ -44,7 +47,6 @@ import org.apache.solr.util.SolrCLI.StatusTool;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
|
||||
public class PackageTool extends SolrCLI.ToolBase {
|
||||
|
||||
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
|
||||
|
|
@ -92,6 +94,10 @@ public class PackageTool extends SolrCLI.ToolBase {
|
|||
repositoryManager.addRepository(repoName, repoUrl);
|
||||
PackageUtils.printGreen("Added repository: " + repoName);
|
||||
break;
|
||||
case "add-key":
|
||||
String keyFilename = cli.getArgs()[1];
|
||||
repositoryManager.addKey(FileUtils.readFileToByteArray(new File(keyFilename)), Paths.get(keyFilename).getFileName().toString());
|
||||
break;
|
||||
case "list-installed":
|
||||
PackageUtils.printGreen("Installed packages:\n-----");
|
||||
for (SolrPackageInstance pkg: packageManager.fetchInstalledPackageInstances()) {
|
||||
|
|
|
|||
|
|
@ -44,8 +44,8 @@ import org.apache.solr.common.params.CommonParams;
|
|||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.packagemanager.PackageUtils;
|
||||
import org.apache.solr.util.LogLevel;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.server.ByteBufferInputStream;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
|
@ -75,8 +75,9 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
|
|||
try {
|
||||
|
||||
byte[] derFile = readFile("cryptokeys/pub_key512.der");
|
||||
cluster.getZkClient().makePath("/keys/exe", true);
|
||||
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);
|
||||
// cluster.getZkClient().makePath("/keys/exe", true);
|
||||
// cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
|
||||
try {
|
||||
postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"),
|
||||
|
|
@ -247,7 +248,16 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
|
|||
return rsp;
|
||||
}
|
||||
|
||||
|
||||
public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster cluster) throws Exception {
|
||||
JettySolrRunner jetty = cluster.getRandomJetty(random());
|
||||
try(HttpSolrClient client = (HttpSolrClient) jetty.newClient()) {
|
||||
PackageUtils.uploadKey(bytes, path, jetty.getCoreContainer().getResourceLoader().getInstancePath(), client);
|
||||
Object resp = Utils.executeGET(client.getHttpClient(), jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true", null);
|
||||
System.out.println("sync resp: "+jetty.getBaseURLV2().toString() + "/node/files" + path + "?sync=true"+" ,is: "+resp);
|
||||
}
|
||||
waitForAllNodesHaveFile(cluster,path, Utils.makeMap(":files:" + path + ":name", (Predicate<Object>) Objects::nonNull),
|
||||
false);
|
||||
}
|
||||
|
||||
public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig)
|
||||
throws SolrServerException, IOException {
|
||||
|
|
|
|||
|
|
@ -50,9 +50,9 @@ import org.apache.solr.common.params.MapSolrParams;
|
|||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.util.Utils;
|
||||
import org.apache.solr.filestore.PackageStoreAPI;
|
||||
import org.apache.solr.filestore.TestDistribPackageStore;
|
||||
import org.apache.solr.util.LogLevel;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.data.Stat;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
|
|
@ -63,6 +63,7 @@ import static org.apache.solr.common.params.CommonParams.JAVABIN;
|
|||
import static org.apache.solr.common.params.CommonParams.WT;
|
||||
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
|
||||
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
|
||||
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
|
||||
import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile;
|
||||
|
||||
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
|
||||
|
|
@ -95,8 +96,9 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
String EXPR1 = "/mypkg/expressible.jar";
|
||||
String COLLECTION_NAME = "testPluginLoadingColl";
|
||||
byte[] derFile = readFile("cryptokeys/pub_key512.der");
|
||||
cluster.getZkClient().makePath("/keys/exe", true);
|
||||
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);
|
||||
// cluster.getZkClient().makePath("/keys/exe", true);
|
||||
// cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE1,
|
||||
"L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
|
||||
|
||||
|
|
@ -459,7 +461,7 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
.build();
|
||||
|
||||
//the files is not yet there. The command should fail with error saying "No such file"
|
||||
expectError(req, cluster.getSolrClient(), errPath, "No such file :");
|
||||
expectError(req, cluster.getSolrClient(), errPath, "No such file:");
|
||||
|
||||
|
||||
//post the jar file. No signature is sent
|
||||
|
|
@ -471,20 +473,19 @@ public class TestPackages extends SolrCloudTestCase {
|
|||
FILE1 + " has no signature");
|
||||
//now we upload the keys
|
||||
byte[] derFile = readFile("cryptokeys/pub_key512.der");
|
||||
cluster.getZkClient().makePath("/keys/exe", true);
|
||||
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
uploadKey(derFile, PackageStoreAPI.KEYS_DIR+"/pub_key512.der", cluster);
|
||||
//and upload the same file with a different name but it has proper signature
|
||||
postFileAndWait(cluster, "runtimecode/runtimelibs.jar.bin", FILE2,
|
||||
"L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ==");
|
||||
// with correct signature
|
||||
//after uploading the file, let's delete the keys to see if we get proper error message
|
||||
cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true);
|
||||
// cluster.getZkClient().delete("/keys/exe/pub_key512.der", -1, true);
|
||||
add.files = Arrays.asList(new String[]{FILE2});
|
||||
expectError(req, cluster.getSolrClient(), errPath,
|
||||
"ZooKeeper does not have any public keys");
|
||||
/*expectError(req, cluster.getSolrClient(), errPath,
|
||||
"ZooKeeper does not have any public keys");*/
|
||||
|
||||
//Now lets' put the keys back
|
||||
cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
// cluster.getZkClient().create("/keys/exe/pub_key512.der", derFile, CreateMode.PERSISTENT, true);
|
||||
|
||||
//this time we have a file with proper signature, public keys are in ZK
|
||||
// so the add {} command should succeed
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ The package manager (CLI) internally uses various Solr APIs to install, deploy a
|
|||
|
||||
At the heart of the system, we have classloader isolation. To achieve this, the system is simplified into two layered classloaders:
|
||||
The root classloader which has all the jars from Solr classpath. This requires Solr node restart to change anything.
|
||||
A set of named classloaders that inherit from the root classloader. The life cycles of the named classloaders are tied to the package configuration in ZK. As soon as the configuration is modified, the corresponding classloaders are reloaded and components are asked to reload.
|
||||
A set of named classloaders that inherit from the root classloader. The life cycles of the named classloaders are tied to the package configuration in ZooKeeper. As soon as the configuration is modified, the corresponding classloaders are reloaded and components are asked to reload.
|
||||
|
||||
== Package Loading Security
|
||||
Packages are disabled by default. Start all your nodes with the system property `-Denable.packages=true` to use this feature.
|
||||
|
|
@ -28,7 +28,7 @@ $ bin/solr -c -Denable.packages=true
|
|||
----
|
||||
|
||||
=== Upload Your Keys
|
||||
Package binaries must be signed with your private keys and ensure your public keys are published in ZooKeeper.
|
||||
Package binaries must be signed with your private keys and ensure your public keys are published in package store's trusted store.
|
||||
|
||||
*Example*
|
||||
[source,bash]
|
||||
|
|
@ -36,9 +36,8 @@ Package binaries must be signed with your private keys and ensure your public ke
|
|||
$ openssl genrsa -out my_key.pem 512
|
||||
# create the public key in .der format
|
||||
$ openssl rsa -in my_key.pem -pubout -outform DER -out my_key.der
|
||||
# upload to ZooKeeper
|
||||
$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd makepath /keys/exe/
|
||||
$ server/scripts/cloud-scripts/zkcli.sh -zkhost 127.0.0.1:9983 -cmd putfile /keys/exe/my_key.der my_key.der
|
||||
# upload key to package store
|
||||
$ bin/solr package add-key my_key.der
|
||||
----
|
||||
|
||||
== Package Store
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
The package manager in Solr allows installation and update of Solr-specific packages in distributed and standalone environments.
|
||||
The package manager in Solr allows installation and updating of Solr-specific packages in Solr's cluster environment.
|
||||
|
||||
In this system, a _package_ is a set of Java jar files (usually one) containing one or more <<solr-plugins.adoc#solr-plugins,Solr plugins>>. Each jar file is also accompanied by a signature string (which can be verified against a supplied public key).
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ A key design aspect of this system is the ability to install or update packages
|
|||
|
||||
Other elements of the design include the ability to install from a remote repository; package standardization; a command line interface (CLI); and a package store.
|
||||
|
||||
This section will focus on how to use the package manager to install and update plugins.
|
||||
This section will focus on how to use the package manager to install and update packages.
|
||||
For technical details, see the section <<package-manager-internals.adoc#package-manager-internals,Package Manager internals>>.
|
||||
|
||||
== Interacting with the Package Manager
|
||||
|
|
@ -158,6 +158,6 @@ You can run the `list-deployed` command to verify that this collection is using
|
|||
|
||||
== Security
|
||||
|
||||
As noted above in the section <<Add Trusted Repositories>>, the `add-repo` step should only be executed using an HTTPS endpoint in Solr (all other steps can be executed using HTTP - see also <<package-manager-internals.adoc#package-manager-internals,Package Manager Internals>>). This step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to ZooKeeper. It is critical to protect ZooKeeper from unauthorized write access.
|
||||
The `add-repo` step should only be executed using HTTPS enabled repository urls only so as to prevent against MITM attacks when Solr is fetching the public key for the repository. This `add-repo` step registers the public key of the trusted repository, and hence can only be executed using the package manager (CLI) having direct write access to the trusted store of the package store (a special location in the package store that cannot be written to using the package store APIs). Also, it is critical to protect ZooKeeper from unauthorized write access.
|
||||
|
||||
Also, keep in mind, that it is possible to install *any* package from a repository once it has been added. If you want to use some packages in production, a best practice is to setup your own repository and add that to Solr instead of adding a generic third-party repository that is beyond your administrative control.
|
||||
Also, keep in mind, that it is possible to install *any* package from a repository once it has been added. If you want to use some packages in production, a best practice is to setup your own repository and add that to Solr instead of adding a generic third-party repository that is beyond your administrative control. You might want to re-sign packages from a third-party repository using your own private keys and host them at your own repository.
|
||||
|
|
|
|||
Loading…
Reference in New Issue