SOLR-14223 Create RSAKeyPair from disk (#1217)

* Create properties for PublicKeyHandler to read existing keys from disk
* Move pregenerated keys from core/test-files to test-framework
* Update tests to use existing keys instead of new keys each run
This commit is contained in:
Mike 2020-02-24 12:07:10 -08:00 committed by GitHub
parent 7ba9d4d756
commit 1770797387
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 214 additions and 44 deletions

View File

@ -16,6 +16,8 @@ Improvements
----------------------
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
* SOLR-14223: PKI Auth can bootstrap from existing key files instead of creating new keys on startup (Mike Drob)
Other Changes
----------------------
* SOLR-10288: Remove non-minified JavaScript from the webapp. (Erik Hatcher, marcussorealheis)

View File

@ -46,10 +46,14 @@ public class CloudConfig {
private final boolean createCollectionCheckLeaderActive;
private final String pkiHandlerPrivateKeyPath;
private final String pkiHandlerPublicKeyPath;
CloudConfig(String zkHost, int zkClientTimeout, int hostPort, String hostName, String hostContext, boolean useGenericCoreNames,
int leaderVoteWait, int leaderConflictResolveWait, int autoReplicaFailoverWaitAfterExpiration,
String zkCredentialsProviderClass, String zkACLProviderClass, int createCollectionWaitTimeTillActive,
boolean createCollectionCheckLeaderActive) {
boolean createCollectionCheckLeaderActive, String pkiHandlerPrivateKeyPath, String pkiHandlerPublicKeyPath) {
this.zkHost = zkHost;
this.zkClientTimeout = zkClientTimeout;
this.hostPort = hostPort;
@ -63,6 +67,8 @@ public class CloudConfig {
this.zkACLProviderClass = zkACLProviderClass;
this.createCollectionWaitTimeTillActive = createCollectionWaitTimeTillActive;
this.createCollectionCheckLeaderActive = createCollectionCheckLeaderActive;
this.pkiHandlerPrivateKeyPath = pkiHandlerPrivateKeyPath;
this.pkiHandlerPublicKeyPath = pkiHandlerPublicKeyPath;
if (this.hostPort == -1)
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "'hostPort' must be configured to run SolrCloud");
@ -122,6 +128,14 @@ public class CloudConfig {
return createCollectionCheckLeaderActive;
}
public String getPkiHandlerPrivateKeyPath() {
return pkiHandlerPrivateKeyPath;
}
public String getPkiHandlerPublicKeyPath() {
return pkiHandlerPublicKeyPath;
}
public static class CloudConfigBuilder {
private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 45000;
@ -145,6 +159,8 @@ public class CloudConfig {
private String zkACLProviderClass;
private int createCollectionWaitTimeTillActive = DEFAULT_CREATE_COLLECTION_ACTIVE_WAIT;
private boolean createCollectionCheckLeaderActive = DEFAULT_CREATE_COLLECTION_CHECK_LEADER_ACTIVE;
private String pkiHandlerPrivateKeyPath;
private String pkiHandlerPublicKeyPath;
public CloudConfigBuilder(String hostName, int hostPort) {
this(hostName, hostPort, null);
@ -206,10 +222,20 @@ public class CloudConfig {
return this;
}
public CloudConfigBuilder setPkiHandlerPrivateKeyPath(String pkiHandlerPrivateKeyPath) {
this.pkiHandlerPrivateKeyPath = pkiHandlerPrivateKeyPath;
return this;
}
public CloudConfigBuilder setPkiHandlerPublicKeyPath(String pkiHandlerPublicKeyPath) {
this.pkiHandlerPublicKeyPath = pkiHandlerPublicKeyPath;
return this;
}
public CloudConfig build() {
return new CloudConfig(zkHost, zkClientTimeout, hostPort, hostName, hostContext, useGenericCoreNames, leaderVoteWait,
leaderConflictResolveWait, autoReplicaFailoverWaitAfterExpiration, zkCredentialsProviderClass, zkACLProviderClass, createCollectionWaitTimeTillActive,
createCollectionCheckLeaderActive);
createCollectionCheckLeaderActive, pkiHandlerPrivateKeyPath, pkiHandlerPublicKeyPath);
}
}
}

View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.spec.InvalidKeySpecException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@ -331,10 +332,14 @@ public class CoreContainer {
}
public CoreContainer(NodeConfig config, Properties properties, CoresLocator locator, boolean asyncSolrCoreLoad) {
this.cfg = requireNonNull(config);
this.loader = config.getSolrResourceLoader();
this.solrHome = loader.getInstancePath().toString();
containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler());
this.cfg = requireNonNull(config);
try {
containerHandlers.put(PublicKeyHandler.PATH, new PublicKeyHandler(cfg.getCloudConfig()));
} catch (IOException | InvalidKeySpecException e) {
throw new RuntimeException("Bad PublicKeyHandler configuration.", e);
}
if (null != this.cfg.getBooleanQueryMaxClauseCount()) {
IndexSearcher.setMaxClauseCount(this.cfg.getBooleanQueryMaxClauseCount());
}

View File

@ -408,6 +408,12 @@ public class SolrXmlConfig {
case "createCollectionCheckLeaderActive":
builder.setCreateCollectionCheckLeaderActive(Boolean.parseBoolean(value));
break;
case "pkiHandlerPrivateKeyPath":
builder.setPkiHandlerPrivateKeyPath(value);
break;
case "pkiHandlerPublicKeyPath":
builder.setPkiHandlerPublicKeyPath(value);
break;
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown configuration parameter in <solrcloud> section of solr.xml: " + name);
}

View File

@ -17,14 +17,47 @@
package org.apache.solr.security;
import com.google.common.annotations.VisibleForTesting;
import org.apache.solr.common.StringUtils;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.CryptoKeys;
import java.io.IOException;
import java.net.URL;
import java.security.spec.InvalidKeySpecException;
public class PublicKeyHandler extends RequestHandlerBase {
public static final String PATH = "/admin/info/key";
final CryptoKeys.RSAKeyPair keyPair = new CryptoKeys.RSAKeyPair();
final CryptoKeys.RSAKeyPair keyPair;
@VisibleForTesting
public PublicKeyHandler() {
keyPair = new CryptoKeys.RSAKeyPair();
}
public PublicKeyHandler(CloudConfig config) throws IOException, InvalidKeySpecException {
keyPair = createKeyPair(config);
}
private CryptoKeys.RSAKeyPair createKeyPair(CloudConfig config) throws IOException, InvalidKeySpecException {
if (config == null) {
return new CryptoKeys.RSAKeyPair();
}
String publicKey = config.getPkiHandlerPublicKeyPath();
String privateKey = config.getPkiHandlerPrivateKeyPath();
// If both properties unset, then we fall back to generating a new key pair
if (StringUtils.isEmpty(publicKey) && StringUtils.isEmpty(privateKey)) {
return new CryptoKeys.RSAKeyPair();
}
return new CryptoKeys.RSAKeyPair(new URL(privateKey), new URL(publicKey));
}
public String getPublicKey() {
return keyPair.getPublicKeyStr();

View File

@ -24,6 +24,7 @@ import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
@ -35,9 +36,10 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.PublicKey;
import java.security.SecureRandom;
import java.security.Signature;
import java.security.SignatureException;
import java.security.spec.InvalidKeySpecException;
import java.security.spec.PKCS8EncodedKeySpec;
import java.security.spec.X509EncodedKeySpec;
import java.util.Arrays;
import java.util.HashMap;
@ -52,7 +54,7 @@ import org.slf4j.LoggerFactory;
/**A utility class to verify signatures
*
*/
public final class CryptoKeys implements CLIO {
public final class CryptoKeys {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final Map<String, PublicKey> keys;
private Exception exception;
@ -112,10 +114,14 @@ public final class CryptoKeys implements CLIO {
* Create PublicKey from a .DER file
*/
public static PublicKey getX509PublicKey(byte[] buf)
throws Exception {
throws InvalidKeySpecException {
X509EncodedKeySpec spec = new X509EncodedKeySpec(buf);
KeyFactory kf = KeyFactory.getInstance("RSA");
return kf.generatePublic(spec);
try {
KeyFactory kf = KeyFactory.getInstance("RSA");
return kf.generatePublic(spec);
} catch (NoSuchAlgorithmException e) {
throw new AssertionError("JVM spec is required to support RSA", e);
}
}
/**
@ -243,7 +249,7 @@ public final class CryptoKeys implements CLIO {
public static String decodeAES(String base64CipherTxt, String pwd, final int keySizeBits) {
final Charset ASCII = Charset.forName("ASCII");
final Charset ASCII = StandardCharsets.US_ASCII;
final int INDEX_KEY = 0;
final int INDEX_IV = 1;
final int ITERATIONS = 1;
@ -309,7 +315,7 @@ public final class CryptoKeys implements CLIO {
}
public static byte[] decryptRSA(byte[] buffer, PublicKey pubKey) throws InvalidKeyException, BadPaddingException, IllegalBlockSizeException {
Cipher rsaCipher = null;
Cipher rsaCipher;
try {
rsaCipher = Cipher.getInstance("RSA/ECB/nopadding");
} catch (Exception e) {
@ -317,36 +323,60 @@ public final class CryptoKeys implements CLIO {
}
rsaCipher.init(Cipher.DECRYPT_MODE, pubKey);
return rsaCipher.doFinal(buffer, 0, buffer.length);
}
public static class RSAKeyPair {
private final String pubKeyStr;
private final PublicKey publicKey;
private final PrivateKey privateKey;
private final SecureRandom random = new SecureRandom();
// If this ever comes back to haunt us see the discussion at
// SOLR-9609 for background and code allowing this to go
// into security.json. Also see SOLR-12103.
private static final int DEFAULT_KEYPAIR_LENGTH = 2048;
/**
* Create an RSA key pair with newly generated keys.
*/
public RSAKeyPair() {
KeyPairGenerator keyGen = null;
KeyPairGenerator keyGen;
try {
keyGen = KeyPairGenerator.getInstance("RSA");
} catch (NoSuchAlgorithmException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
throw new AssertionError("JVM spec is required to support RSA", e);
}
keyGen.initialize(DEFAULT_KEYPAIR_LENGTH);
java.security.KeyPair keyPair = keyGen.genKeyPair();
privateKey = keyPair.getPrivate();
publicKey = keyPair.getPublic();
pubKeyStr = Base64.byteArrayToBase64(publicKey.getEncoded());
}
X509EncodedKeySpec x509EncodedKeySpec = new X509EncodedKeySpec(
publicKey.getEncoded());
/**
* Initialize an RSA key pair from previously saved keys. The formats listed below have been tested, other formats may
* also be acceptable but are not guaranteed to work.
* @param privateKeyResourceName path to private key file, encoded as a PKCS#8 in a PEM file
* @param publicKeyResourceName path to public key file, encoded as X509 in a DER file
* @throws IOException if an I/O error occurs reading either key file
* @throws InvalidKeySpecException if either key file is inappropriate for an RSA key
*/
public RSAKeyPair(URL privateKeyResourceName, URL publicKeyResourceName) throws IOException, InvalidKeySpecException {
try (InputStream inPrivate = privateKeyResourceName.openStream()) {
String privateString = new String(inPrivate.readAllBytes(), StandardCharsets.UTF_8)
.replaceAll("-----[A-Z ]*-----", "")
.replaceAll("\\n", "");
pubKeyStr = Base64.byteArrayToBase64(x509EncodedKeySpec.getEncoded());
PKCS8EncodedKeySpec privateSpec = new PKCS8EncodedKeySpec(Base64.base64ToByteArray(privateString));
KeyFactory rsaFactory = KeyFactory.getInstance("RSA");
privateKey = rsaFactory.generatePrivate(privateSpec);
} catch (NoSuchAlgorithmException e) {
throw new AssertionError("JVM spec is required to support RSA", e);
}
try (InputStream inPublic = publicKeyResourceName.openStream()) {
publicKey = getX509PublicKey(inPublic.readAllBytes());
pubKeyStr = Base64.byteArrayToBase64(publicKey.getEncoded());
}
}
public String getPublicKeyStr() {
@ -359,6 +389,8 @@ public final class CryptoKeys implements CLIO {
public byte[] encrypt(ByteBuffer buffer) {
try {
// This is better than nothing, but still not very secure
// See: https://crypto.stackexchange.com/questions/20085/which-attacks-are-possible-against-raw-textbook-rsa
Cipher rsaCipher = Cipher.getInstance("RSA/ECB/nopadding");
rsaCipher.init(Cipher.ENCRYPT_MODE, privateKey);
return rsaCipher.doFinal(buffer.array(),buffer.position(), buffer.limit());
@ -380,17 +412,4 @@ public final class CryptoKeys implements CLIO {
}
}
public static void main(String[] args) throws Exception {
RSAKeyPair keyPair = new RSAKeyPair();
CLIO.out(keyPair.getPublicKeyStr());
PublicKey pk = deserializeX509PublicKey(keyPair.getPublicKeyStr());
byte[] payload = "Hello World!".getBytes(StandardCharsets.UTF_8);
byte[] encrypted = keyPair.encrypt(ByteBuffer.wrap(payload));
String cipherBase64 = Base64.byteArrayToBase64(encrypted);
CLIO.out("encrypted: "+ cipherBase64);
CLIO.out("signed: "+ Base64.byteArrayToBase64(keyPair.signSha256(payload)));
CLIO.out("decrypted "+ new String(decryptRSA(encrypted , pk), StandardCharsets.UTF_8));
}
}

View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.cloud;
import org.apache.solr.SolrTestCase;
import org.apache.solr.util.CryptoKeys;
import org.junit.Test;
import java.net.URL;
import java.nio.ByteBuffer;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.not;
public class TestRSAKeyPair extends SolrTestCase {
@Test
public void testGenKeyPair() throws Exception {
testRoundTrip(new CryptoKeys.RSAKeyPair());
}
@Test
public void testReadKeysFromDisk() throws Exception {
URL privateKey = getClass().getClassLoader().getResource("cryptokeys/priv_key512_pkcs8.pem");
URL publicKey = getClass().getClassLoader().getResource("cryptokeys/pub_key512.der");
testRoundTrip(new CryptoKeys.RSAKeyPair(privateKey, publicKey));
}
private void testRoundTrip(CryptoKeys.RSAKeyPair kp) throws Exception {
final byte[] plaintext = new byte[random().nextInt(64)];
random().nextBytes(plaintext);
byte[] encrypted = kp.encrypt(ByteBuffer.wrap(plaintext));
assertThat(plaintext, not(equalTo(encrypted)));
byte[] decrypted = CryptoKeys.decryptRSA(encrypted, kp.getPublicKey());
assertTrue("Decrypted text is shorter than original text.", decrypted.length >= plaintext.length);
// Pad with null bytes because RSAKeyPair uses RSA/ECB/NoPadding
int pad = decrypted.length - plaintext.length;
final byte[] padded = new byte[decrypted.length];
System.arraycopy(plaintext, 0, padded, pad, plaintext.length);
assertArrayEquals(padded, decrypted);
}
}

View File

@ -17,8 +17,8 @@
package org.apache.solr.filestore;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
@ -52,6 +52,7 @@ import org.junit.Before;
import static org.apache.solr.common.util.Utils.JAVABINCONSUMER;
import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.filestore.PackageStoreAPI=DEBUG;org.apache.solr.filestore.DistribPackageStore=DEBUG")
public class TestDistribPackageStore extends SolrCloudTestCase {
@ -86,7 +87,7 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
);
fail("should have failed because of wrong signature ");
} catch (RemoteExecutionException e) {
assertTrue(e.getMessage().contains("Signature does not match"));
assertThat(e.getMessage(), containsString("Signature does not match"));
}
postFile(cluster.getSolrClient(), getFileContent("runtimecode/runtimelibs.jar.bin"),
@ -275,12 +276,15 @@ public class TestDistribPackageStore extends SolrCloudTestCase {
assertEquals(name, rsp.getResponse().get(CommonParams.FILE));
}
/**
* Read and return the contents of the file-like resource
* @param fname the name of the resource to read
* @return the bytes of the resource
* @throws IOException if there is an I/O error reading the contents
*/
public static byte[] readFile(String fname) throws IOException {
byte[] buf = null;
try (FileInputStream fis = new FileInputStream(getFile(fname))) {
buf = new byte[fis.available()];
fis.read(buf);
try (InputStream is = TestDistribPackageStore.class.getClassLoader().getResourceAsStream(fname)) {
return is.readAllBytes();
}
return buf;
}
}

View File

@ -65,6 +65,7 @@ import static org.apache.solr.core.TestDynamicLoading.getFileContent;
import static org.apache.solr.filestore.TestDistribPackageStore.readFile;
import static org.apache.solr.filestore.TestDistribPackageStore.uploadKey;
import static org.apache.solr.filestore.TestDistribPackageStore.waitForAllNodesHaveFile;
import static org.hamcrest.CoreMatchers.containsString;
@LogLevel("org.apache.solr.pkg.PackageLoader=DEBUG;org.apache.solr.pkg.PackageAPI=DEBUG")
//@org.apache.lucene.util.LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13822") // leaks files
@ -587,9 +588,7 @@ public class TestPackages extends SolrCloudTestCase {
fail("should have failed with message : " + expectErrorMsg);
} catch (BaseHttpSolrClient.RemoteExecutionException e) {
String msg = e.getMetaData()._getStr(errPath, "");
assertTrue("should have failed with message: " + expectErrorMsg + "actual message : " + msg,
msg.contains(expectErrorMsg)
);
assertThat(msg, containsString(expectErrorMsg));
}
}
}

View File

@ -293,6 +293,10 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.setProperty("solr.clustering.enabled", "false");
System.setProperty("solr.peerSync.useRangeVersions", String.valueOf(random().nextBoolean()));
System.setProperty("solr.cloud.wait-for-updates-with-stale-state-pause", "500");
System.setProperty("pkiHandlerPrivateKeyPath", SolrTestCaseJ4.class.getClassLoader().getResource("cryptokeys/priv_key512_pkcs8.pem").toExternalForm());
System.setProperty("pkiHandlerPublicKeyPath", SolrTestCaseJ4.class.getClassLoader().getResource("cryptokeys/pub_key512.der").toExternalForm());
System.setProperty(ZK_WHITELIST_PROPERTY, "*");
startTrackingSearchers();
ignoreException("ignore_exception");

View File

@ -113,6 +113,8 @@ public class MiniSolrCloudCluster {
" <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
" <str name=\"zkCredentialsProvider\">${zkCredentialsProvider:org.apache.solr.common.cloud.DefaultZkCredentialsProvider}</str> \n" +
" <str name=\"zkACLProvider\">${zkACLProvider:org.apache.solr.common.cloud.DefaultZkACLProvider}</str> \n" +
" <str name=\"pkiHandlerPrivateKeyPath\">${pkiHandlerPrivateKeyPath:cryptokeys/priv_key512_pkcs8.pem}</str> \n" +
" <str name=\"pkiHandlerPublicKeyPath\">${pkiHandlerPublicKeyPath:cryptokeys/pub_key512.der}</str> \n" +
" </solrcloud>\n" +
" <metrics>\n" +
" <reporter name=\"default\" class=\"org.apache.solr.metrics.reporters.SolrJmxReporter\">\n" +

View File

@ -0,0 +1,10 @@
-----BEGIN PRIVATE KEY-----
MIIBUwIBADANBgkqhkiG9w0BAQEFAASCAT0wggE5AgEAAkEAyCZJV+X4TY2P+goA
/ZNx5aHXumTpSCzDkDmcsf454wH9Z5YlmD80QN1rxJd9AMRVGbDk/7YhnKk8BLN8
KtzrIQIDAQABAkBtkXATO2TL59RKuFFEgAQZBplXg8ilZ0QD31Ylppu/5ixY3lRR
3vL4S+7nSaEfdFAjFB35HpJizWR706UeVXZxAiEA/zfruDJtUS7YD4KDV8T29sd+
ceu6ukNCuH2vRkqihO0CIQDIwzG+UeVKHpDZxRNLG7JQRpB835bh596GB8hYFWXM
hQIgQcExnSp42cK87foNRu67RkeNv2IhoN21cf0HzI9sId0CIBpXejRlnHcwMYNR
V2m4dZoQ2C56S9rSSKE/bisYi6XdAiA2plITMDZqhB00+XmgN+SGoAaOzmlvuouC
2Zcm9WGL9A==
-----END PRIVATE KEY-----

View File

@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.util.RevertDefaultThreadHandlerRule;
@ -32,7 +32,7 @@ import org.junit.rules.RuleChain;
import org.junit.rules.TestRule;
@LuceneTestCase.SuppressSysoutChecks(bugUrl = "Solr logs to JUL")
public class MiniSolrCloudClusterTest extends SolrTestCase {
public class MiniSolrCloudClusterTest extends SolrTestCaseJ4 {
@ClassRule
public static TestRule solrClassRules = RuleChain.outerRule(