From 6dbad15e56be90bf0808e86c3fe33586ba229b97 Mon Sep 17 00:00:00 2001 From: jaymode Date: Wed, 7 Oct 2015 06:51:24 -0400 Subject: [PATCH] always sign messages when message signing is enabled This change allows for messages to be signed when message signing is enabled and a system key is not present. This is accomplished by generating a random key on startup and then using HKDF with HmacSHA1 to generate the keying material to be used to sign the messages. The random key from the originating node is added to the signed message so that the signing key can be derived on the receiving node. When a system key is present, the system key is used for signing and the preexisting behavior is maintained. Closes elastic/elasticsearch#711 Original commit: elastic/x-pack-elasticsearch@c41fdc0ac371180a44a9e5ee89d7bb8f66d96db1 --- shield/docs/public/release-notes.asciidoc | 7 + .../shield/crypto/CryptoService.java | 4 +- .../shield/crypto/InternalCryptoService.java | 255 +++++++++++++++--- .../crypto/InternalCryptoServiceTests.java | 60 ++++- 4 files changed, 286 insertions(+), 40 deletions(-) diff --git a/shield/docs/public/release-notes.asciidoc b/shield/docs/public/release-notes.asciidoc index c3eba5ea7b2..39828e56d46 100644 --- a/shield/docs/public/release-notes.asciidoc +++ b/shield/docs/public/release-notes.asciidoc @@ -42,6 +42,13 @@ version of Shield. We recommend copying the changes listed below to your `roles. [[changelist]] === Change List +[float] +==== 2.0.0 + +.breaking changes +* All files that Shield uses must be kept in the <> due to the enhanced security of Elasticsearch 2.0. +* The network format has been changed from all previous versions of Shield and a full cluster restart is required to upgrade to Shield 2.0. + [float] ==== 2.0.0-rc1 diff --git a/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java b/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java index ff8471fef3c..40b8859f12a 100644 --- a/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java +++ b/shield/src/main/java/org/elasticsearch/shield/crypto/CryptoService.java @@ -30,8 +30,10 @@ public interface CryptoService { * Signs the given text and returns the signed text (original text + signature) * @param text the string to sign * @param key the key to sign the text with + * @param systemKey the system key. This is optional and if the key != systemKey then the format of the + * message will change */ - String sign(String text, SecretKey key) throws IOException; + String sign(String text, SecretKey key, SecretKey systemKey) throws IOException; /** * Unsigns the given signed text, verifies the original text with the attached signature and if valid returns diff --git a/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java b/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java index c0eb28d0232..22fc33b8388 100644 --- a/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java +++ b/shield/src/main/java/org/elasticsearch/shield/crypto/InternalCryptoService.java @@ -7,6 +7,7 @@ package org.elasticsearch.shield.crypto; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Base64; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -21,15 +22,18 @@ import javax.crypto.*; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.InvalidKeyException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.regex.Pattern; @@ -51,8 +55,10 @@ public class InternalCryptoService extends AbstractLifecycleComponentRFC 5869 + */ + private static class HmacSHA1HKDF { + private static final int HMAC_SHA1_BYTE_LENGTH = 20; + private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; + + /** + * This method performs the extract and expand steps of HKDF in one call with the given + * data. The output of the extract step is used as the input to the expand step + * + * @param salt optional salt value (a non-secret random value); if not provided, it is set to a string of HashLen zeros. + * @param ikm the input keying material + * @param info optional context and application specific information; if not provided a zero length byte[] is used + * @param outputLength length of output keying material in octets (<= 255*HashLen) + * @return the output keying material + */ + static byte[] extractAndExpand(@Nullable SecretKey salt, byte[] ikm, @Nullable byte[] info, int outputLength) { + // arg checking + Objects.requireNonNull(ikm, "the input keying material must not be null"); + if (outputLength < 1) { + throw new IllegalArgumentException("output length must be positive int >= 1"); + } + if (outputLength > 255 * HMAC_SHA1_BYTE_LENGTH) { + throw new IllegalArgumentException("output length must be <= 255*" + HMAC_SHA1_BYTE_LENGTH); + } + if (salt == null) { + salt = new SecretKeySpec(new byte[HMAC_SHA1_BYTE_LENGTH], HMAC_SHA1_ALGORITHM); + } + if (info == null) { + info = new byte[0]; + } + + // extract + Mac mac = createMac(salt); + byte[] keyBytes = mac.doFinal(ikm); + final SecretKey pseudoRandomKey = new SecretKeySpec(keyBytes, HMAC_SHA1_ALGORITHM); + + /* + * The output OKM is calculated as follows: + * N = ceil(L/HashLen) + * T = T(1) | T(2) | T(3) | ... | T(N) + * OKM = first L octets of T + * + * where: + * T(0) = empty string (zero length) + * T(1) = HMAC-Hash(PRK, T(0) | info | 0x01) + * T(2) = HMAC-Hash(PRK, T(1) | info | 0x02) + * T(3) = HMAC-Hash(PRK, T(2) | info | 0x03) + * ... + * + * (where the constant concatenated to the end of each T(n) is a single octet.) + */ + int n = (outputLength % HMAC_SHA1_BYTE_LENGTH == 0) ? + outputLength / HMAC_SHA1_BYTE_LENGTH : + (outputLength / HMAC_SHA1_BYTE_LENGTH) + 1; + + byte[] hashRound = new byte[0]; + + ByteBuffer generatedBytes = ByteBuffer.allocate(Math.multiplyExact(n, HMAC_SHA1_BYTE_LENGTH)); + try { + // initiliaze the mac with the new key + mac.init(pseudoRandomKey); + } catch (InvalidKeyException e) { + throw new ElasticsearchException("failed to initialize the mac", e); + } + for (int roundNum = 1; roundNum <= n; roundNum++) { + mac.reset(); + mac.update(hashRound); + mac.update(info); + mac.update((byte) roundNum); + hashRound = mac.doFinal(); + generatedBytes.put(hashRound); + } + + byte[] result = new byte[outputLength]; + generatedBytes.rewind(); + generatedBytes.get(result, 0, outputLength); + return result; + } + } } diff --git a/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java b/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java index 4f0f72dc76d..21f2c931e68 100644 --- a/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java +++ b/shield/src/test/java/org/elasticsearch/shield/crypto/InternalCryptoServiceTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.shield.crypto; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -16,6 +17,7 @@ import org.junit.Before; import org.junit.Test; import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -62,6 +64,8 @@ public class InternalCryptoServiceTests extends ESTestCase { @Test public void testSigned() throws Exception { + // randomize whether to use a system key or not + Settings settings = randomBoolean() ? this.settings : Settings.EMPTY; InternalCryptoService service = new InternalCryptoService(settings, env, watcherService).start(); String text = randomAsciiOfLength(10); String signed = service.sign(text); @@ -81,11 +85,12 @@ public class InternalCryptoServiceTests extends ESTestCase { @Test public void testSignAndUnsign_NoKeyFile() throws Exception { InternalCryptoService service = new InternalCryptoService(Settings.EMPTY, env, watcherService).start(); - String text = randomAsciiOfLength(10); + final String text = randomAsciiOfLength(10); String signed = service.sign(text); - assertThat(text, equalTo(signed)); - text = service.unsignAndVerify(signed); - assertThat(text, equalTo(signed)); + // we always have some sort of key to sign with + assertThat(text, not(equalTo(signed))); + String unsigned = service.unsignAndVerify(signed); + assertThat(unsigned, equalTo(text)); } @Test @@ -439,6 +444,53 @@ public class InternalCryptoServiceTests extends ESTestCase { } } + @Test + public void testSigningOnKeyDeleted() throws Exception { + final InternalCryptoService service = new InternalCryptoService(settings, env, watcherService).start(); + final String text = randomAsciiOfLength(10); + final String signed = service.sign(text); + assertThat(text, not(equalTo(signed))); + + final CountDownLatch latch = new CountDownLatch(1); + service.register(new CryptoService.Listener() { + @Override + public void onKeyChange(SecretKey oldSystemKey, SecretKey oldEncryptionKey) { + final String plainText = service.unsignAndVerify(signed, oldSystemKey); + assertThat(plainText, equalTo(text)); + try { + final String newSigned = service.sign(plainText); + assertThat(newSigned, not(equalTo(signed))); + assertThat(newSigned, not(equalTo(plainText))); + assertThat(service.unsignAndVerify(newSigned), equalTo(plainText)); + latch.countDown(); + } catch (IOException e) { + throw new ElasticsearchException("unexpected exception while signing", e); + } + } + }); + + // we need to sleep to ensure the timestamp of the file will definitely change + // and so the resource watcher will pick up the change. + Thread.sleep(1000); + + Files.delete(keyFile); + if (!latch.await(10, TimeUnit.SECONDS)) { + fail("waiting too long for test to complete. Expected callback is not called or finished running"); + } + } + + @Test + public void testSigningKeyCanBeRecomputedConsistently() { + final SecretKey systemKey = new SecretKeySpec(InternalCryptoService.generateKey(), InternalCryptoService.KEY_ALGO); + final SecretKey randomKey = InternalCryptoService.generateSecretKey(InternalCryptoService.RANDOM_KEY_SIZE); + int iterations = randomInt(100); + final SecretKey signingKey = InternalCryptoService.createSigningKey(systemKey, randomKey); + for (int i = 0; i < iterations; i++) { + SecretKey regenerated = InternalCryptoService.createSigningKey(systemKey, randomKey); + assertThat(regenerated, equalTo(signingKey)); + } + } + private static byte[] randomByteArray() { return randomByteArray(0); }