Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12400. Provide a way for NN to drain the local key cache before re-encryption.

This commit is contained in:
Xiao Chen 2017-09-07 20:50:03 -07:00
parent fa6137501c
commit b3a4d7d2a0
5 changed files with 56 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
View File

@ -554,6 +554,16 @@ public class KeyProviderCryptoExtension extends
return getExtension().reencryptEncryptedKey(ekv);
}
/**
* Calls {@link CryptoExtension#drain(String)} for the given key name on the
* underlying {@link CryptoExtension}.
*
* @param keyName
*/
public void drain(String keyName) {
getExtension().drain(keyName);
}
/**
* Batched version of {@link #reencryptEncryptedKey(EncryptedKeyVersion)}.
* <p>

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
View File

@ -32,7 +32,6 @@ import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
@ -698,9 +697,7 @@ final class FSDirEncryptionZoneOp {
// drain the local cache of the key provider.
// Do not invalidateCache on the server, since that's the responsibility
// when rolling the key version.
if (dir.getProvider() instanceof CryptoExtension) {
((CryptoExtension) dir.getProvider()).drain(keyName);
}
dir.getProvider().drain(keyName);
final EncryptedKeyVersion edek;
try {
edek = dir.getProvider().generateEncryptedKey(keyName);

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -7111,6 +7111,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (keyVersionName == null) {
throw new IOException("Failed to get key version name for " + zone);
}
LOG.info("Re-encryption using key version " + keyVersionName
+ " for zone " + zone);
}
writeLock();
try {

103
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
View File

@ -91,7 +91,7 @@ public class TestReencryption {
private FileSystemTestHelper fsHelper;
private MiniDFSCluster cluster;
private HdfsAdmin dfsAdmin;
protected HdfsAdmin dfsAdmin;
private DistributedFileSystem fs;
private FSNamesystem fsn;
private File testRootDir;
@ -199,8 +199,7 @@ public class TestReencryption {
verifyZoneStatus(zone, null, 0);
// test re-encrypt after keyroll
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(2);
FileEncryptionInfo fei1 = getFileEncryptionInfo(encFile1);
@ -316,8 +315,7 @@ public class TestReencryption {
final Path notReencrypted = new Path(zone, "f0");
final FileEncryptionInfo fei = getFileEncryptionInfo(lastReencryptedFile);
final FileEncryptionInfo feiLast = getFileEncryptionInfo(notReencrypted);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// mark pause after first checkpoint (5 files)
getEzManager().pauseForTestingAfterNthSubmission(1);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -363,8 +361,7 @@ public class TestReencryption {
0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// test zone deleted during re-encrypt's checkpointing
getEzManager().pauseForTestingAfterNthSubmission(1);
getEzManager().resetMetricsForTesting();
@ -409,8 +406,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@ -443,8 +439,7 @@ public class TestReencryption {
fsWrapper.rename(new Path(zone, "f"), new Path(zone, "f1"));
// re-encrypt
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@ -495,8 +490,7 @@ public class TestReencryption {
final Path encFile9 = new Path(subdir, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// mark pause after first checkpoint (5 files)
getEzManager().pauseForTestingAfterNthSubmission(1);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -540,8 +534,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@ -585,8 +578,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// disable re-encrypt for testing, and issue a command
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -645,8 +637,7 @@ public class TestReencryption {
final Path encFile9 = new Path(zone, "9");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// disable re-encrypt for testing, and issue a command
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -770,8 +761,7 @@ public class TestReencryption {
0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Disable re-encrypt, send re-encrypt on '/', verify queue
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zoneRoot, ReencryptAction.START);
@ -816,8 +806,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -883,8 +872,7 @@ public class TestReencryption {
.createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -930,8 +918,7 @@ public class TestReencryption {
.createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -984,8 +971,7 @@ public class TestReencryption {
0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1029,8 +1015,7 @@ public class TestReencryption {
0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1071,8 +1056,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1122,8 +1106,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1162,8 +1145,7 @@ public class TestReencryption {
.createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1220,8 +1202,7 @@ public class TestReencryption {
.createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// Issue the command re-encrypt and pause it
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1283,8 +1264,7 @@ public class TestReencryption {
// test re-encrypt on snapshot dir
final Path encFile1 = new Path(zone, "0");
final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile1);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
try {
dfsAdmin.reencryptEncryptionZone(zoneSnap, ReencryptAction.START);
fail("Reencrypt command on snapshot path should fail.");
@ -1423,8 +1403,7 @@ public class TestReencryption {
fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// disable, test basic
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1442,8 +1421,7 @@ public class TestReencryption {
assertExceptionContains("not under re-encryption", expected);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// test cancelling half-way
getEzManager().pauseForTestingAfterNthSubmission(1);
getEzManager().resumeReencryptForTesting();
@ -1537,8 +1515,7 @@ public class TestReencryption {
// re-encrypt 10 files, so 2 callables. Hang 1, pause the updater so the
// callable is taken from the executor but not processed.
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
getEzManager().pauseReencryptForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForQueuedZones(1);
@ -1593,8 +1570,7 @@ public class TestReencryption {
fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// disable, test basic
getEzManager().pauseReencryptUpdaterForTesting();
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1625,8 +1601,7 @@ public class TestReencryption {
}
// re-encrypt the zone
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@ -1678,8 +1653,7 @@ public class TestReencryption {
0xFEED);
}
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
// mark pause after first checkpoint (5 files)
getEzManager().pauseForTestingAfterNthSubmission(1);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
@ -1736,9 +1710,7 @@ public class TestReencryption {
}
// re-encrypt the zone
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
assertEquals(0, injector.exceptionCount);
@ -1790,9 +1762,7 @@ public class TestReencryption {
}
// re-encrypt the zone
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
assertEquals(0, injector.exceptionCount);
@ -1845,9 +1815,7 @@ public class TestReencryption {
}
// re-encrypt the zone
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
assertEquals(0, injector.exceptionCount);
@ -1899,9 +1867,7 @@ public class TestReencryption {
}
// re-encrypt the zone
fsn.getProvider().rollNewVersion(TEST_KEY);
fsn.getProvider().flush();
rollKey(TEST_KEY);
Whitebox.setInternalState(getUpdater(), "faultRetryInterval", 50);
dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
waitForReencryptedZones(1);
@ -1929,4 +1895,11 @@ public class TestReencryption {
return (ReencryptionUpdater) Whitebox
.getInternalState(getHandler(), "reencryptionUpdater");
}
protected void rollKey(final String keyName) throws Exception {
dfsAdmin.getKeyProvider().rollNewVersion(keyName);
// need to flush for jceks provider to make the key version it returned
// after NN restart consistent.
dfsAdmin.getKeyProvider().flush();
}
}

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionWithKMS.java
View File

@ -88,4 +88,9 @@ public class TestReencryptionWithKMS extends TestReencryption{
KMSWebApp.getACLs().run();
testReencryptionBasic();
}
@Override
protected void rollKey(final String keyName) throws Exception {
dfsAdmin.getKeyProvider().rollNewVersion(keyName);
}
}