HADOOP-14949. TestKMS#testACLs fails intermittently.

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSACLs.java

@@ -326,4 +326,8 @@ public boolean isACLPresent(String keyName, KeyOpType opType) {
         || whitelistKeyAcls.containsKey(opType));
   }
 
+  @VisibleForTesting
+  void forceNextReloadForTesting() {
+    lastReload = 0;
+  }
 }

hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java

@@ -20,6 +20,8 @@
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.net.MalformedURLException;
@@ -31,6 +33,8 @@
 @InterfaceAudience.Private
 public class KMSConfiguration {
 
+  static final Logger LOG = LoggerFactory.getLogger(KMSConfiguration.class);
+
   public static final String KMS_CONFIG_DIR = "kms.config.dir";
   public static final String KMS_DEFAULT_XML = "kms-default.xml";
   public static final String KMS_SITE_XML = "kms-site.xml";
@@ -138,6 +142,8 @@ public static boolean isACLsFileNewer(long time) {
             "' must be an absolute path: " + confDir);
       }
       File f = new File(confDir, KMS_ACLS_XML);
+      LOG.trace("Checking file {}, modification time is {}, last reload time is"
+          + " {}", f.getPath(), f.lastModified(), time);
       // at least 100ms newer than time, we do this to ensure the file
       // has been properly closed/flushed
       newer = f.lastModified() - time > 100;

hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java

@@ -54,6 +54,7 @@
 import org.junit.rules.Timeout;
 import org.mockito.Mockito;
 import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.event.Level;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -1637,13 +1638,12 @@ public Void run() throws Exception {
         //stop the reloader, to avoid running while we are writing the new file
         KMSWebApp.getACLs().stopReloader();
 
+        GenericTestUtils.setLogLevel(KMSConfiguration.LOG, Level.TRACE);
         // test ACL reloading
-        Thread.sleep(10); // to ensure the ACLs file modifiedTime is newer
         conf.set(KMSACLs.Type.CREATE.getAclConfigKey(), "foo");
         conf.set(KMSACLs.Type.GENERATE_EEK.getAclConfigKey(), "foo");
         writeConf(testDir, conf);
-        Thread.sleep(1000);
-
+        KMSWebApp.getACLs().forceNextReloadForTesting();
         KMSWebApp.getACLs().run(); // forcing a reload by hand.
 
         // should not be able to create a key now