Clearing the realm caches on file updates

- Changed the behaviour of esusers realm so that whenever the `users` or the `users_roles` file are updated, the realm's cache expunges
- Changed LDAP realm such that when the `role_mapping.yml` file is updated, the realm's cache expunges

Also, cleaned up unused code (mainly around esusers and the different stores)

Original commit: elastic/x-pack-elasticsearch@3f093207da

src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersModule.java

@@ -7,13 +7,8 @@ package org.elasticsearch.shield.authc.esusers;
 
 import org.elasticsearch.common.inject.util.Providers;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.shield.authc.Realm;
-import org.elasticsearch.shield.authc.support.UserPasswdStore;
-import org.elasticsearch.shield.authc.support.UserRolesStore;
 import org.elasticsearch.shield.support.AbstractShieldModule;
 
-import static org.elasticsearch.common.inject.name.Names.named;
-
 /**
  *
  */
@@ -30,8 +25,8 @@ public class ESUsersModule extends AbstractShieldModule.Node {
     protected void configureNode() {
         if (enabled) {
             bind(ESUsersRealm.class).asEagerSingleton();
-            bind(UserPasswdStore.class).annotatedWith(named("file")).to(FileUserPasswdStore.class).asEagerSingleton();
+            bind(FileUserPasswdStore.class).asEagerSingleton();
-            bind(UserRolesStore.class).annotatedWith(named("file")).to(FileUserRolesStore.class).asEagerSingleton();
+            bind(FileUserRolesStore.class).asEagerSingleton();
         } else {
             bind(ESUsersRealm.class).toProvider(Providers.<ESUsersRealm>of(null));
         }

src/main/java/org/elasticsearch/shield/authc/esusers/ESUsersRealm.java

@@ -5,17 +5,14 @@
  */
 package org.elasticsearch.shield.authc.esusers;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.inject.name.Named;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.authc.AuthenticationToken;
 import org.elasticsearch.shield.authc.support.CachingUsernamePasswordRealm;
-import org.elasticsearch.shield.authc.support.UserPasswdStore;
+import org.elasticsearch.shield.authc.support.RefreshListener;
-import org.elasticsearch.shield.authc.support.UserRolesStore;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.transport.TransportMessage;
 
@@ -26,15 +23,18 @@ public class ESUsersRealm extends CachingUsernamePasswordRealm {
 
     public static final String TYPE = "esusers";
 
-    final UserPasswdStore userPasswdStore;
+    final FileUserPasswdStore userPasswdStore;
-    final UserRolesStore userRolesStore;
+    final FileUserRolesStore userRolesStore;
 
     @Inject
-    public ESUsersRealm(Settings settings, @Named("file") UserPasswdStore userPasswdStore,
+    public ESUsersRealm(Settings settings, FileUserPasswdStore userPasswdStore,
-                        @Named("file") UserRolesStore userRolesStore, RestController restController) {
+                        FileUserRolesStore userRolesStore, RestController restController) {
         super(settings);
+        Listener listener = new Listener();
         this.userPasswdStore = userPasswdStore;
+        userPasswdStore.addListener(listener);
         this.userRolesStore = userRolesStore;
+        userRolesStore.addListener(listener);
         restController.registerRelevantHeaders(UsernamePasswordToken.BASIC_AUTH_HEADER);
     }
 
@@ -60,16 +60,17 @@ public class ESUsersRealm extends CachingUsernamePasswordRealm {
 
     @Override
     protected User doAuthenticate(UsernamePasswordToken token) {
-        if (userPasswdStore == null) {
-            return null;
-        }
         if (!userPasswdStore.verifyPassword(token.principal(), token.credentials())) {
             return null;
         }
-        String[] roles = Strings.EMPTY_ARRAY;
+        String[] roles = userRolesStore.roles(token.principal());
-        if (userRolesStore != null) {
-            roles = userRolesStore.roles(token.principal());
-        }
         return new User.Simple(token.principal(), roles);
     }
+
+    class Listener implements RefreshListener {
+        @Override
+        public void onRefresh() {
+            expireAll();
+        }
+    }
 }

src/main/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStore.java

@@ -15,8 +15,8 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.authc.support.SecuredString;
-import org.elasticsearch.shield.authc.support.UserPasswdStore;
 import org.elasticsearch.shield.plugin.ShieldPlugin;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
@@ -32,25 +32,26 @@ import java.nio.file.StandardOpenOption;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 /**
  *
  */
-public class FileUserPasswdStore extends AbstractComponent implements UserPasswdStore {
+public class FileUserPasswdStore extends AbstractComponent {
 
     private final Path file;
     final Hasher hasher = Hasher.HTPASSWD;
 
     private volatile ImmutableMap<String, char[]> esUsers;
 
-    private final Listener listener;
+    private CopyOnWriteArrayList<RefreshListener> listeners;
 
     @Inject
     public FileUserPasswdStore(Settings settings, Environment env, ResourceWatcherService watcherService) {
-        this(settings, env, watcherService, Listener.NOOP);
+        this(settings, env, watcherService, null);
     }
 
-    FileUserPasswdStore(Settings settings, Environment env, ResourceWatcherService watcherService, Listener listener) {
+    FileUserPasswdStore(Settings settings, Environment env, ResourceWatcherService watcherService, RefreshListener listener) {
         super(settings);
         file = resolveFile(settings, env);
         esUsers = parseFile(file, logger);
@@ -60,10 +61,16 @@ public class FileUserPasswdStore extends AbstractComponent implements UserPasswd
         FileWatcher watcher = new FileWatcher(file.getParent().toFile());
         watcher.addListener(new FileListener());
         watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH);
-        this.listener = listener;
+        listeners = new CopyOnWriteArrayList<>();
+        if (listener != null) {
+            listeners.add(listener);
+        }
+    }
+
+    void addListener(RefreshListener listener) {
+        listeners.add(listener);
     }
 
-    @Override
     public boolean verifyPassword(String username, SecuredString password) {
         if (esUsers == null) {
             return false;
@@ -139,20 +146,23 @@ public class FileUserPasswdStore extends AbstractComponent implements UserPasswd
         }
     }
 
+    protected void notifyRefresh() {
+        for (RefreshListener listener : listeners) {
+            listener.onRefresh();
+        }
+    }
+
     private class FileListener extends FileChangesListener {
         @Override
         public void onFileCreated(File file) {
-            if (file.equals(FileUserPasswdStore.this.file.toFile())) {
+            onFileChanged(file);
-                esUsers = parseFile(file.toPath(), logger);
-                listener.onRefresh();
-            }
         }
 
         @Override
         public void onFileDeleted(File file) {
             if (file.equals(FileUserPasswdStore.this.file.toFile())) {
                 esUsers = ImmutableMap.of();
-                listener.onRefresh();
+                notifyRefresh();
             }
         }
 
@@ -160,19 +170,8 @@ public class FileUserPasswdStore extends AbstractComponent implements UserPasswd
         public void onFileChanged(File file) {
             if (file.equals(FileUserPasswdStore.this.file.toFile())) {
                 esUsers = parseFile(file.toPath(), logger);
-                listener.onRefresh();
+                notifyRefresh();
             }
         }
     }
-
-    public static interface Listener {
-
-        final Listener NOOP = new Listener() {
-            @Override
-            public void onRefresh() {
-            }
-        };
-
-        void onRefresh();
-    }
 }

src/main/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStore.java

@@ -15,7 +15,7 @@ import org.elasticsearch.common.inject.internal.Nullable;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
-import org.elasticsearch.shield.authc.support.UserRolesStore;
+import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.plugin.ShieldPlugin;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
@@ -29,12 +29,13 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.regex.Pattern;
 
 /**
  *
  */
-public class FileUserRolesStore extends AbstractComponent implements UserRolesStore {
+public class FileUserRolesStore extends AbstractComponent {
 
     private static final Pattern USERS_DELIM = Pattern.compile("\\s*,\\s*");
 
@@ -42,21 +43,28 @@ public class FileUserRolesStore extends AbstractComponent implements UserRolesSt
 
     private volatile ImmutableMap<String, String[]> userRoles;
 
-    private final Listener listener;
+    private CopyOnWriteArrayList<RefreshListener> listeners;
 
     @Inject
     public FileUserRolesStore(Settings settings, Environment env, ResourceWatcherService watcherService) {
-        this(settings, env, watcherService, Listener.NOOP);
+        this(settings, env, watcherService, null);
     }
 
-    FileUserRolesStore(Settings settings, Environment env, ResourceWatcherService watcherService, Listener listener) {
+    FileUserRolesStore(Settings settings, Environment env, ResourceWatcherService watcherService, RefreshListener listener) {
         super(settings);
         file = resolveFile(settings, env);
         userRoles = parseFile(file, logger);
         FileWatcher watcher = new FileWatcher(file.getParent().toFile());
         watcher.addListener(new FileListener());
         watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH);
-        this.listener = listener;
+        listeners = new CopyOnWriteArrayList<>();
+        if (listener != null) {
+            listeners.add(listener);
+        }
+    }
+
+    synchronized void addListener(RefreshListener listener) {
+        listeners.add(listener);
     }
 
     public String[] roles(String username) {
@@ -175,20 +183,23 @@ public class FileUserRolesStore extends AbstractComponent implements UserRolesSt
         }
     }
 
+    public void notifyRefresh() {
+        for (RefreshListener listener : listeners) {
+            listener.onRefresh();
+        }
+    }
+
     private class FileListener extends FileChangesListener {
         @Override
         public void onFileCreated(File file) {
-            if (file.equals(FileUserRolesStore.this.file.toFile())) {
+            onFileChanged(file);
-                userRoles = parseFile(file.toPath(), logger);
-                listener.onRefresh();
-            }
         }
 
         @Override
         public void onFileDeleted(File file) {
             if (file.equals(FileUserRolesStore.this.file.toFile())) {
                 userRoles = ImmutableMap.of();
-                listener.onRefresh();
+                notifyRefresh();
             }
         }
 
@@ -196,19 +207,8 @@ public class FileUserRolesStore extends AbstractComponent implements UserRolesSt
         public void onFileChanged(File file) {
             if (file.equals(FileUserRolesStore.this.file.toFile())) {
                 userRoles = parseFile(file.toPath(), logger);
-                listener.onRefresh();
+                notifyRefresh();
             }
         }
     }
-
-    public static interface Listener {
-
-        static final Listener NOOP = new Listener() {
-            @Override
-            public void onRefresh() {
-            }
-        };
-
-        void onRefresh();
-    }
 }

src/main/java/org/elasticsearch/shield/authc/ldap/LdapGroupToRoleMapper.java

@@ -13,6 +13,7 @@ import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.plugin.ShieldPlugin;
 import org.elasticsearch.watcher.FileChangesListener;
 import org.elasticsearch.watcher.FileWatcher;
@@ -27,6 +28,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
 
 /**
  * This class loads and monitors the file defining the mappings of LDAP Group DNs to internal ES Roles.
@@ -38,16 +40,17 @@ public class LdapGroupToRoleMapper extends AbstractComponent {
     public static final String USE_UNMAPPED_GROUPS_AS_ROLES_SETTING = "unmapped_groups_as_roles";
 
     private final Path file;
-    private final Listener listener;
     private final boolean useUnmappedGroupsAsRoles;
     private volatile ImmutableMap<LdapName, Set<String>> groupRoles;
 
+    private CopyOnWriteArrayList<RefreshListener> listeners;
+
     @Inject
     public LdapGroupToRoleMapper(Settings settings, Environment env, ResourceWatcherService watcherService) {
-        this(settings, env, watcherService, Listener.NOOP);
+        this(settings, env, watcherService, null);
     }
 
-    LdapGroupToRoleMapper(Settings settings, Environment env, ResourceWatcherService watcherService, Listener listener) {
+    LdapGroupToRoleMapper(Settings settings, Environment env, ResourceWatcherService watcherService, RefreshListener listener) {
         super(settings);
         useUnmappedGroupsAsRoles = componentSettings.getAsBoolean(USE_UNMAPPED_GROUPS_AS_ROLES_SETTING, false);
         file = resolveFile(componentSettings, env);
@@ -55,7 +58,14 @@ public class LdapGroupToRoleMapper extends AbstractComponent {
         FileWatcher watcher = new FileWatcher(file.getParent().toFile());
         watcher.addListener(new FileListener());
         watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH);
-        this.listener = listener;
+        listeners = new CopyOnWriteArrayList<>();
+        if (listener != null) {
+            listeners.add(listener);
+        }
+    }
+
+    synchronized void addListener(RefreshListener listener) {
+        listeners.add(listener);
     }
 
     public static Path resolveFile(Settings settings, Environment env) {
@@ -121,6 +131,12 @@ public class LdapGroupToRoleMapper extends AbstractComponent {
         return (String) groupLdapName.getRdn(groupLdapName.size() - 1).getValue();
     }
 
+    protected void notifyRefresh() {
+        for (RefreshListener listener : listeners) {
+            listener.onRefresh();
+        }
+    }
+
     private class FileListener extends FileChangesListener {
         @Override
         public void onFileCreated(File file) {
@@ -136,7 +152,7 @@ public class LdapGroupToRoleMapper extends AbstractComponent {
         public void onFileChanged(File file) {
             if (file.equals(LdapGroupToRoleMapper.this.file.toFile())) {
                 groupRoles = parseFile(file.toPath(), logger);
-                listener.onRefresh();
+                notifyRefresh();
             }
         }
     }

src/main/java/org/elasticsearch/shield/authc/ldap/LdapRealm.java

@@ -13,6 +13,7 @@ import org.elasticsearch.shield.User;
 import org.elasticsearch.shield.authc.AuthenticationToken;
 import org.elasticsearch.shield.authc.Realm;
 import org.elasticsearch.shield.authc.support.CachingUsernamePasswordRealm;
+import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.transport.TransportMessage;
 
@@ -34,6 +35,7 @@ public class LdapRealm extends CachingUsernamePasswordRealm implements Realm<Use
         super(settings);
         this.connectionFactory = ldap;
         this.roleMapper = roleMapper;
+        roleMapper.addListener(new Listener());
         restController.registerRelevantHeaders(UsernamePasswordToken.BASIC_AUTH_HEADER);
     }
 
@@ -60,12 +62,17 @@ public class LdapRealm extends CachingUsernamePasswordRealm implements Realm<Use
         try (LdapConnection session = connectionFactory.bind(token.principal(), token.credentials())) {
             List<String> groupDNs = session.getGroups();
             Set<String> roles = roleMapper.mapRoles(groupDNs);
-            User.Simple user = new User.Simple(token.principal(), roles.toArray(new String[roles.size()]));
+            return new User.Simple(token.principal(), roles.toArray(new String[roles.size()]));
-            return user;
         } catch (ShieldException e){
             logger.info("Authentication Failed for user [{}]", e, token.principal());
             return null;
         }
     }
 
+    class Listener implements RefreshListener {
+        @Override
+        public void onRefresh() {
+            expireAll();
+        }
+    }
 }

src/main/java/org/elasticsearch/shield/authc/support/CachingUserPasswdStore.java

@@ -1,119 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.authc.support;
-
-import org.elasticsearch.common.cache.CacheBuilder;
-import org.elasticsearch.common.cache.CacheLoader;
-import org.elasticsearch.common.cache.LoadingCache;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.shield.authc.AuthenticationException;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A base class for username/password stores that caches the users and their pwd hashes in-memory. The cache
- * has an expiration time (defaults to 1hr, but it's configurable and can also be disabled by setting the cache
- * ttl to 0).
- */
-public abstract class CachingUserPasswdStore extends AbstractComponent implements UserPasswdStore {
-
-    private static final TimeValue DEFAULT_TTL = TimeValue.timeValueHours(1);
-
-    private final LoadingCache<String, PasswordHash> cache;
-
-    protected CachingUserPasswdStore(Settings settings) {
-        super(settings);
-        TimeValue ttl = componentSettings.getAsTime("cache.ttl", DEFAULT_TTL);
-        if (ttl.millis() > 0) {
-            cache = CacheBuilder.newBuilder()
-                    .expireAfterWrite(ttl.getMillis(), TimeUnit.MILLISECONDS)
-                    .build(new CacheLoader<String, PasswordHash>() {
-                        @Override
-                        public PasswordHash load(String username) throws Exception {
-                            PasswordHash hash = passwordHash(username);
-                            if (hash == null) {
-                                throw new AuthenticationException("Authentication failed");
-                            }
-                            return hash;
-                        }
-                    });
-        } else {
-            cache = null;
-        }
-    }
-
-    protected final void expire(String username) {
-        if (cache != null) {
-            cache.invalidate(username);
-        }
-    }
-
-    protected final void expireAll() {
-        if (cache != null) {
-            cache.invalidateAll();
-        }
-    }
-
-    @Override
-    public final boolean verifyPassword(final String username, final SecuredString password) {
-        if (cache == null) {
-            return doVerifyPassword(username, password);
-        }
-
-        try {
-
-            PasswordHash hash = cache.get(username);
-            return hash.verify(password);
-
-        } catch (ExecutionException ee) {
-            return false;
-        }
-    }
-
-    /**
-     * Verifies the given password. Both the given username, and if the username is verified, then the
-     * given password. This method is used when the caching is disabled.
-     */
-    protected abstract boolean doVerifyPassword(String username, SecuredString password);
-
-    protected abstract PasswordHash passwordHash(String username);
-
-
-    public static abstract class Writable extends CachingUserPasswdStore implements UserPasswdStore.Writable {
-
-        protected Writable(Settings settings) {
-            super(settings);
-        }
-
-        @Override
-        public final void store(String username, SecuredString password) {
-            doStore(username, password);
-            expire(username);
-        }
-
-        @Override
-        public void remove(String username) {
-            doRemove(username);
-            expire(username);
-        }
-
-        protected abstract void doStore(String username, SecuredString password);
-
-        protected abstract void doRemove(String username);
-    }
-
-    /**
-     * Represents a hash of a password.
-     */
-    static interface PasswordHash {
-
-        boolean verify(SecuredString password);
-
-    }
-}

src/main/java/org/elasticsearch/shield/authc/support/CachingUserRolesStore.java

@@ -1,113 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.authc.support;
-
-import org.elasticsearch.common.cache.CacheBuilder;
-import org.elasticsearch.common.cache.CacheLoader;
-import org.elasticsearch.common.cache.LoadingCache;
-import org.elasticsearch.common.component.AbstractComponent;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
-import org.elasticsearch.shield.authc.AuthenticationException;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A base class for user roles store that caches the roles per username in-memory. The cache
- * has an expiration time (defaults to 1hr, but it's configurable and can also be disabled by setting the cache
- * ttl to 0).
- */
-public abstract class CachingUserRolesStore extends AbstractComponent implements UserRolesStore {
-
-    private static final TimeValue DEFAULT_TTL = TimeValue.timeValueHours(1);
-
-    private final LoadingCache<String, String[]> cache;
-
-    protected CachingUserRolesStore(Settings settings) {
-        super(settings);
-        TimeValue ttl = componentSettings.getAsTime("cache.ttl", DEFAULT_TTL);
-        if (ttl.millis() > 0) {
-            cache = CacheBuilder.newBuilder()
-                    .expireAfterWrite(ttl.getMillis(), TimeUnit.MILLISECONDS)
-                    .build(new CacheLoader<String, String[]>() {
-                        @Override
-                        public String[] load(String username) throws Exception {
-                            return doLoadRoles(username);
-                        }
-                    });
-        } else {
-            cache = null;
-        }
-    }
-
-    protected final void expire(String username) {
-        if (cache != null) {
-            cache.invalidate(username);
-        }
-    }
-
-    protected final void expireAll() {
-        if (cache != null) {
-            cache.invalidateAll();
-        }
-    }
-
-    @Override
-    public String[] roles(final String username) {
-        if (cache == null) {
-            return doLoadRoles(username);
-        }
-        try {
-            return cache.get(username);
-        } catch (ExecutionException ee) {
-            throw new AuthenticationException("Could not load user roles", ee);
-        }
-    }
-
-    protected abstract String[] doLoadRoles(String username);
-
-    public static abstract class Writable extends CachingUserRolesStore implements UserRolesStore.Writable {
-
-        protected Writable(Settings settings) {
-            super(settings);
-        }
-
-        @Override
-        public void setRoles(String username, String... roles) {
-            doSetRoles(username, roles);
-            expire(username);
-        }
-
-        @Override
-        public void addRoles(String username, String... roles) {
-            doAddRoles(username, roles);
-            expire(username);
-        }
-
-        @Override
-        public void removeRoles(String username, String... roles) {
-            doRemoveRoles(username, roles);
-            expire(username);
-        }
-
-        @Override
-        public void removeUser(String username) {
-            doRemoveUser(username);
-            expire(username);
-        }
-
-        public abstract void doSetRoles(String username, String... roles);
-
-        public abstract void doAddRoles(String username, String... roles);
-
-        public abstract void doRemoveRoles(String username, String... roles);
-
-        public abstract void doRemoveUser(String username);
-
-    }
-
-}

src/main/java/org/elasticsearch/shield/authc/support/RefreshListener.java

@@ -8,16 +8,14 @@ package org.elasticsearch.shield.authc.support;
 /**
  *
  */
-public interface UserPasswdStore {
+public interface RefreshListener {
 
-    boolean verifyPassword(String username, SecuredString password);
+    static final RefreshListener NOOP = new RefreshListener() {
+        @Override
+        public void onRefresh() {
+        }
+    };
 
-    static interface Writable extends UserPasswdStore {
+    void onRefresh();
-
-        void store(String username, SecuredString password);
-
-        void remove(String username);
-
-    }
 
 }

src/main/java/org/elasticsearch/shield/authc/support/UserRolesStore.java

@@ -1,25 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.shield.authc.support;
-
-/**
- *
- */
-public interface UserRolesStore {
-
-    String[] roles(String username);
-
-    static interface Writable extends UserRolesStore {
-
-        void setRoles(String username, String... roles);
-
-        void addRoles(String username, String... roles);
-
-        void removeRoles(String username, String... roles);
-
-        void removeUser(String username);
-    }
-}

src/test/java/org/elasticsearch/shield/authc/esusers/ESUsersRealmTests.java

@@ -17,14 +17,18 @@ import org.elasticsearch.client.IndicesAdminClient;
 import org.elasticsearch.common.collect.ImmutableSet;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.shield.User;
-import org.elasticsearch.shield.authc.support.*;
+import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.SecuredStringTests;
+import org.elasticsearch.shield.authc.support.UsernamePasswordToken;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.elasticsearch.transport.TransportRequest;
+import org.elasticsearch.watcher.ResourceWatcherService;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 import org.junit.Test;
@@ -42,28 +46,31 @@ public class ESUsersRealmTests extends ElasticsearchTestCase {
     private RestController restController;
     private Client client;
     private AdminClient adminClient;
+    private FileUserPasswdStore userPasswdStore;
+    private FileUserRolesStore userRolesStore;
+
     @Before
     public void init() throws Exception {
         client = mock(Client.class);
         adminClient = mock(AdminClient.class);
         restController = mock(RestController.class);
+        userPasswdStore = mock(FileUserPasswdStore.class);
+        userRolesStore = mock(FileUserRolesStore.class);
     }
 
     @Test
     public void testRestHeaderRegistration() {
-        new ESUsersRealm(ImmutableSettings.EMPTY, mock(UserPasswdStore.class), mock(UserRolesStore.class), restController);
+        new ESUsersRealm(ImmutableSettings.EMPTY, mock(FileUserPasswdStore.class), mock(FileUserRolesStore.class), restController);
         verify(restController).registerRelevantHeaders(UsernamePasswordToken.BASIC_AUTH_HEADER);
     }
 
     @Test
     public void testAuthenticate() throws Exception {
         Settings settings = ImmutableSettings.builder().build();
-        MockUserPasswdStore userPasswdStore = new MockUserPasswdStore("user1", "test123");
+        when(userPasswdStore.verifyPassword("user1", SecuredStringTests.build("test123"))).thenReturn(true);
-        MockUserRolesStore userRolesStore = new MockUserRolesStore("user1", "role1", "role2");
+        when(userRolesStore.roles("user1")).thenReturn(new String[] { "role1", "role2" });
         ESUsersRealm realm = new ESUsersRealm(settings, userPasswdStore, userRolesStore, restController);
         User user = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
-        assertTrue(userPasswdStore.called);
-        assertTrue(userRolesStore.called);
         assertThat(user, notNullValue());
         assertThat(user.principal(), equalTo("user1"));
         assertThat(user.roles(), notNullValue());
@@ -76,19 +83,40 @@ public class ESUsersRealmTests extends ElasticsearchTestCase {
         Settings settings = ImmutableSettings.builder()
                 .put("shield.authc.esusers.cache.hash_algo", Hasher.values()[randomIntBetween(0, Hasher.values().length - 1)].name().toLowerCase(Locale.ROOT))
                 .build();
-        MockUserPasswdStore userPasswdStore = new MockUserPasswdStore("user1", "test123");
+        when(userPasswdStore.verifyPassword("user1", SecuredStringTests.build("test123"))).thenReturn(true);
-        MockUserRolesStore userRolesStore = new MockUserRolesStore("user1", "role1", "role2");
+        when(userRolesStore.roles("user1")).thenReturn(new String[] { "role1", "role2" });
         ESUsersRealm realm = new ESUsersRealm(settings, userPasswdStore, userRolesStore, restController);
         User user1 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
         User user2 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
         assertThat(user1, sameInstance(user2));
     }
 
+    public void testAuthenticate_Caching_Refresh() throws Exception {
+        userPasswdStore = spy(new UserPasswdStore());
+        userRolesStore = spy(new UserRolesStore());
+        doReturn(true).when(userPasswdStore).verifyPassword("user1", SecuredStringTests.build("test123"));
+        doReturn(new String[] { "role1", "role2" }).when(userRolesStore).roles("user1");
+        ESUsersRealm realm = new ESUsersRealm(ImmutableSettings.EMPTY, userPasswdStore, userRolesStore, restController);
+        User user1 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
+        User user2 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
+        assertThat(user1, sameInstance(user2));
+        userPasswdStore.notifyRefresh();
+        User user3 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
+        assertThat(user2, not(sameInstance(user3)));
+        User user4 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
+        assertThat(user3, sameInstance(user4));
+        userRolesStore.notifyRefresh();
+        User user5 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
+        assertThat(user4, not(sameInstance(user5)));
+        User user6 = realm.authenticate(new UsernamePasswordToken("user1", SecuredStringTests.build("test123")));
+        assertThat(user5, sameInstance(user6));
+    }
+
     @Test
     public void testToken() throws Exception {
         Settings settings = ImmutableSettings.builder().build();
-        MockUserPasswdStore userPasswdStore = new MockUserPasswdStore("user1", "test123");
+        when(userPasswdStore.verifyPassword("user1", SecuredStringTests.build("test123"))).thenReturn(true);
-        MockUserRolesStore userRolesStore = new MockUserRolesStore("user1", "role1", "role2");
+        when(userRolesStore.roles("user1")).thenReturn(new String[] { "role1", "role2" });
         ESUsersRealm realm = new ESUsersRealm(settings, userPasswdStore, userRolesStore, restController);
 
         TransportRequest request = new TransportRequest() {};
@@ -101,51 +129,10 @@ public class ESUsersRealmTests extends ElasticsearchTestCase {
         assertThat(new String(token.credentials().internalChars()), equalTo("test123"));
     }
 
-
-    private static class MockUserPasswdStore implements UserPasswdStore {
-
-        final String username;
-        final String password;
-        boolean called = false;
-
-        private MockUserPasswdStore(String username, String password) {
-            this.username = username;
-            this.password = password;
-        }
-
-        @Override
-        public boolean verifyPassword(String username, SecuredString password) {
-            called = true;
-            assertThat(username, equalTo(this.username));
-            assertThat(new String(password.internalChars()), equalTo(this.password));
-            return true;
-        }
-    }
-
-
-    private static class MockUserRolesStore implements UserRolesStore {
-
-        final String username;
-        final String[] roles;
-        boolean called = false;
-
-        private MockUserRolesStore(String username, String... roles) {
-            this.username = username;
-            this.roles = roles;
-        }
-
-        @Override
-        public String[] roles(String username) {
-            called = true;
-            assertThat(username, equalTo(this.username));
-            return roles;
-        }
-    }
-
     @Test @SuppressWarnings("unchecked")
     public void testRestHeadersAreCopied() throws Exception {
         // the required header will be registered only if ESUsersRealm is actually used.
-        new ESUsersRealm(ImmutableSettings.EMPTY, null, null, restController);
+        new ESUsersRealm(ImmutableSettings.EMPTY, new UserPasswdStore(), new UserRolesStore(), restController);
         when(restController.relevantHeaders()).thenReturn(ImmutableSet.of(UsernamePasswordToken.BASIC_AUTH_HEADER));
         when(client.admin()).thenReturn(adminClient);
         when(adminClient.cluster()).thenReturn(mock(ClusterAdminClient.class));
@@ -171,4 +158,18 @@ public class ESUsersRealmTests extends ElasticsearchTestCase {
         handler.handleRequest(restRequest, channel);
         assertThat((String) request.getHeader(UsernamePasswordToken.BASIC_AUTH_HEADER), Matchers.equalTo("foobar"));
     }
+
+    static class UserPasswdStore extends FileUserPasswdStore {
+
+        public UserPasswdStore() {
+            super(ImmutableSettings.EMPTY, new Environment(ImmutableSettings.EMPTY), mock(ResourceWatcherService.class));
+        }
+    }
+
+    static class UserRolesStore extends FileUserRolesStore {
+
+        public UserRolesStore() {
+            super(ImmutableSettings.EMPTY, new Environment(ImmutableSettings.EMPTY), mock(ResourceWatcherService.class));
+        }
+    }
 }

src/test/java/org/elasticsearch/shield/authc/esusers/FileUserPasswdStoreTests.java

@@ -12,6 +12,7 @@ import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.shield.authc.support.Hasher;
+import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.shield.authc.support.SecuredStringTests;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -31,6 +32,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.*;
+import static org.mockito.Mockito.contains;
 import static org.mockito.Mockito.*;
 
 /**
@@ -79,7 +81,7 @@ public class FileUserPasswdStoreTests extends ElasticsearchTestCase {
             threadPool = new ThreadPool("test");
             watcherService = new ResourceWatcherService(settings, threadPool);
             final CountDownLatch latch = new CountDownLatch(1);
-            FileUserPasswdStore store = new FileUserPasswdStore(settings, env, watcherService, new FileUserPasswdStore.Listener() {
+            FileUserPasswdStore store = new FileUserPasswdStore(settings, env, watcherService, new RefreshListener() {
                 @Override
                 public void onRefresh() {
                     latch.countDown();

src/test/java/org/elasticsearch/shield/authc/esusers/FileUserRolesStoreTests.java

@@ -10,6 +10,7 @@ import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.shield.authc.support.RefreshListener;
 import org.elasticsearch.test.ElasticsearchTestCase;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
@@ -73,7 +74,7 @@ public class FileUserRolesStoreTests extends ElasticsearchTestCase {
             threadPool = new ThreadPool("test");
             watcherService = new ResourceWatcherService(settings, threadPool);
             final CountDownLatch latch = new CountDownLatch(1);
-            FileUserRolesStore store = new FileUserRolesStore(settings, env, watcherService, new FileUserRolesStore.Listener() {
+            FileUserRolesStore store = new FileUserRolesStore(settings, env, watcherService, new RefreshListener() {
                 @Override
                 public void onRefresh() {
                     latch.countDown();

src/test/java/org/elasticsearch/shield/authc/ldap/LdapRealmTest.java

@@ -41,7 +41,7 @@ public class LdapRealmTest extends LdapTest {
     }
 
     @Test
-    public void testAuthenticate_subTreeGroupSearch(){
+    public void testAuthenticate_SubTreeGroupSearch(){
         String groupSearchBase = "o=sevenSeas";
         boolean isSubTreeSearch = true;
         String userTemplate = VALID_USER_TEMPLATE;
@@ -55,7 +55,7 @@ public class LdapRealmTest extends LdapTest {
     }
 
     @Test
-    public void testAuthenticate_oneLevelGroupSearch(){
+    public void testAuthenticate_OneLevelGroupSearch(){
         String groupSearchBase = "ou=crews,ou=groups,o=sevenSeas";
         boolean isSubTreeSearch = false;
         String userTemplate = VALID_USER_TEMPLATE;
@@ -70,7 +70,7 @@ public class LdapRealmTest extends LdapTest {
     }
 
     @Test
-    public void testAuthenticate_caching(){
+    public void testAuthenticate_Caching(){
         String groupSearchBase = "o=sevenSeas";
         boolean isSubTreeSearch = true;
         String userTemplate = VALID_USER_TEMPLATE;
@@ -87,7 +87,33 @@ public class LdapRealmTest extends LdapTest {
     }
 
     @Test
-    public void testAuthenticate_noncaching(){
+    public void testAuthenticate_Caching_Refresh(){
+        String groupSearchBase = "o=sevenSeas";
+        boolean isSubTreeSearch = true;
+        String userTemplate = VALID_USER_TEMPLATE;
+        StandardLdapConnectionFactory ldapFactory = new StandardLdapConnectionFactory(
+                buildLdapSettings( apacheDsRule.getUrl(), userTemplate, groupSearchBase, isSubTreeSearch) );
+
+        LdapGroupToRoleMapper roleMapper = buildGroupAsRoleMapper();
+
+        ldapFactory = spy(ldapFactory);
+        LdapRealm ldap = new LdapRealm( buildCachingSettings(), ldapFactory, roleMapper, restController);
+        User user = ldap.authenticate( new UsernamePasswordToken(VALID_USERNAME, SecuredStringTests.build(PASSWORD)));
+        user = ldap.authenticate( new UsernamePasswordToken(VALID_USERNAME, SecuredStringTests.build(PASSWORD)));
+
+        //verify one and only one bind -> caching is working
+        verify(ldapFactory, times(1)).bind(anyString(), any(SecuredString.class));
+
+        roleMapper.notifyRefresh();
+
+        user = ldap.authenticate( new UsernamePasswordToken(VALID_USERNAME, SecuredStringTests.build(PASSWORD)));
+
+        //we need to bind again
+        verify(ldapFactory, times(2)).bind(anyString(), any(SecuredString.class));
+    }
+
+    @Test
+    public void testAuthenticate_Noncaching(){
         String groupSearchBase = "o=sevenSeas";
         boolean isSubTreeSearch = true;
         String userTemplate = VALID_USER_TEMPLATE;
@@ -102,4 +128,6 @@ public class LdapRealmTest extends LdapTest {
         //verify two and only two binds -> caching is disabled
         verify(ldapFactory, times(2)).bind(anyString(), any(SecuredString.class));
     }
+
+
 }