Issue #2609 - Uniform Find Expired Sessions (#5097)

* Issue #2609 Make all session stores use same algorithm to find expired sessions

Signed-off-by: Jan Bartel <janb@webtide.com>

jetty-documentation/src/main/asciidoc/distribution-guide/sessions/session-configuration-hazelcast.adoc

@@ -84,7 +84,7 @@ Opening the `start.d/session-store-hazelcast-remote.ini` will show a list of all
 #jetty.session.hazelcast.mapName=jetty_sessions
 #jetty.session.hazelcast.onlyClient=true
 #jetty.session.hazelcast.configurationLocation=
-jetty.session.hazelcast.scavengeZombies=false
+jetty.session.hazelcast.useQueries=false
 #jetty.session.gracePeriod.seconds=3600
 #jetty.session.savePeriod.seconds=0
 ----
@@ -95,8 +95,8 @@ jetty.session.hazelcast.onlyClient::
 Hazelcast instance will be configured in client mode
 jetty.session.hazelcast.configurationLocation::
 Path to an an Hazelcast xml configuration file
-jetty.session.hazelcast.scavengeZombies::
-True/False. `False` by default. If `true`, jetty will use hazelcast queries to find sessions that are no longer being used on any jetty node and whose expiry time has passed. If you enable this option, and your session stores attributes that reference classes from inside your webapp, or jetty classes, you will need to ensure that these classes are available on each of your hazelcast instances.
+jetty.session.hazelcast.useQueries::
+True/False. `False` by default. If `true`, jetty will use hazelcast queries to find sessions that are no longer being used on any jetty node and whose expiry time has passed.
 jetty.session.gracePeriod.seconds::
 Amount of time, in seconds, to wait for other nodes to be checked to verify an expired session is in fact expired throughout the cluster before closing it.
 jetty.session.savePeriod.seconds=0::
@@ -110,7 +110,7 @@ In a clustered environment, there is a risk of the last access time of the sessi
 This allows the possibility that a node may prematurely expire the session, even though it is in use by another node.
 Thorough consideration of the `maxIdleTime` of the session when setting the `savePeriod` is imperative - there is no point in setting a `savePeriod` that is larger than the `maxIdleTime`.
 
-Be aware using the `scavengeZombies` option that if your session attributes contain classes from inside your webapp (or jetty classes) then you will need to put these classes onto the classpath of all of your hazelcast instances.
+Be aware that if your session attributes contain classes from inside your webapp (or jetty classes) then you will need to put these classes onto the classpath of all of your hazelcast instances.
 ____
 
 ==== Configuring Embedded Hazelcast Clustering
@@ -178,8 +178,8 @@ jetty.session.hazelcast.mapName::
 Name of the Map in Hazelcast where sessions will be stored.
 jetty.session.hazelcast.configurationLocation::
 Path to an an Hazelcast xml configuration file
-jetty.session.hazelcast.scavengeZombies::
-True/False. `False` by default. If `true`, jetty will use hazelcast queries to find sessions that are no longer being used on any jetty node and whose expiry time has passed. If you enable this option, and your sessions contain attributes that reference classes from inside your webapp (or jetty classes) you will need to ensure that these classes are available on each of your hazelcast instances.
+jetty.session.hazelcast.useQueries::
+True/False. `False` by default. If `true`, jetty will use hazelcast queries to find sessions that are no longer being used on any jetty node and whose expiry time has passed. 
 jetty.session.gracePeriod.seconds::
 Amount of time, in seconds, to wait for other nodes to be checked to verify an expired session is in fact expired throughout the cluster before closing it.
 jetty.session.savePeriod.seconds=0::
@@ -193,5 +193,5 @@ In a clustered environment, there is a risk of the last access time of the sessi
 This allows the possibility that a node may prematurely expire the session, even though it is in use by another node.
 Thorough consideration of the `maxIdleTime` of the session when setting the `savePeriod` is imperative - there is no point in setting a `savePeriod` that is larger than the `maxIdleTime`.
 
-Be aware using the `scavengeZombies` option that if your session attributes contain classes from inside your webapp (or jetty classes) then you will need to put these classes onto the classpath of all of your hazelcast instances. In the cast of embedded hazelcast, as it is started before your webapp, it will NOT have access to your webapp's classes - you will need to extract these classes and put them onto the jetty server's classpath.
+If your session attributes contain classes from inside your webapp (or jetty classes) then you will need to put these classes onto the classpath of all of your hazelcast instances. In the case of embedded hazelcast, as it is started before your webapp, it will NOT have access to your webapp's classes - you will need to extract these classes and put them onto the jetty server's classpath.
 ____

jetty-gcloud/jetty-gcloud-session-manager/src/main/java/org/eclipse/jetty/gcloud/session/GCloudSessionDataStore.java

@@ -22,6 +22,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.ObjectOutputStream;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import com.google.cloud.datastore.Blob;
 import com.google.cloud.datastore.BlobValue;
@@ -65,12 +66,10 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
     protected int _maxResults = DEFAULT_MAX_QUERY_RESULTS;
     protected int _maxRetries = DEFAULT_MAX_RETRIES;
     protected int _backoff = DEFAULT_BACKOFF_MS;
-
     protected boolean _dsProvided = false;
     protected boolean _indexesPresent = false;
     protected EntityDataModel _model;
     protected boolean _modelProvided;
-
     private String _namespace;
 
     /**
@@ -92,7 +91,6 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
         public static final String MAXINACTIVE = "maxInactive";
         public static final String ATTRIBUTES = "attributes";
         public static final String LASTSAVED = "lastSaved";
-
         public static final String KIND = "GCloudSession";
         protected String _kind = KIND;
         protected String _id = ID;
@@ -351,18 +349,24 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
     {
         String _id;
         String _lastNode;
+        String _contextPath;
+        String _vhost;
         long _expiry;
 
         /**
          * @param id session id
          * @param lastNode last node id to manage the session
          * @param expiry timestamp of expiry
+         * @param contextPath  context path for session
+         * @param vhost vhost of context for session
          */
-        public ExpiryInfo(String id, String lastNode, long expiry)
+        public ExpiryInfo(String id, String lastNode, long expiry, String contextPath, String vhost)
         {
             _id = id;
             _lastNode = lastNode;
             _expiry = expiry;
+            _contextPath = contextPath;
+            _vhost = vhost;
         }
 
         /**
@@ -388,6 +392,26 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
         {
             return _expiry;
         }
+        
+        public String getContextPath()
+        {
+            return _contextPath;
+        }
+
+        public void setContextPath(String contextPath)
+        {
+            _contextPath = contextPath;
+        }
+
+        public String getVhost()
+        {
+            return _vhost;
+        }
+
+        public void setVhost(String vhost)
+        {
+            _vhost = vhost;
+        }
     }
 
     public void setEntityDataModel(EntityDataModel model)
@@ -527,56 +551,34 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
     {
-        long now = System.currentTimeMillis();
         Set<String> expired = new HashSet<String>();
 
         try
-        {
+        {    
             Set<ExpiryInfo> info = null;
             if (_indexesPresent)
-                info = queryExpiryByIndex();
+                info = queryExpiryByIndex(time);
             else
-                info = queryExpiryByEntity();
+                info = queryExpiryByEntity(time);
 
             for (ExpiryInfo item : info)
             {
-                if (StringUtil.isBlank(item.getLastNode()))
-                {
-                    expired.add(item.getId()); //nobody managing it
-                }
-                else
-                {
-                    if (_context.getWorkerName().equals(item.getLastNode()))
-                    {
-                        expired.add(item.getId()); //we're managing it, we can expire it
-                    }
-                    else
-                    {
-                        if (_lastExpiryCheckTime <= 0)
-                        {
-                            //our first check, just look for sessions that we managed by another node that
-                            //expired at least 3 graceperiods ago
-                            if (item.getExpiry() < (now - (1000L * (3 * _gracePeriodSec))))
-                                expired.add(item.getId());
-                        }
-                        else
-                        {
-                            //another node was last managing it, only expire it if it expired a graceperiod ago
-                            if (item.getExpiry() < (now - (1000L * _gracePeriodSec)))
-                                expired.add(item.getId());
-                        }
-                    }
-                }
-            }
+                if (StringUtil.isBlank(item.getLastNode()) || !(_context.getWorkerName().equals(item.getLastNode())))
+                    continue; //we're not its manager so skip it
+                
+                if (StringUtil.isBlank(item.getContextPath()) || !(_context.getCanonicalContextPath().equals(item.getContextPath())))
+                    continue; // session is not for this context
+                expired.add(item.getId());
+            }   
 
             //reconcile against ids that the SessionCache thinks are expired
             Set<String> tmp = new HashSet<String>(candidates);
             tmp.removeAll(expired);
             if (!tmp.isEmpty())
             {
-                //sessionstore thinks these are expired, but they are either no
+                //sessioncache thinks these are expired, but they are either no
                 //longer in the db or not expired in the db, or we exceeded the
                 //number of records retrieved by the expiry query, so check them
                 //individually
@@ -600,12 +602,69 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
             }
 
             return expired;
+            
         }
         catch (Exception e)
         {
             LOG.warn("Unable to get expired", e);
             return expired; //return what we got
+        } 
+    }
+    
+    @Override
+    public Set<String> doGetExpired(long time)
+    {
+        // Get sessions managed by any node that expired at or before the given
+        // time limit
+        Set<String> expired = new HashSet<String>();
+        try
+        {    
+            Set<ExpiryInfo> info = null;
+            if (_indexesPresent)
+                info = queryExpiryByIndex(time);
+            else
+                info = queryExpiryByEntity(time);
+
+            for (ExpiryInfo item:info)
+            {
+                expired.add(item.getId());
+            }
+            return expired;
+        }     
+        catch (Exception e)
+        {
+            LOG.warn("Error querying expired sessions", e);
+            return expired; //return what we got
+        }   
+    }
+
+    @Override
+    public void doCleanOrphans(long timeLimit)
+    {
+        // Gcloud datastore does not support DELETE statements with query params.
+        // Therefore need to do a query, and then a separate operation to delete keys 
+        //returned.
+        try
+        {    
+            Set<ExpiryInfo> info = null;
+            if (_indexesPresent)
+                info = queryExpiryByIndex(timeLimit);
+            else
+                info = queryExpiryByEntity(timeLimit);
+
+            //iterate over each of the returned infos,
+            //make a key for each, then do the delete
+            Set<Key> keys = info.stream().map(i ->
+            {
+                return makeKey(i.getId(), i.getContextPath(), i.getVhost());
+            }).collect(Collectors.toSet());
+            
+            _datastore.delete(keys.toArray(new Key[keys.size()]));
         }
+        catch (Exception e)
+        {
+            LOG.warn("Error deleting orphaned sessions", e);
+        }   
     }
 
     /**
@@ -618,12 +677,25 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
      */
     protected Set<ExpiryInfo> queryExpiryByEntity() throws Exception
     {
-        Set<ExpiryInfo> info = new HashSet<>();
+        return queryExpiryByEntity(System.currentTimeMillis());
+    }
 
+    /**
+     * A less efficient query to find sessions whose expiry time is before the
+     * given timeLimit.
+     * 
+     * @param timeLimit time since the epoch
+     * 
+     * @return set of ExpiryInfo representing the id,lastNode and expiry time
+     * @throws Exception
+     */
+    protected Set<ExpiryInfo> queryExpiryByEntity(long timeLimit) throws Exception
+    {
+        Set<ExpiryInfo> infos = new HashSet<>();
         //get up to maxResult number of sessions that have expired
         Query<Entity> query = Query.newEntityQueryBuilder()
             .setKind(_model.getKind())
-            .setFilter(CompositeFilter.and(PropertyFilter.gt(_model.getExpiry(), 0), PropertyFilter.le(_model.getExpiry(), System.currentTimeMillis())))
+            .setFilter(CompositeFilter.and(PropertyFilter.gt(_model.getExpiry(), 0), PropertyFilter.le(_model.getExpiry(), timeLimit)))
             .setLimit(_maxResults)
             .build();
 
@@ -636,15 +708,21 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
         }
         else
             results = _datastore.run(query);
+        
         while (results.hasNext())
         {
             Entity entity = results.next();
-            info.add(new ExpiryInfo(entity.getString(_model.getId()), entity.getString(_model.getLastNode()), entity.getLong(_model.getExpiry())));
+            ExpiryInfo info = new ExpiryInfo(entity.getString(_model.getId()),
+                                             entity.getString(_model.getLastNode()), 
+                                             entity.getLong(_model.getExpiry()),
+                                             entity.getString(_model.getContextPath()),
+                                             entity.getString(_model.getVhost()));
+            
+            infos.add(info);
         }
-
-        return info;
+        return infos;
     }
-
+    
     /**
      * An efficient query to find sessions whose expiry time has passed:
      * uses a projection query, which requires indexes to be uploaded.
@@ -654,12 +732,24 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
      */
     protected Set<ExpiryInfo> queryExpiryByIndex() throws Exception
     {
-        long now = System.currentTimeMillis();
-        Set<ExpiryInfo> info = new HashSet<>();
+        return queryExpiryByIndex(System.currentTimeMillis());       
+    }
+
+    /**
+     * An efficient query to find sessions whose expiry time is before the given timeLimit:
+     * uses a projection query, which requires indexes to be uploaded.
+     * 
+     * @param timeLimit the upper limit of expiry time to check
+     * @return  id,lastnode and expiry time of sessions that have expired
+     * @throws Exception
+     */
+    protected Set<ExpiryInfo>  queryExpiryByIndex(long timeLimit) throws Exception
+    {
+        Set<ExpiryInfo> infos = new HashSet<>();
         Query<ProjectionEntity> query = Query.newProjectionEntityQueryBuilder()
             .setKind(_model.getKind())
-            .setProjection(_model.getId(), _model.getLastNode(), _model.getExpiry())
-            .setFilter(CompositeFilter.and(PropertyFilter.gt(_model.getExpiry(), 0), PropertyFilter.le(_model.getExpiry(), now)))
+            .setProjection(_model.getId(), _model.getLastNode(), _model.getExpiry(), _model.getContextPath(), _model.getVhost())
+            .setFilter(CompositeFilter.and(PropertyFilter.gt(_model.getExpiry(), 0), PropertyFilter.le(_model.getExpiry(), timeLimit)))
             .setLimit(_maxResults)
             .build();
 
@@ -677,14 +767,19 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
         while (presults.hasNext())
         {
             ProjectionEntity pe = presults.next();
-            info.add(new ExpiryInfo(pe.getString(_model.getId()), pe.getString(_model.getLastNode()), pe.getLong(_model.getExpiry())));
+            ExpiryInfo info = new ExpiryInfo(pe.getString(_model.getId()),
+                                             pe.getString(_model.getLastNode()), 
+                                             pe.getLong(_model.getExpiry()),
+                                             pe.getString(_model.getContextPath()),
+                                             pe.getString(_model.getVhost()));
+            infos.add(info);
         }
 
-        return info;
+        return infos; 
     }
-
+   
     @Override
-    public boolean exists(String id) throws Exception
+    public boolean doExists(String id) throws Exception
     {
         if (_indexesPresent)
         {
@@ -819,7 +914,12 @@ public class GCloudSessionDataStore extends AbstractSessionDataStore
      */
     protected Key makeKey(String id, SessionContext context)
     {
-        String key = context.getCanonicalContextPath() + "_" + context.getVhost() + "_" + id;
+        return makeKey(id, context.getCanonicalContextPath(), context.getVhost());
+    }
+    
+    protected Key makeKey(String id, String canonicalContextPath, String canonicalVHost)
+    {
+        String key = canonicalContextPath + "_" + canonicalVHost + "_" + id;
         return _keyFactory.newKey(key);
     }
 

jetty-hazelcast/pom.xml

@@ -11,23 +11,10 @@
   <name>Jetty :: Hazelcast Session Manager</name>
 
   <properties>
-    <hazelcast.version>3.12.6</hazelcast.version>
     <bundle-symbolic-name>${project.groupId}.hazelcast</bundle-symbolic-name>
   </properties>
 
   <dependencies>
-    <dependency>
-      <groupId>com.hazelcast</groupId>
-      <artifactId>hazelcast</artifactId>
-      <version>${hazelcast.version}</version>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.hazelcast</groupId>
-      <artifactId>hazelcast-client</artifactId>
-      <version>${hazelcast.version}</version>
-    </dependency>
     <dependency>
       <groupId>com.hazelcast</groupId>
       <artifactId>hazelcast</artifactId>

jetty-hazelcast/src/main/config/etc/sessions/hazelcast/default.xml

@@ -13,7 +13,7 @@
       <New id="sessionDataStoreFactory" class="org.eclipse.jetty.hazelcast.session.HazelcastSessionDataStoreFactory">
         <Set name="mapName" property="jetty.session.hazelcast.mapName"/>
         <Set name="hazelcastInstanceName" property="jetty.session.hazelcast.hazelcastInstanceName"/>
-        <Set name="scavengeZombies" property="jetty.session.hazelcast.scavengeZombies"/>
+        <Set name="useQueries" property="jetty.session.hazelcast.useQueries"/>
         <Set name="gracePeriodSec"><Property name="jetty.session.gracePeriod.seconds" default="3600" /></Set>
         <Set name="savePeriodSec"><Property name="jetty.session.savePeriod.seconds" default="0" /></Set>
         <Set name="configurationLocation"><Property name="jetty.session.hazelcast.configurationLocation" default="" /></Set>

jetty-hazelcast/src/main/config/etc/sessions/hazelcast/remote.xml

@@ -16,8 +16,8 @@
         <Set name="hazelcastInstanceName">
           <Property name="jetty.session.hazelcast.hazelcastInstanceName" default="JETTY_DISTRIBUTED_SESSION_INSTANCE" />
         </Set>
-        <Set name="scavengeZombies">
-          <Property name="jetty.session.hazelcast.scavengeZombies" default="false" />
+        <Set name="useQueries">
+          <Property name="jetty.session.hazelcast.useQueries" default="false" />
         </Set>
         <Set name="gracePeriodSec">
           <Property name="jetty.session.gracePeriod.seconds" default="3600" />

jetty-hazelcast/src/main/config/modules/session-store-hazelcast-embedded.mod

@@ -13,7 +13,7 @@ session-store
 sessions
 
 [files]
-maven://com.hazelcast/hazelcast/3.12.6|lib/hazelcast/hazelcast-3.12.6.jar
+maven://com.hazelcast/hazelcast/4.0.1|lib/hazelcast/hazelcast-4.0.1.jar
 
 [xml]
 etc/sessions/hazelcast/default.xml
@@ -31,7 +31,7 @@ http://www.apache.org/licenses/LICENSE-2.0.html
 [ini-template]
 jetty.session.hazelcast.mapName=jetty-distributed-session-map
 jetty.session.hazelcast.hazelcastInstanceName=JETTY_DISTRIBUTED_SESSION_INSTANCE
-jetty.session.hazelcast.scavengeZombies=false
+jetty.session.hazelcast.useQueries=false
 jetty.session.gracePeriod.seconds=3600
 jetty.session.savePeriod.seconds=0
 #jetty.session.hazelcast.configurationLocation

jetty-hazelcast/src/main/config/modules/session-store-hazelcast-remote.mod

@@ -13,8 +13,7 @@ session-store
 sessions
 
 [files]
-maven://com.hazelcast/hazelcast/3.12.6|lib/hazelcast/hazelcast-3.12.6.jar
-maven://com.hazelcast/hazelcast-client/3.12.6|lib/hazelcast/hazelcast-client-3.12.6.jar
+maven://com.hazelcast/hazelcast/4.0.1|lib/hazelcast/hazelcast-4.0.1.jar
 
 [xml]
 etc/sessions/hazelcast/remote.xml
@@ -33,7 +32,7 @@ http://www.apache.org/licenses/LICENSE-2.0.html
 jetty.session.hazelcast.mapName=jetty-distributed-session-map
 jetty.session.hazelcast.hazelcastInstanceName=JETTY_DISTRIBUTED_SESSION_INSTANCE
 jetty.session.hazelcast.onlyClient=true
-jetty.session.hazelcast.scavengeZombies=false
+jetty.session.hazelcast.useQueries=false
 jetty.session.gracePeriod.seconds=3600
 jetty.session.savePeriod.seconds=0
 #jetty.session.hazelcast.configurationLocation

jetty-hazelcast/src/main/java/org/eclipse/jetty/hazelcast/session/HazelcastSessionDataStore.java

@@ -19,15 +19,19 @@
 package org.eclipse.jetty.hazelcast.session;
 
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import com.hazelcast.core.IMap;
-import com.hazelcast.query.EntryObject;
+import com.hazelcast.config.IndexConfig;
+import com.hazelcast.config.IndexType;
+import com.hazelcast.map.IMap;
 import com.hazelcast.query.Predicate;
 import com.hazelcast.query.PredicateBuilder;
+import com.hazelcast.query.PredicateBuilder.EntryObject;
+import com.hazelcast.query.Predicates;
 import org.eclipse.jetty.server.session.AbstractSessionDataStore;
 import org.eclipse.jetty.server.session.SessionContext;
 import org.eclipse.jetty.server.session.SessionData;
@@ -41,8 +45,7 @@ import org.slf4j.LoggerFactory;
  * Session data stored in Hazelcast
  */
 @ManagedObject
-public class HazelcastSessionDataStore
-    extends AbstractSessionDataStore
+public class HazelcastSessionDataStore extends AbstractSessionDataStore
     implements SessionDataStore
 {
 
@@ -50,7 +53,7 @@ public class HazelcastSessionDataStore
 
     private IMap<String, SessionData> sessionDataMap;
 
-    private boolean _scavengeZombies;
+    private boolean _useQueries;
 
     public HazelcastSessionDataStore()
     {
@@ -58,9 +61,9 @@ public class HazelcastSessionDataStore
 
     /**
      * Control whether or not to execute queries to find
-     * "zombie" sessions - ie sessions that are no longer
-     * actively referenced by any jetty instance and should
-     * be expired.
+     * expired sessions - ie sessions for this context 
+     * that are no longer actively referenced by any jetty 
+     * instance and should be expired.
      *
      * If you use this feature, be aware that if your session
      * stores any attributes that use classes from within your
@@ -68,18 +71,18 @@ public class HazelcastSessionDataStore
      * those classes are available to all of your hazelcast
      * instances, whether embedded or remote.
      *
-     * @param scavengeZombies true means unreferenced sessions
+     * @param useQueries true means unreferenced sessions
      * will be actively sought and expired. False means that they
      * will remain in hazelcast until some other mechanism removes them.
      */
-    public void setScavengeZombieSessions(boolean scavengeZombies)
+    public void setUseQueries(boolean useQueries)
     {
-        _scavengeZombies = scavengeZombies;
+        _useQueries = useQueries;
     }
 
-    public boolean isScavengeZombies()
+    public boolean isUseQueries()
     {
-        return _scavengeZombies;
+        return _useQueries;
     }
 
     @Override
@@ -127,8 +130,8 @@ public class HazelcastSessionDataStore
         throws Exception
     {
         super.initialize(context);
-        if (isScavengeZombies())
-            sessionDataMap.addIndex("expiry", true);
+        if (isUseQueries())
+            sessionDataMap.addIndex(new IndexConfig(IndexType.SORTED, "expiry"));
     }
 
     @Override
@@ -143,15 +146,80 @@ public class HazelcastSessionDataStore
     {
         return true;
     }
+    
+    @Override
+    public void doCleanOrphans(long timeLimit)
+    {
+        if (!isUseQueries())
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Hazelcast useQueries=false, cannot clean orphaned sessions");
+            return;
+        }
+
+        EntryObject eo = Predicates.newPredicateBuilder().getEntryObject();
+        @SuppressWarnings("unchecked")
+        Predicate<String, SessionData> predicate = eo.get("expiry").greaterThan(0)
+            .and(eo.get("expiry").lessEqual(timeLimit));
+        sessionDataMap.removeAll(predicate);
+    }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doGetExpired(long time)
     {
-        long now = System.currentTimeMillis();
+        if (!isUseQueries())
+        {
+            if (LOG.isDebugEnabled())
+                LOG.debug("Hazelcast useQueries=false, cannot search for expired sessions");
+            return Collections.emptySet();
+        }
 
+        //Now find other sessions for our context in hazelcast that have expired
+        final AtomicReference<Set<String>> reference = new AtomicReference<>();
+        final AtomicReference<Exception> exception = new AtomicReference<>();
+
+        _context.run(() ->
+        {
+            try
+            {
+                Set<String> ids = new HashSet<>();
+                EntryObject eo = Predicates.newPredicateBuilder().getEntryObject();
+                Predicate<String, SessionData> predicate = eo.get("expiry").greaterThan(0)
+                    .and(eo.get("expiry").lessEqual(time))
+                    .and(eo.get("contextPath").equal(_context.getCanonicalContextPath()));
+                Collection<SessionData> results = sessionDataMap.values(predicate);
+                if (results != null)
+                {
+                    for (SessionData sd : results)
+                    {
+                        ids.add(sd.getId());
+                    }
+                }
+                reference.set(ids);
+            }
+            catch (Exception e)
+            {
+                exception.set(e);
+            }
+        });
+
+        if (exception.get() != null)
+        {
+            LOG.warn("Error querying for expired sessions {}", exception.get());
+            return Collections.emptySet();
+        }
+
+        if (reference.get() == null)
+            return Collections.emptySet();
+           
+        return reference.get();
+    }
+    
+    @Override
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
+    {
         Set<String> expiredSessionIds = candidates.stream().filter(candidate ->
         {
-
             if (LOG.isDebugEnabled())
                 LOG.debug("Checking expiry for candidate {}", candidate);
 
@@ -173,7 +241,7 @@ public class HazelcastSessionDataStore
                     if (_context.getWorkerName().equals(sd.getLastNode()))
                     {
                         //we are its manager, add it to the expired set if it is expired now
-                        if ((sd.getExpiry() > 0) && sd.getExpiry() <= now)
+                        if ((sd.getExpiry() > 0) && sd.getExpiry() <= time)
                         {
                             if (LOG.isDebugEnabled())
                             {
@@ -182,27 +250,6 @@ public class HazelcastSessionDataStore
                             return true;
                         }
                     }
-                    else
-                    {
-                        //if we are not the session's manager, only expire it iff:
-                        // this is our first expiryCheck and the session expired a long time ago
-                        //or
-                        //the session expired at least one graceperiod ago
-                        if (_lastExpiryCheckTime <= 0)
-                        {
-                            if ((sd.getExpiry() > 0) && sd.getExpiry() < (now - (1000L * (3 * _gracePeriodSec))))
-                            {
-                                return true;
-                            }
-                        }
-                        else
-                        {
-                            if ((sd.getExpiry() > 0) && sd.getExpiry() < (now - (1000L * _gracePeriodSec)))
-                            {
-                                return true;
-                            }
-                        }
-                    }
                 }
             }
             catch (Exception e)
@@ -213,56 +260,15 @@ public class HazelcastSessionDataStore
             return false;
         }).collect(Collectors.toSet());
 
-        if (isScavengeZombies())
-        {
-            //Now find other sessions in hazelcast that have expired
-            final AtomicReference<Set<String>> reference = new AtomicReference<>();
-            final AtomicReference<Exception> exception = new AtomicReference<>();
-
-            _context.run(() ->
-            {
-                try
-                {
-                    Set<String> ids = new HashSet<>();
-                    EntryObject eo = new PredicateBuilder().getEntryObject();
-                    Predicate<?, ?> predicate = eo.get("expiry").greaterThan(0).and(eo.get("expiry").lessEqual(now));
-                    Collection<SessionData> results = sessionDataMap.values(predicate);
-                    if (results != null)
-                    {
-                        for (SessionData sd : results)
-                        {
-                            ids.add(sd.getId());
-                        }
-                    }
-                    reference.set(ids);
-                }
-                catch (Exception e)
-                {
-                    exception.set(e);
-                }
-            });
-
-            if (exception.get() != null)
-            {
-                LOG.warn("Error querying for expired sessions", exception.get());
-                return expiredSessionIds;
-            }
-
-            if (reference.get() != null)
-            {
-                expiredSessionIds.addAll(reference.get());
-            }
-        }
-
         return expiredSessionIds;
     }
 
     @Override
-    public boolean exists(String id)
+    public boolean doExists(String id)
         throws Exception
     {
         //TODO find way to do query without pulling in whole session data
-        SessionData sd = load(id);
+        SessionData sd = doLoad(id);
         if (sd == null)
             return false;
 

jetty-hazelcast/src/main/java/org/eclipse/jetty/hazelcast/session/HazelcastSessionDataStoreFactory.java

@@ -57,18 +57,18 @@ public class HazelcastSessionDataStoreFactory
 
     private MapConfig mapConfig;
 
-    private boolean scavengeZombies = false;
+    private boolean useQueries = false;
 
     private String addresses;
 
-    public boolean isScavengeZombies()
+    public boolean isUseQueries()
     {
-        return scavengeZombies;
+        return useQueries;
     }
 
-    public void setScavengeZombies(boolean scavengeZombies)
+    public void setUseQueries(boolean useQueries)
     {
-        this.scavengeZombies = scavengeZombies;
+        this.useQueries = useQueries;
     }
 
     @Override
@@ -145,7 +145,7 @@ public class HazelcastSessionDataStoreFactory
         hazelcastSessionDataStore.setSessionDataMap(hazelcastInstance.getMap(mapName));
         hazelcastSessionDataStore.setGracePeriodSec(getGracePeriodSec());
         hazelcastSessionDataStore.setSavePeriodSec(getSavePeriodSec());
-        hazelcastSessionDataStore.setScavengeZombieSessions(scavengeZombies);
+        hazelcastSessionDataStore.setUseQueries(isUseQueries());
         return hazelcastSessionDataStore;
     }
 

jetty-hazelcast/src/test/java/org/eclipse/jetty/hazelcast/session/TestHazelcastSessions.java

@@ -65,7 +65,6 @@ public class TestHazelcastSessions
             else if ("get".equals(arg))
             {
                 s = req.getSession(false);
-                System.err.println("GET: s=" + s + ",id=" + (s != null ? s.getId() : ""));
             }
             else if ("del".equals(arg))
             {

jetty-infinispan/infinispan-common/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanKeyBuilder.java

@@ -0,0 +1,31 @@
+//
+// ========================================================================
+// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
+//
+// This program and the accompanying materials are made available under
+// the terms of the Eclipse Public License 2.0 which is available at
+// https://www.eclipse.org/legal/epl-2.0
+//
+// This Source Code may also be made available under the following
+// Secondary Licenses when the conditions for such availability set
+// forth in the Eclipse Public License, v. 2.0 are satisfied:
+// the Apache License v2.0 which is available at
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
+// ========================================================================
+//
+
+package org.eclipse.jetty.session.infinispan;
+
+/**
+ *
+ *
+ */
+public class InfinispanKeyBuilder
+{
+    public static String build(String contextPath, String vhost, String id)
+    {
+        return contextPath + "_" + vhost + "_" + id;
+    }
+}

jetty-infinispan/infinispan-common/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionDataStore.java

@@ -18,6 +18,7 @@
 
 package org.eclipse.jetty.session.infinispan;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
@@ -35,6 +36,8 @@ import org.slf4j.LoggerFactory;
 
 /**
  * InfinispanSessionDataStore
+ *
+ *
  */
 @ManagedObject
 public class InfinispanSessionDataStore extends AbstractSessionDataStore
@@ -44,44 +47,31 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
     /**
      * Clustered cache of sessions
      */
-    private BasicCache<String, SessionData> _cache;
-
+    private BasicCache<String, InfinispanSessionData> _cache;
     private int _infinispanIdleTimeoutSec;
-
     private QueryManager _queryManager;
-
     private boolean _passivating;
-
+    
     /**
      * Get the clustered cache instance.
-     *
+     * 
      * @return the cache
      */
-    public BasicCache<String, SessionData> getCache()
+    public BasicCache<String, InfinispanSessionData> getCache() 
     {
         return _cache;
     }
-
+    
     /**
      * Set the clustered cache instance.
-     *
+     * 
      * @param cache the cache
      */
-    public void setCache(BasicCache<String, SessionData> cache)
+    public void setCache(BasicCache<String, InfinispanSessionData> cache) 
     {
         this._cache = cache;
     }
 
-    public QueryManager getQueryManager()
-    {
-        return _queryManager;
-    }
-
-    public void setQueryManager(QueryManager queryManager)
-    {
-        _queryManager = queryManager;
-    }
-
     @Override
     protected void doStart() throws Exception
     {
@@ -89,7 +79,7 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
         if (_cache == null)
             throw new IllegalStateException("No cache");
 
-        try
+        try 
         {
             _passivating = false;
             Class<?> remoteClass = InfinispanSessionDataStore.class.getClassLoader().loadClass("org.infinispan.client.hotrod.RemoteCache");
@@ -103,15 +93,25 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
         }
     }
 
+    public QueryManager getQueryManager()
+    {
+        return _queryManager;
+    }
+
+    public void setQueryManager(QueryManager queryManager)
+    {
+        _queryManager = queryManager;
+    }
+
     @Override
     public SessionData doLoad(String id) throws Exception
-    {
+    {  
         try
         {
             if (LOG.isDebugEnabled())
                 LOG.debug("Loading session {} from infinispan", id);
 
-            InfinispanSessionData sd = (InfinispanSessionData)_cache.get(getCacheKey(id));
+            InfinispanSessionData sd = _cache.get(getCacheKey(id));
             if (isPassivating() && sd != null)
             {
                 if (LOG.isDebugEnabled())
@@ -136,97 +136,77 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
     {
-
-        long now = System.currentTimeMillis();
+        if (candidates == null  || candidates.isEmpty())
+            return candidates;
 
         Set<String> expired = new HashSet<>();
-
-        /*
-         * 1. Select sessions managed by this node for our context that have expired
-         */
-        if (candidates != null)
+   
+        for (String candidate:candidates)
         {
-            for (String candidate : candidates)
+            if (LOG.isDebugEnabled())
+                LOG.debug("Checking expiry for candidate {}", candidate);
+            try
             {
-                if (LOG.isDebugEnabled())
-                    LOG.debug("Checking expiry for candidate {}", candidate);
-                try
-                {
-                    SessionData sd = load(candidate);
+                SessionData sd = load(candidate);
 
-                    //if the session no longer exists
-                    if (sd == null)
-                    {
-                        expired.add(candidate);
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("Session {} does not exist in infinispan", candidate);
-                    }
-                    else
-                    {
-                        if (_context.getWorkerName().equals(sd.getLastNode()))
-                        {
-                            //we are its manager, add it to the expired set if it is expired now
-                            if ((sd.getExpiry() > 0) && sd.getExpiry() <= now)
-                            {
-                                expired.add(candidate);
-                                if (LOG.isDebugEnabled())
-                                    LOG.debug("Session {} managed by {} is expired", candidate, _context.getWorkerName());
-                            }
-                        }
-                        else
-                        {
-                            //if we are not the session's manager, only expire it iff:
-                            // this is our first expiryCheck and the session expired a long time ago
-                            //or
-                            //the session expired at least one graceperiod ago
-                            if (_lastExpiryCheckTime <= 0)
-                            {
-                                if ((sd.getExpiry() > 0) && sd.getExpiry() < (now - (1000L * (3 * _gracePeriodSec))))
-                                    expired.add(candidate);
-                            }
-                            else
-                            {
-                                if ((sd.getExpiry() > 0) && sd.getExpiry() < (now - (1000L * _gracePeriodSec)))
-                                    expired.add(candidate);
-                            }
-                        }
-                    }
-                }
-                catch (Exception e)
+                //if the session no longer exists
+                if (sd == null)
                 {
-                    LOG.warn("Error checking if candidate {} is expired", candidate, e);
+                    expired.add(candidate);
+                    if (LOG.isDebugEnabled())
+                        LOG.debug("Session {} does not exist in infinispan", candidate);
+                }
+                else
+                {
+                    if (_context.getWorkerName().equals(sd.getLastNode()))
+                    {
+                        //we are its manager, add it to the expired set if it is expired now
+                        if ((sd.getExpiry() > 0) && sd.getExpiry() <= time)
+                        {
+                            expired.add(candidate);
+                            if (LOG.isDebugEnabled())
+                                LOG.debug("Session {} managed by {} is expired", candidate, _context.getWorkerName());
+                        }
+                    }
                 }
             }
+            catch (Exception e)
+            {
+                LOG.warn("Error checking if candidate {} is expired", candidate, e);
+            }
         }
-
-
-        /*
-         *  2. Select sessions for any node or context that have expired
-         *  at least 1 graceperiod since the last expiry check. If we haven't done previous expiry checks, then check
-         *  those that have expired at least 3 graceperiod ago.
-         */
+        return expired;
+    }
+    
+    @Override
+    public Set<String> doGetExpired(long time)
+    {
+        //If there is a query manager, find the sessions for our context that expired before the time limit
         if (_queryManager != null)
         {
-            long upperBound = now;
-            if (_lastExpiryCheckTime <= 0)
-                upperBound = (now - (3 * (1000L * _gracePeriodSec)));
-            else
-                upperBound = _lastExpiryCheckTime - (1000L * _gracePeriodSec);
-
-            if (LOG.isDebugEnabled())
-                LOG.debug("{}- Pass 2: Searching for sessions expired before {}", _context.getWorkerName(), upperBound);
-
-            for (String sessionId : _queryManager.queryExpiredSessions(upperBound))
+            Set<String> expired = new HashSet<>();
+            for (String sessionId : _queryManager.queryExpiredSessions(_context, time))
             {
                 expired.add(sessionId);
                 if (LOG.isDebugEnabled())
                     LOG.debug("{}- Found expired sessionId={}", _context.getWorkerName(), sessionId);
             }
+            return expired;
         }
+        return Collections.emptySet();
+    }
 
-        return expired;
+    @Override
+    public void doCleanOrphans(long timeLimit)
+    {
+        //if there is a query manager, find the sessions for any context that expired before the time limit and delete
+        if (_queryManager != null)
+            _queryManager.deleteOrphanSessions(timeLimit);
+        else
+            if (LOG.isDebugEnabled())
+                LOG.debug("Unable to clean orphans, no QueryManager");
     }
 
     @Override
@@ -244,10 +224,10 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
         if (LOG.isDebugEnabled())
             LOG.debug("Session {} saved to infinispan, expires {} ", id, data.getExpiry());
     }
-
+    
     public String getCacheKey(String id)
     {
-        return _context.getCanonicalContextPath() + "_" + _context.getVhost() + "_" + id;
+        return InfinispanKeyBuilder.build(_context.getCanonicalContextPath(), _context.getVhost(), id);
     }
 
     @ManagedAttribute(value = "does store serialize sessions", readonly = true)
@@ -258,46 +238,24 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public boolean exists(String id) throws Exception
+    public boolean doExists(String id) throws Exception
     {
-        // TODO find a better way to do this that does not pull into memory the
-        // whole session object
-        final AtomicBoolean reference = new AtomicBoolean();
-        final AtomicReference<Exception> exception = new AtomicReference<>();
-
-        Runnable load = new Runnable()
+        //if we have a query manager we can do a query with a projection to check
+        //if there is an unexpired session with the given id
+        if (_queryManager != null)
+            return _queryManager.exists(_context, id);
+        else
         {
-            @Override
-            public void run()
-            {
-                try
-                {
-                    SessionData sd = load(id);
-                    if (sd == null)
-                    {
-                        reference.set(false);
-                        return;
-                    }
+            //no query manager, load the entire session data object
+            SessionData sd = doLoad(id);
+            if (sd == null)
+                return false;
 
-                    if (sd.getExpiry() <= 0)
-                        reference.set(true); //never expires
-                    else
-                        reference.set(sd.getExpiry() > System.currentTimeMillis()); //not expired yet
-                }
-                catch (Exception e)
-                {
-                    exception.set(e);
-                }
-            }
-        };
-
-        //ensure the load runs in the context classloader scope
-        _context.run(load);
-
-        if (exception.get() != null)
-            throw exception.get();
-
-        return reference.get();
+            if (sd.getExpiry() <= 0)
+                return true;
+            else
+                return (sd.getExpiry() > System.currentTimeMillis()); //not expired yet
+        }
     }
 
     @Override
@@ -312,8 +270,8 @@ public class InfinispanSessionDataStore extends AbstractSessionDataStore
     public void setInfinispanIdleTimeoutSec(int sec)
     {
         _infinispanIdleTimeoutSec = sec;
-    }
-
+    } 
+    
     @ManagedAttribute(value = "infinispan idle timeout sec", readonly = true)
     public int getInfinispanIdleTimeoutSec()
     {

jetty-infinispan/infinispan-common/src/main/java/org/eclipse/jetty/session/infinispan/InfinispanSessionDataStoreFactory.java

@@ -30,7 +30,7 @@ import org.infinispan.commons.api.BasicCache;
 public class InfinispanSessionDataStoreFactory extends AbstractSessionDataStoreFactory
 {
     int _infinispanIdleTimeoutSec;
-    BasicCache<String, SessionData> _cache;
+    BasicCache<String, InfinispanSessionData> _cache;
     protected QueryManager _queryManager;
 
     /**
@@ -66,7 +66,7 @@ public class InfinispanSessionDataStoreFactory extends AbstractSessionDataStoreF
      *
      * @return the cache
      */
-    public BasicCache<String, SessionData> getCache()
+    public BasicCache<String, InfinispanSessionData> getCache()
     {
         return _cache;
     }
@@ -76,7 +76,7 @@ public class InfinispanSessionDataStoreFactory extends AbstractSessionDataStoreF
      *
      * @param cache the cache
      */
-    public void setCache(BasicCache<String, SessionData> cache)
+    public void setCache(BasicCache<String, InfinispanSessionData> cache)
     {
         this._cache = cache;
     }

jetty-infinispan/infinispan-common/src/main/java/org/eclipse/jetty/session/infinispan/NullQueryManagerFactory.java

@@ -18,7 +18,6 @@
 
 package org.eclipse.jetty.session.infinispan;
 
-import org.eclipse.jetty.server.session.SessionData;
 import org.infinispan.commons.api.BasicCache;
 
 /**
@@ -29,7 +28,7 @@ import org.infinispan.commons.api.BasicCache;
 public class NullQueryManagerFactory implements QueryManagerFactory
 {
     @Override
-    public QueryManager getQueryManager(BasicCache<String, SessionData> cache)
+    public QueryManager getQueryManager(BasicCache<String, InfinispanSessionData> cache)
     {
         return null;
     }

jetty-infinispan/infinispan-common/src/main/java/org/eclipse/jetty/session/infinispan/QueryManager.java

@@ -20,9 +20,13 @@ package org.eclipse.jetty.session.infinispan;
 
 import java.util.Set;
 
+import org.eclipse.jetty.server.session.SessionContext;
+
 public interface QueryManager
 {
-    Set<String> queryExpiredSessions();
+    Set<String> queryExpiredSessions(SessionContext sessionContext, long currentTime);
 
-    Set<String> queryExpiredSessions(long currentTime);
+    public void deleteOrphanSessions(long time);
+    
+    public boolean exists(SessionContext sessionContext, String id);
 }

jetty-infinispan/infinispan-common/src/main/java/org/eclipse/jetty/session/infinispan/QueryManagerFactory.java

@@ -18,10 +18,9 @@
 
 package org.eclipse.jetty.session.infinispan;
 
-import org.eclipse.jetty.server.session.SessionData;
 import org.infinispan.commons.api.BasicCache;
 
 public interface QueryManagerFactory
 {
-    public QueryManager getQueryManager(BasicCache<String, SessionData> cache);
+    public QueryManager getQueryManager(BasicCache<String, InfinispanSessionData> cache);
 }

jetty-infinispan/infinispan-embedded-query/src/main/config-template/etc/sessions/infinispan/infinispan-embedded-query.xml

@@ -18,7 +18,7 @@
         <New class="org.hibernate.search.cfg.SearchMapping">
           <Call name="entity">
             <Arg>
-              <Get class="org.eclipse.jetty.server.session.SessionData" name="class"/>
+              <Get class="org.eclipse.jetty.server.session.infinispan.InfinispanSessionData" name="class"/>
             </Arg>
             <Call name="indexed">
               <Call name="providedId">
@@ -57,7 +57,7 @@
           </Arg>
           <Call name="addIndexedEntity">
             <Arg>
-              <Get class="org.eclipse.jetty.server.session.SessionData" name="class"/>
+              <Get class="org.eclipse.jetty.server.session.infinispan.InfinispanSessionData" name="class"/>
             </Arg>
             <Call name="withProperties">
               <Arg>

jetty-infinispan/infinispan-embedded-query/src/main/java/org/eclipse/jetty/session/infinispan/EmbeddedQueryManager.java

@@ -18,43 +18,88 @@
 
 package org.eclipse.jetty.session.infinispan;
 
-import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
-import org.eclipse.jetty.server.session.SessionData;
+import org.eclipse.jetty.server.session.SessionContext;
 import org.infinispan.Cache;
 import org.infinispan.query.Search;
 import org.infinispan.query.dsl.Query;
 import org.infinispan.query.dsl.QueryFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.stream.Collectors.toSet;
 
 public class EmbeddedQueryManager implements QueryManager
 {
-    private Cache<String, SessionData> _cache;
+    private static final Logger LOG = LoggerFactory.getLogger(EmbeddedQueryManager.class);
+    
+    private Cache<String, InfinispanSessionData> _cache;
 
-    public EmbeddedQueryManager(Cache<String, SessionData> cache)
+    public EmbeddedQueryManager(Cache<String, InfinispanSessionData> cache)
     {
         _cache = cache;
     }
 
     @Override
-    public Set<String> queryExpiredSessions(long time)
+    public Set<String> queryExpiredSessions(SessionContext sessionContext, long time)
     {
+        Objects.requireNonNull(sessionContext);
         QueryFactory qf = Search.getQueryFactory(_cache);
-        Query q = qf.from(SessionData.class).select("id").having("expiry").lte(time).build();
+        Query q = qf.from(InfinispanSessionData.class)
+            .select("id")
+            .having("contextPath").eq(sessionContext.getCanonicalContextPath())
+            .and()
+            .having("expiry").lte(time)
+            .and().having("expiry").gt(0)
+            .build();
 
         List<Object[]> list = q.list();
-        Set<String> ids = new HashSet<>();
-        for (Object[] sl : list)
-        {
-            ids.add((String)sl[0]);
-        }
+        Set<String> ids = list.stream().map(a -> (String)a[0]).collect(toSet());
         return ids;
     }
 
-    @Override
-    public Set<String> queryExpiredSessions()
+    public void deleteOrphanSessions(long time)
     {
-        return queryExpiredSessions(System.currentTimeMillis());
+        QueryFactory qf = Search.getQueryFactory(_cache);
+        Query q = qf.from(InfinispanSessionData.class)
+            .select("id", "contextPath", "vhost")
+            .having("expiry").lte(time)
+            .and().having("expiry").gt(0)
+            .build();
+        List<Object[]> list = q.list();
+        list.stream().forEach(a ->
+        {
+            String key = InfinispanKeyBuilder.build((String)a[1], (String)a[2], (String)a[0]);
+            try
+            {
+                _cache.remove(key);
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Error deleting {}", key, e);
+            }
+        });
+    }
+
+    @Override
+    public boolean exists(SessionContext sessionContext, String id)
+    {
+        Objects.requireNonNull(sessionContext);
+        QueryFactory qf = Search.getQueryFactory(_cache);
+        Query q = qf.from(InfinispanSessionData.class)
+            .select("id")
+            .having("id").eq(id)
+            .and()
+            .having("contextPath").eq(sessionContext.getCanonicalContextPath())
+            .and()
+            .having("expiry").gt(System.currentTimeMillis())
+            .or()
+            .having("expiry").lte(0)
+            .build();
+        List<Object[]> list = q.list();
+        return !list.isEmpty();
     }
 }

jetty-infinispan/infinispan-embedded-query/src/main/java/org/eclipse/jetty/session/infinispan/EmbeddedQueryManagerFactory.java

@@ -26,11 +26,11 @@ public class EmbeddedQueryManagerFactory implements QueryManagerFactory
 {
 
     @Override
-    public QueryManager getQueryManager(BasicCache<String, SessionData> cache)
+    public QueryManager getQueryManager(BasicCache<String, InfinispanSessionData> cache)
     {
         if (!(cache instanceof Cache))
             throw new IllegalArgumentException("Argument was not of type Cache");
 
-        return new EmbeddedQueryManager((Cache<String, SessionData>)cache);
+        return new EmbeddedQueryManager((Cache<String, InfinispanSessionData>)cache);
     }
 }

jetty-infinispan/infinispan-embedded-query/src/test/java/org/eclipse/jetty/server/session/infinispan/EmbeddedQueryManagerTest.java

@@ -24,8 +24,11 @@ import java.util.Properties;
 import java.util.Random;
 import java.util.Set;
 
+import org.eclipse.jetty.server.handler.ContextHandler;
+import org.eclipse.jetty.server.session.SessionContext;
 import org.eclipse.jetty.server.session.SessionData;
 import org.eclipse.jetty.session.infinispan.EmbeddedQueryManager;
+import org.eclipse.jetty.session.infinispan.InfinispanSessionData;
 import org.eclipse.jetty.session.infinispan.QueryManager;
 import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
 import org.hibernate.search.cfg.Environment;
@@ -39,12 +42,15 @@ import org.infinispan.manager.DefaultCacheManager;
 import org.infinispan.manager.EmbeddedCacheManager;
 import org.junit.jupiter.api.Test;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class EmbeddedQueryManagerTest
 {
     public static final String DEFAULT_CACHE_NAME = "session_test_cache";
+    private static final Random r = new Random();
+    private static final int NUM_SESSIONS = 10;
+    private static int count = 0;
+    private static final int MAX_EXPIRY_TIME = 1000;
 
     @Test
     public void test()
@@ -54,7 +60,8 @@ public class EmbeddedQueryManagerTest
 
         //TODO verify that this is being indexed properly, if you change expiry to something that is not a valid field it still passes the tests
         SearchMapping mapping = new SearchMapping();
-        mapping.entity(SessionData.class).indexed().providedId().property("expiry", ElementType.FIELD).field();
+        mapping.entity(InfinispanSessionData.class).indexed().property("expiry", ElementType.FIELD).field();
+
         Properties properties = new Properties();
         properties.put(Environment.MODEL_MAPPING, mapping);
         properties.put("hibernate.search.default.indexBase", MavenTestingUtils.getTargetTestingDir().getAbsolutePath());
@@ -64,43 +71,64 @@ public class EmbeddedQueryManagerTest
         if (dcc != null)
             b = b.read(dcc);
 
-        b.indexing().index(Index.ALL).addIndexedEntity(SessionData.class).withProperties(properties);
+        b.indexing().index(Index.ALL).addIndexedEntity(InfinispanSessionData.class).withProperties(properties);
         Configuration c = b.build();
 
         cacheManager.defineConfiguration(name, c);
-        Cache<String, SessionData> cache = cacheManager.getCache(name);
+        Cache<String, InfinispanSessionData> cache = cacheManager.getCache(name);
 
-        //put some sessions into the cache
-        int numSessions = 10;
-        long currentTime = 500;
-        int maxExpiryTime = 1000;
-        Set<String> expiredSessions = new HashSet<>();
-        Random r = new Random();
+        //put some sessions into the cache for "foo" context
+        ContextHandler fooHandler = new ContextHandler();
+        fooHandler.setContextPath("/foo");
+        SessionContext fooSessionContext = new SessionContext("w0", fooHandler.getServletContext());
+        Set<InfinispanSessionData> fooSessions = createSessions(cache, fooSessionContext);
 
-        for (int i = 0; i < numSessions; i++)
+        //put some sessions into the cache for "bar" context
+        ContextHandler barHandler = new ContextHandler();
+        barHandler.setContextPath("/bar");
+        SessionContext barSessionContext = new SessionContext("w0", barHandler.getServletContext());
+        Set<InfinispanSessionData> barSessions = createSessions(cache, barSessionContext);
+
+        int time = 500;
+
+        //run the query for "foo" context
+        checkResults(cache, fooSessionContext, time, fooSessions);
+
+        //run the query for the "bar" context
+        checkResults(cache, barSessionContext, time, barSessions);
+    }
+
+    private Set<InfinispanSessionData> createSessions(Cache<String, InfinispanSessionData> cache, SessionContext sessionContext)
+    {
+        Set<InfinispanSessionData> sessions = new HashSet<>();
+
+        for (int i = 0; i < NUM_SESSIONS; i++)
         {
             //create new sessiondata with random expiry time
-            long expiryTime = r.nextInt(maxExpiryTime);
-            SessionData sd = new SessionData("sd" + i, "", "", 0, 0, 0, 0);
+            long expiryTime = r.nextInt(MAX_EXPIRY_TIME);
+            String id = "sd" + count;
+            ++count;
+            InfinispanSessionData sd = new InfinispanSessionData(id, sessionContext.getCanonicalContextPath(), sessionContext.getVhost(), 0, 0, 0, 0);
             sd.setExpiry(expiryTime);
-
-            //if this entry has expired add it to expiry list
-            if (expiryTime <= currentTime)
-                expiredSessions.add("sd" + i);
-
+            sessions.add(sd);
             //add to cache
-            cache.put("sd" + i, sd);
+            cache.put(id, sd);
         }
+        return sessions;
+    }
 
-        //run the query
+    private void checkResults(Cache<String, InfinispanSessionData> cache, SessionContext sessionContext, int time, Set<InfinispanSessionData> sessions)
+    {
         QueryManager qm = new EmbeddedQueryManager(cache);
-        Set<String> queryResult = qm.queryExpiredSessions(currentTime);
+        Set<String> queryResult = qm.queryExpiredSessions(sessionContext, time);
 
-        // Check that the result is correct
-        assertEquals(expiredSessions.size(), queryResult.size());
-        for (String s : expiredSessions)
+        for (SessionData s : sessions)
         {
-            assertTrue(queryResult.contains(s));
+            if (s.getExpiry() > 0 && s.getExpiry() <= time)
+            {
+                assertTrue(queryResult.remove(s.getId()));
+            }
         }
+        assertTrue(queryResult.isEmpty()); //check we got them all
     }
 }

jetty-infinispan/infinispan-remote-query/src/main/config-template/etc/sessions/infinispan/infinispan-remote-query.xml

@@ -8,7 +8,7 @@
   <!-- ===================================================================== -->
   <New id="mapping" class="org.hibernate.search.cfg.SearchMapping">
     <Call name="entity">
-      <Arg><Call class="java.lang.Class" name="forName"><Arg>org.eclipse.jetty.server.session.SessionData</Arg></Call></Arg>
+      <Arg><Call class="java.lang.Class" name="forName"><Arg>org.eclipse.jetty.server.session.infinispan.InfinispanSessionData</Arg></Call></Arg>
       <Call name="indexed">
         <Call name="providedId">
           <Call name="property">

jetty-infinispan/infinispan-remote-query/src/main/java/org/eclipse/jetty/session/infinispan/RemoteQueryManager.java

@@ -18,15 +18,19 @@
 
 package org.eclipse.jetty.session.infinispan;
 
-import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
 import java.util.Set;
 
-import org.eclipse.jetty.server.session.SessionData;
+import org.eclipse.jetty.server.session.SessionContext;
 import org.infinispan.client.hotrod.RemoteCache;
 import org.infinispan.client.hotrod.Search;
 import org.infinispan.query.dsl.Query;
 import org.infinispan.query.dsl.QueryFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.stream.Collectors.toSet;
 
 /**
  * RemoteQueryManager
@@ -35,33 +39,75 @@ import org.infinispan.query.dsl.QueryFactory;
  */
 public class RemoteQueryManager implements QueryManager
 {
-    private RemoteCache<String, SessionData> _cache;
+    private static final Logger LOG = LoggerFactory.getLogger(RemoteQueryManager.class);
+    private RemoteCache<String, InfinispanSessionData> _cache;
 
-    public RemoteQueryManager(RemoteCache<String, SessionData> cache)
+    public RemoteQueryManager(RemoteCache<String, InfinispanSessionData> cache)
     {
         _cache = cache;
     }
 
     @Override
-    public Set<String> queryExpiredSessions(long time)
+    public Set<String> queryExpiredSessions(SessionContext sessionContext, long time)
     {
-        // TODO can the QueryFactory be created only once
+        Objects.requireNonNull(sessionContext);
         QueryFactory qf = Search.getQueryFactory(_cache);
-        Query q = qf.from(InfinispanSessionData.class).select("id").having("expiry").lte(time).build();
+        Query q = qf.from(InfinispanSessionData.class)
+            .select("id")
+            .having("contextPath").eq(sessionContext.getCanonicalContextPath())
+            .and()
+            .having("expiry").lte(time)
+            .and()
+            .having("expiry").gt(0)
+            .build();
 
         List<Object[]> list = q.list();
-        Set<String> ids = new HashSet<>();
-        for (Object[] sl : list)
-        {
-            ids.add((String)sl[0]);
-        }
-
+        Set<String> ids = list.stream().map(a -> (String)a[0]).collect(toSet());
         return ids;
     }
 
     @Override
-    public Set<String> queryExpiredSessions()
+    public void deleteOrphanSessions(long time)
     {
-        return queryExpiredSessions(System.currentTimeMillis());
+        QueryFactory qf = Search.getQueryFactory(_cache);
+        Query q = qf.from(InfinispanSessionData.class)
+            .select("id", "contextPath", "vhost")
+            .having("expiry").lte(time)
+            .and()
+            .having("expiry").gt(0)
+            .build();
+        List<Object[]> list = q.list();
+        list.stream().forEach(a ->
+        {
+            String key = InfinispanKeyBuilder.build((String)a[1], (String)a[2], (String)a[0]);
+            try
+            {
+                _cache.remove(key);
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Error deleting {}", key, e);
+            }
+        });  
+    }
+
+    @Override
+    public boolean exists(SessionContext sessionContext, String id)
+    {
+        Objects.requireNonNull(sessionContext);
+        QueryFactory qf = Search.getQueryFactory(_cache);
+        Query q = qf.from(InfinispanSessionData.class)
+            .select("id")
+            .having("id").eq(id)
+            .and()
+            .having("contextPath").eq(sessionContext.getCanonicalContextPath())
+            .and()
+            .having("expiry").gt(System.currentTimeMillis())
+            .or()
+            .having("expiry").lte(0)
+            .build();
+
+        List<Object[]> list = q.list();
+        return !list.isEmpty();
     }
 }

jetty-infinispan/infinispan-remote-query/src/main/java/org/eclipse/jetty/session/infinispan/RemoteQueryManagerFactory.java

@@ -26,12 +26,11 @@ public class RemoteQueryManagerFactory implements QueryManagerFactory
 {
 
     @Override
-    public QueryManager getQueryManager(BasicCache<String, SessionData> cache)
+    public QueryManager getQueryManager(BasicCache<String, InfinispanSessionData> cache)
     {
-        System.err.println(cache.getClass().getName());
         if (!RemoteCache.class.isAssignableFrom(cache.getClass()))
             throw new IllegalArgumentException("Argument is not of type RemoteCache");
 
-        return new RemoteQueryManager((RemoteCache<String, SessionData>)cache);
+        return new RemoteQueryManager((RemoteCache<String, InfinispanSessionData>)cache);
     }
 }

jetty-infinispan/infinispan-remote-query/src/test/java/org/eclipse/jetty/server/session/infinispan/RemoteQueryManagerTest.java

@@ -26,6 +26,8 @@ import java.util.Properties;
 import java.util.Random;
 import java.util.Set;
 
+import org.eclipse.jetty.server.handler.ContextHandler;
+import org.eclipse.jetty.server.session.SessionContext;
 import org.eclipse.jetty.server.session.SessionData;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionData;
 import org.eclipse.jetty.session.infinispan.QueryManager;
@@ -63,9 +65,14 @@ public class RemoteQueryManagerTest
     private static final Logger INFINISPAN_LOG =
         LoggerFactory.getLogger("org.eclipse.jetty.server.session.infinispan.infinispanLogs");
 
+    private static final Random r = new Random();
+    private static final int NUM_SESSIONS = 10;
+    private static final int MAX_EXPIRY_TIME = 1000;
+    private static final String NODE_ID = "w0";
+    private static int count;
     private String host;
     private int port;
-
+    
     GenericContainer infinispan =
         new GenericContainer(System.getProperty("infinispan.docker.image.name", "jboss/infinispan-server") +
                                  ":" + System.getProperty("infinispan.docker.image.version", "9.4.8.Final"))
@@ -130,42 +137,61 @@ public class RemoteQueryManagerTest
             remoteCacheManager.getCache("___protobuf_metadata").put("session.proto", content);
         }
 
-        RemoteCache<String, SessionData> cache = remoteCacheManager.getCache(DEFAULT_CACHE_NAME);
-
-        //put some sessions into the remote cache
-        int numSessions = 10;
-        long currentTime = 500;
-        int maxExpiryTime = 1000;
-        Set<String> expiredSessions = new HashSet<>();
-        Random r = new Random();
-
-        for (int i = 0; i < numSessions; i++)
+        RemoteCache<String, InfinispanSessionData> cache = remoteCacheManager.getCache(DEFAULT_CACHE_NAME);   
+        
+        //put some sessions into the cache for "foo" context
+        ContextHandler fooHandler = new ContextHandler();
+        fooHandler.setContextPath("/foo");
+        SessionContext fooSessionContext = new SessionContext(NODE_ID, fooHandler.getServletContext());
+        Set<SessionData> fooSessions = createSessions(cache, fooSessionContext);
+ 
+        //put some sessions into the cache for "bar" context
+        ContextHandler barHandler = new ContextHandler();
+        barHandler.setContextPath("/bar");
+        SessionContext barSessionContext = new SessionContext(NODE_ID, barHandler.getServletContext());
+        Set<SessionData> barSessions = createSessions(cache, barSessionContext);
+        
+        int time = 500;
+        
+        //run the query for "foo" context
+        checkResults(cache, fooSessionContext, time, fooSessions);
+        
+        //run the query for the "bar" context
+        checkResults(cache, barSessionContext, time, barSessions);
+    }
+    
+    private Set<SessionData> createSessions(RemoteCache<String, InfinispanSessionData> cache, SessionContext sessionContext)
+    {
+        Set<SessionData> sessions = new HashSet<>();
+        
+        for (int i = 0; i < NUM_SESSIONS; i++)
         {
-            String id = "sd" + i;
             //create new sessiondata with random expiry time
-            long expiryTime = r.nextInt(maxExpiryTime);
-            InfinispanSessionData sd = new InfinispanSessionData(id, "", "", 0, 0, 0, 0);
-            sd.setLastNode("lastNode");
+            long expiryTime = r.nextInt(MAX_EXPIRY_TIME);
+            String id = "sd" + count;
+            count++;
+            InfinispanSessionData sd = new InfinispanSessionData(id, sessionContext.getCanonicalContextPath(), sessionContext.getVhost(), 0, 0, 0, 0);
+            sd.setLastNode(sessionContext.getWorkerName());
             sd.setExpiry(expiryTime);
-
-            //if this entry has expired add it to expiry list
-            if (expiryTime <= currentTime)
-                expiredSessions.add(id);
-
+            sessions.add(sd);
             //add to cache
             cache.put(id, sd);
-            assertNotNull(cache.get(id));
         }
-
-        //run the query
+        return sessions;
+    }
+    
+    private void checkResults(RemoteCache<String, InfinispanSessionData> cache, SessionContext sessionContext, int time, Set<SessionData> sessions)
+    {
         QueryManager qm = new RemoteQueryManager(cache);
-        Set<String> queryResult = qm.queryExpiredSessions(currentTime);
+        Set<String> queryResult = qm.queryExpiredSessions(sessionContext, time);
 
-        // Check that the result is correct
-        assertEquals(expiredSessions.size(), queryResult.size());
-        for (String s : expiredSessions)
+        for (SessionData s : sessions)
         {
-            assertTrue(queryResult.contains(s));
+            if (s.getExpiry() > 0 && s.getExpiry() <= time)
+            {
+                assertTrue(queryResult.remove(s.getId()));
+            }
         }
+        assertTrue(queryResult.isEmpty()); //check we got them all
     }
 }

jetty-memcached/jetty-memcached-sessions/src/test/java/org/eclipse/jetty/memcached/session/TestMemcachedSessions.java

@@ -64,7 +64,6 @@ public class TestMemcachedSessions
             else if ("get".equals(arg))
             {
                 s = req.getSession(false);
-                System.err.println("GET: s=" + s);
             }
             else if ("del".equals(arg))
             {

jetty-nosql/src/main/java/org/eclipse/jetty/nosql/mongodb/MongoSessionDataStore.java

@@ -277,7 +277,6 @@ public class MongoSessionDataStore extends NoSqlSessionDataStore
          */
         BasicDBObject mongoKey = new BasicDBObject(__ID, id);
 
-        //DBObject sessionDocument = _dbSessions.findOne(mongoKey,_version1);
         DBObject sessionDocument = _dbSessions.findOne(new BasicDBObject(__ID, id));
 
         if (sessionDocument != null)
@@ -320,7 +319,7 @@ public class MongoSessionDataStore extends NoSqlSessionDataStore
     }
 
     @Override
-    public boolean exists(String id) throws Exception
+    public boolean doExists(String id) throws Exception
     {
         DBObject fields = new BasicDBObject();
         fields.put(__EXPIRY, 1);
@@ -351,18 +350,16 @@ public class MongoSessionDataStore extends NoSqlSessionDataStore
     }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
     {
-        long now = System.currentTimeMillis();
-        long upperBound = now;
         Set<String> expiredSessions = new HashSet<>();
 
         //firstly ask mongo to verify if these candidate ids have expired - all of
         //these candidates will be for our node
         BasicDBObject query = new BasicDBObject();
         query.append(__ID, new BasicDBObject("$in", candidates));
-        query.append(__EXPIRY, new BasicDBObject("$gt", 0).append("$lt", upperBound));
-
+        query.append(__EXPIRY, new BasicDBObject("$gt", 0).append("$lte", time));
+        
         DBCursor verifiedExpiredSessions = null;
         try
         {
@@ -380,46 +377,11 @@ public class MongoSessionDataStore extends NoSqlSessionDataStore
             if (verifiedExpiredSessions != null)
                 verifiedExpiredSessions.close();
         }
+        
 
-        //now ask mongo to find sessions last managed by any nodes that expired a while ago 
-        //if this is our first expiry check, make sure that we only grab really old sessions
-        if (_lastExpiryCheckTime <= 0)
-            upperBound = (now - (3 * (1000L * _gracePeriodSec)));
-        else
-            upperBound = _lastExpiryCheckTime - (1000L * _gracePeriodSec);
-
-        query = new BasicDBObject();
-        BasicDBObject gt = new BasicDBObject(__EXPIRY, new BasicDBObject("$gt", 0));
-        BasicDBObject lt = new BasicDBObject(__EXPIRY, new BasicDBObject("$lt", upperBound));
-        BasicDBList list = new BasicDBList();
-        list.add(gt);
-        list.add(lt);
-        query.append("$and", list);
-
-        DBCursor oldExpiredSessions = null;
-        try
-        {
-            BasicDBObject bo = new BasicDBObject(__ID, 1);
-            bo.append(__EXPIRY, 1);
-
-            oldExpiredSessions = _dbSessions.find(query, bo);
-            for (DBObject session : oldExpiredSessions)
-            {
-                String id = (String)session.get(__ID);
-                if (LOG.isDebugEnabled())
-                    LOG.debug("{} Mongo found old expired session {} exp={}", _context, id, session.get(__EXPIRY));
-                expiredSessions.add(id);
-            }
-        }
-        finally
-        {
-            if (oldExpiredSessions != null)
-                oldExpiredSessions.close();
-        }
-
-        //check through sessions that were candidates, but not found as expired.
+        //check through sessions that were candidates, but not found as expired. 
         //they may no longer be persisted, in which case they are treated as expired.
-        for (String c : candidates)
+        for (String c:candidates)
         {
             if (!expiredSessions.contains(c))
             {
@@ -437,6 +399,59 @@ public class MongoSessionDataStore extends NoSqlSessionDataStore
         return expiredSessions;
     }
 
+    @Override
+    public Set<String> doGetExpired(long timeLimit)
+    {
+        // now ask mongo to find sessions for this context, last managed by any
+        // node, that expired before timeLimit
+        Set<String> expiredSessions = new HashSet<>();
+
+        BasicDBObject query = new BasicDBObject();
+        BasicDBObject gt = new BasicDBObject(__EXPIRY, new BasicDBObject("$gt", 0));
+        BasicDBObject lt = new BasicDBObject(__EXPIRY, new BasicDBObject("$lte", timeLimit));
+        BasicDBList list = new BasicDBList();
+        list.add(gt);
+        list.add(lt);
+        query.append("$and", list);
+
+        DBCursor oldExpiredSessions = null;
+        try
+        {
+            BasicDBObject bo = new BasicDBObject(__ID, 1);
+            bo.append(__EXPIRY, 1);
+
+            oldExpiredSessions = _dbSessions.find(query, bo);
+            for (DBObject session : oldExpiredSessions)
+            {
+                String id = (String)session.get(__ID);
+                
+                //TODO we should verify if there is a session for my context, not any context
+                expiredSessions.add(id);
+            }
+        }
+        finally
+        {
+            if (oldExpiredSessions != null)
+                oldExpiredSessions.close();
+        }
+
+        return expiredSessions;
+    }
+
+    @Override
+    public void doCleanOrphans(long timeLimit)
+    {
+        //Delete all session documents where the expiry time (which is always the most
+        //up-to-date expiry of all contexts sharing that session id) has already past as
+        //at the timeLimit.
+        BasicDBObject query = new BasicDBObject();
+        query.append(__EXPIRY, new BasicDBObject("$gt", 0).append("$lte", timeLimit));
+        _dbSessions.remove(query, WriteConcern.SAFE);
+    }
+
+    /**
+     * @see org.eclipse.jetty.server.session.SessionDataStore#initialize(org.eclipse.jetty.server.session.SessionContext)
+     */
     public void initialize(SessionContext context) throws Exception
     {
         if (isStarted())
@@ -539,7 +554,8 @@ public class MongoSessionDataStore extends NoSqlSessionDataStore
             .add("sparse", false)
             .add("unique", true)
             .get());
-        LOG.debug("done ensure Mongodb indexes existing");
+        if (LOG.isDebugEnabled())
+            LOG.debug("Done ensure Mongodb indexes existing");
         //TODO perhaps index on expiry time?
     }
 

jetty-server/src/main/java/org/eclipse/jetty/server/session/AbstractSessionDataStore.java

@@ -18,9 +18,9 @@
 
 package org.eclipse.jetty.server.session;
 
+import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
 
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
 import org.eclipse.jetty.util.annotation.ManagedObject;
@@ -39,8 +39,52 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem
     protected SessionContext _context; //context associated with this session data store
     protected int _gracePeriodSec = 60 * 60; //default of 1hr 
     protected long _lastExpiryCheckTime = 0; //last time in ms that getExpired was called
+    protected long _lastOrphanSweepTime = 0; //last time in ms that we deleted orphaned sessions
     protected int _savePeriodSec = 0; //time in sec between saves
-
+    
+    /**
+     * Small utility class to allow us to
+     * return a result and an Exception
+     * from invocation of Runnables.
+     *
+     * @param <V> the type of the result.
+     */
+    private class Result<V>
+    {
+        private V _result;
+        private Exception _exception;
+        
+        public void setResult(V result)
+        {
+            _result = result;
+        }
+        
+        public void setException(Exception exception)
+        {
+            _exception = exception;
+        }
+        
+        private void throwIfException() throws Exception
+        {
+            if (_exception != null)
+                throw _exception;
+        }
+        
+        public V getOrThrow() throws Exception
+        {
+            throwIfException();
+            return _result;
+        }
+    }
+    
+    /**
+     * Check if a session for the given id exists.
+     * 
+     * @param id the session id to check
+     * @return true if the session exists in the persistent store, false otherwise
+     */
+    public abstract boolean doExists(String id) throws Exception;
+    
     /**
      * Store the session data persistently.
      *
@@ -61,13 +105,43 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem
     public abstract SessionData doLoad(String id) throws Exception;
 
     /**
-     * Implemented by subclasses to resolve which sessions this node
-     * should attempt to expire.
+     * Implemented by subclasses to resolve which sessions in this context 
+     * that are being managed by this node that should be expired.
      *
-     * @param candidates the ids of sessions the SessionDataStore thinks has expired
-     * @return the reconciled set of session ids that this node should attempt to expire
+     * @param candidates the ids of sessions the SessionCache thinks has expired
+     * @param time the time at which to check for expiry
+     * @return the reconciled set of session ids that have been checked in the store
      */
-    public abstract Set<String> doGetExpired(Set<String> candidates);
+    public abstract Set<String> doCheckExpired(Set<String> candidates, long time);
+    
+    /**
+     * Implemented by subclasses to find sessions for this context in the store
+     * that expired at or before the time limit and thus not being actively
+     * managed by any node. This method is only called periodically (the period
+     * is configurable) to avoid putting too much load on the store.
+     * 
+     * @param before the upper limit of expiry times to check. Sessions expired
+     *            at or before this timestamp will match.
+     * 
+     * @return the empty set if there are no sessions expired as at the time, or
+     *         otherwise a set of session ids.
+     */
+    public abstract Set<String> doGetExpired(long before);
+    
+    /**
+     * Implemented by subclasses to delete sessions for other contexts that
+     * expired at or before the timeLimit. These are 'orphaned' sessions that
+     * are no longer being actively managed by any node. These are explicitly
+     * sessions that do NOT belong to this context (other mechanisms such as
+     * doGetExpired take care of those). As they don't belong to this context,
+     * they cannot be loaded by us.
+     * 
+     * This is called only periodically to avoid placing excessive load on the
+     * store.
+     * 
+     * @param time the upper limit of the expiry time to check in msec
+     */
+    public abstract void doCleanOrphans(long time);
 
     @Override
     public void initialize(SessionContext context) throws Exception
@@ -76,6 +150,22 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem
             throw new IllegalStateException("Context set after SessionDataStore started");
         _context = context;
     }
+    
+    /**
+     * Remove all sessions for any context that expired at or before the given time.
+     * @param timeLimit the time before which the sessions must have expired.
+     */
+    public void cleanOrphans(long timeLimit)
+    {
+        if (!isStarted())
+            throw new IllegalStateException("Not started");
+
+        Runnable r = () ->
+        {
+            doCleanOrphans(timeLimit);
+        };
+        _context.run(r);
+    }
 
     @Override
     public SessionData load(String id) throws Exception
@@ -83,26 +173,22 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem
         if (!isStarted())
             throw new IllegalStateException("Not started");
 
-        final AtomicReference<SessionData> reference = new AtomicReference<SessionData>();
-        final AtomicReference<Exception> exception = new AtomicReference<Exception>();
+        final Result<SessionData> result = new Result<>();
 
         Runnable r = () ->
         {
             try
             {
-                reference.set(doLoad(id));
+                result.setResult(doLoad(id));
             }
             catch (Exception e)
             {
-                exception.set(e);
+                result.setException(e);
             }
         };
 
         _context.run(r);
-        if (exception.get() != null)
-            throw exception.get();
-
-        return reference.get();
+        return result.getOrThrow();
     }
 
     @Override
@@ -114,49 +200,61 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem
         if (data == null)
             return;
 
-        final AtomicReference<Exception> exception = new AtomicReference<Exception>();
+        long lastSave = data.getLastSaved();
+        long savePeriodMs = (_savePeriodSec <= 0 ? 0 : TimeUnit.SECONDS.toMillis(_savePeriodSec));
 
-        Runnable r = new Runnable()
+        if (LOG.isDebugEnabled())
         {
-            @Override
-            public void run()
+            LOG.debug("Store: id={}, mdirty={}, dirty={}, lsave={}, period={}, elapsed={}", id, data.isMetaDataDirty(),
+                data.isDirty(), data.getLastSaved(), savePeriodMs, (System.currentTimeMillis() - lastSave));
+        }
+
+        //save session if attribute changed, never been saved or metadata changed (eg expiry time) and save interval exceeded
+        if (data.isDirty() || (lastSave <= 0) ||
+            (data.isMetaDataDirty() && ((System.currentTimeMillis() - lastSave) >= savePeriodMs)))
+        {
+            //set the last saved time to now
+            data.setLastSaved(System.currentTimeMillis());
+            
+            final Result<Object> result = new Result<>();
+            Runnable r = () ->
             {
-                long lastSave = data.getLastSaved();
-                long savePeriodMs = (_savePeriodSec <= 0 ? 0 : TimeUnit.SECONDS.toMillis(_savePeriodSec));
-
-                if (LOG.isDebugEnabled())
+                try
                 {
-                    LOG.debug("Store: id={}, mdirty={}, dirty={}, lsave={}, period={}, elapsed={}", id, data.isMetaDataDirty(),
-                        data.isDirty(), data.getLastSaved(), savePeriodMs, (System.currentTimeMillis() - lastSave));
+                    //call the specific store method, passing in previous save time
+                    doStore(id, data, lastSave);
+                    data.clean(); //unset all dirty flags
                 }
-
-                //save session if attribute changed, never been saved or metadata changed (eg expiry time) and save interval exceeded
-                if (data.isDirty() || (lastSave <= 0) ||
-                    (data.isMetaDataDirty() && ((System.currentTimeMillis() - lastSave) >= savePeriodMs)))
+                catch (Exception e)
                 {
-                    //set the last saved time to now
-                    data.setLastSaved(System.currentTimeMillis());
-                    try
-                    {
-                        //call the specific store method, passing in previous save time
-                        doStore(id, data, lastSave);
-                        data.clean(); //unset all dirty flags
-                    }
-                    catch (Exception e)
-                    {
-                        //reset last save time if save failed
-                        data.setLastSaved(lastSave);
-                        exception.set(e);
-                    }
+                    //reset last save time if save failed
+                    data.setLastSaved(lastSave);
+                    result.setException(e);
                 }
+            };
+            _context.run(r);
+            result.throwIfException();
+        }
+    }
+
+    @Override
+    public boolean exists(String id) throws Exception
+    {
+        Result<Boolean> result = new Result<>();
+        Runnable r = () ->
+        {
+            try
+            {
+                result.setResult(doExists(id));
+            }
+            catch (Exception e)
+            {
+                result.setException(e);
             }
-
-            ;
         };
 
         _context.run(r);
-        if (exception.get() != null)
-            throw exception.get();
+        return result.getOrThrow();
     }
 
     @Override
@@ -165,14 +263,79 @@ public abstract class AbstractSessionDataStore extends ContainerLifeCycle implem
         if (!isStarted())
             throw new IllegalStateException("Not started");
         
+        long now = System.currentTimeMillis();
+        final Set<String> expired = new HashSet<>();
+
+        // 1. always verify the set of candidates we've been given
+        //by the sessioncache
+        Runnable r = () ->
+        {
+            Set<String> expiredCandidates = doCheckExpired(candidates, now);
+            if (expiredCandidates != null)
+                expired.addAll(expiredCandidates);
+        };
+        _context.run(r);
+
+        // 2. check the backing store to find other sessions
+        // in THIS context that expired long ago (ie cannot be actively managed
+        //by any node)
         try
         {
-            return doGetExpired(candidates);
+            long t = 0;
+
+            // if we have never checked for old expired sessions, then only find
+            // those that are very old so we don't find sessions that other nodes
+            // that are also starting up find
+            if (_lastExpiryCheckTime <= 0)
+                t = now - TimeUnit.SECONDS.toMillis(_gracePeriodSec * 3);
+            else
+            {
+                // only do the check once every gracePeriod to avoid expensive searches,
+                // and find sessions that expired at least one gracePeriod ago
+                if (now > (_lastExpiryCheckTime + TimeUnit.SECONDS.toMillis(_gracePeriodSec)))
+                    t = now - TimeUnit.SECONDS.toMillis(_gracePeriodSec);
+            }
+
+            if (t > 0)
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Searching for sessions expired before {} for context {}", t, _context.getCanonicalContextPath());
+
+                final long expiryTime = t;
+                r = () ->
+                {
+                    Set<String> tmp = doGetExpired(expiryTime);
+                    if (tmp != null)
+                        expired.addAll(tmp);
+                };
+                _context.run(r);
+            }
         }
         finally
         {
-            _lastExpiryCheckTime = System.currentTimeMillis();
+            _lastExpiryCheckTime = now;
         }
+
+        // 3. Periodically but infrequently comb the backing store to delete sessions for 
+        // OTHER contexts that expired a very long time ago (ie not being actively
+        // managed by any node). As these sessions are not for our context, we 
+        // can't load them, so they must just be forcibly deleted.
+        try
+        {
+            if (now > (_lastOrphanSweepTime + TimeUnit.SECONDS.toMillis(10 * _gracePeriodSec)))
+            {
+                if (LOG.isDebugEnabled())
+                    LOG.debug("Cleaning orphans at {}, last sweep at {}", now, _lastOrphanSweepTime);
+                
+                cleanOrphans(now - TimeUnit.SECONDS.toMillis(10 * _gracePeriodSec));
+            }
+        }
+        finally
+        {
+            _lastOrphanSweepTime = now;
+        }
+
+        return expired;
     }
 
     @Override

jetty-server/src/main/java/org/eclipse/jetty/server/session/FileSessionDataStore.java

@@ -150,73 +150,84 @@ public class FileSessionDataStore extends AbstractSessionDataStore
      * that are not currently loaded into the SessionCache
      */
     @Override
-    public Set<String> doGetExpired(final Set<String> candidates)
+    public Set<String> doCheckExpired(final Set<String> candidates, long time)
     {
-        final long now = System.currentTimeMillis();
-        HashSet<String> expired = new HashSet<String>();
+        HashSet<String> expired = new HashSet<>();
 
-        //iterate over the files and work out which have expired
-        for (String filename : _sessionFileMap.values())
+        //check the candidates
+        for (String id:candidates)
+        {
+            String filename = _sessionFileMap.get(getIdWithContext(id));
+            // no such file, therefore no longer any such session, it can be expired
+            if (filename == null)
+                expired.add(id);
+            else
+            {
+                try
+                {
+                    long expiry = getExpiryFromFilename(filename);
+                    if (expiry > 0 && expiry <= time)
+                        expired.add(id);
+                }
+                catch (Exception e)
+                {
+                    LOG.warn("Error finding expired sessions", e);
+                }
+            }
+        }
+
+        return expired;
+    }
+
+    @Override
+    public Set<String> doGetExpired(long timeLimit)
+    {
+        HashSet<String> expired = new HashSet<>();
+
+        // iterate over the files and work out which expired at or
+        // before the time limit
+        for (String filename:_sessionFileMap.values())
         {
             try
             {
                 long expiry = getExpiryFromFilename(filename);
-                if (expiry > 0 && expiry < now)
+                if (expiry > 0 && expiry <= timeLimit)
                     expired.add(getIdFromFilename(filename));
             }
             catch (Exception e)
             {
-                LOG.warn("Unable to get expired for {}", filename, e);
+                LOG.warn("Error finding sessions expired before {}", timeLimit, e);
             }
         }
-
-        //check candidates that were not found to be expired, perhaps 
-        //because they no longer exist and they should be expired
-        for (String c : candidates)
-        {
-            if (!expired.contains(c))
-            {
-                //if it doesn't have a file then the session doesn't exist
-                String filename = _sessionFileMap.get(getIdWithContext(c));
-                if (filename == null)
-                    expired.add(c);
-            }
-        }
-
-        //Infrequently iterate over all files in the store, and delete those
-        //that expired a long time ago, even if they belong to
-        //another context. This ensures that files that
-        //belong to defunct contexts are cleaned up. 
-        //If the graceperiod is disabled, don't do the sweep!
-        if ((_gracePeriodSec > 0) && ((_lastSweepTime == 0) || ((now - _lastSweepTime) >= (5 * TimeUnit.SECONDS.toMillis(_gracePeriodSec)))))
-        {
-            _lastSweepTime = now;
-            sweepDisk();
-        }
+        
         return expired;
     }
 
-    /**
-     * Check all session files that do not belong to this context and
-     * remove any that expired long ago (ie at least 5 gracePeriods ago).
-     */
-    public void sweepDisk()
+    @Override
+    public void doCleanOrphans(long time)
     {
-        //iterate over the files in the store dir and check expiry times
-        long now = System.currentTimeMillis();
+        sweepDisk(time);
+    }
+
+    /**
+     * Check all session files for any context and remove any
+     * that expired at or before the time limit.
+     */
+    protected void sweepDisk(long time)
+    {        
+        // iterate over the files in the store dir and check expiry times
         if (LOG.isDebugEnabled())
-            LOG.debug("Sweeping {} for old session files", _storeDir);
+            LOG.debug("Sweeping {} for old session files at {}", _storeDir, time);
         try
         {
             Files.walk(_storeDir.toPath(), 1, FileVisitOption.FOLLOW_LINKS)
-                .filter(p -> !Files.isDirectory(p)).filter(p -> !isOurContextSessionFilename(p.getFileName().toString()))
+                .filter(p -> !Files.isDirectory(p))
                 .filter(p -> isSessionFilename(p.getFileName().toString()))
                 .forEach(p ->
                 {
-
                     try
                     {
-                        sweepFile(now, p);
+                        sweepFile(time, p);
                     }
                     catch (Exception e)
                     {
@@ -231,15 +242,13 @@ public class FileSessionDataStore extends AbstractSessionDataStore
     }
 
     /**
-     * Check to see if the expiry on the file is very old, and
-     * delete the file if so. "Old" means that it expired at least
-     * 5 gracePeriods ago. The session can belong to any context.
+     * Delete file (from any context) that expired at or before the given time
      *
-     * @param now the time now in msec
+     * @param time the time in msec
      * @param p the file to check
      * @throws Exception indicating error in sweep
      */
-    public void sweepFile(long now, Path p)
+    protected void sweepFile(long time, Path p)
         throws Exception
     {
         if (p == null)
@@ -249,7 +258,7 @@ public class FileSessionDataStore extends AbstractSessionDataStore
         {
             long expiry = getExpiryFromFilename(p.getFileName().toString());
             //files with 0 expiry never expire
-            if (expiry > 0 && ((now - expiry) >= (5 * TimeUnit.SECONDS.toMillis(_gracePeriodSec))))
+            if (expiry > 0 && expiry <= time)
             {
                 Files.deleteIfExists(p);
                 if (LOG.isDebugEnabled())
@@ -371,7 +380,7 @@ public class FileSessionDataStore extends AbstractSessionDataStore
                     //context they are for
                     try
                     {
-                        sweepFile(now, p);
+                        sweepFile(now - (10 * TimeUnit.SECONDS.toMillis(getGracePeriodSec())), p);
                     }
                     catch (Exception x)
                     {
@@ -438,7 +447,7 @@ public class FileSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public boolean exists(String id) throws Exception
+    public boolean doExists(String id) throws Exception
     {
         String idWithContext = getIdWithContext(id);
         String filename = _sessionFileMap.get(idWithContext);

jetty-server/src/main/java/org/eclipse/jetty/server/session/JDBCSessionDataStore.java

@@ -385,19 +385,7 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
             statement.setLong(4, expiry);
             return statement;
         }
-
-        public PreparedStatement getAllAncientExpiredSessionsStatement(Connection connection)
-            throws SQLException
-        {
-            if (_dbAdaptor == null)
-                throw new IllegalStateException("No DB adaptor");
-
-            PreparedStatement statement = connection.prepareStatement("select " + getIdColumn() + ", " + getContextPathColumn() + ", " + getVirtualHostColumn() +
-                " from " + getSchemaTableName() +
-                " where " + getExpiryTimeColumn() + " >0 and " + getExpiryTimeColumn() + " <= ?");
-            return statement;
-        }
-
+    
         public PreparedStatement getCheckSessionExistsStatement(Connection connection, SessionContext context)
             throws SQLException
         {
@@ -483,6 +471,20 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
             return statement;
         }
 
+        public PreparedStatement getCleanOrphansStatement(Connection connection, long timeLimit)
+            throws Exception
+        { 
+            if (_dbAdaptor == null)
+                throw new IllegalStateException("No DB adaptor");
+
+            PreparedStatement statement = connection.prepareStatement("delete from " + getSchemaTableName() +
+                " where " +
+                getExpiryTimeColumn() + " > 0 and " + getExpiryTimeColumn() + " <= ?");
+            statement.setLong(1, timeLimit);
+            return statement;
+
+        }
+
         /**
          * Set up the tables in the database
          *
@@ -795,26 +797,23 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
     {
         if (LOG.isDebugEnabled())
-            LOG.debug("Getting expired sessions at time {}", System.currentTimeMillis());
-
-        long now = System.currentTimeMillis();
-
+            LOG.debug("Getting expired sessions at time {}", time);
+        
         Set<String> expiredSessionKeys = new HashSet<>();
         try (Connection connection = _dbAdaptor.getConnection())
         {
             connection.setAutoCommit(true);
 
-            /*
-             * 1. Select sessions managed by this node for our context that have expired
-             */
-            long upperBound = now;
+            //Select sessions managed by this node for our context that have expired
+            long upperBound = time;
             if (LOG.isDebugEnabled())
-                LOG.debug("{}- Pass 1: Searching for sessions for context {} managed by me and expired before {}", _context.getWorkerName(), _context.getCanonicalContextPath(), upperBound);
+                LOG.debug("{} - Searching for sessions for context {} managed by me and expired before {}",  
+                    _context.getWorkerName(), _context.getCanonicalContextPath(), upperBound);
 
-            try (PreparedStatement statement = _sessionTableSchema.getExpiredSessionsStatement(connection, _context.getCanonicalContextPath(), _context.getVhost(), upperBound))
+            try (PreparedStatement statement = _sessionTableSchema.getMyExpiredSessionsStatement(connection, _context, upperBound))
             {
                 try (ResultSet result = statement.executeQuery())
                 {
@@ -824,37 +823,8 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
                         long exp = result.getLong(_sessionTableSchema.getExpiryTimeColumn());
                         expiredSessionKeys.add(sessionId);
                         if (LOG.isDebugEnabled())
-                            LOG.debug("{}- Found expired sessionId={}",_context.getCanonicalContextPath(), sessionId);
-                    }
-                }
-            }
-
-            /*
-             *  2. Select sessions for any node or context that have expired
-             *  at least 1 graceperiod since the last expiry check. If we haven't done previous expiry checks, then check
-             *  those that have expired at least 3 graceperiod ago.
-             */
-            try (PreparedStatement selectExpiredSessions = _sessionTableSchema.getAllAncientExpiredSessionsStatement(connection))
-            {
-                if (_lastExpiryCheckTime <= 0)
-                    upperBound = (now - (3 * (1000L * _gracePeriodSec)));
-                else
-                    upperBound = _lastExpiryCheckTime - (1000L * _gracePeriodSec);
-
-                if (LOG.isDebugEnabled())
-                    LOG.debug("{}- Pass 2: Searching for sessions expired before {}", _context.getWorkerName(), upperBound);
-
-                selectExpiredSessions.setLong(1, upperBound);
-                try (ResultSet result = selectExpiredSessions.executeQuery())
-                {
-                    while (result.next())
-                    {
-                        String sessionId = result.getString(_sessionTableSchema.getIdColumn());
-                        String ctxtpth = result.getString(_sessionTableSchema.getContextPathColumn());
-                        String vh = result.getString(_sessionTableSchema.getVirtualHostColumn());
-                        expiredSessionKeys.add(sessionId);
-                        if (LOG.isDebugEnabled())
-                            LOG.debug("{}- Found expired sessionId={}", _context.getWorkerName(), sessionId);
+                            LOG.debug("{} - Found expired sessionId={}, in context={}, expiry={}",
+                                _context.getWorkerName(), sessionId, _context.getCanonicalContextPath(),exp);
                     }
                 }
             }
@@ -862,16 +832,18 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
             Set<String> notExpiredInDB = new HashSet<>();
             for (String k : candidates)
             {
-                //there are some keys that the session store thought had expired, but were not
-                //found in our sweep either because it is no longer in the db, or its
+                //there are some keys that the sessioncache thought had expired, but were not
+                //found in our query either because it is no longer in the db, or its
                 //expiry time was updated
                 if (!expiredSessionKeys.contains(k))
                     notExpiredInDB.add(k);
             }
 
+            //Check the candidates that were not reported as expired in the db: they
+            //either do not exist, or they weren't expired (which means some other node
+            //must be managing it)
             if (!notExpiredInDB.isEmpty())
             {
-                //we have some sessions to check
                 try (PreparedStatement checkSessionExists = _sessionTableSchema.getCheckSessionExistsStatement(connection, _context))
                 {
                     for (String k : notExpiredInDB)
@@ -884,7 +856,11 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
                                 //session doesn't exist any more, can be expired
                                 expiredSessionKeys.add(k);
                             }
-                            //else its expiry time has not been reached
+                            else
+                            {
+                                if (LOG.isDebugEnabled())
+                                    LOG.debug("{} Session {} expiry fresher in db than cache, another node must be managing it", _context.getWorkerName(), k);
+                            }
                         }
                         catch (Exception e)
                         {
@@ -903,6 +879,61 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
         }
     }
 
+    @Override
+    public Set<String> doGetExpired(long timeLimit)
+    {
+        Set<String> expired = new HashSet<>();
+        
+        //Get sessions for my context but managed by any node that expired at or before the timeLimit   
+        try (Connection connection = _dbAdaptor.getConnection())
+        {
+            connection.setAutoCommit(true);
+            try (PreparedStatement selectExpiredSessions = _sessionTableSchema.getExpiredSessionsStatement(connection, _context.getCanonicalContextPath(),
+                                                                                                           _context.getVhost(), timeLimit))
+            {
+                if (LOG.isDebugEnabled()) 
+                    LOG.debug("{}- Searching for sessions for context {} expired before {}",_context.getWorkerName(),_context.getCanonicalContextPath(), timeLimit);
+
+                try (ResultSet result = selectExpiredSessions.executeQuery())
+                {
+                    while (result.next())
+                    {
+                        String sessionId = result.getString(_sessionTableSchema.getIdColumn());
+                        long exp = result.getLong(_sessionTableSchema.getExpiryTimeColumn());
+                        expired.add(sessionId);
+                        if (LOG.isDebugEnabled())
+                            LOG.debug("{}- Found expired sessionId={} for context={} expiry={}",
+                                _context.getWorkerName(),sessionId,_context.getCanonicalContextPath(), exp);
+                    }
+                }
+            }
+            return expired;
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Error finding sessions expired before {}", timeLimit, e);
+            return expired; //return whatever we got
+        } 
+    }
+
+    @Override
+    public void doCleanOrphans(long time)
+    {
+        //Harshly delete sessions for any node and context that expired at or before the timeLimit
+        try (Connection connection = _dbAdaptor.getConnection();
+             PreparedStatement statement = _sessionTableSchema.getCleanOrphansStatement(connection, time))
+        {
+            connection.setAutoCommit(true);
+            int rows = statement.executeUpdate();
+            if (LOG.isDebugEnabled())
+                LOG.debug("Deleted {} orphaned sessions",rows);
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Error cleaning orphan sessions", e);
+        } 
+    }
+
     public void setDatabaseAdaptor(DatabaseAdaptor dbAdaptor)
     {
         checkStarted();
@@ -926,7 +957,7 @@ public class JDBCSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public boolean exists(String id)
+    public boolean doExists(String id)
         throws Exception
     {
         try (Connection connection = _dbAdaptor.getConnection())

jetty-server/src/main/java/org/eclipse/jetty/server/session/NullSessionDataStore.java

@@ -18,6 +18,7 @@
 
 package org.eclipse.jetty.server.session;
 
+import java.util.Collections;
 import java.util.Set;
 
 import org.eclipse.jetty.util.annotation.ManagedAttribute;
@@ -31,7 +32,6 @@ import org.eclipse.jetty.util.annotation.ManagedObject;
 @ManagedObject
 public class NullSessionDataStore extends AbstractSessionDataStore
 {
-
     @Override
     public SessionData doLoad(String id) throws Exception
     {
@@ -57,11 +57,20 @@ public class NullSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
     {
         return candidates; //whatever is suggested we accept
     }
 
+    @Override
+    public Set<String> doGetExpired(long timeLimit)
+    {
+        return Collections.emptySet();
+    }
+    
+    /** 
+     * @see org.eclipse.jetty.server.session.SessionDataStore#isPassivating()
+     */
     @ManagedAttribute(value = "does this store serialize sessions", readonly = true)
     @Override
     public boolean isPassivating()
@@ -70,8 +79,14 @@ public class NullSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public boolean exists(String id)
+    public boolean doExists(String id)
     {
         return false;
     }
+
+    @Override
+    public void doCleanOrphans(long timeLimit)
+    {
+        //noop
+    }
 }

jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionData.java

@@ -147,6 +147,7 @@ public class SessionData implements Serializable
                 boolean isServerClassLoader = in.readBoolean(); //use server or webapp classloader to load
                 if (LOG.isDebugEnabled())
                     LOG.debug("Deserialize {} isServerLoader={} serverLoader={} tccl={}", name, isServerClassLoader, serverLoader, contextLoader);
+                
                 Object value = ((ClassLoadingObjectInputStream)in).readObject(isServerClassLoader ? serverLoader : contextLoader);
                 data._attributes.put(name, value);
             }

pom.xml

@@ -28,6 +28,7 @@
     <websocket.api.version>1.1.2</websocket.api.version>
     <jsp.version>9.0.29</jsp.version>
     <infinispan.version>9.4.8.Final</infinispan.version>
+    <hazelcast.version>4.0.1</hazelcast.version>
     <conscrypt.version>2.4.0</conscrypt.version>
     <asm.version>8.0.1</asm.version>
     <jmh.version>1.21</jmh.version>

tests/test-sessions/test-file-sessions/src/test/java/org/eclipse/jetty/server/session/ClusteredOrphanedSessionTest.java

@@ -27,22 +27,24 @@ import org.junit.jupiter.api.Test;
  */
 public class ClusteredOrphanedSessionTest extends AbstractClusteredOrphanedSessionTest
 {
+    FileTestHelper _helper;
+    
     @BeforeEach
     public void before() throws Exception
     {
-        FileTestHelper.setup();
+        _helper = new FileTestHelper();
     }
 
     @AfterEach
     public void after()
     {
-        FileTestHelper.teardown();
+        _helper.teardown();
     }
 
     @Override
     public SessionDataStoreFactory createSessionDataStoreFactory()
     {
-        return FileTestHelper.newSessionDataStoreFactory();
+        return _helper.newSessionDataStoreFactory();
     }
 
     @Test

tests/test-sessions/test-file-sessions/src/test/java/org/eclipse/jetty/server/session/FileSessionDataStoreTest.java

@@ -20,42 +20,44 @@ package org.eclipse.jetty.server.session;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
 
 /**
  * FileSessionDataStoreTest
  */
 public class FileSessionDataStoreTest extends AbstractSessionDataStoreTest
 {
+    private FileTestHelper _helper;
+    
     @BeforeEach
     public void before() throws Exception
     {
-        FileTestHelper.setup();
+        _helper = new FileTestHelper();
     }
 
     @AfterEach
     public void after()
     {
-        FileTestHelper.teardown();
+        _helper.teardown();
+        _helper = null;
     }
 
     @Override
     public SessionDataStoreFactory createSessionDataStoreFactory()
     {
-        return FileTestHelper.newSessionDataStoreFactory();
+        return _helper.newSessionDataStoreFactory();
     }
 
     @Override
     public void persistSession(SessionData data) throws Exception
     {
-        FileTestHelper.createFile(data.getId(), data.getContextPath(), data.getVhost(), data.getLastNode(), data.getCreated(),
+        _helper.createFile(data.getId(), data.getContextPath(), data.getVhost(), data.getLastNode(), data.getCreated(),
             data.getAccessed(), data.getLastAccessed(), data.getMaxInactiveMs(), data.getExpiry(), data.getCookieSet(), data.getAllAttributes());
     }
 
     @Override
     public void persistUnreadableSession(SessionData data) throws Exception
     {
-        FileTestHelper.createFile(data.getId(), data.getContextPath(), data.getVhost(), data.getLastNode(), data.getCreated(),
+        _helper.createFile(data.getId(), data.getContextPath(), data.getVhost(), data.getLastNode(), data.getCreated(),
             data.getAccessed(), data.getLastAccessed(), data.getMaxInactiveMs(), data.getExpiry(), data.getCookieSet(), null);
     }
 
@@ -66,7 +68,7 @@ public class FileSessionDataStoreTest extends AbstractSessionDataStoreTest
         Thread.currentThread().setContextClassLoader(_contextClassLoader);
         try
         {
-            return (FileTestHelper.getFile(data.getId()) != null);
+            return (_helper.getFile(data.getId()) != null);
         }
         finally
         {
@@ -81,7 +83,7 @@ public class FileSessionDataStoreTest extends AbstractSessionDataStoreTest
         Thread.currentThread().setContextClassLoader(_contextClassLoader);
         try
         {
-            return FileTestHelper.checkSessionPersisted(data);
+            return _helper.checkSessionPersisted(data);
         }
         catch (Throwable e)
         {
@@ -93,11 +95,4 @@ public class FileSessionDataStoreTest extends AbstractSessionDataStoreTest
             Thread.currentThread().setContextClassLoader(old);
         }
     }
-
-    @Override
-    @Test
-    public void testStoreSession() throws Exception
-    {
-        super.testStoreSession();
-    }
 }

tests/test-sessions/test-file-sessions/src/test/java/org/eclipse/jetty/server/session/FileTestHelper.java

@@ -41,26 +41,25 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
  */
 public class FileTestHelper
 {
-    static int __workers = 0;
-    static File _tmpDir;
+    File _tmpDir;
 
-    public static void setup()
+    public  FileTestHelper()
         throws Exception
     {
 
-        _tmpDir = File.createTempFile("file", null);
+        _tmpDir = File.createTempFile("file", "test");
         _tmpDir.delete();
         _tmpDir.mkdirs();
         _tmpDir.deleteOnExit();
     }
 
-    public static void teardown()
+    public void teardown()
     {
         IO.delete(_tmpDir);
         _tmpDir = null;
     }
 
-    public static void assertStoreDirEmpty(boolean isEmpty)
+    public void assertStoreDirEmpty(boolean isEmpty)
     {
         assertNotNull(_tmpDir);
         assertTrue(_tmpDir.exists());
@@ -77,7 +76,7 @@ public class FileTestHelper
         }
     }
 
-    public static File getFile(String sessionId)
+    public File getFile(String sessionId)
     {
         assertNotNull(_tmpDir);
         assertTrue(_tmpDir.exists());
@@ -98,7 +97,7 @@ public class FileTestHelper
         return null;
     }
 
-    public static void assertSessionExists(String sessionId, boolean exists)
+    public void assertSessionExists(String sessionId, boolean exists)
     {
         assertNotNull(_tmpDir);
         assertTrue(_tmpDir.exists());
@@ -121,7 +120,7 @@ public class FileTestHelper
             assertFalse(found);
     }
 
-    public static void assertFileExists(String filename, boolean exists)
+    public void assertFileExists(String filename, boolean exists)
     {
         assertNotNull(_tmpDir);
         assertTrue(_tmpDir.exists());
@@ -132,7 +131,7 @@ public class FileTestHelper
             assertFalse(file.exists());
     }
 
-    public static void createFile(String filename)
+    public void createFile(String filename)
         throws IOException
     {
         assertNotNull(_tmpDir);
@@ -143,7 +142,7 @@ public class FileTestHelper
         file.createNewFile();
     }
 
-    public static void createFile(String id, String contextPath, String vhost,
+    public void createFile(String id, String contextPath, String vhost,
                                   String lastNode, long created, long accessed,
                                   long lastAccessed, long maxIdle, long expiry,
                                   long cookieSet, Map<String, Object> attributes)
@@ -174,7 +173,7 @@ public class FileTestHelper
         }
     }
 
-    public static boolean checkSessionPersisted(SessionData data)
+    public boolean checkSessionPersisted(SessionData data)
         throws Exception
     {
         String filename = "" + data.getExpiry() + "_" + data.getContextPath() + "_" + data.getVhost() + "_" + data.getId();
@@ -224,7 +223,7 @@ public class FileTestHelper
         return true;
     }
 
-    public static void deleteFile(String sessionId)
+    public void deleteFile(String sessionId)
     {
         assertNotNull(_tmpDir);
         assertTrue(_tmpDir.exists());
@@ -247,7 +246,7 @@ public class FileTestHelper
         }
     }
 
-    public static FileSessionDataStoreFactory newSessionDataStoreFactory()
+    public FileSessionDataStoreFactory newSessionDataStoreFactory()
     {
         FileSessionDataStoreFactory storeFactory = new FileSessionDataStoreFactory();
         storeFactory.setStoreDir(_tmpDir);

tests/test-sessions/test-file-sessions/src/test/java/org/eclipse/jetty/server/session/TestFileSessions.java

@@ -37,22 +37,25 @@ import static org.junit.jupiter.api.Assertions.fail;
  */
 public class TestFileSessions extends AbstractTestBase
 {
+    FileTestHelper _helper;
+
     @BeforeEach
     public void before() throws Exception
     {
-        FileTestHelper.setup();
+        _helper = new FileTestHelper();
     }
 
     @AfterEach
     public void after()
     {
-        FileTestHelper.teardown();
+        _helper.teardown();
+        _helper = null;
     }
 
     @Override
     public SessionDataStoreFactory createSessionDataStoreFactory()
     {
-        return FileTestHelper.newSessionDataStoreFactory();
+        return _helper.newSessionDataStoreFactory();
     }
 
     /**
@@ -73,7 +76,7 @@ public class TestFileSessions extends AbstractTestBase
         store.initialize(sessionContext);
 
         //make a file for foobar context
-        FileTestHelper.createFile((System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) + "__foobar_0.0.0.0_1234");
+        _helper.createFile((System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) + "__foobar_0.0.0.0_1234");
 
         store.start();
 
@@ -107,7 +110,7 @@ public class TestFileSessions extends AbstractTestBase
 
         try
         {
-            long ll = store.getExpiryFromFilename("nonnumber__test_0.0.0.0_1234");
+            store.getExpiryFromFilename("nonnumber__test_0.0.0.0_1234");
             fail("Should be non numeric");
         }
         catch (Exception e)
@@ -117,7 +120,7 @@ public class TestFileSessions extends AbstractTestBase
 
         try
         {
-            long ll = store.getExpiryFromFilename(null);
+            store.getExpiryFromFilename(null);
             fail("Should throw ISE");
         }
         catch (Exception e)
@@ -127,7 +130,7 @@ public class TestFileSessions extends AbstractTestBase
 
         try
         {
-            long ll = store.getExpiryFromFilename("thisisnotavalidsessionfilename");
+            store.getExpiryFromFilename("thisisnotavalidsessionfilename");
             fail("Should throw ISE");
         }
         catch (IllegalStateException e)
@@ -183,7 +186,7 @@ public class TestFileSessions extends AbstractTestBase
 
         try
         {
-            long ll = store.getExpiryFromFilename("nonnumber__0.0.0.0_1234");
+            store.getExpiryFromFilename("nonnumber__0.0.0.0_1234");
             fail("Should be non numeric");
         }
         catch (Exception e)
@@ -207,12 +210,12 @@ public class TestFileSessions extends AbstractTestBase
     @Test
     public void testSweep() throws Exception
     {
-
+        int gracePeriodSec = 10;
         //create the SessionDataStore
         ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
         context.setContextPath("/test");
         SessionDataStoreFactory factory = createSessionDataStoreFactory();
-        ((AbstractSessionDataStoreFactory)factory).setGracePeriodSec(10);
+        ((AbstractSessionDataStoreFactory)factory).setGracePeriodSec(gracePeriodSec);
         SessionDataStore store = factory.getSessionDataStore(context.getSessionHandler());
         SessionContext sessionContext = new SessionContext("foo", context.getServletContext());
         store.initialize(sessionContext);
@@ -220,58 +223,58 @@ public class TestFileSessions extends AbstractTestBase
         store.start();
 
         //create file not for our context that expired long ago and should be removed by sweep
-        FileTestHelper.createFile("101__foobar_0.0.0.0_sessiona");
-        FileTestHelper.assertSessionExists("sessiona", true);
+        _helper.createFile("101__foobar_0.0.0.0_sessiona");
+        _helper.assertSessionExists("sessiona", true);
 
         //create a file not for our context that is not expired and should be ignored
         String nonExpiredForeign = (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) + "__foobar_0.0.0.0_sessionb";
-        FileTestHelper.createFile(nonExpiredForeign);
-        FileTestHelper.assertFileExists(nonExpiredForeign, true);
+        _helper.createFile(nonExpiredForeign);
+        _helper.assertFileExists(nonExpiredForeign, true);
 
         //create a file not for our context that is recently expired, a thus ignored by sweep
         String expiredForeign = (System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(1)) + "__foobar_0.0.0.0_sessionc";
-        FileTestHelper.createFile(expiredForeign);
-        FileTestHelper.assertFileExists(expiredForeign, true);
+        _helper.createFile(expiredForeign);
+        _helper.assertFileExists(expiredForeign, true);
 
         //create a file that is not a session file, it should be ignored
-        FileTestHelper.createFile("whatever.txt");
-        FileTestHelper.assertFileExists("whatever.txt", true);
+        _helper.createFile("whatever.txt");
+        _helper.assertFileExists("whatever.txt", true);
 
         //create a file that is not a valid session filename, should be ignored
-        FileTestHelper.createFile("nonNumber__0.0.0.0_spuriousFile");
-        FileTestHelper.assertFileExists("nonNumber__0.0.0.0_spuriousFile", true);
+        _helper.createFile("nonNumber__0.0.0.0_spuriousFile");
+        _helper.assertFileExists("nonNumber__0.0.0.0_spuriousFile", true);
 
         //create a file that is a non-expired session file for our context that should be ignored
         String nonExpired = (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) + "__test_0.0.0.0_sessionb";
-        FileTestHelper.createFile(nonExpired);
-        FileTestHelper.assertFileExists(nonExpired, true);
+        _helper.createFile(nonExpired);
+        _helper.assertFileExists(nonExpired, true);
 
         //create a file that is a never-expire session file for our context that should be ignored
         String neverExpired = "0__test_0.0.0.0_sessionc";
-        FileTestHelper.createFile(neverExpired);
-        FileTestHelper.assertFileExists(neverExpired, true);
+        _helper.createFile(neverExpired);
+        _helper.assertFileExists(neverExpired, true);
 
         //create a file that is a never-expire session file for another context that should be ignored
         String foreignNeverExpired = "0__other_0.0.0.0_sessionc";
-        FileTestHelper.createFile(foreignNeverExpired);
-        FileTestHelper.assertFileExists(foreignNeverExpired, true);
+        _helper.createFile(foreignNeverExpired);
+        _helper.assertFileExists(foreignNeverExpired, true);
 
         //sweep - we're expecting a debug log with exception stacktrace due to file named 
         //nonNumber__0.0.0.0_spuriousFile so suppress it
         try (StacklessLogging ignored = new StacklessLogging(TestFileSessions.class.getPackage()))
         {
-            ((FileSessionDataStore)store).sweepDisk();
+            ((FileSessionDataStore)store).sweepDisk(System.currentTimeMillis() - (10 * TimeUnit.SECONDS.toMillis(gracePeriodSec)));
         }
 
         //check results
-        FileTestHelper.assertSessionExists("sessiona", false);
-        FileTestHelper.assertFileExists("whatever.txt", true);
-        FileTestHelper.assertFileExists("nonNumber__0.0.0.0_spuriousFile", true);
-        FileTestHelper.assertFileExists(nonExpired, true);
-        FileTestHelper.assertFileExists(nonExpiredForeign, true);
-        FileTestHelper.assertFileExists(expiredForeign, true);
-        FileTestHelper.assertFileExists(neverExpired, true);
-        FileTestHelper.assertFileExists(foreignNeverExpired, true);
+        _helper.assertSessionExists("sessiona", false);
+        _helper.assertFileExists("whatever.txt", true);
+        _helper.assertFileExists("nonNumber__0.0.0.0_spuriousFile", true);
+        _helper.assertFileExists(nonExpired, true);
+        _helper.assertFileExists(nonExpiredForeign, true);
+        _helper.assertFileExists(expiredForeign, true);
+        _helper.assertFileExists(neverExpired, true);
+        _helper.assertFileExists(foreignNeverExpired, true);
     }
 
     /**
@@ -291,54 +294,54 @@ public class TestFileSessions extends AbstractTestBase
         store.initialize(sessionContext);
 
         //create file not for our context that expired long ago and should be removed 
-        FileTestHelper.createFile("101_foobar_0.0.0.0_sessiona");
-        FileTestHelper.assertSessionExists("sessiona", true);
+        _helper.createFile("101_foobar_0.0.0.0_sessiona");
+        _helper.assertSessionExists("sessiona", true);
 
         //create a file not for our context that is not expired and should be ignored
         String nonExpiredForeign = (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) + "_foobar_0.0.0.0_sessionb";
-        FileTestHelper.createFile(nonExpiredForeign);
-        FileTestHelper.assertFileExists(nonExpiredForeign, true);
+        _helper.createFile(nonExpiredForeign);
+        _helper.assertFileExists(nonExpiredForeign, true);
 
         //create a file not for our context that is recently expired, a thus ignored 
         String expiredForeign = (System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(1)) + "_foobar_0.0.0.0_sessionc";
-        FileTestHelper.createFile(expiredForeign);
-        FileTestHelper.assertFileExists(expiredForeign, true);
+        _helper.createFile(expiredForeign);
+        _helper.assertFileExists(expiredForeign, true);
 
         //create a file that is not a session file, it should be ignored
-        FileTestHelper.createFile("whatever.txt");
-        FileTestHelper.assertFileExists("whatever.txt", true);
+        _helper.createFile("whatever.txt");
+        _helper.assertFileExists("whatever.txt", true);
 
         //create a file that is not a valid session filename, should be ignored
-        FileTestHelper.createFile("nonNumber_0.0.0.0_spuriousFile");
-        FileTestHelper.assertFileExists("nonNumber_0.0.0.0_spuriousFile", true);
+        _helper.createFile("nonNumber_0.0.0.0_spuriousFile");
+        _helper.assertFileExists("nonNumber_0.0.0.0_spuriousFile", true);
 
         //create a file that is a non-expired session file for our context that should be ignored
         String nonExpired = (System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1)) + "_test_0.0.0.0_sessionb";
-        FileTestHelper.createFile(nonExpired);
-        FileTestHelper.assertFileExists(nonExpired, true);
+        _helper.createFile(nonExpired);
+        _helper.assertFileExists(nonExpired, true);
 
         //create a file that is a never-expire session file for our context that should be ignored
         String neverExpired = "0_test_0.0.0.0_sessionc";
-        FileTestHelper.createFile(neverExpired);
-        FileTestHelper.assertFileExists(neverExpired, true);
+        _helper.createFile(neverExpired);
+        _helper.assertFileExists(neverExpired, true);
 
         //create a file that is a never-expire session file for another context that should be ignored
         String foreignNeverExpired = "0_test_0.0.0.0_sessionc";
-        FileTestHelper.createFile(foreignNeverExpired);
-        FileTestHelper.assertFileExists(foreignNeverExpired, true);
+        _helper.createFile(foreignNeverExpired);
+        _helper.assertFileExists(foreignNeverExpired, true);
 
         //walk all files in the store
         ((FileSessionDataStore)store).initializeStore();
 
         //check results
-        FileTestHelper.assertSessionExists("sessiona", false);
-        FileTestHelper.assertFileExists("whatever.txt", true);
-        FileTestHelper.assertFileExists("nonNumber_0.0.0.0_spuriousFile", true);
-        FileTestHelper.assertFileExists(nonExpired, true);
-        FileTestHelper.assertFileExists(nonExpiredForeign, true);
-        FileTestHelper.assertFileExists(expiredForeign, true);
-        FileTestHelper.assertFileExists(neverExpired, true);
-        FileTestHelper.assertFileExists(foreignNeverExpired, true);
+        _helper.assertSessionExists("sessiona", false);
+        _helper.assertFileExists("whatever.txt", true);
+        _helper.assertFileExists("nonNumber_0.0.0.0_spuriousFile", true);
+        _helper.assertFileExists(nonExpired, true);
+        _helper.assertFileExists(nonExpiredForeign, true);
+        _helper.assertFileExists(expiredForeign, true);
+        _helper.assertFileExists(neverExpired, true);
+        _helper.assertFileExists(foreignNeverExpired, true);
     }
 
     /**
@@ -360,8 +363,8 @@ public class TestFileSessions extends AbstractTestBase
         store.initialize(sessionContext);
 
         String expectedFilename = (System.currentTimeMillis() + 10000) + "__test_0.0.0.0_validFile123";
-        FileTestHelper.createFile(expectedFilename);
-        FileTestHelper.assertFileExists(expectedFilename, true);
+        _helper.createFile(expectedFilename);
+        _helper.assertFileExists(expectedFilename, true);
 
         store.start();
 
@@ -375,7 +378,7 @@ public class TestFileSessions extends AbstractTestBase
             //expected exception
         }
 
-        FileTestHelper.assertFileExists(expectedFilename, false);
+        _helper.assertFileExists(expectedFilename, false);
     }
 
     /**
@@ -402,22 +405,22 @@ public class TestFileSessions extends AbstractTestBase
         //create a file for session abc that expired 5sec ago
         long exp = now - 5000L;
         String name1 = Long.toString(exp) + "__test_0.0.0.0_abc";
-        FileTestHelper.createFile(name1);
+        _helper.createFile(name1);
 
         //create a file for same session that expired 4 sec ago
         exp = now - 4000L;
         String name2 = Long.toString(exp) + "__test_0.0.0.0_abc";
-        FileTestHelper.createFile(name2);
+        _helper.createFile(name2);
 
         //make a file for same session that expired 3 sec ago
         exp = now - 3000L;
         String name3 = Long.toString(exp) + "__test_0.0.0.0_abc";
-        FileTestHelper.createFile(name3);
+        _helper.createFile(name3);
 
         store.start();
 
-        FileTestHelper.assertFileExists(name1, false);
-        FileTestHelper.assertFileExists(name2, false);
-        FileTestHelper.assertFileExists(name3, true);
+        _helper.assertFileExists(name1, false);
+        _helper.assertFileExists(name2, false);
+        _helper.assertFileExists(name3, true);
     }
 }

tests/test-sessions/test-gcloud-sessions/src/test/java/org/eclipse/jetty/gcloud/session/ClusteredOrphanedSessionTest.java

@@ -22,7 +22,6 @@ import org.eclipse.jetty.server.session.AbstractClusteredOrphanedSessionTest;
 import org.eclipse.jetty.server.session.SessionDataStoreFactory;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Test;
 
 /**
  * ClusteredOrphanedSessionTest
@@ -51,11 +50,4 @@ public class ClusteredOrphanedSessionTest extends AbstractClusteredOrphanedSessi
     {
         return GCloudSessionTestSupport.newSessionDataStoreFactory(__testSupport.getDatastore());
     }
-
-    @Test
-    @Override
-    public void testOrphanedSession() throws Exception
-    {
-        super.testOrphanedSession();
-    }
 }

tests/test-sessions/test-gcloud-sessions/src/test/java/org/eclipse/jetty/gcloud/session/InvalidationSessionTest.java

@@ -28,7 +28,6 @@ import org.junit.jupiter.api.BeforeAll;
  */
 public class InvalidationSessionTest extends AbstractClusteredInvalidationSessionTest
 {
-
     public static GCloudSessionTestSupport __testSupport;
 
     @BeforeAll

tests/test-sessions/test-hazelcast-sessions/pom.xml

@@ -52,6 +52,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <configuration>
+           <argLine>--add-modules java.se --add-exports java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED</argLine>
+          <useSystemClassLoader>true</useSystemClassLoader>
           <forkedProcessExitTimeoutInSeconds>45</forkedProcessExitTimeoutInSeconds>
           <forkedProcessTimeoutInSeconds>240</forkedProcessTimeoutInSeconds>
         </configuration>
@@ -88,6 +90,17 @@
       <artifactId>jetty-hazelcast</artifactId>
       <version>${project.version}</version>
       <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>com.hazelcast</groupId>
+          <artifactId>hazelcast</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+       <groupId>com.hazelcast</groupId>
+       <artifactId>hazelcast-all</artifactId>
+       <version>${hazelcast.version}</version>
     </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/ClusteredOrphanedSessionTest.java

@@ -29,7 +29,6 @@ import org.junit.jupiter.api.BeforeEach;
 public class ClusteredOrphanedSessionTest
     extends AbstractClusteredOrphanedSessionTest
 {
-
     HazelcastSessionDataStoreFactory factory;
 
     HazelcastTestHelper _testHelper;

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/ClusteredSessionScavengingTest.java

@@ -29,7 +29,6 @@ import org.junit.jupiter.api.BeforeEach;
 public class ClusteredSessionScavengingTest
     extends AbstractClusteredSessionScavengingTest
 {
-
     HazelcastSessionDataStoreFactory factory;
 
     HazelcastTestHelper _testHelper;

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/HazelcastSessionDataStoreTest.java

@@ -37,7 +37,6 @@ import static org.junit.jupiter.api.Assertions.fail;
  */
 public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
 {
-
     HazelcastTestHelper _testHelper;
 
     @Override
@@ -75,40 +74,29 @@ public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
     {
         return _testHelper.checkSessionExists(data);
     }
-
+    
+    @Override
     @Test
-    @Override
-    public void testGetExpiredDifferentNode() throws Exception
-    {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle 
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
-    }
-
-    @Test
-    @Override
-    public void testGetExpiredPersistedAndExpiredOnly() throws Exception
-    {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle 
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
-    }
-
-    @Override
     public void testStoreSession() throws Exception
     {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle 
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
+        /*
+         * This test does not work with hazelcast, because it uses session attributes
+         * that are classes that are only on the webapp's classloader. Unfortunately
+         * it seems impossible to get hazelcast to use the thread context classloader
+         * when deserializing sessions: it is only using the System classloader.
+         */
+    }
+
+    @Override
+    @Test
+    public void testStoreObjectAttributes() throws Exception
+    {
+        /*
+         * This test does not work with hazelcast, because it uses session attributes
+         * that are classes that are only on the webapp's classloader. Unfortunately
+         * it seems impossible to get hazelcast to use the thread context classloader
+         * when deserializing sessions: it is only using the System classloader.
+         */
     }
 
     /**
@@ -130,7 +118,7 @@ public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
 
         // persist a session
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("222", 100, now, now - 1, -1);
+        SessionData data = store.newSessionData("ggg", 100, now, now - 1, -1);
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 
@@ -141,7 +129,7 @@ public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
         // test that loading it fails
         try
         {
-            store.load("222");
+            store.load("ggg");
             fail("Session should be unreadable");
         }
         catch (UnreadableSessionDataException e)

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/HazelcastTestHelper.java

@@ -66,7 +66,7 @@ public class HazelcastTestHelper
         Config config = new Config();
         config.setInstanceName(_hazelcastInstanceName);
         config.setNetworkConfig(new NetworkConfig().setJoin(new JoinConfig().setMulticastConfig(new MulticastConfig().setEnabled(false))));
-        config.addMapConfig(new MapConfig().setName(_name));
+        config.addMapConfig(new MapConfig().setName(_name)).setClassLoader(null);
         config.getSerializationConfig().addSerializerConfig(_serializerConfig);
         _instance = Hazelcast.getOrCreateHazelcastInstance(config);
     }
@@ -81,6 +81,7 @@ public class HazelcastTestHelper
         HazelcastSessionDataStoreFactory factory = new HazelcastSessionDataStoreFactory();
         factory.setOnlyClient(onlyClient);
         factory.setMapName(_name);
+        factory.setUseQueries(true);
         if (onlyClient)
         {
             ClientNetworkConfig clientNetworkConfig = new ClientNetworkConfig()
@@ -109,7 +110,7 @@ public class HazelcastTestHelper
 
     public void createSession(SessionData data)
     {
-        _instance.getMap(_name).put(data.getContextPath() + "_" + data.getVhost() + "_" + data.getId(), data);
+        Object o = _instance.getMap(_name).put(data.getContextPath() + "_" + data.getVhost() + "_" + data.getId(), data);
     }
 
     public boolean checkSessionExists(SessionData data)

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/client/ClientOrphanedSessionTest.java

@@ -27,7 +27,6 @@ import org.junit.jupiter.api.BeforeEach;
 public class ClientOrphanedSessionTest
     extends AbstractClusteredOrphanedSessionTest
 {
-
     HazelcastTestHelper _testHelper;
 
     @Override

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/client/ClientSessionScavengingTest.java

@@ -27,7 +27,6 @@ import org.junit.jupiter.api.BeforeEach;
 public class ClientSessionScavengingTest
     extends AbstractClusteredSessionScavengingTest
 {
-
     HazelcastTestHelper _testHelper;
 
     @Override

tests/test-sessions/test-hazelcast-sessions/src/test/java/org/eclipse/jetty/hazelcast/session/client/HazelcastSessionDataStoreTest.java

@@ -39,7 +39,6 @@ import static org.junit.jupiter.api.Assertions.fail;
  */
 public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
 {
-
     HazelcastTestHelper _testHelper;
 
     @Override
@@ -80,48 +79,26 @@ public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
 
     @Test
     @Override
-    public void testGetExpiredDifferentNode() throws Exception
+    public void testStoreSession() throws Exception
     {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle 
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
+       /*
+        * This test does not work with hazelcast, because it uses session attributes
+        * that are classes that are only on the webapp's classloader. Unfortunately
+        * it seems impossible to get hazelcast to use the thread context classloader
+        * when deserializing sessions: it is only using the System classloader.
+        */
     }
 
     @Test
-    @Override
-    public void testGetExpiredPersistedAndExpiredOnly() throws Exception
-    {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle 
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
-    }
-
-    @Override
-    public void testStoreSession() throws Exception
-    {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle 
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
-    }
-
     @Override
     public void testStoreObjectAttributes() throws Exception
     {
-        //This test will not work for hazelcast because we can't enable
-        //HazelcastSessionDataStore.setScavengeZombieSessions, as it's
-        //too difficult to get the required classes onto the embedded
-        //hazelcast instance: these classes are required  to handle
-        //the serialization/deserialization that hazelcast performs when querying
-        //to find zombie sessions.
+        /*
+         * This test does not work with hazelcast, because it uses session attributes
+         * that are classes that are only on the webapp's classloader. Unfortunately
+         * it seems impossible to get hazelcast to use the thread context classloader
+         * when deserializing sessions: it is only using the System classloader.
+         */
     }
 
     /**
@@ -143,7 +120,7 @@ public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
 
         // persist a session
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("222", 100, now, now - 1, -1);
+        SessionData data = store.newSessionData("fff", 100, now, now - 1, -1);
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 
@@ -154,7 +131,7 @@ public class HazelcastSessionDataStoreTest extends AbstractSessionDataStoreTest
         // test that loading it fails
         try
         {
-            store.load("222");
+            store.load("fff");
             fail("Session should be unreadable");
         }
         catch (UnreadableSessionDataException e)

tests/test-sessions/test-infinispan-sessions/pom.xml

@@ -94,6 +94,12 @@
       <version>${project.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>infinispan-embedded-query</artifactId>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.eclipse.jetty</groupId>
       <artifactId>jetty-jmx</artifactId>

tests/test-sessions/test-infinispan-sessions/src/test/java/org/eclipse/jetty/server/session/InfinispanFileSessionDataStoreTest.java

@@ -25,7 +25,6 @@ import org.junit.jupiter.api.BeforeEach;
  */
 public class InfinispanFileSessionDataStoreTest extends InfinispanSessionDataStoreTest
 {
-
     @BeforeEach
     public void setup() throws Exception
     {
@@ -33,5 +32,4 @@ public class InfinispanFileSessionDataStoreTest extends InfinispanSessionDataSto
         _testSupport.setUseFileStore(true);
         _testSupport.setup();
     }
-
 }

tests/test-sessions/test-infinispan-sessions/src/test/java/org/eclipse/jetty/server/session/InfinispanSessionDataStoreTest.java

@@ -19,9 +19,11 @@
 package org.eclipse.jetty.server.session;
 
 import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.session.infinispan.EmbeddedQueryManager;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionData;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionDataStore;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionDataStoreFactory;
+import org.eclipse.jetty.session.infinispan.QueryManager;
 import org.infinispan.query.Search;
 import org.infinispan.query.dsl.Query;
 import org.infinispan.query.dsl.QueryFactory;
@@ -62,6 +64,8 @@ public class InfinispanSessionDataStoreTest extends AbstractSessionDataStoreTest
     {
         InfinispanSessionDataStoreFactory factory = new InfinispanSessionDataStoreFactory();
         factory.setCache(_testSupport.getCache());
+        QueryManager qm = new EmbeddedQueryManager(_testSupport.getCache());
+        factory.setQueryManager(qm);
         return factory;
     }
 
@@ -113,31 +117,6 @@ public class InfinispanSessionDataStoreTest extends AbstractSessionDataStoreTest
         assertThrows(UnreadableSessionDataException.class, () -> store.load("222"));
     }
 
-    /**
-     * This test currently won't work for Infinispan - there is currently no
-     * means to query it to find sessions that have expired.
-     *
-     * @see org.eclipse.jetty.server.session.AbstractSessionDataStoreTest#testGetExpiredPersistedAndExpiredOnly()
-     */
-    @Override
-    public void testGetExpiredPersistedAndExpiredOnly() throws Exception
-    {
-
-    }
-
-    /**
-     * This test won't work for Infinispan - there is currently no
-     * means to query infinispan to find other expired sessions.
-     */
-    @Override
-    public void testGetExpiredDifferentNode() throws Exception
-    {
-        //Ignore
-    }
-
-    /**
-     *
-     */
     @Override
     public boolean checkSessionPersisted(SessionData data) throws Exception
     {
@@ -152,7 +131,7 @@ public class InfinispanSessionDataStoreTest extends AbstractSessionDataStoreTest
             Thread.currentThread().setContextClassLoader(old);
         }
     }
-
+    
     @Test
     public void testQuery() throws Exception
     {

tests/test-sessions/test-infinispan-sessions/src/test/java/org/eclipse/jetty/server/session/SerializedInfinispanSessionDataStoreTest.java

@@ -19,9 +19,11 @@
 package org.eclipse.jetty.server.session;
 
 import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.session.infinispan.EmbeddedQueryManager;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionData;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionDataStore;
 import org.eclipse.jetty.session.infinispan.InfinispanSessionDataStoreFactory;
+import org.eclipse.jetty.session.infinispan.QueryManager;
 import org.infinispan.query.Search;
 import org.infinispan.query.dsl.Query;
 import org.infinispan.query.dsl.QueryFactory;
@@ -37,7 +39,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
  */
 public class SerializedInfinispanSessionDataStoreTest extends AbstractSessionDataStoreTest
 {
-
     public InfinispanTestSupport _testSupport;
 
     @BeforeEach
@@ -59,6 +60,8 @@ public class SerializedInfinispanSessionDataStoreTest extends AbstractSessionDat
     {
         InfinispanSessionDataStoreFactory factory = new InfinispanSessionDataStoreFactory();
         factory.setCache(_testSupport.getCache());
+        QueryManager qm = new EmbeddedQueryManager(_testSupport.getCache());
+        factory.setQueryManager(qm);
         return factory;
     }
 
@@ -110,31 +113,6 @@ public class SerializedInfinispanSessionDataStoreTest extends AbstractSessionDat
         assertThrows(UnreadableSessionDataException.class,() -> store.load("222"));
     }
 
-    /**
-     * This test currently won't work for Infinispan - there is currently no
-     * means to query it to find sessions that have expired.
-     *
-     * @see org.eclipse.jetty.server.session.AbstractSessionDataStoreTest#testGetExpiredPersistedAndExpiredOnly()
-     */
-    @Override
-    public void testGetExpiredPersistedAndExpiredOnly() throws Exception
-    {
-
-    }
-
-    /**
-     * This test won't work for Infinispan - there is currently no
-     * means to query infinispan to find other expired sessions.
-     */
-    @Override
-    public void testGetExpiredDifferentNode() throws Exception
-    {
-        //Ignore
-    }
-
-    /**
-     *
-     */
     @Override
     public boolean checkSessionPersisted(SessionData data) throws Exception
     {

tests/test-sessions/test-infinispan-sessions/src/test/java/org/eclipse/jetty/server/session/remote/RemoteInfinispanSessionDataStoreTest.java

@@ -78,7 +78,7 @@ public class RemoteInfinispanSessionDataStoreTest extends AbstractSessionDataSto
     @Override
     public void persistSession(SessionData data) throws Exception
     {
-        __testSupport.createSession(data);
+        __testSupport.createSession((InfinispanSessionData)data);
     }
 
     @Override
@@ -90,7 +90,7 @@ public class RemoteInfinispanSessionDataStoreTest extends AbstractSessionDataSto
     @Override
     public boolean checkSessionExists(SessionData data) throws Exception
     {
-        return __testSupport.checkSessionExists(data);
+        return __testSupport.checkSessionExists((InfinispanSessionData)data);
     }
 
     @Override
@@ -126,7 +126,7 @@ public class RemoteInfinispanSessionDataStoreTest extends AbstractSessionDataSto
 
         //persist a session
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("222", 100, now, now - 1, -1);
+        InfinispanSessionData data = (InfinispanSessionData)store.newSessionData("222", 100, now, now - 1, -1);
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 

tests/test-sessions/test-infinispan-sessions/src/test/java/org/eclipse/jetty/server/session/remote/RemoteInfinispanTestSupport.java

@@ -54,7 +54,7 @@ public class RemoteInfinispanTestSupport
 {
     private static final Logger LOG = LoggerFactory.getLogger(RemoteInfinispanTestSupport.class);
     public static final String DEFAULT_CACHE_NAME = "session_test_cache";
-    public RemoteCache<String, SessionData> _cache;
+    public RemoteCache<String, InfinispanSessionData> _cache;
     private String _name;
     public static RemoteCacheManager _manager;
     private static final Logger INFINISPAN_LOG =
@@ -88,7 +88,7 @@ public class RemoteInfinispanTestSupport
             LOG.info("Infinispan container started for {}:{} - {}ms", host, port,
                      System.currentTimeMillis() - start);
             SearchMapping mapping = new SearchMapping();
-            mapping.entity(SessionData.class).indexed().providedId()
+            mapping.entity(InfinispanSessionData.class).indexed().providedId()
                 .property("expiry", ElementType.METHOD).field();
 
             Properties properties = new Properties();
@@ -154,7 +154,7 @@ public class RemoteInfinispanTestSupport
         _name = cacheName;
     }
 
-    public RemoteCache<String, SessionData> getCache()
+    public RemoteCache<String, InfinispanSessionData> getCache()
     {
         return _cache;
     }
@@ -169,18 +169,18 @@ public class RemoteInfinispanTestSupport
         _cache.clear();
     }
 
-    public void createSession(SessionData data)
+    public void createSession(InfinispanSessionData data)
         throws Exception
     {
         _cache.put(data.getContextPath() + "_" + data.getVhost() + "_" + data.getId(), data);
     }
 
-    public void createUnreadableSession(SessionData data)
+    public void createUnreadableSession(InfinispanSessionData data)
     {
 
     }
 
-    public boolean checkSessionExists(SessionData data)
+    public boolean checkSessionExists(InfinispanSessionData data)
         throws Exception
     {
         return (_cache.get(data.getContextPath() + "_" + data.getVhost() + "_" + data.getId()) != null);

tests/test-sessions/test-jdbc-sessions/src/test/java/org/eclipse/jetty/server/session/ClusteredInvalidationSessionTest.java

@@ -27,7 +27,6 @@ import org.testcontainers.junit.jupiter.Testcontainers;
 @Testcontainers(disabledWithoutDocker = true)
 public class ClusteredInvalidationSessionTest extends AbstractClusteredInvalidationSessionTest
 {
-
     @AfterEach
     public void tearDown() throws Exception
     {

tests/test-sessions/test-jdbc-sessions/src/test/java/org/eclipse/jetty/server/session/ClusteredSessionMigrationTest.java

@@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
 @Testcontainers(disabledWithoutDocker = true)
 public class ClusteredSessionMigrationTest extends AbstractTestBase
 {
-
     @Override
     public SessionDataStoreFactory createSessionDataStoreFactory()
     {

tests/test-sessions/test-jdbc-sessions/src/test/java/org/eclipse/jetty/server/session/JDBCSessionDataStoreTest.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.server.session;
 
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
 /**
@@ -28,7 +29,6 @@ import org.testcontainers.junit.jupiter.Testcontainers;
 @Testcontainers(disabledWithoutDocker = true)
 public class JDBCSessionDataStoreTest extends AbstractSessionDataStoreTest
 {
-
     @BeforeEach
     public void setUp() throws Exception
     {
@@ -62,6 +62,12 @@ public class JDBCSessionDataStoreTest extends AbstractSessionDataStoreTest
             data.getMaxInactiveMs(), data.getExpiry(), data.getCookieSet(),
             data.getLastSaved());
     }
+    
+    @Test
+    public void testCleanOrphans() throws Exception
+    {
+        super.testCleanOrphans();
+    }
 
     @Override
     public boolean checkSessionExists(SessionData data) throws Exception

tests/test-sessions/test-jdbc-sessions/src/test/java/org/eclipse/jetty/server/session/WebAppObjectInSessionTest.java

@@ -29,7 +29,6 @@ import org.testcontainers.junit.jupiter.Testcontainers;
 @Testcontainers(disabledWithoutDocker = true)
 public class WebAppObjectInSessionTest extends AbstractWebAppObjectInSessionTest
 {
-
     @Override
     public SessionDataStoreFactory createSessionDataStoreFactory()
     {

tests/test-sessions/test-memcached-sessions/src/test/java/org/eclipse/jetty/memcached/sessions/CachingSessionDataStoreTest.java

@@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
  */
 public class CachingSessionDataStoreTest
 {
-
     @Test
     public void testSessionCRUD() throws Exception
     {

tests/test-sessions/test-memcached-sessions/src/test/java/org/eclipse/jetty/memcached/sessions/MemcachedTestHelper.java

@@ -20,6 +20,7 @@ package org.eclipse.jetty.memcached.sessions;
 
 import java.net.InetSocketAddress;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -57,7 +58,7 @@ public class MemcachedTestHelper
         }
 
         @Override
-        public boolean exists(String id) throws Exception
+        public boolean doExists(String id) throws Exception
         {
             return _store.get(id) != null;
         }
@@ -92,10 +93,9 @@ public class MemcachedTestHelper
         }
 
         @Override
-        public Set<String> doGetExpired(Set<String> candidates)
+        public Set<String> doCheckExpired(Set<String> candidates, long time)
         {
             Set<String> expiredIds = new HashSet<>();
-            long now = System.currentTimeMillis();
             if (candidates != null)
             {
                 for (String id : candidates)
@@ -103,7 +103,7 @@ public class MemcachedTestHelper
                     SessionData sd = _store.get(id);
                     if (sd == null)
                         expiredIds.add(id);
-                    else if (sd.isExpiredAt(now))
+                    else if (sd.isExpiredAt(time))
                         expiredIds.add(id);
                 }
             }
@@ -111,12 +111,24 @@ public class MemcachedTestHelper
             for (String id : _store.keySet())
             {
                 SessionData sd = _store.get(id);
-                if (sd.isExpiredAt(now))
+                if (sd.isExpiredAt(time))
                     expiredIds.add(id);
             }
 
             return expiredIds;
         }
+        
+        @Override
+        public Set<String> doGetExpired(long timeLimit)
+        {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public void doCleanOrphans(long timeLimit)
+        {
+            //noop
+        }
 
         @Override
         protected void doStop() throws Exception
@@ -139,6 +151,7 @@ public class MemcachedTestHelper
 
     private static final Logger MEMCACHED_LOG = LoggerFactory.getLogger("org.eclipse.jetty.memcached.sessions.MemcachedLogs");
 
+    @SuppressWarnings({"rawtypes", "unchecked"})
     static GenericContainer memcached =
         new GenericContainer("memcached:" + System.getProperty("memcached.docker.version", "1.6.6"))
             .withLogConsumer(new Slf4jLogConsumer(MEMCACHED_LOG));

tests/test-sessions/test-mongodb-sessions/src/test/java/org/eclipse/jetty/nosql/mongodb/AttributeNameTest.java

@@ -50,7 +50,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
  */
 public class AttributeNameTest
 {
-
     @BeforeAll
     public static void beforeClass() throws Exception
     {

tests/test-sessions/test-mongodb-sessions/src/test/java/org/eclipse/jetty/nosql/mongodb/ClusteredInvalidateSessionTest.java

@@ -25,7 +25,6 @@ import org.junit.jupiter.api.BeforeAll;
 
 public class ClusteredInvalidateSessionTest extends AbstractClusteredInvalidationSessionTest
 {
-
     @BeforeAll
     public static void beforeClass() throws Exception
     {

tests/test-sessions/test-mongodb-sessions/src/test/java/org/eclipse/jetty/nosql/mongodb/ClusteredOrphanedSessionTest.java

@@ -29,7 +29,6 @@ import org.junit.jupiter.api.Test;
  */
 public class ClusteredOrphanedSessionTest extends AbstractClusteredOrphanedSessionTest
 {
-
     @BeforeAll
     public static void beforeClass() throws Exception
     {

tests/test-sessions/test-mongodb-sessions/src/test/java/org/eclipse/jetty/nosql/mongodb/ClusteredSessionScavengingTest.java

@@ -25,7 +25,6 @@ import org.junit.jupiter.api.BeforeAll;
 
 public class ClusteredSessionScavengingTest extends AbstractClusteredSessionScavengingTest
 {
-
     @BeforeAll
     public static void beforeClass() throws Exception
     {

tests/test-sessions/test-mongodb-sessions/src/test/java/org/eclipse/jetty/nosql/mongodb/MongoSessionDataStoreTest.java

@@ -42,7 +42,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
  */
 public class MongoSessionDataStoreTest extends AbstractSessionDataStoreTest
 {
-
     @BeforeEach
     public void beforeEach() throws Exception
     {

tests/test-sessions/test-sessions-common/src/main/java/org/eclipse/jetty/server/session/AbstractClusteredInvalidationSessionTest.java

@@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.fail;
  */
 public abstract class AbstractClusteredInvalidationSessionTest extends AbstractTestBase
 {
-
     @Test
     public void testInvalidation() throws Exception
     {

tests/test-sessions/test-sessions-common/src/main/java/org/eclipse/jetty/server/session/AbstractClusteredOrphanedSessionTest.java

@@ -45,7 +45,6 @@ import static org.junit.jupiter.api.Assertions.assertNull;
  */
 public abstract class AbstractClusteredOrphanedSessionTest extends AbstractTestBase
 {
-
     /**
      * @throws Exception on test failure
      */

tests/test-sessions/test-sessions-common/src/main/java/org/eclipse/jetty/server/session/AbstractClusteredSessionScavengingTest.java

@@ -51,7 +51,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
  */
 public abstract class AbstractClusteredSessionScavengingTest extends AbstractTestBase
 {
-
     public void pause(int secs)
         throws InterruptedException
     {

tests/test-sessions/test-sessions-common/src/main/java/org/eclipse/jetty/server/session/AbstractSessionDataStoreTest.java

@@ -34,6 +34,7 @@ import java.util.concurrent.TimeUnit;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.toolchain.test.IO;
 import org.eclipse.jetty.toolchain.test.MavenTestingUtils;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 
 import static org.hamcrest.MatcherAssert.assertThat;
@@ -59,6 +60,7 @@ public abstract class AbstractSessionDataStoreTest
      */
     public static final long ANCIENT_TIMESTAMP = 100L;
     public static final long RECENT_TIMESTAMP = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(3 * GRACE_PERIOD_SEC);
+    protected static File extraClasses;
 
     protected URLClassLoader _contextClassLoader = new URLClassLoader(new URL[]{}, Thread.currentThread().getContextClassLoader());
 
@@ -72,6 +74,33 @@ public abstract class AbstractSessionDataStoreTest
 
     public abstract boolean checkSessionPersisted(SessionData data) throws Exception;
 
+    @BeforeAll
+    public static void beforeAll()
+        throws Exception
+    {
+        InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("Foo.clazz");
+        extraClasses = new File(MavenTestingUtils.getTargetDir(), "extraClasses");
+        extraClasses.mkdirs();
+        File fooclass = new File(extraClasses, "Foo.class");
+        IO.copy(is, new FileOutputStream(fooclass));
+        is.close();
+
+        is = Thread.currentThread().getContextClassLoader().getResourceAsStream("Proxyable.clazz");
+        File proxyableClass = new File(extraClasses, "Proxyable.class");
+        IO.copy(is, new FileOutputStream(proxyableClass));
+        is.close();
+
+        is = Thread.currentThread().getContextClassLoader().getResourceAsStream("ProxyableInvocationHandler.clazz");
+        File pihClass = new File(extraClasses, "ProxyableInvocationHandler.class");
+        IO.copy(is, new FileOutputStream(pihClass));
+        is.close();
+
+        is = Thread.currentThread().getContextClassLoader().getResourceAsStream("ProxyableFactory.clazz");
+        File factoryClass = new File(extraClasses, "ProxyableFactory.class");
+        IO.copy(is, new FileOutputStream(factoryClass));
+        is.close();
+    }
+
     /**
      * Test that the store can persist a session. The session uses an attribute
      * class that is only known to the webapp classloader. This tests that
@@ -81,16 +110,7 @@ public abstract class AbstractSessionDataStoreTest
     public void testStoreSession() throws Exception
     {
         //Use a class that would only be known to the webapp classloader
-        InputStream foostream = Thread.currentThread().getContextClassLoader().getResourceAsStream("Foo.clazz");
-        File foodir = new File(MavenTestingUtils.getTargetDir(), "foo");
-        foodir.mkdirs();
-        File fooclass = new File(foodir, "Foo.class");
-        IO.copy(foostream, new FileOutputStream(fooclass));
-
-        assertTrue(fooclass.exists());
-        assertTrue(fooclass.length() != 0);
-
-        URL[] foodirUrls = new URL[]{foodir.toURI().toURL()};
+        URL[] foodirUrls = new URL[]{extraClasses.toURI().toURL()};
         _contextClassLoader = new URLClassLoader(foodirUrls, Thread.currentThread().getContextClassLoader());
 
         //create the SessionDataStore
@@ -115,7 +135,7 @@ public abstract class AbstractSessionDataStoreTest
             Class fooclazz = Class.forName("Foo", true, _contextClassLoader);
             //create a session
             long now = System.currentTimeMillis();
-            data = store.newSessionData("1234", 100, now, now - 1, -1);//never expires
+            data = store.newSessionData("aaa1", 100, now, now - 1, -1);//never expires
             data.setLastNode(sessionContext.getWorkerName());
 
             //Make an attribute that uses the class only known to the webapp classloader
@@ -139,7 +159,7 @@ public abstract class AbstractSessionDataStoreTest
             {
                 try
                 {
-                    store.store("1234", finalData);
+                    store.store("aaa1", finalData);
                 }
                 catch (Exception e)
                 {
@@ -175,7 +195,7 @@ public abstract class AbstractSessionDataStoreTest
 
         //create a session
         final long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("1234", 100, 200, 199, -1);//never expires
+        SessionData data = store.newSessionData("aaa2", 100, 200, 199, -1);//never expires
         data.setAttribute("a", "b");
         data.setLastNode(sessionContext.getWorkerName());
         data.setLastSaved(400); //make it look like it was previously saved by the store
@@ -188,7 +208,7 @@ public abstract class AbstractSessionDataStoreTest
         data.setAccessed(now);
         data.setMaxInactiveMs(TimeUnit.MINUTES.toMillis(2));
         data.setAttribute("a", "c");
-        store.store("1234", data);
+        store.store("aaa2", data);
 
         assertTrue(checkSessionPersisted(data));
     }
@@ -200,34 +220,8 @@ public abstract class AbstractSessionDataStoreTest
     @Test
     public void testStoreObjectAttributes() throws Exception
     {
-
         //Use classes that would only be known to the webapp classloader
-        InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream("Proxyable.clazz");
-        File proxyabledir = new File(MavenTestingUtils.getTargetDir(), "proxyable");
-        proxyabledir.mkdirs();
-        File proxyableClass = new File(proxyabledir, "Proxyable.class");
-        IO.copy(is, new FileOutputStream(proxyableClass));
-        is.close();
-
-        assertTrue(proxyableClass.exists());
-        assertTrue(proxyableClass.length() != 0);
-
-        is = Thread.currentThread().getContextClassLoader().getResourceAsStream("ProxyableInvocationHandler.clazz");
-        File pihClass = new File(proxyabledir, "ProxyableInvocationHandler.class");
-        IO.copy(is, new FileOutputStream(pihClass));
-        is.close();
-
-        is = Thread.currentThread().getContextClassLoader().getResourceAsStream("ProxyableFactory.clazz");
-        File factoryClass = new File(proxyabledir, "ProxyableFactory.class");
-        IO.copy(is, new FileOutputStream(factoryClass));
-        is.close();
-
-        is = Thread.currentThread().getContextClassLoader().getResourceAsStream("Foo.clazz");
-        File fooClass = new File(proxyabledir, "Foo.class");
-        IO.copy(is, new FileOutputStream(fooClass));
-        is.close();
-
-        URL[] proxyabledirUrls = new URL[]{proxyabledir.toURI().toURL()};
+        URL[] proxyabledirUrls = new URL[]{extraClasses.toURI().toURL()};
         _contextClassLoader = new URLClassLoader(proxyabledirUrls, Thread.currentThread().getContextClassLoader());
 
         //create the SessionDataStore 
@@ -253,7 +247,7 @@ public abstract class AbstractSessionDataStoreTest
             Class factoryclazz = Class.forName("ProxyableFactory", true, _contextClassLoader);
             //create a session
             long now = System.currentTimeMillis();
-            data = store.newSessionData("1234", 100, now, now - 1, -1);//never expires
+            data = store.newSessionData("aaa3", 100, now, now - 1, -1);//never expires
             data.setLastNode(sessionContext.getWorkerName());
             Method m = factoryclazz.getMethod("newProxyable", ClassLoader.class);
             Object proxy = m.invoke(null, _contextClassLoader);
@@ -288,7 +282,7 @@ public abstract class AbstractSessionDataStoreTest
             {
                 try
                 {
-                    store.store("1234", finalData);
+                    store.store("aaa3", finalData);
                 }
                 catch (Exception e)
                 {
@@ -300,7 +294,7 @@ public abstract class AbstractSessionDataStoreTest
         Thread t = new Thread(r, "saver");
         t.start();
         t.join(TimeUnit.SECONDS.toMillis(10));
-
+        
         //check that the store contains all of the session data
         assertTrue(checkSessionPersisted(data));
     }
@@ -322,16 +316,16 @@ public abstract class AbstractSessionDataStoreTest
 
         //persist a session that is not expired
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("1234", 100, now, now - 1, -1);//never expires
+        SessionData data = store.newSessionData("aaa4", 100, now, now - 1, -1);//never expires
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 
         store.start();
 
         //test that we can retrieve it
-        SessionData loaded = store.load("1234");
+        SessionData loaded = store.load("aaa4");
         assertNotNull(loaded);
-        assertEquals("1234", loaded.getId());
+        assertEquals("aaa4", loaded.getId());
         assertEquals(100, loaded.getCreated());
         assertEquals(now, loaded.getAccessed());
         assertEquals(now - 1, loaded.getLastAccessed());
@@ -355,7 +349,7 @@ public abstract class AbstractSessionDataStoreTest
 
         //persist a session that is expired
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("678", 100, now - 20, now - 30, 10);//10 sec max idle
+        SessionData data = store.newSessionData("aaa5", 100, now - 20, now - 30, 10);//10 sec max idle
         data.setLastNode(sessionContext.getWorkerName());
         data.setExpiry(RECENT_TIMESTAMP); //make it expired recently
         persistSession(data);
@@ -363,9 +357,9 @@ public abstract class AbstractSessionDataStoreTest
         store.start();
 
         //test we can retrieve it
-        SessionData loaded = store.load("678");
+        SessionData loaded = store.load("aaa5");
         assertNotNull(loaded);
-        assertEquals("678", loaded.getId());
+        assertEquals("aaa5", loaded.getId());
         assertEquals(100, loaded.getCreated());
         assertEquals(now - 20, loaded.getAccessed());
         assertEquals(now - 30, loaded.getLastAccessed());
@@ -448,13 +442,13 @@ public abstract class AbstractSessionDataStoreTest
         
         //persist a session that has no attributes
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("222", 100, now, now - 1, -1);
+        SessionData data = store.newSessionData("aaa6", 100, now, now - 1, -1);
         data.setLastNode(sessionContext.getWorkerName());
         //persistSession(data);
-        store.store("222", data);
+        store.store("aaa6", data);
 
         //test that we can retrieve it
-        SessionData savedSession = store.load("222");
+        SessionData savedSession = store.load("aaa6");
         assertEquals(0, savedSession.getAllAttributes().size());
     }
     
@@ -475,21 +469,21 @@ public abstract class AbstractSessionDataStoreTest
         
         //persist a session that has attributes
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("222", 100, now, now - 1, -1);
+        SessionData data = store.newSessionData("aaa7", 100, now, now - 1, -1);
         data.setAttribute("foo", "bar");
         data.setLastNode(sessionContext.getWorkerName());
-        store.store("222", data);
+        store.store("aaa7", data);
 
         //test that we can retrieve it
-        SessionData savedSession = store.load("222");
+        SessionData savedSession = store.load("aaa7");
         assertEquals("bar", savedSession.getAttribute("foo"));
         
         //now modify so there are no attributes
         savedSession.setAttribute("foo", null);
-        store.store("222", savedSession);
+        store.store("aaa7", savedSession);
         
         //check its still readable
-        savedSession = store.load("222");
+        savedSession = store.load("aaa7");
         assertEquals(0, savedSession.getAllAttributes().size());
     }
     
@@ -510,14 +504,14 @@ public abstract class AbstractSessionDataStoreTest
 
         //persist a session that is not expired
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("1234", 100, now, now - 1, -1);
+        SessionData data = store.newSessionData("aaa8", 100, now, now - 1, -1);
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 
         store.start();
 
         //delete the session via the store
-        store.delete("1234");
+        store.delete("aaa8");
 
         //check the session is no longer exists
         assertFalse(checkSessionExists(data));
@@ -560,22 +554,22 @@ public abstract class AbstractSessionDataStoreTest
         store.initialize(sessionContext);
 
         //persist a session that is expired
-        SessionData data = store.newSessionData("1234", 100, 101, 101, 10);
+        SessionData data = store.newSessionData("aaa9", 100, 101, 101, 10);
         data.setLastNode(sessionContext.getWorkerName());
         data.setExpiry(RECENT_TIMESTAMP); //make it expired recently so FileSessionDataStore doesn't eliminate it on startup
         persistSession(data);
 
         //persist another session that is expired
-        SessionData data2 = store.newSessionData("5678", 100, 100, 101, 30);
+        SessionData data2 = store.newSessionData("aaa10", 100, 100, 101, 30);
         data2.setLastNode(sessionContext.getWorkerName());
         data2.setExpiry(RECENT_TIMESTAMP); //make it expired recently so FileSessionDataStore doesn't eliminate it on startup
         persistSession(data2);
 
         store.start();
 
-        Set<String> candidates = new HashSet<>(Arrays.asList(new String[]{"1234", "5678"}));
+        Set<String> candidates = new HashSet<>(Arrays.asList(new String[]{"aaa9", "aaa10"}));
         Set<String> expiredIds = store.getExpired(candidates);
-        assertThat(expiredIds, containsInAnyOrder("1234", "5678"));
+        assertThat(expiredIds, containsInAnyOrder("aaa9", "aaa10"));
     }
 
     /**
@@ -596,18 +590,18 @@ public abstract class AbstractSessionDataStoreTest
 
         long now = System.currentTimeMillis();
         //persist a session that is not expired
-        SessionData data = store.newSessionData("1234", 100, now, now - 1, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa11", 100, now, now - 1, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 
         //persist another session that is not expired
-        SessionData data2 = store.newSessionData("5678", 100, now, now - 1, TimeUnit.MINUTES.toMillis(60));
+        SessionData data2 = store.newSessionData("aaa12", 100, now, now - 1, TimeUnit.MINUTES.toMillis(60));
         data2.setLastNode(sessionContext.getWorkerName());
         persistSession(data2);
 
         store.start();
 
-        Set<String> candidates = new HashSet<>(Arrays.asList(new String[]{"1234", "5678"}));
+        Set<String> candidates = new HashSet<>(Arrays.asList(new String[]{"aaa11", "aaa12"}));
         Set<String> expiredIds = store.getExpired(candidates);
         assertEquals(0, expiredIds.size());
     }
@@ -629,9 +623,9 @@ public abstract class AbstractSessionDataStoreTest
         store.initialize(sessionContext);
         store.start();
 
-        Set<String> candidates = new HashSet<>(Arrays.asList(new String[]{"1234", "5678"}));
+        Set<String> candidates = new HashSet<>(Arrays.asList(new String[]{"a", "b"}));
         Set<String> expiredIds = store.getExpired(candidates);
-        assertThat(expiredIds, containsInAnyOrder("1234", "5678"));
+        assertThat(expiredIds, containsInAnyOrder("a", "b"));
     }
 
     /**
@@ -652,13 +646,13 @@ public abstract class AbstractSessionDataStoreTest
         store.initialize(sessionContext);
 
         //persist a session that is expired
-        SessionData data = store.newSessionData("1234", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa13", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
         data.setExpiry(RECENT_TIMESTAMP); //must be recently expired, or FileSessionDataStore will eliminate it on startup
         persistSession(data);
 
         //persist another session that is expired
-        SessionData data2 = store.newSessionData("5678", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        SessionData data2 = store.newSessionData("aaa14", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
         data2.setLastNode(sessionContext.getWorkerName());
         data2.setExpiry(RECENT_TIMESTAMP); //must be recently expired, or FileSessionDataStore will eliminate it on startup
         persistSession(data2);
@@ -667,7 +661,7 @@ public abstract class AbstractSessionDataStoreTest
 
         Set<String> candidates = new HashSet<>();
         Set<String> expiredIds = store.getExpired(candidates);
-        assertThat(expiredIds, containsInAnyOrder("1234", "5678"));
+        assertThat(expiredIds, containsInAnyOrder("aaa13", "aaa14"));
     }
 
     /**
@@ -688,7 +682,7 @@ public abstract class AbstractSessionDataStoreTest
         store.initialize(sessionContext);
 
         //persist a session that is expired for a different node
-        SessionData data = store.newSessionData("1234", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa15", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode("other");
         data.setExpiry(RECENT_TIMESTAMP); //must be recently expired, or FileSessionDataStore will eliminate it on startup
         persistSession(data);
@@ -697,7 +691,104 @@ public abstract class AbstractSessionDataStoreTest
 
         Set<String> candidates = new HashSet<>();
         Set<String> expiredIds = store.getExpired(candidates);
-        assertThat(expiredIds, containsInAnyOrder("1234"));
+        assertThat(expiredIds, containsInAnyOrder("aaa15"));
+    }
+    
+    @Test
+    public void testCleanOrphans() throws Exception
+    {
+        //create the SessionDataStore
+        ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
+        context.setContextPath("/test");
+        SessionDataStoreFactory factory = createSessionDataStoreFactory();
+        ((AbstractSessionDataStoreFactory)factory).setGracePeriodSec(GRACE_PERIOD_SEC);
+        SessionDataStore store = factory.getSessionDataStore(context.getSessionHandler());
+        SessionContext sessionContext = new SessionContext("foo", context.getServletContext());
+        store.initialize(sessionContext);
+
+        long now = System.currentTimeMillis();
+        
+        //persist a long ago expired session for our context
+        SessionData oldSession = store.newSessionData("001", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        oldSession.setExpiry(200);
+        oldSession.setLastNode("me");
+        persistSession(oldSession);
+        assertTrue(checkSessionExists(oldSession));
+        
+        //persist a recently expired session for our context
+        SessionData expiredSession = store.newSessionData("002", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        expiredSession.setExpiry(RECENT_TIMESTAMP);
+        expiredSession.setLastNode("me");
+        persistSession(expiredSession);
+        assertTrue(checkSessionExists(expiredSession));
+
+        //persist a non expired session for our context
+        SessionData unexpiredSession = store.newSessionData("003", 100, now + 10, now + 5, TimeUnit.MINUTES.toMillis(60));
+        unexpiredSession.setExpiry(now + TimeUnit.MINUTES.toMillis(10));
+        unexpiredSession.setLastNode("me");
+        persistSession(unexpiredSession);
+        assertTrue(checkSessionExists(unexpiredSession));
+
+        //persist an immortal session for our context
+        SessionData immortalSession = store.newSessionData("004", 100, now + 10, now + 5, TimeUnit.MINUTES.toMillis(60));
+        immortalSession.setExpiry(0);
+        immortalSession.setLastNode("me");
+        persistSession(immortalSession);
+        assertTrue(checkSessionExists(immortalSession));
+        
+        //create sessions for a different context
+        //persist a long ago expired session for a different context
+        SessionData oldForeignSession = store.newSessionData("005", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        oldForeignSession.setContextPath("_other");
+        oldForeignSession.setExpiry(200);
+        oldForeignSession.setLastNode("me");
+        persistSession(oldForeignSession);
+        assertTrue(checkSessionExists(oldForeignSession));
+        
+        //persist a recently expired session for our context
+        SessionData expiredForeignSession = store.newSessionData("006", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        expiredForeignSession.setContextPath("_other");
+        expiredForeignSession.setExpiry(RECENT_TIMESTAMP);
+        expiredForeignSession.setLastNode("me");
+        persistSession(expiredForeignSession);
+        assertTrue(checkSessionExists(expiredForeignSession));
+        
+        //persist a non expired session for our context
+        SessionData unexpiredForeignSession = store.newSessionData("007", 100, now + 10, now + 5, TimeUnit.MINUTES.toMillis(60));
+        unexpiredForeignSession.setContextPath("_other");
+        unexpiredForeignSession.setExpiry(now + TimeUnit.MINUTES.toMillis(10));
+        unexpiredForeignSession.setLastNode("me");
+        persistSession(unexpiredForeignSession);
+        assertTrue(checkSessionExists(unexpiredForeignSession));
+        
+        //persist an immortal session for our context
+        SessionData immortalForeignSession = store.newSessionData("008", 100, now + 10, now + 5, TimeUnit.MINUTES.toMillis(60));
+        immortalForeignSession.setContextPath("_other");
+        immortalForeignSession.setExpiry(0);
+        immortalForeignSession.setLastNode("me");
+        persistSession(immortalForeignSession);
+        assertTrue(checkSessionExists(immortalForeignSession));
+        
+        store.start();
+        
+        ((AbstractSessionDataStore)store).cleanOrphans(now - TimeUnit.SECONDS.toMillis(10 * GRACE_PERIOD_SEC));
+
+        //old session should be gone
+        assertFalse(checkSessionExists(oldSession));
+        //recently expired session should still be there
+        assertTrue(checkSessionExists(expiredSession));
+        //unexpired session should still be there
+        assertTrue(checkSessionExists(unexpiredSession));
+        //immortal session should still exist
+        assertTrue(checkSessionExists(immortalSession));
+        //old foreign session should be gone
+        assertFalse(checkSessionExists(oldSession));
+        //recently expired foreign session should still be there
+        assertTrue(checkSessionExists(expiredSession));
+        //unexpired foreign session should still be there
+        assertTrue(checkSessionExists(unexpiredSession));
+        //immortal foreign session should still exist
+        assertTrue(checkSessionExists(immortalSession));
     }
 
     /**
@@ -717,13 +808,13 @@ public abstract class AbstractSessionDataStoreTest
 
         long now = System.currentTimeMillis();
         //persist a session that is not expired
-        SessionData data = store.newSessionData("1234", 100, now, now - 1, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa16", 100, now, now - 1, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
 
         store.start();
 
-        assertTrue(store.exists("1234"));
+        assertTrue(store.exists("aaa16"));
     }
 
     /**
@@ -742,14 +833,14 @@ public abstract class AbstractSessionDataStoreTest
         store.initialize(sessionContext);
 
         //persist a session that is expired
-        SessionData data = store.newSessionData("1234", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa17", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
         data.setExpiry(RECENT_TIMESTAMP);
         persistSession(data);
 
         store.start();
 
-        assertFalse(store.exists("1234"));
+        assertFalse(store.exists("aaa17"));
     }
 
     /**
@@ -785,7 +876,7 @@ public abstract class AbstractSessionDataStoreTest
         store.initialize(sessionContext);
 
         //persist a session for a different context
-        SessionData data = store.newSessionData("1234", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa18", 100, 101, 100, TimeUnit.MINUTES.toMillis(60));
         data.setContextPath("_other");
         data.setLastNode(sessionContext.getWorkerName());
         persistSession(data);
@@ -793,7 +884,7 @@ public abstract class AbstractSessionDataStoreTest
         store.start();
 
         //check that session does not exist for this context
-        assertFalse(store.exists("1234"));
+        assertFalse(store.exists("aaa18"));
     }
 
     /**
@@ -817,7 +908,7 @@ public abstract class AbstractSessionDataStoreTest
         long now = System.currentTimeMillis();
 
         //persist a session that is not expired, and has been saved before
-        SessionData data = store.newSessionData("1234", 100, now - 10, now - 20, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa19", 100, now - 10, now - 20, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
         data.setLastSaved(now - 100);
         persistSession(data);
@@ -827,7 +918,7 @@ public abstract class AbstractSessionDataStoreTest
         data.setAccessed(now - 1);
 
         //test that a save does not change the stored data
-        store.store("1234", data);
+        store.store("aaa19", data);
 
         //reset the times for a check
         data.setLastAccessed(now - 20);
@@ -856,10 +947,10 @@ public abstract class AbstractSessionDataStoreTest
 
         long now = System.currentTimeMillis();
         //create a session that is not expired, and has never been saved before
-        SessionData data = store.newSessionData("1234", 100, now - 10, now - 20, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa20", 100, now - 10, now - 20, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
 
-        store.store("1234", data);
+        store.store("aaa20", data);
 
         checkSessionPersisted(data);
     }
@@ -885,7 +976,7 @@ public abstract class AbstractSessionDataStoreTest
 
         //persist a session that is not expired
         long now = System.currentTimeMillis();
-        SessionData data = store.newSessionData("1234", 100, now - 10, now - 20, TimeUnit.MINUTES.toMillis(60));
+        SessionData data = store.newSessionData("aaa21", 100, now - 10, now - 20, TimeUnit.MINUTES.toMillis(60));
         data.setLastNode(sessionContext.getWorkerName());
         data.setLastSaved(now - 100);
         data.setAttribute("wibble", "wobble");
@@ -896,7 +987,7 @@ public abstract class AbstractSessionDataStoreTest
         data.setLastAccessed(now - 5);
         data.setAccessed(now - 1);
 
-        store.store("1234", data);
+        store.store("aaa21", data);
 
         checkSessionPersisted(data);
     }

tests/test-sessions/test-sessions-common/src/main/java/org/eclipse/jetty/server/session/AbstractWebAppObjectInSessionTest.java

@@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
  */
 public abstract class AbstractWebAppObjectInSessionTest extends AbstractTestBase
 {
-
     @Test
     public void testWebappObjectInSession() throws Exception
     {

tests/test-sessions/test-sessions-common/src/main/java/org/eclipse/jetty/server/session/TestSessionDataStore.java

@@ -54,7 +54,7 @@ public class TestSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public boolean exists(String id) throws Exception
+    public boolean doExists(String id) throws Exception
     {
         return _map.containsKey(id);
     }
@@ -84,7 +84,7 @@ public class TestSessionDataStore extends AbstractSessionDataStore
     }
 
     @Override
-    public Set<String> doGetExpired(Set<String> candidates)
+    public Set<String> doCheckExpired(Set<String> candidates, long time)
     {
         HashSet<String> set = new HashSet<>();
         long now = System.currentTimeMillis();
@@ -96,4 +96,23 @@ public class TestSessionDataStore extends AbstractSessionDataStore
         }
         return set;
     }
+
+    @Override
+    public Set<String> doGetExpired(long timeLimit)
+    {
+        Set<String> set =  new HashSet<>();
+        
+        for (SessionData d:_map.values())
+        {
+            if (d.getExpiry() > 0 && d.getExpiry() <= timeLimit)
+                set.add(d.getId());
+        }
+        return set;
+    }
+
+    @Override
+    public void doCleanOrphans(long timeLimit)
+    {
+        //noop
+    }
 }

tests/test-sessions/test-sessions-common/src/test/java/org/eclipse/jetty/server/session/AbstractSessionCacheTest.java

@@ -64,7 +64,7 @@ public abstract class AbstractSessionCacheTest
         }
 
         @Override
-        public boolean exists(String id) throws Exception
+        public boolean doExists(String id) throws Exception
         {
             return _data != null;
         }
@@ -92,10 +92,21 @@ public abstract class AbstractSessionCacheTest
         }
 
         @Override
-        public Set<String> doGetExpired(Set<String> candidates)
+        public Set<String> doCheckExpired(Set<String> candidates, long time)
         {
             return null;
         }
+
+        @Override
+        public Set<String> doGetExpired(long before)
+        {
+            return null;
+        }
+
+        @Override
+        public void doCleanOrphans(long timeLimit)
+        {
+        }
     }
 
     public static class TestSessionActivationListener implements HttpSessionActivationListener

tests/test-sessions/test-sessions-common/src/test/java/org/eclipse/jetty/server/session/DeleteUnloadableSessionTest.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.server.session;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -62,7 +63,7 @@ public class DeleteUnloadableSessionTest
         }
 
         @Override
-        public boolean exists(String id)
+        public boolean doExists(String id)
         {
             return o != null;
         }
@@ -92,10 +93,23 @@ public class DeleteUnloadableSessionTest
         }
 
         @Override
-        public Set<String> doGetExpired(Set<String> candidates)
+        public Set<String> doCheckExpired(Set<String> candidates, long timeLimit)
         {
-            return null;
+            return Collections.emptySet();
         }
+
+        @Override
+        public Set<String> doGetExpired(long timeLimit)
+        {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public void doCleanOrphans(long timeLimit)
+        {
+           //noop
+        }
+        
     }
 
     public static class DelSessionDataStoreFactory extends AbstractSessionDataStoreFactory

tests/test-sessions/test-sessions-common/src/test/java/org/eclipse/jetty/server/session/IdleSessionTest.java

@@ -66,19 +66,18 @@ public class IdleSessionTest
     {
         String contextPath = "";
         String servletMapping = "/server";
-        int inactivePeriod = 20;
-        int scavengePeriod = 3;
-        int evictionSec = 5; //evict from cache if idle for 5 sec
+        int inactivePeriod = 10;
+        int scavengePeriod = 1;
+        int evictionSec = 2; //evict from cache if idle for 2 sec
 
         DefaultSessionCacheFactory cacheFactory = new DefaultSessionCacheFactory();
         cacheFactory.setEvictionPolicy(evictionSec);
+        cacheFactory.setFlushOnResponseCommit(true);
         SessionDataStoreFactory storeFactory = new TestSessionDataStoreFactory();
 
         _server1 = new TestServer(0, inactivePeriod, scavengePeriod, cacheFactory, storeFactory);
         ServletHolder holder = new ServletHolder(_servlet);
         ServletContextHandler contextHandler = _server1.addContext(contextPath);
-        TestHttpChannelCompleteListener scopeListener = new TestHttpChannelCompleteListener();
-        _server1.getServerConnector().addBean(scopeListener);
         contextHandler.addServlet(holder, servletMapping);
         _server1.start();
         int port1 = _server1.getPort();
@@ -90,15 +89,11 @@ public class IdleSessionTest
             String url = "http://localhost:" + port1 + contextPath + servletMapping;
 
             //make a request to set up a session on the server
-            CountDownLatch synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
+
             ContentResponse response = client.GET(url + "?action=init");
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
             String sessionCookie = response.getHeaders().get("Set-Cookie");
             assertNotNull(sessionCookie);
-            
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
 
             //and wait until the session should be passivated out
             pause(evictionSec * 2);
@@ -109,15 +104,10 @@ public class IdleSessionTest
             assertTrue(contextHandler.getSessionHandler().getSessionCache().getSessionDataStore().exists(id));
 
             //make another request to reactivate the session
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             Request request = client.newRequest(url + "?action=test");
             ContentResponse response2 = request.send();
             assertEquals(HttpServletResponse.SC_OK, response2.getStatus());
             
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
-            
             //check session reactivated
             assertTrue(contextHandler.getSessionHandler().getSessionCache().contains(id));
 
@@ -133,27 +123,17 @@ public class IdleSessionTest
             ((TestSessionDataStore)contextHandler.getSessionHandler().getSessionCache().getSessionDataStore())._map.clear();
 
             //make a request
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             request = client.newRequest(url + "?action=testfail");
             response2 = request.send();
             assertEquals(HttpServletResponse.SC_OK, response2.getStatus());
-
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
             
             //Test trying to reactivate an expired session (ie before the scavenger can get to it)
             //make a request to set up a session on the server
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             response = client.GET(url + "?action=init");
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
             sessionCookie = response.getHeaders().get("Set-Cookie");
             assertNotNull(sessionCookie);
             id = TestServer.extractSessionId(sessionCookie);
-            
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
 
             //and wait until the session should be idled out
             pause(evictionSec * 2);
@@ -170,15 +150,9 @@ public class IdleSessionTest
             pause(inactivePeriod + (3 * scavengePeriod));
 
             //make another request to reactivate the session
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             request = client.newRequest(url + "?action=testfail");
             response2 = request.send();
             assertEquals(HttpServletResponse.SC_OK, response2.getStatus());
-            
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
-
             assertFalse(contextHandler.getSessionHandler().getSessionCache().contains(id));
             assertFalse(contextHandler.getSessionHandler().getSessionCache().getSessionDataStore().exists(id));
         }
@@ -198,13 +172,12 @@ public class IdleSessionTest
         int scavengePeriod = 3;
 
         NullSessionCacheFactory cacheFactory = new NullSessionCacheFactory();
+        cacheFactory.setFlushOnResponseCommit(true);
         SessionDataStoreFactory storeFactory = new TestSessionDataStoreFactory();
 
         _server1 = new TestServer(0, inactivePeriod, scavengePeriod, cacheFactory, storeFactory);
         ServletHolder holder = new ServletHolder(_servlet);
         ServletContextHandler contextHandler = _server1.addContext(contextPath);
-        TestHttpChannelCompleteListener scopeListener = new TestHttpChannelCompleteListener();
-        _server1.getServerConnector().addBean(scopeListener);
         contextHandler.addServlet(holder, servletMapping);
         _server1.start();
         int port1 = _server1.getPort();
@@ -216,15 +189,10 @@ public class IdleSessionTest
             String url = "http://localhost:" + port1 + contextPath + servletMapping;
 
             //make a request to set up a session on the server
-            CountDownLatch synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             ContentResponse response = client.GET(url + "?action=init");
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
             String sessionCookie = response.getHeaders().get("Set-Cookie");
             assertNotNull(sessionCookie);
-            
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
 
             //the session should never be cached
             String id = TestServer.extractSessionId(sessionCookie);
@@ -232,15 +200,10 @@ public class IdleSessionTest
             assertTrue(contextHandler.getSessionHandler().getSessionCache().getSessionDataStore().exists(id));
 
             //make another request to reactivate the session
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             Request request = client.newRequest(url + "?action=test");
             ContentResponse response2 = request.send();
             assertEquals(HttpServletResponse.SC_OK, response2.getStatus());
             
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
-            
             //check session still not in the cache
             assertFalse(contextHandler.getSessionHandler().getSessionCache().contains(id));
             assertTrue(contextHandler.getSessionHandler().getSessionCache().getSessionDataStore().exists(id));
@@ -250,29 +213,18 @@ public class IdleSessionTest
             ((TestSessionDataStore)contextHandler.getSessionHandler().getSessionCache().getSessionDataStore())._map.clear();
 
             //make a request
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             request = client.newRequest(url + "?action=testfail");
             response2 = request.send();
             assertEquals(HttpServletResponse.SC_OK, response2.getStatus());
-
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
             
             //Test trying to reactivate an expired session (ie before the scavenger can get to it)
             //make a request to set up a session on the server
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             response = client.GET(url + "?action=init");
             assertEquals(HttpServletResponse.SC_OK, response.getStatus());
             sessionCookie = response.getHeaders().get("Set-Cookie");
             assertNotNull(sessionCookie);
             id = TestServer.extractSessionId(sessionCookie);
             
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
-            
-
             //stop the scavenger
             if (_server1.getHouseKeeper() != null)
                 _server1.getHouseKeeper().stop();
@@ -285,15 +237,10 @@ public class IdleSessionTest
             pause(inactivePeriod + (3 * scavengePeriod));
 
             //make another request to reactivate the session
-            synchronizer = new CountDownLatch(1);
-            scopeListener.setExitSynchronizer(synchronizer);
             request = client.newRequest(url + "?action=testfail");
             response2 = request.send();
             assertEquals(HttpServletResponse.SC_OK, response2.getStatus());
             
-            //ensure request has finished being handled
-            synchronizer.await(5, TimeUnit.SECONDS);
-            
             assertFalse(contextHandler.getSessionHandler().getSessionCache().contains(id));
             assertFalse(contextHandler.getSessionHandler().getSessionCache().getSessionDataStore().exists(id));
         }

tests/test-sessions/test-sessions-common/src/test/java/org/eclipse/jetty/server/session/SessionEvictionFailureTest.java

@@ -19,6 +19,7 @@
 package org.eclipse.jetty.server.session;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import javax.servlet.ServletException;
@@ -45,7 +46,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
  */
 public class SessionEvictionFailureTest
 {
-
     /**
      * MockSessionDataStore
      */
@@ -66,7 +66,7 @@ public class SessionEvictionFailureTest
         }
 
         @Override
-        public boolean exists(String id) throws Exception
+        public boolean doExists(String id) throws Exception
         {
             return true;
         }
@@ -93,10 +93,22 @@ public class SessionEvictionFailureTest
         }
 
         @Override
-        public Set<String> doGetExpired(Set<String> candidates)
+        public Set<String> doCheckExpired(Set<String> candidates, long timeLimit)
         {
             return candidates;
         }
+
+        @Override
+        public Set<String> doGetExpired(long timeLimit)
+        {
+            return Collections.emptySet();
+        }
+
+        @Override
+        public void doCleanOrphans(long timeLimit)
+        {
+            //noop
+        }
     }
 
     /**