diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index e8189a2b945..123a6de6a40 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -4946,6 +4946,19 @@ public class YarnConfiguration extends Configuration { public static final String DEFAULT_YARN_WORKFLOW_ID_TAG_PREFIX = "workflowid:"; + public static final String APPS_CACHE_ENABLE = YARN_PREFIX + "apps.cache.enable"; + public static final boolean DEFAULT_APPS_CACHE_ENABLE = false; + + // The size of cache for RMWebServices.getApps when yarn.apps.cache.enable = true, + // default is 1000 + public static final String APPS_CACHE_SIZE = YARN_PREFIX + "apps.cache.size"; + public static final int DEFAULT_APPS_CACHE_SIZE = 1000; + + // The expire time of cache for RMWebServices.getApps when yarn.apps.cache.enable = true, + // default is 30s + public static final String APPS_CACHE_EXPIRE = YARN_PREFIX + "apps.cache.expire"; + public static final String DEFAULT_APPS_CACHE_EXPIRE = "30s"; + public YarnConfiguration() { super(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AppsCacheKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AppsCacheKey.java new file mode 100644 index 00000000000..a0df35cb5f5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AppsCacheKey.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Set; + +public class AppsCacheKey { + private static final Logger LOG = LoggerFactory.getLogger(AppsCacheKey.class.getName()); + + private UserGroupInformation ugi; + private String stateQuery; + private String finalStatusQuery; + private String userQuery; + private String queueQuery; + private String limit; + private String startedBegin; + private String startedEnd; + private String finishBegin; + private String finishEnd; + private String name; + private Set unselectedFields; + + private Set applicationTags; + private Set applicationTypes; + private Set statesQuery; + + @SuppressWarnings("checkstyle:ParameterNumber") + public AppsCacheKey(UserGroupInformation ugi, String stateQuery, Set statesQuery, + String finalStatusQuery, String userQuery, String queueQuery, String limit, + String startedBegin, String startedEnd, String finishBegin, String finishEnd, + Set applicationTypes, Set applicationTags, String name, + Set unselectedFields) { + this.ugi = ugi; + this.stateQuery = stateQuery; + this.statesQuery = statesQuery; + this.finalStatusQuery = finalStatusQuery; + this.userQuery = userQuery; + this.queueQuery = queueQuery; + this.limit = limit; + this.startedBegin = startedBegin; + this.startedEnd = startedEnd; + this.finishBegin = finishBegin; + this.finishEnd = finishEnd; + this.applicationTypes = applicationTypes; + this.applicationTags = applicationTags; + this.name = name; + this.unselectedFields = unselectedFields; + } + + @SuppressWarnings("checkstyle:ParameterNumber") + public static AppsCacheKey newInstance(String stateQuery, + Set statesQuery, String finalStatusQuery, String userQuery, String queueQuery, + String limit, String startedBegin, String startedEnd, String finishBegin, String finishEnd, + Set applicationTypes, Set applicationTags, String name, + Set unselectedFields) { + + UserGroupInformation ugi = null; + try { + ugi = UserGroupInformation.getCurrentUser(); + } catch (IOException e) { + LOG.error("unable to get ugi", e); + } + + return new AppsCacheKey(ugi, stateQuery, statesQuery, finalStatusQuery, userQuery, queueQuery, + limit, startedBegin, startedEnd, finishBegin, finishEnd, applicationTypes, applicationTags, + name, unselectedFields); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AppsCacheKey that = (AppsCacheKey) o; + + return new EqualsBuilder() + .append(this.ugi.getUserName(), that.ugi.getUserName()) + .append(this.stateQuery, that.stateQuery) + .append(this.statesQuery, that.statesQuery) + .append(this.finalStatusQuery, that.finalStatusQuery) + .append(this.userQuery, that.userQuery) + .append(this.queueQuery, that.queueQuery) + .append(this.limit, that.limit) + .append(this.startedBegin, that.startedBegin) + .append(this.startedEnd, that.startedEnd) + .append(this.finishBegin, that.finishBegin) + .append(this.finishEnd, that.finishEnd) + .append(this.applicationTypes, that.applicationTypes) + .append(this.applicationTags, that.applicationTags) + .append(this.name, that.name) + .append(this.unselectedFields, that.unselectedFields) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder() + .append(this.ugi.getUserName()) + .append(this.stateQuery) + .append(this.statesQuery) + .append(this.finalStatusQuery) + .append(this.userQuery) + .append(this.queueQuery) + .append(this.limit) + .append(this.startedBegin) + .append(this.startedEnd) + .append(this.finishBegin) + .append(this.finishEnd) + .append(this.applicationTypes) + .append(this.applicationTags) + .append(this.name) + .append(this.unselectedFields) + .toHashCode(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CacheNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CacheNode.java new file mode 100644 index 00000000000..a436ded96db --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/CacheNode.java @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.apache.hadoop.util.Time; + +public class CacheNode{ + private V value; + private long cacheTime; + + public CacheNode(V value){ + this.value = value; + cacheTime = Time.now(); + } + + public V get(){ + return value; + } + + public long getCacheTime(){ + return cacheTime; + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LRUCache.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LRUCache.java new file mode 100644 index 00000000000..3c0009b0945 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/LRUCache.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.util.Time; + +import java.util.Map; + +public class LRUCache { + + private final long expireTimeMs; + private final Map> cache; + + public LRUCache(int capacity) { + this(capacity, -1); + } + + public LRUCache(int capacity, long expireTimeMs) { + cache = new LRUCacheHashMap<>(capacity, true); + this.expireTimeMs = expireTimeMs; + } + + public synchronized V get(K key) { + CacheNode cacheNode = cache.get(key); + if (cacheNode != null) { + if (expireTimeMs > 0 && Time.now() > cacheNode.getCacheTime() + expireTimeMs) { + cache.remove(key); + return null; + } + } + return cacheNode == null ? null : cacheNode.get(); + } + + public synchronized V put(K key, V value) { + cache.put(key, new CacheNode<>(value)); + return value; + } + + @VisibleForTesting + public void clear(){ + cache.clear(); + } + + public int size() { + return cache.size(); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 5e11c6526e8..d89f048ed3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -5261,4 +5261,33 @@ false + + yarn.apps.cache.enable + false + + Optional. + To enable cache for RMWebServices.getApps + + + + + yarn.apps.cache.size + 1000 + + Optional. + The size of cache for RMWebServices.getApps when + yarn.apps.cache.enable = true, Default is 1000 + + + + + yarn.apps.cache.expire + 30s + + Optional. + The expire time of cache for RMWebServices.getApps when + yarn.apps.cache.enable = true, Default is 30s + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCache.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCache.java new file mode 100644 index 00000000000..f3a14f15170 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestLRUCache.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; +import org.junit.Assert; +import org.junit.Test; + +public class TestLRUCache { + public static final int CACHE_EXPIRE_TIME = 200; + @Test + public void testLRUCache() throws InterruptedException { + LRUCache lruCache = new LRUCache<>(3, CACHE_EXPIRE_TIME); + lruCache.put("1", 1); + lruCache.put("2", 1); + lruCache.put("3", 3); + lruCache.put("4", 4); + Assert.assertEquals(lruCache.size(), 3); + Assert.assertNull(lruCache.get("1")); + Assert.assertNotNull(lruCache.get("2")); + Assert.assertNotNull(lruCache.get("3")); + Assert.assertNotNull(lruCache.get("3")); + lruCache.clear(); + + lruCache.put("1", 1); + Thread.sleep(201); + Assert.assertEquals(lruCache.size(), 1); + lruCache.get("1"); + Assert.assertEquals(lruCache.size(), 0); + lruCache.put("2", 2); + Assert.assertEquals(lruCache.size(), 1); + lruCache.put("3", 3); + Assert.assertEquals(lruCache.size(), 2); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index ae2fab45266..357f9aadadc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -37,6 +37,8 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; @@ -211,7 +213,9 @@ import org.apache.hadoop.yarn.server.webapp.WebServices; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.AdHocLogDumper; +import org.apache.hadoop.yarn.util.AppsCacheKey; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.LRUCache; import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.BadRequestException; @@ -257,6 +261,10 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { private boolean filterAppsByUser = false; private boolean filterInvalidXMLChars = false; private boolean enableRestAppSubmissions = true; + private LRUCache appsLRUCache; + private AtomicLong getAppsSuccessTimes = new AtomicLong(0); + private AtomicLong hitAppsCacheTimes = new AtomicLong(0); + private boolean enableAppsCache = false; public final static String DELEGATION_TOKEN_HEADER = "Hadoop-YARN-RM-Delegation-Token"; @@ -278,6 +286,15 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { this.enableRestAppSubmissions = conf.getBoolean( YarnConfiguration.ENABLE_REST_APP_SUBMISSIONS, YarnConfiguration.DEFAULT_ENABLE_REST_APP_SUBMISSIONS); + this.enableAppsCache = this.conf.getBoolean(YarnConfiguration.APPS_CACHE_ENABLE, + YarnConfiguration.DEFAULT_APPS_CACHE_ENABLE); + if (enableAppsCache) { + int cacheSize = this.conf.getInt(YarnConfiguration.APPS_CACHE_SIZE, + YarnConfiguration.DEFAULT_APPS_CACHE_SIZE); + long appsCacheTimeMs = this.conf.getTimeDuration(YarnConfiguration.APPS_CACHE_EXPIRE, + YarnConfiguration.DEFAULT_APPS_CACHE_EXPIRE, TimeUnit.MILLISECONDS); + appsLRUCache = new LRUCache<>(cacheSize, appsCacheTimeMs); + } } RMWebServices(ResourceManager rm, Configuration conf, @@ -625,6 +642,23 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { @QueryParam(RMWSConsts.NAME) String name, @QueryParam(RMWSConsts.DESELECTS) Set unselectedFields) { + AppsCacheKey cacheKey = AppsCacheKey.newInstance(stateQuery, new HashSet<>(statesQuery), + finalStatusQuery, userQuery, queueQuery, limit, startedBegin, startedEnd, finishBegin, + finishEnd, new HashSet<>(applicationTypes), new HashSet<>(applicationTags), name, + unselectedFields); + if (this.enableAppsCache) { + long successTimes = getAppsSuccessTimes.incrementAndGet(); + if (successTimes % 1000 == 0) { + LOG.debug("hit cache info: getAppsSuccessTimes={}, hitAppsCacheTimes={}", + successTimes, hitAppsCacheTimes.get()); + } + AppsInfo appsInfo = appsLRUCache.get(cacheKey); + if (appsInfo != null) { + hitAppsCacheTimes.getAndIncrement(); + return appsInfo; + } + } + initForReadableEndpoints(); GetApplicationsRequest request = @@ -695,6 +729,10 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { } } + if (enableAppsCache) { + appsLRUCache.put(cacheKey, allApps); + getAppsSuccessTimes.getAndIncrement(); + } return allApps; } @@ -2981,4 +3019,9 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol { ResourceScheduler rs = rm.getResourceScheduler(); return new SchedulerOverviewInfo(rs); } + + @VisibleForTesting + public LRUCache getAppsLRUCache(){ + return appsLRUCache; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java index b416947c550..8322553a9a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java @@ -40,6 +40,7 @@ import java.util.Set; import java.util.HashMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeoutException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -49,6 +50,7 @@ import javax.ws.rs.core.Response; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -72,6 +74,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.*; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -90,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterUserInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.AdHocLogDumper; +import org.apache.hadoop.yarn.util.AppsCacheKey; +import org.apache.hadoop.yarn.util.LRUCache; import org.apache.hadoop.yarn.util.YarnVersionInfo; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; @@ -1157,4 +1162,73 @@ public class TestRMWebServices extends JerseyTestBase { int applicationPriority = json.getInt("applicationPriority"); assertEquals(0, applicationPriority); } + + @Test + public void testGetAppsCache() throws YarnException, InterruptedException, TimeoutException { + // mock up an RM that returns app reports for apps that don't exist + // in the RMApps list + ApplicationId appId = ApplicationId.newInstance(1, 1); + ApplicationReport mockReport = mock(ApplicationReport.class); + when(mockReport.getApplicationId()).thenReturn(appId); + GetApplicationsResponse mockAppsResponse = + mock(GetApplicationsResponse.class); + when(mockAppsResponse.getApplicationList()) + .thenReturn(Arrays.asList(new ApplicationReport[]{mockReport})); + + ClientRMService mockClientSvc = mock(ClientRMService.class); + when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class))) + .thenReturn(mockAppsResponse); + ResourceManager mockRM = mock(ResourceManager.class); + RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, null, null, + null, null); + when(mockRM.getRMContext()).thenReturn(rmContext); + when(mockRM.getClientRMService()).thenReturn(mockClientSvc); + rmContext.setNodeLabelManager(mock(RMNodeLabelsManager.class)); + + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.APPS_CACHE_ENABLE, true); + conf.setInt(YarnConfiguration.APPS_CACHE_SIZE, 2); + conf.setInt(YarnConfiguration.APPS_CACHE_EXPIRE, 100); + RMWebServices webSvc = new RMWebServices(mockRM, conf, + mock(HttpServletResponse.class)); + final Set emptySet = + Collections.unmodifiableSet(Collections.emptySet()); + + // verify we don't get any apps when querying + HttpServletRequest mockHsr = mock(HttpServletRequest.class); + AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null, + "mock_user", "mock_queue", null, null, null, null, null, emptySet, + emptySet, null, null); + LRUCache cache = webSvc.getAppsLRUCache(); + Assert.assertEquals(1, cache.size()); + AppsCacheKey appsCacheKey = AppsCacheKey.newInstance(null, emptySet, + null, "mock_user", "mock_queue", null, null, null, null, null, emptySet, + emptySet, null, null); + Assert.assertEquals(appsInfo, cache.get(appsCacheKey)); + + AppsInfo appsInfo1 = webSvc.getApps(mockHsr, null, emptySet, null, + "mock_user1", "mock_queue", null, null, null, null, null, emptySet, + emptySet, null, null); + Assert.assertEquals(2, cache.size()); + AppsCacheKey appsCacheKey1 = AppsCacheKey.newInstance(null, emptySet, + null, "mock_user1", "mock_queue", null, null, null, null, null, emptySet, + emptySet, null, null); + Assert.assertEquals(appsInfo1, cache.get(appsCacheKey1)); + + AppsInfo appsInfo2 = webSvc.getApps(mockHsr, null, emptySet, null, + "mock_user2", "mock_queue", null, null, null, null, null, emptySet, + emptySet, null, null); + Assert.assertEquals(2, cache.size()); + AppsCacheKey appsCacheKey2 = AppsCacheKey.newInstance(null, emptySet, + null, "mock_user2", "mock_queue", null, null, null, null, null, emptySet, + emptySet, null, null); + Assert.assertEquals(appsInfo2, cache.get(appsCacheKey2)); + // appsCacheKey have removed + Assert.assertNull(cache.get(appsCacheKey)); + + GenericTestUtils.waitFor(() -> cache.get(appsCacheKey1) == null, + 300, 1000); + GenericTestUtils.waitFor(() -> cache.get(appsCacheKey2) == null, + 300, 1000); + } } \ No newline at end of file