Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-10793. Upgrade Junit from 4 to 5 in hadoop-yarn-server-applicationhistoryservice (#4603) 

Co-authored-by: Ashutosh Gupta <ashugpt@amazon.com>
Signed-off-by: Akira Ajisaka <aajisaka@apache.org>
This commit is contained in:
Ashutosh Gupta 2022-08-07 08:15:47 +01:00 committed by GitHub
parent 52c2d99889
commit 1cda2dcb6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1391 additions and 1356 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
View File

@ -34,6 +34,8 @@
<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir> <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
</properties> </properties>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>javax.servlet</groupId> <groupId>javax.servlet</groupId>
@ -194,6 +196,21 @@
<artifactId>bcprov-jdk15on</artifactId> <artifactId>bcprov-jdk15on</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>de.ruedigermoeller</groupId> <groupId>de.ruedigermoeller</groupId>
<artifactId>fst</artifactId> <artifactId>fst</artifactId>

93
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
View File

@ -21,6 +21,9 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@ -49,9 +52,11 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.junit.Assert;
import org.junit.BeforeClass; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TestApplicationHistoryClientService { public class TestApplicationHistoryClientService {
@ -59,7 +64,7 @@ public class TestApplicationHistoryClientService {
private static TimelineDataManager dataManager; private static TimelineDataManager dataManager;
private final static int MAX_APPS = 2; private final static int MAX_APPS = 2;
@BeforeClass @BeforeAll
public static void setup() throws Exception { public static void setup() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
TimelineStore store = TimelineStore store =
@ -78,7 +83,7 @@ public class TestApplicationHistoryClientService {
} }
@Test @Test
public void testApplicationNotFound() throws IOException, YarnException { void testApplicationNotFound() throws IOException, YarnException {
ApplicationId appId = null; ApplicationId appId = null;
appId = ApplicationId.newInstance(0, MAX_APPS + 1); appId = ApplicationId.newInstance(0, MAX_APPS + 1);
GetApplicationReportRequest request = GetApplicationReportRequest request =
@ -87,18 +92,18 @@ public class TestApplicationHistoryClientService {
@SuppressWarnings("unused") @SuppressWarnings("unused")
GetApplicationReportResponse response = GetApplicationReportResponse response =
clientService.getApplicationReport(request); clientService.getApplicationReport(request);
Assert.fail("Exception should have been thrown before we reach here."); fail("Exception should have been thrown before we reach here.");
} catch (ApplicationNotFoundException e) { } catch (ApplicationNotFoundException e) {
//This exception is expected. //This exception is expected.
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"doesn't exist in the timeline store")); "doesn't exist in the timeline store"));
} catch (Exception e) { } catch (Exception e) {
Assert.fail("Undesired exception caught"); fail("Undesired exception caught");
} }
} }
@Test @Test
public void testApplicationAttemptNotFound() throws IOException, YarnException { void testApplicationAttemptNotFound() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, MAX_APPS + 1); ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
@ -108,19 +113,19 @@ public class TestApplicationHistoryClientService {
@SuppressWarnings("unused") @SuppressWarnings("unused")
GetApplicationAttemptReportResponse response = GetApplicationAttemptReportResponse response =
clientService.getApplicationAttemptReport(request); clientService.getApplicationAttemptReport(request);
Assert.fail("Exception should have been thrown before we reach here."); fail("Exception should have been thrown before we reach here.");
} catch (ApplicationAttemptNotFoundException e) { } catch (ApplicationAttemptNotFoundException e) {
//This Exception is expected //This Exception is expected
System.out.println(e.getMessage()); System.out.println(e.getMessage());
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"doesn't exist in the timeline store")); "doesn't exist in the timeline store"));
} catch (Exception e) { } catch (Exception e) {
Assert.fail("Undesired exception caught"); fail("Undesired exception caught");
} }
} }
@Test @Test
public void testContainerNotFound() throws IOException, YarnException { void testContainerNotFound() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -134,15 +139,15 @@ public class TestApplicationHistoryClientService {
clientService.getContainerReport(request); clientService.getContainerReport(request);
} catch (ContainerNotFoundException e) { } catch (ContainerNotFoundException e) {
//This exception is expected //This exception is expected
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"doesn't exist in the timeline store")); "doesn't exist in the timeline store"));
} catch (Exception e) { } catch (Exception e) {
Assert.fail("Undesired exception caught"); fail("Undesired exception caught");
} }
} }
@Test @Test
public void testApplicationReport() throws IOException, YarnException { void testApplicationReport() throws IOException, YarnException {
ApplicationId appId = null; ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1); appId = ApplicationId.newInstance(0, 1);
GetApplicationReportRequest request = GetApplicationReportRequest request =
@ -150,20 +155,20 @@ public class TestApplicationHistoryClientService {
GetApplicationReportResponse response = GetApplicationReportResponse response =
clientService.getApplicationReport(request); clientService.getApplicationReport(request);
ApplicationReport appReport = response.getApplicationReport(); ApplicationReport appReport = response.getApplicationReport();
Assert.assertNotNull(appReport); assertNotNull(appReport);
Assert.assertEquals(123, appReport.getApplicationResourceUsageReport() assertEquals(123, appReport.getApplicationResourceUsageReport()
.getMemorySeconds()); .getMemorySeconds());
Assert.assertEquals(345, appReport.getApplicationResourceUsageReport() assertEquals(345, appReport.getApplicationResourceUsageReport()
.getVcoreSeconds()); .getVcoreSeconds());
Assert.assertEquals("application_0_0001", appReport.getApplicationId() assertEquals("application_0_0001", appReport.getApplicationId()
.toString()); .toString());
Assert.assertEquals("test app type", assertEquals("test app type",
appReport.getApplicationType().toString()); appReport.getApplicationType().toString());
Assert.assertEquals("test queue", appReport.getQueue().toString()); assertEquals("test queue", appReport.getQueue().toString());
} }
@Test @Test
public void testApplications() throws IOException, YarnException { void testApplications() throws IOException, YarnException {
ApplicationId appId = null; ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1); appId = ApplicationId.newInstance(0, 1);
ApplicationId appId1 = ApplicationId.newInstance(0, 2); ApplicationId appId1 = ApplicationId.newInstance(0, 2);
@ -171,9 +176,9 @@ public class TestApplicationHistoryClientService {
GetApplicationsResponse response = GetApplicationsResponse response =
clientService.getApplications(request); clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList(); List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport); assertNotNull(appReport);
Assert.assertEquals(appId, appReport.get(1).getApplicationId()); assertEquals(appId, appReport.get(1).getApplicationId());
Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); assertEquals(appId1, appReport.get(0).getApplicationId());
// Create a historyManager, and set the max_apps can be loaded // Create a historyManager, and set the max_apps can be loaded
// as 1. // as 1.
@ -189,14 +194,14 @@ public class TestApplicationHistoryClientService {
new ApplicationHistoryClientService(historyManager2); new ApplicationHistoryClientService(historyManager2);
response = clientService2.getApplications(request); response = clientService2.getApplications(request);
appReport = response.getApplicationList(); appReport = response.getApplicationList();
Assert.assertNotNull(appReport); assertNotNull(appReport);
Assert.assertTrue(appReport.size() == 1); assertTrue(appReport.size() == 1);
// Expected to get the appReport for application with appId1 // Expected to get the appReport for application with appId1
Assert.assertEquals(appId1, appReport.get(0).getApplicationId()); assertEquals(appId1, appReport.get(0).getApplicationId());
} }
@Test @Test
public void testApplicationAttemptReport() throws IOException, YarnException { void testApplicationAttemptReport() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -206,13 +211,13 @@ public class TestApplicationHistoryClientService {
clientService.getApplicationAttemptReport(request); clientService.getApplicationAttemptReport(request);
ApplicationAttemptReport attemptReport = ApplicationAttemptReport attemptReport =
response.getApplicationAttemptReport(); response.getApplicationAttemptReport();
Assert.assertNotNull(attemptReport); assertNotNull(attemptReport);
Assert.assertEquals("appattempt_0_0001_000001", attemptReport assertEquals("appattempt_0_0001_000001", attemptReport
.getApplicationAttemptId().toString()); .getApplicationAttemptId().toString());
} }
@Test @Test
public void testApplicationAttempts() throws IOException, YarnException { void testApplicationAttempts() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -224,15 +229,15 @@ public class TestApplicationHistoryClientService {
clientService.getApplicationAttempts(request); clientService.getApplicationAttempts(request);
List<ApplicationAttemptReport> attemptReports = List<ApplicationAttemptReport> attemptReports =
response.getApplicationAttemptList(); response.getApplicationAttemptList();
Assert.assertNotNull(attemptReports); assertNotNull(attemptReports);
Assert.assertEquals(appAttemptId, attemptReports.get(0) assertEquals(appAttemptId, attemptReports.get(0)
.getApplicationAttemptId()); .getApplicationAttemptId());
Assert.assertEquals(appAttemptId1, attemptReports.get(1) assertEquals(appAttemptId1, attemptReports.get(1)
.getApplicationAttemptId()); .getApplicationAttemptId());
} }
@Test @Test
public void testContainerReport() throws IOException, YarnException { void testContainerReport() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -242,15 +247,15 @@ public class TestApplicationHistoryClientService {
GetContainerReportResponse response = GetContainerReportResponse response =
clientService.getContainerReport(request); clientService.getContainerReport(request);
ContainerReport container = response.getContainerReport(); ContainerReport container = response.getContainerReport();
Assert.assertNotNull(container); assertNotNull(container);
Assert.assertEquals(containerId, container.getContainerId()); assertEquals(containerId, container.getContainerId());
Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" + assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
"test host:100/container_0_0001_01_000001/" + "test host:100/container_0_0001_01_000001/" +
"container_0_0001_01_000001/user1", container.getLogUrl()); "container_0_0001_01_000001/user1", container.getLogUrl());
} }
@Test @Test
public void testContainers() throws IOException, YarnException { void testContainers() throws IOException, YarnException {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -261,8 +266,8 @@ public class TestApplicationHistoryClientService {
GetContainersResponse response = GetContainersResponse response =
clientService.getContainers(request); clientService.getContainers(request);
List<ContainerReport> containers = response.getContainerList(); List<ContainerReport> containers = response.getContainerList();
Assert.assertNotNull(containers); assertNotNull(containers);
Assert.assertEquals(containerId, containers.get(0).getContainerId()); assertEquals(containerId, containers.get(0).getContainerId());
Assert.assertEquals(containerId1, containers.get(1).getContainerId()); assertEquals(containerId1, containers.get(1).getContainerId());
} }
} }

44
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerImpl.java
View File

@ -21,22 +21,26 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException; import java.io.IOException;
import java.util.Map; import java.util.Map;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.Before; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertNull;
public class TestApplicationHistoryManagerImpl extends public class TestApplicationHistoryManagerImpl extends
ApplicationHistoryStoreTestUtils { ApplicationHistoryStoreTestUtils {
ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null; private ApplicationHistoryManagerImpl applicationHistoryManagerImpl = null;
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
Configuration config = new Configuration(); Configuration config = new Configuration();
config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE, config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
@ -47,13 +51,13 @@ public class TestApplicationHistoryManagerImpl extends
store = applicationHistoryManagerImpl.getHistoryStore(); store = applicationHistoryManagerImpl.getHistoryStore();
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
applicationHistoryManagerImpl.stop(); applicationHistoryManagerImpl.stop();
} }
@Test @Test
public void testApplicationReport() throws IOException, YarnException { void testApplicationReport() throws IOException, YarnException {
ApplicationId appId = null; ApplicationId appId = null;
appId = ApplicationId.newInstance(0, 1); appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId); writeApplicationStartData(appId);
@ -64,17 +68,17 @@ public class TestApplicationHistoryManagerImpl extends
writeApplicationAttemptFinishData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId);
ApplicationReport appReport = ApplicationReport appReport =
applicationHistoryManagerImpl.getApplication(appId); applicationHistoryManagerImpl.getApplication(appId);
Assert.assertNotNull(appReport); assertNotNull(appReport);
Assert.assertEquals(appId, appReport.getApplicationId()); assertEquals(appId, appReport.getApplicationId());
Assert.assertEquals(appAttemptId, assertEquals(appAttemptId,
appReport.getCurrentApplicationAttemptId()); appReport.getCurrentApplicationAttemptId());
Assert.assertEquals(appAttemptId.toString(), appReport.getHost()); assertEquals(appAttemptId.toString(), appReport.getHost());
Assert.assertEquals("test type", appReport.getApplicationType().toString()); assertEquals("test type", appReport.getApplicationType().toString());
Assert.assertEquals("test queue", appReport.getQueue().toString()); assertEquals("test queue", appReport.getQueue().toString());
} }
@Test @Test
public void testApplications() throws IOException { void testApplications() throws IOException {
ApplicationId appId1 = ApplicationId.newInstance(0, 1); ApplicationId appId1 = ApplicationId.newInstance(0, 1);
ApplicationId appId2 = ApplicationId.newInstance(0, 2); ApplicationId appId2 = ApplicationId.newInstance(0, 2);
ApplicationId appId3 = ApplicationId.newInstance(0, 3); ApplicationId appId3 = ApplicationId.newInstance(0, 3);
@ -86,10 +90,10 @@ public class TestApplicationHistoryManagerImpl extends
writeApplicationFinishData(appId3); writeApplicationFinishData(appId3);
Map<ApplicationId, ApplicationReport> reports = Map<ApplicationId, ApplicationReport> reports =
applicationHistoryManagerImpl.getApplications(2, 2000L, 5000L); applicationHistoryManagerImpl.getApplications(2, 2000L, 5000L);
Assert.assertNotNull(reports); assertNotNull(reports);
Assert.assertEquals(2, reports.size()); assertEquals(2, reports.size());
Assert.assertNull(reports.get("1")); assertNull(reports.get("1"));
Assert.assertNull(reports.get("2")); assertNull(reports.get("2"));
Assert.assertNull(reports.get("3")); assertNull(reports.get("3"));
} }
} }

463
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
View File

@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the * to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance * "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at * with the License. You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -26,6 +26,11 @@ import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod; import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
@ -56,16 +61,13 @@ import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class) import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TestApplicationHistoryManagerOnTimelineStore { public class TestApplicationHistoryManagerOnTimelineStore {
private static final int SCALE = 5; private static final int SCALE = 5;
@ -75,91 +77,78 @@ public class TestApplicationHistoryManagerOnTimelineStore {
private UserGroupInformation callerUGI; private UserGroupInformation callerUGI;
private Configuration conf; private Configuration conf;
@BeforeClass @BeforeAll
public static void prepareStore() throws Exception { public static void prepareStore() throws Exception {
store = createStore(SCALE); store = createStore(SCALE);
TimelineEntities entities = new TimelineEntities(); TimelineEntities entities = new TimelineEntities();
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(
ApplicationId.newInstance(0, SCALE + 1), true, true, false, false, createApplicationTimelineEntity(ApplicationId.newInstance(0, SCALE + 1), true, true, false,
YarnApplicationState.FINISHED)); false, YarnApplicationState.FINISHED));
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(
ApplicationId.newInstance(0, SCALE + 2), true, false, true, false, createApplicationTimelineEntity(ApplicationId.newInstance(0, SCALE + 2), true, false, true,
YarnApplicationState.FINISHED)); false, YarnApplicationState.FINISHED));
store.put(entities); store.put(entities);
} }
public static TimelineStore createStore(int scale) throws Exception { public static TimelineStore createStore(int scale) throws Exception {
TimelineStore store = new MemoryTimelineStore(); store = new MemoryTimelineStore();
prepareTimelineStore(store, scale); prepareTimelineStore(scale);
return store; return store;
} }
@Before @AfterEach
public void setup() throws Exception {
// Only test the ACLs of the generic history
TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
aclsManager.setTimelineStore(store);
TimelineDataManager dataManager =
new TimelineDataManager(store, aclsManager);
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
historyManager =
new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
historyManager.init(conf);
historyManager.start();
}
@After
public void tearDown() { public void tearDown() {
if (historyManager != null) { if (historyManager != null) {
historyManager.stop(); historyManager.stop();
} }
} }
@Parameters
public static Collection<Object[]> callers() { public static Collection<Object[]> callers() {
// user1 is the owner // user1 is the owner
// user2 is the authorized user // user2 is the authorized user
// user3 is the unauthorized user // user3 is the unauthorized user
// admin is the admin acl // admin is the admin acl
return Arrays.asList( return Arrays.asList(new Object[][] {{""}, {"user1"}, {"user2"}, {"user3"}, {"admin"}});
new Object[][] { { "" }, { "user1" }, { "user2" }, { "user3" }, { "admin" } });
} }
public TestApplicationHistoryManagerOnTimelineStore(String caller) { public void initTestApplicationHistoryManagerOnTimelineStore(String caller) {
conf = new YarnConfiguration(); conf = new YarnConfiguration();
if (!caller.equals("")) { if (!caller.equals("")) {
callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE); callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE);
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
} }
TimelineACLsManager aclsManager = new TimelineACLsManager(new YarnConfiguration());
aclsManager.setTimelineStore(store);
TimelineDataManager dataManager = new TimelineDataManager(store, aclsManager);
dataManager.init(conf);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager, appAclsManager);
historyManager.init(conf);
historyManager.start();
} }
private static void prepareTimelineStore(TimelineStore store, int scale) private static void prepareTimelineStore(int scale) throws Exception {
throws Exception {
for (int i = 1; i <= scale; ++i) { for (int i = 1; i <= scale; ++i) {
TimelineEntities entities = new TimelineEntities(); TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.newInstance(0, i); ApplicationId appId = ApplicationId.newInstance(0, i);
if (i == 2) { if (i == 2) {
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(appId, true, false, false, true,
appId, true, false, false, true, YarnApplicationState.FINISHED)); YarnApplicationState.FINISHED));
} else if (i == 3) { } else if (i == 3) {
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false,
appId, false, false, false, false, YarnApplicationState.FINISHED, YarnApplicationState.FINISHED, true, false));
true, false));
} else if (i == SCALE + 1) { } else if (i == SCALE + 1) {
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false,
appId, false, false, false, false, YarnApplicationState.FINISHED, YarnApplicationState.FINISHED, false, true));
false, true));
} else { } else {
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(appId, false, false, false, false,
appId, false, false, false, false, YarnApplicationState.FINISHED)); YarnApplicationState.FINISHED));
} }
store.put(entities); store.put(entities);
for (int j = 1; j <= scale; ++j) { for (int j = 1; j <= scale; ++j) {
entities = new TimelineEntities(); entities = new TimelineEntities();
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
ApplicationAttemptId.newInstance(appId, j);
entities.addEntity(createAppAttemptTimelineEntity(appAttemptId)); entities.addEntity(createAppAttemptTimelineEntity(appAttemptId));
store.put(entities); store.put(entities);
for (int k = 1; k <= scale; ++k) { for (int k = 1; k <= scale; ++k) {
@ -172,129 +161,119 @@ public class TestApplicationHistoryManagerOnTimelineStore {
} }
TimelineEntities entities = new TimelineEntities(); TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.newInstance(1234, 1); ApplicationId appId = ApplicationId.newInstance(1234, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
ApplicationAttemptId.newInstance(appId, 1);
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
entities.addEntity(createApplicationTimelineEntity( entities.addEntity(createApplicationTimelineEntity(appId, true, false, false, false,
appId, true, false, false, false, YarnApplicationState.RUNNING)); YarnApplicationState.RUNNING));
entities.addEntity(createAppAttemptTimelineEntity(appAttemptId)); entities.addEntity(createAppAttemptTimelineEntity(appAttemptId));
entities.addEntity(createContainerEntity(containerId)); entities.addEntity(createContainerEntity(containerId));
store.put(entities); store.put(entities);
} }
@Test @MethodSource("callers")
public void testGetApplicationReport() throws Exception { @ParameterizedTest
void testGetApplicationReport(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
for (int i = 1; i <= 3; ++i) { for (int i = 1; i <= 3; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i); final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app; ApplicationReport app;
if (callerUGI == null) { if (callerUGI == null) {
app = historyManager.getApplication(appId); app = historyManager.getApplication(appId);
} else { } else {
app = app = callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport>() {
callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
@Override @Override
public ApplicationReport run() throws Exception { public ApplicationReport run() throws Exception {
return historyManager.getApplication(appId); return historyManager.getApplication(appId);
} }
}); });
} }
Assert.assertNotNull(app); assertNotNull(app);
Assert.assertEquals(appId, app.getApplicationId()); assertEquals(appId, app.getApplicationId());
Assert.assertEquals("test app", app.getName()); assertEquals("test app", app.getName());
Assert.assertEquals("test app type", app.getApplicationType()); assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser()); assertEquals("user1", app.getUser());
if (i == 2) { if (i == 2) {
// Change event is fired only in case of app with ID 2, hence verify // Change event is fired only in case of app with ID 2, hence verify
// with updated changes. And make sure last updated change is accepted. // with updated changes. And make sure last updated change is accepted.
Assert.assertEquals("changed queue1", app.getQueue()); assertEquals("changed queue1", app.getQueue());
Assert.assertEquals(Priority.newInstance(6), app.getPriority()); assertEquals(Priority.newInstance(6), app.getPriority());
} else { } else {
Assert.assertEquals("test queue", app.getQueue()); assertEquals("test queue", app.getQueue());
Assert.assertEquals(Priority.newInstance(0), app.getPriority()); assertEquals(Priority.newInstance(0), app.getPriority());
} }
Assert.assertEquals(Integer.MAX_VALUE + 2L assertEquals(Integer.MAX_VALUE + 2L + app.getApplicationId().getId(), app.getStartTime());
+ app.getApplicationId().getId(), app.getStartTime()); assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime()); assertEquals(Integer.MAX_VALUE + 3L + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+ +app.getApplicationId().getId(), app.getFinishTime()); assertEquals(2, app.getApplicationTags().size());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001); assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
Assert.assertEquals(2, app.getApplicationTags().size()); assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used. // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app. // Nobody except admin and owner has access to the details of the app.
if ((i != 2 && callerUGI != null && if ((i != 2 && callerUGI != null && callerUGI.getShortUserName().equals("user3")) || (i == 2
callerUGI.getShortUserName().equals("user3")) || && callerUGI != null && (callerUGI.getShortUserName().equals("user2")
(i == 2 && callerUGI != null && || callerUGI.getShortUserName().equals("user3")))) {
(callerUGI.getShortUserName().equals("user2") || assertEquals(ApplicationAttemptId.newInstance(appId, -1),
callerUGI.getShortUserName().equals("user3")))) {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
app.getCurrentApplicationAttemptId()); app.getCurrentApplicationAttemptId());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getHost());
app.getHost()); assertEquals(-1, app.getRpcPort());
Assert.assertEquals(-1, app.getRpcPort()); assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, app.getTrackingUrl());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE, assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getTrackingUrl());
Assert.assertEquals(ApplicationHistoryManagerOnTimelineStore.UNAVAILABLE,
app.getOriginalTrackingUrl()); app.getOriginalTrackingUrl());
Assert.assertEquals("", app.getDiagnostics()); assertEquals("", app.getDiagnostics());
} else { } else {
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1), assertEquals(ApplicationAttemptId.newInstance(appId, 1),
app.getCurrentApplicationAttemptId()); app.getCurrentApplicationAttemptId());
Assert.assertEquals("test host", app.getHost()); assertEquals("test host", app.getHost());
Assert.assertEquals(100, app.getRpcPort()); assertEquals(100, app.getRpcPort());
Assert.assertEquals("test tracking url", app.getTrackingUrl()); assertEquals("test tracking url", app.getTrackingUrl());
Assert.assertEquals("test original tracking url", assertEquals("test original tracking url", app.getOriginalTrackingUrl());
app.getOriginalTrackingUrl()); assertEquals("test diagnostics info", app.getDiagnostics());
Assert.assertEquals("test diagnostics info", app.getDiagnostics());
} }
ApplicationResourceUsageReport applicationResourceUsageReport = ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport(); app.getApplicationResourceUsageReport();
Assert.assertEquals(123, assertEquals(123, applicationResourceUsageReport.getMemorySeconds());
applicationResourceUsageReport.getMemorySeconds()); assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
long expectedPreemptMemSecs = 456; long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789; long expectedPreemptVcoreSecs = 789;
if (i == 3) { if (i == 3) {
expectedPreemptMemSecs = 0; expectedPreemptMemSecs = 0;
expectedPreemptVcoreSecs = 0; expectedPreemptVcoreSecs = 0;
} }
Assert.assertEquals(expectedPreemptMemSecs, assertEquals(expectedPreemptMemSecs,
applicationResourceUsageReport.getPreemptedMemorySeconds()); applicationResourceUsageReport.getPreemptedMemorySeconds());
Assert assertEquals(expectedPreemptVcoreSecs,
.assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport applicationResourceUsageReport.getPreemptedVcoreSeconds());
.getPreemptedVcoreSeconds()); assertEquals(FinalApplicationStatus.UNDEFINED, app.getFinalApplicationStatus());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED, assertEquals(YarnApplicationState.FINISHED, app.getYarnApplicationState());
app.getFinalApplicationStatus());
Assert.assertEquals(YarnApplicationState.FINISHED,
app.getYarnApplicationState());
} }
} }
@Test @MethodSource("callers")
public void testGetApplicationReportWithNotAttempt() throws Exception { @ParameterizedTest
void testGetApplicationReportWithNotAttempt(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
final ApplicationId appId = ApplicationId.newInstance(0, SCALE + 1); final ApplicationId appId = ApplicationId.newInstance(0, SCALE + 1);
ApplicationReport app; ApplicationReport app;
if (callerUGI == null) { if (callerUGI == null) {
app = historyManager.getApplication(appId); app = historyManager.getApplication(appId);
} else { } else {
app = app = callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport>() {
callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
@Override @Override
public ApplicationReport run() throws Exception { public ApplicationReport run() throws Exception {
return historyManager.getApplication(appId); return historyManager.getApplication(appId);
} }
}); });
} }
Assert.assertNotNull(app); assertNotNull(app);
Assert.assertEquals(appId, app.getApplicationId()); assertEquals(appId, app.getApplicationId());
Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1), assertEquals(ApplicationAttemptId.newInstance(appId, -1), app.getCurrentApplicationAttemptId());
app.getCurrentApplicationAttemptId());
} }
@Test @MethodSource("callers")
public void testGetApplicationAttemptReport() throws Exception { @ParameterizedTest
void testGetApplicationAttemptReport(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
final ApplicationAttemptId appAttemptId = final ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
ApplicationAttemptReport appAttempt; ApplicationAttemptReport appAttempt;
@ -302,8 +281,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
appAttempt = historyManager.getApplicationAttempt(appAttemptId); appAttempt = historyManager.getApplicationAttempt(appAttemptId);
} else { } else {
try { try {
appAttempt = appAttempt = callerUGI.doAs(new PrivilegedExceptionAction<ApplicationAttemptReport>() {
callerUGI.doAs(new PrivilegedExceptionAction<ApplicationAttemptReport> () {
@Override @Override
public ApplicationAttemptReport run() throws Exception { public ApplicationAttemptReport run() throws Exception {
return historyManager.getApplicationAttempt(appAttemptId); return historyManager.getApplicationAttempt(appAttemptId);
@ -311,7 +289,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}); });
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
// The exception is expected // The exception is expected
Assert.fail(); fail();
} }
} catch (AuthorizationException e) { } catch (AuthorizationException e) {
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@ -321,32 +299,29 @@ public class TestApplicationHistoryManagerOnTimelineStore {
throw e; throw e;
} }
} }
Assert.assertNotNull(appAttempt); assertNotNull(appAttempt);
Assert.assertEquals(appAttemptId, appAttempt.getApplicationAttemptId()); assertEquals(appAttemptId, appAttempt.getApplicationAttemptId());
Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), assertEquals(ContainerId.newContainerId(appAttemptId, 1), appAttempt.getAMContainerId());
appAttempt.getAMContainerId()); assertEquals("test host", appAttempt.getHost());
Assert.assertEquals("test host", appAttempt.getHost()); assertEquals(100, appAttempt.getRpcPort());
Assert.assertEquals(100, appAttempt.getRpcPort()); assertEquals("test tracking url", appAttempt.getTrackingUrl());
Assert.assertEquals("test tracking url", appAttempt.getTrackingUrl()); assertEquals("test original tracking url", appAttempt.getOriginalTrackingUrl());
Assert.assertEquals("test original tracking url", assertEquals("test diagnostics info", appAttempt.getDiagnostics());
appAttempt.getOriginalTrackingUrl()); assertEquals(YarnApplicationAttemptState.FINISHED, appAttempt.getYarnApplicationAttemptState());
Assert.assertEquals("test diagnostics info", appAttempt.getDiagnostics());
Assert.assertEquals(YarnApplicationAttemptState.FINISHED,
appAttempt.getYarnApplicationAttemptState());
} }
@Test @MethodSource("callers")
public void testGetContainerReport() throws Exception { @ParameterizedTest
final ContainerId containerId = void testGetContainerReport(String caller) throws Exception {
ContainerId.newContainerId(ApplicationAttemptId.newInstance( initTestApplicationHistoryManagerOnTimelineStore(caller);
ApplicationId.newInstance(0, 1), 1), 1); final ContainerId containerId = ContainerId.newContainerId(
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1), 1);
ContainerReport container; ContainerReport container;
if (callerUGI == null) { if (callerUGI == null) {
container = historyManager.getContainer(containerId); container = historyManager.getContainer(containerId);
} else { } else {
try { try {
container = container = callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport>() {
callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport> () {
@Override @Override
public ContainerReport run() throws Exception { public ContainerReport run() throws Exception {
return historyManager.getContainer(containerId); return historyManager.getContainer(containerId);
@ -354,7 +329,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}); });
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
// The exception is expected // The exception is expected
Assert.fail(); fail();
} }
} catch (AuthorizationException e) { } catch (AuthorizationException e) {
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@ -364,53 +339,51 @@ public class TestApplicationHistoryManagerOnTimelineStore {
throw e; throw e;
} }
} }
Assert.assertNotNull(container); assertNotNull(container);
Assert.assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime()); assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime());
Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime()); assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime());
Assert.assertEquals(Resource.newInstance(-1, -1), assertEquals(Resource.newInstance(-1, -1), container.getAllocatedResource());
container.getAllocatedResource()); assertEquals(NodeId.newInstance("test host", 100), container.getAssignedNode());
Assert.assertEquals(NodeId.newInstance("test host", 100), assertEquals(Priority.UNDEFINED, container.getPriority());
container.getAssignedNode()); assertEquals("test diagnostics info", container.getDiagnosticsInfo());
Assert.assertEquals(Priority.UNDEFINED, container.getPriority()); assertEquals(ContainerState.COMPLETE, container.getContainerState());
Assert assertEquals(-1, container.getContainerExitStatus());
.assertEquals("test diagnostics info", container.getDiagnosticsInfo()); assertEquals(
Assert.assertEquals(ContainerState.COMPLETE, container.getContainerState()); "http://0.0.0.0:8188/applicationhistory/logs/" + "test host:100/container_0_0001_01_000001/"
Assert.assertEquals(-1, container.getContainerExitStatus());
Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
"test host:100/container_0_0001_01_000001/"
+ "container_0_0001_01_000001/user1", container.getLogUrl()); + "container_0_0001_01_000001/user1", container.getLogUrl());
} }
@Test @MethodSource("callers")
public void testGetApplications() throws Exception { @ParameterizedTest
void testGetApplications(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
Collection<ApplicationReport> apps = Collection<ApplicationReport> apps =
historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE) historyManager.getApplications(Long.MAX_VALUE, 0L, Long.MAX_VALUE).values();
.values(); assertNotNull(apps);
Assert.assertNotNull(apps); assertEquals(SCALE + 2, apps.size());
Assert.assertEquals(SCALE + 2, apps.size());
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2); ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
for (ApplicationReport app : apps) { for (ApplicationReport app : apps) {
Assert.assertNotEquals(ignoredAppId, app.getApplicationId()); assertNotEquals(ignoredAppId, app.getApplicationId());
} }
// Get apps by given appStartedTime period // Get apps by given appStartedTime period
apps = apps = historyManager.getApplications(Long.MAX_VALUE, 2147483653L, Long.MAX_VALUE).values();
historyManager.getApplications(Long.MAX_VALUE, 2147483653L, assertNotNull(apps);
Long.MAX_VALUE).values(); assertEquals(2, apps.size());
Assert.assertNotNull(apps);
Assert.assertEquals(2, apps.size());
} }
@Test @MethodSource("callers")
public void testGetApplicationAttempts() throws Exception { @ParameterizedTest
void testGetApplicationAttempts(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
final ApplicationId appId = ApplicationId.newInstance(0, 1); final ApplicationId appId = ApplicationId.newInstance(0, 1);
Collection<ApplicationAttemptReport> appAttempts; Collection<ApplicationAttemptReport> appAttempts;
if (callerUGI == null) { if (callerUGI == null) {
appAttempts = historyManager.getApplicationAttempts(appId).values(); appAttempts = historyManager.getApplicationAttempts(appId).values();
} else { } else {
try { try {
appAttempts = callerUGI.doAs( appAttempts =
new PrivilegedExceptionAction<Collection<ApplicationAttemptReport>> () { callerUGI.doAs(new PrivilegedExceptionAction<Collection<ApplicationAttemptReport>>() {
@Override @Override
public Collection<ApplicationAttemptReport> run() throws Exception { public Collection<ApplicationAttemptReport> run() throws Exception {
return historyManager.getApplicationAttempts(appId).values(); return historyManager.getApplicationAttempts(appId).values();
@ -418,7 +391,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}); });
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
// The exception is expected // The exception is expected
Assert.fail(); fail();
} }
} catch (AuthorizationException e) { } catch (AuthorizationException e) {
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@ -428,12 +401,14 @@ public class TestApplicationHistoryManagerOnTimelineStore {
throw e; throw e;
} }
} }
Assert.assertNotNull(appAttempts); assertNotNull(appAttempts);
Assert.assertEquals(SCALE, appAttempts.size()); assertEquals(SCALE, appAttempts.size());
} }
@Test @MethodSource("callers")
public void testGetContainers() throws Exception { @ParameterizedTest
void testGetContainers(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
final ApplicationAttemptId appAttemptId = final ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
Collection<ContainerReport> containers; Collection<ContainerReport> containers;
@ -441,8 +416,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
containers = historyManager.getContainers(appAttemptId).values(); containers = historyManager.getContainers(appAttemptId).values();
} else { } else {
try { try {
containers = callerUGI.doAs( containers = callerUGI.doAs(new PrivilegedExceptionAction<Collection<ContainerReport>>() {
new PrivilegedExceptionAction<Collection<ContainerReport>> () {
@Override @Override
public Collection<ContainerReport> run() throws Exception { public Collection<ContainerReport> run() throws Exception {
return historyManager.getContainers(appAttemptId).values(); return historyManager.getContainers(appAttemptId).values();
@ -450,7 +424,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}); });
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
// The exception is expected // The exception is expected
Assert.fail(); fail();
} }
} catch (AuthorizationException e) { } catch (AuthorizationException e) {
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@ -460,12 +434,14 @@ public class TestApplicationHistoryManagerOnTimelineStore {
throw e; throw e;
} }
} }
Assert.assertNotNull(containers); assertNotNull(containers);
Assert.assertEquals(SCALE, containers.size()); assertEquals(SCALE, containers.size());
} }
@Test @MethodSource("callers")
public void testGetAMContainer() throws Exception { @ParameterizedTest
void testGetAMContainer(String caller) throws Exception {
initTestApplicationHistoryManagerOnTimelineStore(caller);
final ApplicationAttemptId appAttemptId = final ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1); ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
ContainerReport container; ContainerReport container;
@ -473,8 +449,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
container = historyManager.getAMContainer(appAttemptId); container = historyManager.getAMContainer(appAttemptId);
} else { } else {
try { try {
container = container = callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport>() {
callerUGI.doAs(new PrivilegedExceptionAction<ContainerReport> () {
@Override @Override
public ContainerReport run() throws Exception { public ContainerReport run() throws Exception {
return historyManager.getAMContainer(appAttemptId); return historyManager.getAMContainer(appAttemptId);
@ -482,7 +457,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}); });
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
// The exception is expected // The exception is expected
Assert.fail(); fail();
} }
} catch (AuthorizationException e) { } catch (AuthorizationException e) {
if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) { if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
@ -492,24 +467,20 @@ public class TestApplicationHistoryManagerOnTimelineStore {
throw e; throw e;
} }
} }
Assert.assertNotNull(container); assertNotNull(container);
Assert.assertEquals(appAttemptId, container.getContainerId() assertEquals(appAttemptId, container.getContainerId().getApplicationAttemptId());
.getApplicationAttemptId());
} }
private static TimelineEntity createApplicationTimelineEntity( private static TimelineEntity createApplicationTimelineEntity(ApplicationId appId,
ApplicationId appId, boolean emptyACLs, boolean noAttemptId, boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent,
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state) { YarnApplicationState state) {
return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId, return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId, wrongAppId,
wrongAppId, enableUpdateEvent, state, false, false); enableUpdateEvent, state, false, false);
} }
private static TimelineEntity createApplicationTimelineEntity( private static TimelineEntity createApplicationTimelineEntity(ApplicationId appId,
ApplicationId appId, boolean emptyACLs, boolean noAttemptId, boolean emptyACLs, boolean noAttemptId, boolean wrongAppId, boolean enableUpdateEvent,
boolean wrongAppId, boolean enableUpdateEvent, YarnApplicationState state, boolean missingPreemptMetrics, boolean missingQueue) {
YarnApplicationState state, boolean missingPreemptMetrics,
boolean missingQueue) {
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE); entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
if (wrongAppId) { if (wrongAppId) {
@ -518,23 +489,17 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setEntityId(appId.toString()); entity.setEntityId(appId.toString());
} }
entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
entity.addPrimaryFilter( entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
Map<String, Object> entityInfo = new HashMap<String, Object>(); Map<String, Object> entityInfo = new HashMap<String, Object>();
entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app"); entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app");
entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, "test app type");
"test app type");
entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1");
if (!missingQueue) { if (!missingQueue) {
entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
"test queue");
} }
entityInfo.put( entityInfo.put(ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, Integer.MAX_VALUE + 1L);
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
if (!missingPreemptMetrics) { if (!missingPreemptMetrics) {
@ -544,8 +509,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
if (emptyACLs) { if (emptyACLs) {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
} else { } else {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "user2");
"user2");
} }
Set<String> appTags = new HashSet<String>(); Set<String> appTags = new HashSet<String>();
appTags.add("Test_APP_TAGS_1"); appTags.add("Test_APP_TAGS_1");
@ -557,16 +521,13 @@ public class TestApplicationHistoryManagerOnTimelineStore {
tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId()); tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent); entity.addEvent(tEvent);
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
tEvent.setEventType( tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId()); tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>(); Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO, "test diagnostics info");
"test diagnostics info");
eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
FinalApplicationStatus.UNDEFINED.toString()); FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, state.toString());
state.toString());
if (!noAttemptId) { if (!noAttemptId) {
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO, eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
ApplicationAttemptId.newInstance(appId, 1)); ApplicationAttemptId.newInstance(appId, 1));
@ -577,60 +538,51 @@ public class TestApplicationHistoryManagerOnTimelineStore {
// after YARN_APPLICATION_FINISHED // after YARN_APPLICATION_FINISHED
// The final YarnApplicationState should not be changed // The final YarnApplicationState should not be changed
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
tEvent.setEventType( tEvent.setEventType(ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE);
ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 4L + appId.getId()); tEvent.setTimestamp(Integer.MAX_VALUE + 4L + appId.getId());
eventInfo = new HashMap<String, Object>(); eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, YarnApplicationState.KILLED);
YarnApplicationState.KILLED);
tEvent.setEventInfo(eventInfo); tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent); entity.addEvent(tEvent);
if (enableUpdateEvent) { if (enableUpdateEvent) {
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
long updatedTimeIndex = 4L; long updatedTimeIndex = 4L;
createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", 5);
5);
entity.addEvent(tEvent); entity.addEvent(tEvent);
// Change priority alone // Change priority alone
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue", 6);
6);
// Now change queue // Now change queue
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, createAppModifiedEvent(appId, tEvent, updatedTimeIndex++, "changed queue1", 6);
"changed queue1", 6);
entity.addEvent(tEvent); entity.addEvent(tEvent);
} }
return entity; return entity;
} }
private static void createAppModifiedEvent(ApplicationId appId, private static void createAppModifiedEvent(ApplicationId appId, TimelineEvent tEvent,
TimelineEvent tEvent, long updatedTimeIndex, String queue, int priority) { long updatedTimeIndex, String queue, int priority) {
tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE); tEvent.setEventType(ApplicationMetricsConstants.UPDATED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + updatedTimeIndex + appId.getId()); tEvent.setTimestamp(Integer.MAX_VALUE + updatedTimeIndex + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>(); Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, queue); eventInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, queue);
eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, eventInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, priority);
priority);
tEvent.setEventInfo(eventInfo); tEvent.setEventInfo(eventInfo);
} }
private static TimelineEntity createAppAttemptTimelineEntity( private static TimelineEntity createAppAttemptTimelineEntity(ApplicationAttemptId appAttemptId) {
ApplicationAttemptId appAttemptId) {
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE); entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE);
entity.setEntityId(appAttemptId.toString()); entity.setEntityId(appAttemptId.toString());
entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
appAttemptId.getApplicationId().toString()); appAttemptId.getApplicationId().toString());
entity.addPrimaryFilter( entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
TimelineEvent tEvent = new TimelineEvent(); TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 1L); tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
Map<String, Object> eventInfo = new HashMap<String, Object>(); Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, "test tracking url");
"test tracking url");
eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO, eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO,
"test original tracking url"); "test original tracking url");
eventInfo.put(AppAttemptMetricsConstants.HOST_INFO, "test host"); eventInfo.put(AppAttemptMetricsConstants.HOST_INFO, "test host");
@ -643,12 +595,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 2L); tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
eventInfo = new HashMap<String, Object>(); eventInfo = new HashMap<String, Object>();
eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, "test tracking url");
"test tracking url");
eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO, eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO,
"test original tracking url"); "test original tracking url");
eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO, eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO, "test diagnostics info");
"test diagnostics info");
eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO, eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_INFO,
FinalApplicationStatus.UNDEFINED.toString()); FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_INFO, eventInfo.put(AppAttemptMetricsConstants.STATE_INFO,
@ -665,33 +615,26 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID); entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER, entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
containerId.getApplicationAttemptId().toString()); containerId.getApplicationAttemptId().toString());
entity.addPrimaryFilter( entity.addPrimaryFilter(TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
Map<String, Object> entityInfo = new HashMap<String, Object>(); Map<String, Object> entityInfo = new HashMap<String, Object>();
entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, -1); entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, -1);
entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, -1); entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, -1);
entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, "test host");
"test host");
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, 100); entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, 100);
entityInfo entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1);
.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1); entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234");
entityInfo.put(ContainerMetricsConstants
.ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234");
entity.setOtherInfo(entityInfo); entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent(); TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE); tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 1L); tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
entity.addEvent(tEvent); entity.addEvent(tEvent);
;
tEvent = new TimelineEvent(); tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE); tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
tEvent.setTimestamp(Integer.MAX_VALUE + 2L); tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
Map<String, Object> eventInfo = new HashMap<String, Object>(); Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, "test diagnostics info");
"test diagnostics info");
eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_INFO, -1); eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_INFO, -1);
eventInfo.put(ContainerMetricsConstants.STATE_INFO, eventInfo.put(ContainerMetricsConstants.STATE_INFO, ContainerState.COMPLETE.toString());
ContainerState.COMPLETE.toString());
tEvent.setEventInfo(eventInfo); tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent); entity.addEvent(tEvent);
return entity; return entity;

60
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
View File

@ -18,9 +18,17 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice; package org.apache.hadoop.yarn.server.applicationhistoryservice;
import static org.junit.Assert.assertEquals; import java.io.ByteArrayOutputStream;
import static org.junit.Assert.assertNotNull; import java.io.File;
import static org.junit.Assert.fail; import java.io.FileInputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.http.lib.StaticUserWebFilter;
@ -33,23 +41,18 @@ import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore; import org.apache.hadoop.yarn.server.timeline.recovery.MemoryTimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore; import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
import org.junit.Assert;
import org.junit.Test;
import java.io.ByteArrayOutputStream; import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.File; import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.io.FileInputStream; import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.InputStream; import static org.junit.jupiter.api.Assertions.fail;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
public class TestApplicationHistoryServer { public class TestApplicationHistoryServer {
// simple test init/start/stop ApplicationHistoryServer. Status should change. // simple test init/start/stop ApplicationHistoryServer. Status should change.
@Test(timeout = 60000) @Test
public void testStartStopServer() throws Exception { @Timeout(60000)
void testStartStopServer() throws Exception {
ApplicationHistoryServer historyServer = new ApplicationHistoryServer(); ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
Configuration config = new YarnConfiguration(); Configuration config = new YarnConfiguration();
config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE, config.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
@ -65,7 +68,7 @@ public class TestApplicationHistoryServer {
historyServer.start(); historyServer.start();
fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT)); YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT));
} }
config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, config.setInt(YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
@ -89,8 +92,9 @@ public class TestApplicationHistoryServer {
} }
// test launch method // test launch method
@Test(timeout = 60000) @Test
public void testLaunch() throws Exception { @Timeout(60000)
void testLaunch() throws Exception {
ExitUtil.disableSystemExit(); ExitUtil.disableSystemExit();
ApplicationHistoryServer historyServer = null; ApplicationHistoryServer historyServer = null;
try { try {
@ -110,8 +114,9 @@ public class TestApplicationHistoryServer {
} }
//test launch method with -D arguments //test launch method with -D arguments
@Test(timeout = 60000) @Test
public void testLaunchWithArguments() throws Exception { @Timeout(60000)
void testLaunchWithArguments() throws Exception {
ExitUtil.disableSystemExit(); ExitUtil.disableSystemExit();
ApplicationHistoryServer historyServer = null; ApplicationHistoryServer historyServer = null;
try { try {
@ -135,8 +140,10 @@ public class TestApplicationHistoryServer {
} }
} }
} }
@Test(timeout = 240000)
public void testFilterOverrides() throws Exception { @Test
@Timeout(240000)
void testFilterOverrides() throws Exception {
HashMap<String, String> driver = new HashMap<String, String>(); HashMap<String, String> driver = new HashMap<String, String>();
driver.put("", TimelineAuthenticationFilterInitializer.class.getName()); driver.put("", TimelineAuthenticationFilterInitializer.class.getName());
@ -176,8 +183,9 @@ public class TestApplicationHistoryServer {
} }
} }
@Test(timeout = 240000) @Test
public void testHostedUIs() throws Exception { @Timeout(240000)
void testHostedUIs() throws Exception {
ApplicationHistoryServer historyServer = new ApplicationHistoryServer(); ApplicationHistoryServer historyServer = new ApplicationHistoryServer();
Configuration config = new YarnConfiguration(); Configuration config = new YarnConfiguration();
@ -209,8 +217,8 @@ public class TestApplicationHistoryServer {
} finally { } finally {
historyServer.stop(); historyServer.stop();
} }
assertEquals("Web file contents should be the same as on disk contents", assertEquals(diskFileStr, connFileStr,
diskFileStr, connFileStr); "Web file contents should be the same as on disk contents");
} }
private String readInputStream(InputStream input) throws Exception { private String readInputStream(InputStream input) throws Exception {
ByteArrayOutputStream data = new ByteArrayOutputStream(); ByteArrayOutputStream data = new ByteArrayOutputStream();

111
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
View File

@ -22,15 +22,12 @@ import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import org.junit.Assert; import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import static org.mockito.Mockito.any; import org.junit.jupiter.api.Test;
import static org.mockito.Mockito.doReturn; import org.mockito.Mockito;
import static org.mockito.Mockito.doThrow; import org.slf4j.Logger;
import static org.mockito.Mockito.never; import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
@ -45,12 +42,19 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.junit.After;
import org.junit.Before; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.mockito.Mockito; import static org.junit.jupiter.api.Assertions.assertNull;
import org.slf4j.Logger; import static org.junit.jupiter.api.Assertions.assertTrue;
import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
public class TestFileSystemApplicationHistoryStore extends public class TestFileSystemApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils { ApplicationHistoryStoreTestUtils {
@ -61,7 +65,7 @@ public class TestFileSystemApplicationHistoryStore extends
private FileSystem fs; private FileSystem fs;
private Path fsWorkingPath; private Path fsWorkingPath;
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
fs = new RawLocalFileSystem(); fs = new RawLocalFileSystem();
initAndStartStore(fs); initAndStartStore(fs);
@ -75,8 +79,7 @@ public class TestFileSystemApplicationHistoryStore extends
new Path("target", new Path("target",
TestFileSystemApplicationHistoryStore.class.getSimpleName()); TestFileSystemApplicationHistoryStore.class.getSimpleName());
fs.delete(fsWorkingPath, true); fs.delete(fsWorkingPath, true);
conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
fsWorkingPath.toString());
store = new FileSystemApplicationHistoryStore() { store = new FileSystemApplicationHistoryStore() {
@Override @Override
protected FileSystem getFileSystem(Path path, Configuration conf) { protected FileSystem getFileSystem(Path path, Configuration conf) {
@ -87,7 +90,7 @@ public class TestFileSystemApplicationHistoryStore extends
store.start(); store.start();
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
store.stop(); store.stop();
fs.delete(fsWorkingPath, true); fs.delete(fsWorkingPath, true);
@ -95,7 +98,7 @@ public class TestFileSystemApplicationHistoryStore extends
} }
@Test @Test
public void testReadWriteHistoryData() throws IOException { void testReadWriteHistoryData() throws IOException {
LOG.info("Starting testReadWriteHistoryData"); LOG.info("Starting testReadWriteHistoryData");
testWriteHistoryData(5); testWriteHistoryData(5);
testReadHistoryData(5); testReadHistoryData(5);
@ -146,58 +149,56 @@ public class TestFileSystemApplicationHistoryStore extends
int num, boolean missingContainer, boolean missingApplicationAttempt) int num, boolean missingContainer, boolean missingApplicationAttempt)
throws IOException { throws IOException {
// read application history data // read application history data
Assert.assertEquals(num, store.getAllApplications().size()); assertEquals(num, store.getAllApplications().size());
for (int i = 1; i <= num; ++i) { for (int i = 1; i <= num; ++i) {
ApplicationId appId = ApplicationId.newInstance(0, i); ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationHistoryData appData = store.getApplication(appId); ApplicationHistoryData appData = store.getApplication(appId);
Assert.assertNotNull(appData); assertNotNull(appData);
Assert.assertEquals(appId.toString(), appData.getApplicationName()); assertEquals(appId.toString(), appData.getApplicationName());
Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo()); assertEquals(appId.toString(), appData.getDiagnosticsInfo());
// read application attempt history data // read application attempt history data
Assert.assertEquals(num, store.getApplicationAttempts(appId).size()); assertEquals(num, store.getApplicationAttempts(appId).size());
for (int j = 1; j <= num; ++j) { for (int j = 1; j <= num; ++j) {
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, j); ApplicationAttemptId.newInstance(appId, j);
ApplicationAttemptHistoryData attemptData = ApplicationAttemptHistoryData attemptData =
store.getApplicationAttempt(appAttemptId); store.getApplicationAttempt(appAttemptId);
Assert.assertNotNull(attemptData); assertNotNull(attemptData);
Assert.assertEquals(appAttemptId.toString(), attemptData.getHost()); assertEquals(appAttemptId.toString(), attemptData.getHost());
if (missingApplicationAttempt && j == num) { if (missingApplicationAttempt && j == num) {
Assert.assertNull(attemptData.getDiagnosticsInfo()); assertNull(attemptData.getDiagnosticsInfo());
continue; continue;
} else { } else {
Assert.assertEquals(appAttemptId.toString(), assertEquals(appAttemptId.toString(),
attemptData.getDiagnosticsInfo()); attemptData.getDiagnosticsInfo());
} }
// read container history data // read container history data
Assert.assertEquals(num, store.getContainers(appAttemptId).size()); assertEquals(num, store.getContainers(appAttemptId).size());
for (int k = 1; k <= num; ++k) { for (int k = 1; k <= num; ++k) {
ContainerId containerId = ContainerId.newContainerId(appAttemptId, k); ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
ContainerHistoryData containerData = store.getContainer(containerId); ContainerHistoryData containerData = store.getContainer(containerId);
Assert.assertNotNull(containerData); assertNotNull(containerData);
Assert.assertEquals(Priority.newInstance(containerId.getId()), assertEquals(Priority.newInstance(containerId.getId()), containerData.getPriority());
containerData.getPriority());
if (missingContainer && k == num) { if (missingContainer && k == num) {
Assert.assertNull(containerData.getDiagnosticsInfo()); assertNull(containerData.getDiagnosticsInfo());
} else { } else {
Assert.assertEquals(containerId.toString(), assertEquals(containerId.toString(),
containerData.getDiagnosticsInfo()); containerData.getDiagnosticsInfo());
} }
} }
ContainerHistoryData masterContainer = ContainerHistoryData masterContainer =
store.getAMContainer(appAttemptId); store.getAMContainer(appAttemptId);
Assert.assertNotNull(masterContainer); assertNotNull(masterContainer);
Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), assertEquals(ContainerId.newContainerId(appAttemptId, 1), masterContainer.getContainerId());
masterContainer.getContainerId());
} }
} }
} }
@Test @Test
public void testWriteAfterApplicationFinish() throws IOException { void testWriteAfterApplicationFinish() throws IOException {
LOG.info("Starting testWriteAfterApplicationFinish"); LOG.info("Starting testWriteAfterApplicationFinish");
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId); writeApplicationStartData(appId);
@ -207,34 +208,34 @@ public class TestFileSystemApplicationHistoryStore extends
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
try { try {
writeApplicationAttemptStartData(appAttemptId); writeApplicationAttemptStartData(appAttemptId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is not opened")); assertTrue(e.getMessage().contains("is not opened"));
} }
try { try {
writeApplicationAttemptFinishData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is not opened")); assertTrue(e.getMessage().contains("is not opened"));
} }
// write container history data // write container history data
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
try { try {
writeContainerStartData(containerId); writeContainerStartData(containerId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is not opened")); assertTrue(e.getMessage().contains("is not opened"));
} }
try { try {
writeContainerFinishData(containerId); writeContainerFinishData(containerId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is not opened")); assertTrue(e.getMessage().contains("is not opened"));
} }
} }
@Test @Test
public void testMassiveWriteContainerHistoryData() throws IOException { void testMassiveWriteContainerHistoryData() throws IOException {
LOG.info("Starting testMassiveWriteContainerHistoryData"); LOG.info("Starting testMassiveWriteContainerHistoryData");
long mb = 1024 * 1024; long mb = 1024 * 1024;
long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb; long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
@ -249,25 +250,25 @@ public class TestFileSystemApplicationHistoryStore extends
} }
writeApplicationFinishData(appId); writeApplicationFinishData(appId);
long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb; long usedDiskAfter = fs.getContentSummary(fsWorkingPath).getLength() / mb;
Assert.assertTrue((usedDiskAfter - usedDiskBefore) < 20); assertTrue((usedDiskAfter - usedDiskBefore) < 20);
} }
@Test @Test
public void testMissingContainerHistoryData() throws IOException { void testMissingContainerHistoryData() throws IOException {
LOG.info("Starting testMissingContainerHistoryData"); LOG.info("Starting testMissingContainerHistoryData");
testWriteHistoryData(3, true, false); testWriteHistoryData(3, true, false);
testReadHistoryData(3, true, false); testReadHistoryData(3, true, false);
} }
@Test @Test
public void testMissingApplicationAttemptHistoryData() throws IOException { void testMissingApplicationAttemptHistoryData() throws IOException {
LOG.info("Starting testMissingApplicationAttemptHistoryData"); LOG.info("Starting testMissingApplicationAttemptHistoryData");
testWriteHistoryData(3, false, true); testWriteHistoryData(3, false, true);
testReadHistoryData(3, false, true); testReadHistoryData(3, false, true);
} }
@Test @Test
public void testInitExistingWorkingDirectoryInSafeMode() throws Exception { void testInitExistingWorkingDirectoryInSafeMode() throws Exception {
LOG.info("Starting testInitExistingWorkingDirectoryInSafeMode"); LOG.info("Starting testInitExistingWorkingDirectoryInSafeMode");
tearDown(); tearDown();
@ -280,7 +281,7 @@ public class TestFileSystemApplicationHistoryStore extends
try { try {
initAndStartStore(fileSystem); initAndStartStore(fileSystem);
} catch (Exception e) { } catch (Exception e) {
Assert.fail("Exception should not be thrown: " + e); fail("Exception should not be thrown: " + e);
} }
// Make sure that directory creation was not attempted // Make sure that directory creation was not attempted
@ -289,7 +290,7 @@ public class TestFileSystemApplicationHistoryStore extends
} }
@Test @Test
public void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception { void testInitNonExistingWorkingDirectoryInSafeMode() throws Exception {
LOG.info("Starting testInitNonExistingWorkingDirectoryInSafeMode"); LOG.info("Starting testInitNonExistingWorkingDirectoryInSafeMode");
tearDown(); tearDown();
@ -302,7 +303,7 @@ public class TestFileSystemApplicationHistoryStore extends
try { try {
initAndStartStore(fileSystem); initAndStartStore(fileSystem);
Assert.fail("Exception should have been thrown"); fail("Exception should have been thrown");
} catch (Exception e) { } catch (Exception e) {
// Expected failure // Expected failure
} }

87
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
View File

@ -20,7 +20,8 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException; import java.io.IOException;
import org.junit.Assert; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -29,26 +30,29 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData; import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
import org.junit.Before;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TestMemoryApplicationHistoryStore extends public class TestMemoryApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils { ApplicationHistoryStoreTestUtils {
@Before @BeforeEach
public void setup() { public void setup() {
store = new MemoryApplicationHistoryStore(); store = new MemoryApplicationHistoryStore();
} }
@Test @Test
public void testReadWriteApplicationHistory() throws Exception { void testReadWriteApplicationHistory() throws Exception {
// Out of order // Out of order
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
try { try {
writeApplicationFinishData(appId); writeApplicationFinishData(appId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"is stored before the start information")); "is stored before the start information"));
} }
// Normal // Normal
@ -58,41 +62,41 @@ public class TestMemoryApplicationHistoryStore extends
writeApplicationStartData(appId); writeApplicationStartData(appId);
writeApplicationFinishData(appId); writeApplicationFinishData(appId);
} }
Assert.assertEquals(numApps, store.getAllApplications().size()); assertEquals(numApps, store.getAllApplications().size());
for (int i = 1; i <= numApps; ++i) { for (int i = 1; i <= numApps; ++i) {
appId = ApplicationId.newInstance(0, i); appId = ApplicationId.newInstance(0, i);
ApplicationHistoryData data = store.getApplication(appId); ApplicationHistoryData data = store.getApplication(appId);
Assert.assertNotNull(data); assertNotNull(data);
Assert.assertEquals(appId.toString(), data.getApplicationName()); assertEquals(appId.toString(), data.getApplicationName());
Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo()); assertEquals(appId.toString(), data.getDiagnosticsInfo());
} }
// Write again // Write again
appId = ApplicationId.newInstance(0, 1); appId = ApplicationId.newInstance(0, 1);
try { try {
writeApplicationStartData(appId); writeApplicationStartData(appId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored")); assertTrue(e.getMessage().contains("is already stored"));
} }
try { try {
writeApplicationFinishData(appId); writeApplicationFinishData(appId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored")); assertTrue(e.getMessage().contains("is already stored"));
} }
} }
@Test @Test
public void testReadWriteApplicationAttemptHistory() throws Exception { void testReadWriteApplicationAttemptHistory() throws Exception {
// Out of order // Out of order
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
try { try {
writeApplicationAttemptFinishData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"is stored before the start information")); "is stored before the start information"));
} }
// Normal // Normal
@ -103,36 +107,36 @@ public class TestMemoryApplicationHistoryStore extends
writeApplicationAttemptStartData(appAttemptId); writeApplicationAttemptStartData(appAttemptId);
writeApplicationAttemptFinishData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId);
} }
Assert.assertEquals(numAppAttempts, store.getApplicationAttempts(appId) assertEquals(numAppAttempts, store.getApplicationAttempts(appId)
.size()); .size());
for (int i = 1; i <= numAppAttempts; ++i) { for (int i = 1; i <= numAppAttempts; ++i) {
appAttemptId = ApplicationAttemptId.newInstance(appId, i); appAttemptId = ApplicationAttemptId.newInstance(appId, i);
ApplicationAttemptHistoryData data = ApplicationAttemptHistoryData data =
store.getApplicationAttempt(appAttemptId); store.getApplicationAttempt(appAttemptId);
Assert.assertNotNull(data); assertNotNull(data);
Assert.assertEquals(appAttemptId.toString(), data.getHost()); assertEquals(appAttemptId.toString(), data.getHost());
Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo()); assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
} }
writeApplicationFinishData(appId); writeApplicationFinishData(appId);
// Write again // Write again
appAttemptId = ApplicationAttemptId.newInstance(appId, 1); appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
try { try {
writeApplicationAttemptStartData(appAttemptId); writeApplicationAttemptStartData(appAttemptId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored")); assertTrue(e.getMessage().contains("is already stored"));
} }
try { try {
writeApplicationAttemptFinishData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored")); assertTrue(e.getMessage().contains("is already stored"));
} }
} }
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
@Test @Test
public void testReadWriteContainerHistory() throws Exception { void testReadWriteContainerHistory() throws Exception {
// Out of order // Out of order
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
@ -140,9 +144,9 @@ public class TestMemoryApplicationHistoryStore extends
ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
try { try {
writeContainerFinishData(containerId); writeContainerFinishData(containerId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"is stored before the start information")); "is stored before the start information"));
} }
// Normal // Normal
@ -153,39 +157,38 @@ public class TestMemoryApplicationHistoryStore extends
writeContainerStartData(containerId); writeContainerStartData(containerId);
writeContainerFinishData(containerId); writeContainerFinishData(containerId);
} }
Assert assertEquals(numContainers, store.getContainers(appAttemptId).size());
.assertEquals(numContainers, store.getContainers(appAttemptId).size());
for (int i = 1; i <= numContainers; ++i) { for (int i = 1; i <= numContainers; ++i) {
containerId = ContainerId.newContainerId(appAttemptId, i); containerId = ContainerId.newContainerId(appAttemptId, i);
ContainerHistoryData data = store.getContainer(containerId); ContainerHistoryData data = store.getContainer(containerId);
Assert.assertNotNull(data); assertNotNull(data);
Assert.assertEquals(Priority.newInstance(containerId.getId()), assertEquals(Priority.newInstance(containerId.getId()),
data.getPriority()); data.getPriority());
Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo()); assertEquals(containerId.toString(), data.getDiagnosticsInfo());
} }
ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId); ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
Assert.assertNotNull(masterContainer); assertNotNull(masterContainer);
Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), assertEquals(ContainerId.newContainerId(appAttemptId, 1),
masterContainer.getContainerId()); masterContainer.getContainerId());
writeApplicationAttemptFinishData(appAttemptId); writeApplicationAttemptFinishData(appAttemptId);
// Write again // Write again
containerId = ContainerId.newContainerId(appAttemptId, 1); containerId = ContainerId.newContainerId(appAttemptId, 1);
try { try {
writeContainerStartData(containerId); writeContainerStartData(containerId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored")); assertTrue(e.getMessage().contains("is already stored"));
} }
try { try {
writeContainerFinishData(containerId); writeContainerFinishData(containerId);
Assert.fail(); fail();
} catch (IOException e) { } catch (IOException e) {
Assert.assertTrue(e.getMessage().contains("is already stored")); assertTrue(e.getMessage().contains("is already stored"));
} }
} }
@Test @Test
public void testMassiveWriteContainerHistory() throws IOException { void testMassiveWriteContainerHistory() throws IOException {
long mb = 1024 * 1024; long mb = 1024 * 1024;
Runtime runtime = Runtime.getRuntime(); Runtime runtime = Runtime.getRuntime();
long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb; long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
@ -199,7 +202,7 @@ public class TestMemoryApplicationHistoryStore extends
writeContainerFinishData(containerId); writeContainerFinishData(containerId);
} }
long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb; long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400); assertTrue((usedMemoryAfter - usedMemoryBefore) < 400);
} }
} }

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebApp.java
View File

@ -18,10 +18,12 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
import static org.apache.hadoop.yarn.webapp.Params.TITLE;
import static org.mockito.Mockito.mock;
import java.util.Map; import java.util.Map;
import com.google.inject.Injector;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationBaseProtocol; import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@ -38,11 +40,11 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplication
import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.google.inject.Injector; import static org.apache.hadoop.yarn.webapp.Params.TITLE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils { public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
@ -50,25 +52,24 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
this.store = store; this.store = store;
} }
@Before @BeforeEach
public void setup() { public void setup() {
store = new MemoryApplicationHistoryStore(); store = new MemoryApplicationHistoryStore();
} }
@Test @Test
public void testAppControllerIndex() throws Exception { void testAppControllerIndex() throws Exception {
ApplicationHistoryManager ahManager = mock(ApplicationHistoryManager.class); ApplicationHistoryManager ahManager = mock(ApplicationHistoryManager.class);
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationHistoryManager.class, WebAppTests.createMockInjector(ApplicationHistoryManager.class,
ahManager); ahManager);
AHSController controller = injector.getInstance(AHSController.class); AHSController controller = injector.getInstance(AHSController.class);
controller.index(); controller.index();
Assert assertEquals("Application History", controller.get(TITLE, "unknown"));
.assertEquals("Application History", controller.get(TITLE, "unknown"));
} }
@Test @Test
public void testView() throws Exception { void testView() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(5, 1, 1)); mockApplicationHistoryClientService(5, 1, 1));
@ -89,7 +90,7 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
} }
@Test @Test
public void testAPPViewNaturalSortType() throws Exception { void testAPPViewNaturalSortType() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(5, 1, 1)); mockApplicationHistoryClientService(5, 1, 1));
@ -100,11 +101,11 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
Map<String, String> moreParams = Map<String, String> moreParams =
ahsViewInstance.context().requestContext().moreParams(); ahsViewInstance.context().requestContext().moreParams();
String appTableColumnsMeta = moreParams.get("ui.dataTables.apps.init"); String appTableColumnsMeta = moreParams.get("ui.dataTables.apps.init");
Assert.assertTrue(appTableColumnsMeta.indexOf("natural") != -1); assertTrue(appTableColumnsMeta.indexOf("natural") != -1);
} }
@Test @Test
public void testAboutPage() throws Exception { void testAboutPage() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(0, 0, 0)); mockApplicationHistoryClientService(0, 0, 0));
@ -118,7 +119,7 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
} }
@Test @Test
public void testAppPage() throws Exception { void testAppPage() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(1, 5, 1)); mockApplicationHistoryClientService(1, 5, 1));
@ -134,7 +135,7 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
} }
@Test @Test
public void testAppPageNaturalSortType() throws Exception { void testAppPageNaturalSortType() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(1, 5, 1)); mockApplicationHistoryClientService(1, 5, 1));
@ -146,11 +147,11 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
appPageInstance.context().requestContext().moreParams(); appPageInstance.context().requestContext().moreParams();
String attemptsTableColumnsMeta = String attemptsTableColumnsMeta =
moreParams.get("ui.dataTables.attempts.init"); moreParams.get("ui.dataTables.attempts.init");
Assert.assertTrue(attemptsTableColumnsMeta.indexOf("natural") != -1); assertTrue(attemptsTableColumnsMeta.indexOf("natural") != -1);
} }
@Test @Test
public void testAppAttemptPage() throws Exception { void testAppAttemptPage() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(1, 1, 5)); mockApplicationHistoryClientService(1, 1, 5));
@ -168,7 +169,7 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
} }
@Test @Test
public void testAppAttemptPageNaturalSortType() throws Exception { void testAppAttemptPageNaturalSortType() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(1, 1, 5)); mockApplicationHistoryClientService(1, 1, 5));
@ -179,11 +180,11 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
Map<String, String> moreParams = Map<String, String> moreParams =
appAttemptPageInstance.context().requestContext().moreParams(); appAttemptPageInstance.context().requestContext().moreParams();
String tableColumnsMeta = moreParams.get("ui.dataTables.containers.init"); String tableColumnsMeta = moreParams.get("ui.dataTables.containers.init");
Assert.assertTrue(tableColumnsMeta.indexOf("natural") != -1); assertTrue(tableColumnsMeta.indexOf("natural") != -1);
} }
@Test @Test
public void testContainerPage() throws Exception { void testContainerPage() throws Exception {
Injector injector = Injector injector =
WebAppTests.createMockInjector(ApplicationBaseProtocol.class, WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
mockApplicationHistoryClientService(1, 1, 1)); mockApplicationHistoryClientService(1, 1, 1));
@ -230,7 +231,7 @@ public class TestAHSWebApp extends ApplicationHistoryStoreTestUtils {
class MockApplicationHistoryManagerImpl extends ApplicationHistoryManagerImpl { class MockApplicationHistoryManagerImpl extends ApplicationHistoryManagerImpl {
public MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) { MockApplicationHistoryManagerImpl(ApplicationHistoryStore store) {
super(); super();
init(new YarnConfiguration()); init(new YarnConfiguration());
start(); start();

247
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
View File

@ -18,16 +18,6 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp; package org.apache.hadoop.yarn.server.applicationhistoryservice.webapp;
import static org.assertj.core.api.Assertions.assertThat;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.net.URI; import java.net.URI;
import java.net.URL; import java.net.URL;
@ -36,11 +26,31 @@ import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Properties; import java.util.Properties;
import javax.servlet.FilterConfig; import javax.servlet.FilterConfig;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MediaType;
import com.google.inject.Guice;
import com.google.inject.Singleton;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
@ -57,6 +67,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType; import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo; import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
@ -72,36 +83,23 @@ import org.apache.hadoop.yarn.server.webapp.LogServlet;
import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils; import org.apache.hadoop.yarn.server.webapp.LogWebServiceUtils;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams; import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import com.google.inject.Guice; import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import com.google.inject.Singleton; import static org.assertj.core.api.Assertions.assertThat;
import com.google.inject.servlet.ServletModule; import static org.junit.jupiter.api.Assertions.assertEquals;
import com.sun.jersey.api.client.ClientResponse; import static org.junit.jupiter.api.Assertions.assertNotNull;
import com.sun.jersey.api.client.ClientResponse.Status; import static org.junit.jupiter.api.Assertions.assertTrue;
import com.sun.jersey.api.client.GenericType; import static org.junit.jupiter.api.Assertions.fail;
import com.sun.jersey.api.client.UniformInterfaceException; import static org.mockito.ArgumentMatchers.any;
import com.sun.jersey.api.client.WebResource; import static org.mockito.Mockito.doReturn;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import static org.mockito.Mockito.spy;
import com.sun.jersey.test.framework.WebAppDescriptor;
@RunWith(Parameterized.class)
public class TestAHSWebServices extends JerseyTestBase { public class TestAHSWebServices extends JerseyTestBase {
private static ApplicationHistoryClientService historyClientService; private static ApplicationHistoryClientService historyClientService;
@ -115,7 +113,7 @@ public class TestAHSWebServices extends JerseyTestBase {
private static final String NM_WEBADDRESS = "test-nm-web-address:9999"; private static final String NM_WEBADDRESS = "test-nm-web-address:9999";
private static final String NM_ID = "test:1234"; private static final String NM_ID = "test:1234";
@BeforeClass @BeforeAll
public static void setupClass() throws Exception { public static void setupClass() throws Exception {
conf = new YarnConfiguration(); conf = new YarnConfiguration();
TimelineStore store = TimelineStore store =
@ -153,7 +151,7 @@ public class TestAHSWebServices extends JerseyTestBase {
Guice.createInjector(new WebServletModule())); Guice.createInjector(new WebServletModule()));
} }
@AfterClass @AfterAll
public static void tearDownClass() throws Exception { public static void tearDownClass() throws Exception {
if (historyClientService != null) { if (historyClientService != null) {
historyClientService.stop(); historyClientService.stop();
@ -162,7 +160,6 @@ public class TestAHSWebServices extends JerseyTestBase {
fs.delete(new Path(rootLogDir), true); fs.delete(new Path(rootLogDir), true);
} }
@Parameterized.Parameters
public static Collection<Object[]> rounds() { public static Collection<Object[]> rounds() {
return Arrays.asList(new Object[][]{{0}, {1}}); return Arrays.asList(new Object[][]{{0}, {1}});
} }
@ -179,7 +176,7 @@ public class TestAHSWebServices extends JerseyTestBase {
} }
} }
@Before @BeforeEach
public void setUp() throws Exception { public void setUp() throws Exception {
super.setUp(); super.setUp();
GuiceServletConfig.setInjector( GuiceServletConfig.setInjector(
@ -199,19 +196,17 @@ public class TestAHSWebServices extends JerseyTestBase {
} }
} }
private int round; public TestAHSWebServices() {
public TestAHSWebServices(int round) {
super(new WebAppDescriptor.Builder( super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.applicationhistoryservice.webapp") "org.apache.hadoop.yarn.server.applicationhistoryservice.webapp")
.contextListenerClass(GuiceServletConfig.class) .contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class) .filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build()); .contextPath("jersey-guice-filter").servletPath("/").build());
this.round = round;
} }
@Test @MethodSource("rounds")
public void testInvalidApp() { @ParameterizedTest
void testInvalidApp(int round) {
ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1); ApplicationId appId = ApplicationId.newInstance(0, MAX_APPS + 1);
WebResource r = resource(); WebResource r = resource();
ClientResponse response = ClientResponse response =
@ -224,8 +219,9 @@ public class TestAHSWebServices extends JerseyTestBase {
Status.NOT_FOUND, response.getStatusInfo()); Status.NOT_FOUND, response.getStatusInfo());
} }
@Test @MethodSource("rounds")
public void testInvalidAttempt() { @ParameterizedTest
void testInvalidAttempt(int round) {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, MAX_APPS + 1); ApplicationAttemptId.newInstance(appId, MAX_APPS + 1);
@ -245,8 +241,9 @@ public class TestAHSWebServices extends JerseyTestBase {
Status.NOT_FOUND, response.getStatusInfo()); Status.NOT_FOUND, response.getStatusInfo());
} }
@Test @MethodSource("rounds")
public void testInvalidContainer() throws Exception { @ParameterizedTest
void testInvalidContainer(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -269,8 +266,9 @@ public class TestAHSWebServices extends JerseyTestBase {
Status.NOT_FOUND, response.getStatusInfo()); Status.NOT_FOUND, response.getStatusInfo());
} }
@Test @MethodSource("rounds")
public void testInvalidUri() throws JSONException, Exception { @ParameterizedTest
void testInvalidUri(int round) throws JSONException, Exception {
WebResource r = resource(); WebResource r = resource();
String responseStr = ""; String responseStr = "";
try { try {
@ -288,8 +286,9 @@ public class TestAHSWebServices extends JerseyTestBase {
} }
} }
@Test @MethodSource("rounds")
public void testInvalidUri2() throws JSONException, Exception { @ParameterizedTest
void testInvalidUri2(int round) throws JSONException, Exception {
WebResource r = resource(); WebResource r = resource();
String responseStr = ""; String responseStr = "";
try { try {
@ -304,8 +303,9 @@ public class TestAHSWebServices extends JerseyTestBase {
} }
} }
@Test @MethodSource("rounds")
public void testInvalidAccept() throws JSONException, Exception { @ParameterizedTest
void testInvalidAccept(int round) throws JSONException, Exception {
WebResource r = resource(); WebResource r = resource();
String responseStr = ""; String responseStr = "";
try { try {
@ -323,8 +323,9 @@ public class TestAHSWebServices extends JerseyTestBase {
} }
} }
@Test @MethodSource("rounds")
public void testAbout() throws Exception { @ParameterizedTest
void testAbout(int round) throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r ClientResponse response = r
.path("ws").path("v1").path("applicationhistory").path("about") .path("ws").path("v1").path("applicationhistory").path("about")
@ -335,25 +336,26 @@ public class TestAHSWebServices extends JerseyTestBase {
TimelineAbout actualAbout = response.getEntity(TimelineAbout.class); TimelineAbout actualAbout = response.getEntity(TimelineAbout.class);
TimelineAbout expectedAbout = TimelineAbout expectedAbout =
TimelineUtils.createTimelineAbout("Generic History Service API"); TimelineUtils.createTimelineAbout("Generic History Service API");
Assert.assertNotNull( assertNotNull(
"Timeline service about response is null", actualAbout); actualAbout, "Timeline service about response is null");
Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout()); assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
Assert.assertEquals(expectedAbout.getTimelineServiceVersion(), assertEquals(expectedAbout.getTimelineServiceVersion(),
actualAbout.getTimelineServiceVersion()); actualAbout.getTimelineServiceVersion());
Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(), assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
actualAbout.getTimelineServiceBuildVersion()); actualAbout.getTimelineServiceBuildVersion());
Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(), assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
actualAbout.getTimelineServiceVersionBuiltOn()); actualAbout.getTimelineServiceVersionBuiltOn());
Assert.assertEquals(expectedAbout.getHadoopVersion(), assertEquals(expectedAbout.getHadoopVersion(),
actualAbout.getHadoopVersion()); actualAbout.getHadoopVersion());
Assert.assertEquals(expectedAbout.getHadoopBuildVersion(), assertEquals(expectedAbout.getHadoopBuildVersion(),
actualAbout.getHadoopBuildVersion()); actualAbout.getHadoopBuildVersion());
Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(), assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
actualAbout.getHadoopVersionBuiltOn()); actualAbout.getHadoopVersionBuiltOn());
} }
@Test @MethodSource("rounds")
public void testAppsQuery() throws Exception { @ParameterizedTest
void testAppsQuery(int round) throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = ClientResponse response =
r.path("ws").path("v1").path("applicationhistory").path("apps") r.path("ws").path("v1").path("applicationhistory").path("apps")
@ -363,15 +365,16 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject apps = json.getJSONObject("apps"); JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length()); assertEquals(1, apps.length(), "incorrect number of elements");
JSONArray array = apps.getJSONArray("app"); JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", MAX_APPS, array.length()); assertEquals(MAX_APPS, array.length(), "incorrect number of elements");
} }
@Test @MethodSource("rounds")
public void testQueueQuery() throws Exception { @ParameterizedTest
void testQueueQuery(int round) throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = ClientResponse response =
r.path("ws").path("v1").path("applicationhistory").path("apps") r.path("ws").path("v1").path("applicationhistory").path("apps")
@ -382,16 +385,18 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject apps = json.getJSONObject("apps"); JSONObject apps = json.getJSONObject("apps");
assertEquals("incorrect number of elements", 1, apps.length()); assertEquals(1, apps.length(), "incorrect number of elements");
JSONArray array = apps.getJSONArray("app"); JSONArray array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", MAX_APPS - 1, assertEquals(MAX_APPS - 1,
array.length()); array.length(),
"incorrect number of elements");
} }
@Test @MethodSource("rounds")
public void testSingleApp() throws Exception { @ParameterizedTest
void testSingleApp(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
WebResource r = resource(); WebResource r = resource();
ClientResponse response = ClientResponse response =
@ -403,7 +408,7 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject app = json.getJSONObject("app"); JSONObject app = json.getJSONObject("app");
assertEquals(appId.toString(), app.getString("appId")); assertEquals(appId.toString(), app.getString("appId"));
assertEquals("test app", app.get("name")); assertEquals("test app", app.get("name"));
@ -416,14 +421,15 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(FinalApplicationStatus.UNDEFINED.toString(), assertEquals(FinalApplicationStatus.UNDEFINED.toString(),
app.get("finalAppStatus")); app.get("finalAppStatus"));
assertEquals(YarnApplicationState.FINISHED.toString(), app.get("appState")); assertEquals(YarnApplicationState.FINISHED.toString(), app.get("appState"));
assertNotNull("Aggregate resource allocation is null", assertNotNull(app.get("aggregateResourceAllocation"),
app.get("aggregateResourceAllocation")); "Aggregate resource allocation is null");
assertNotNull("Aggregate Preempted Resource Allocation is null", assertNotNull(app.get("aggregatePreemptedResourceAllocation"),
app.get("aggregatePreemptedResourceAllocation")); "Aggregate Preempted Resource Allocation is null");
} }
@Test @MethodSource("rounds")
public void testMultipleAttempts() throws Exception { @ParameterizedTest
void testMultipleAttempts(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
WebResource r = resource(); WebResource r = resource();
ClientResponse response = ClientResponse response =
@ -438,15 +444,16 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject appAttempts = json.getJSONObject("appAttempts"); JSONObject appAttempts = json.getJSONObject("appAttempts");
assertEquals("incorrect number of elements", 1, appAttempts.length()); assertEquals(1, appAttempts.length(), "incorrect number of elements");
JSONArray array = appAttempts.getJSONArray("appAttempt"); JSONArray array = appAttempts.getJSONArray("appAttempt");
assertEquals("incorrect number of elements", MAX_APPS, array.length()); assertEquals(MAX_APPS, array.length(), "incorrect number of elements");
} }
@Test @MethodSource("rounds")
public void testSingleAttempt() throws Exception { @ParameterizedTest
void testSingleAttempt(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -465,7 +472,7 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject appAttempt = json.getJSONObject("appAttempt"); JSONObject appAttempt = json.getJSONObject("appAttempt");
assertEquals(appAttemptId.toString(), appAttempt.getString("appAttemptId")); assertEquals(appAttemptId.toString(), appAttempt.getString("appAttemptId"));
assertEquals("test host", appAttempt.getString("host")); assertEquals("test host", appAttempt.getString("host"));
@ -476,8 +483,9 @@ public class TestAHSWebServices extends JerseyTestBase {
appAttempt.get("appAttemptState")); appAttempt.get("appAttemptState"));
} }
@Test @MethodSource("rounds")
public void testMultipleContainers() throws Exception { @ParameterizedTest
void testMultipleContainers(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -495,15 +503,16 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject containers = json.getJSONObject("containers"); JSONObject containers = json.getJSONObject("containers");
assertEquals("incorrect number of elements", 1, containers.length()); assertEquals(1, containers.length(), "incorrect number of elements");
JSONArray array = containers.getJSONArray("container"); JSONArray array = containers.getJSONArray("container");
assertEquals("incorrect number of elements", MAX_APPS, array.length()); assertEquals(MAX_APPS, array.length(), "incorrect number of elements");
} }
@Test @MethodSource("rounds")
public void testSingleContainer() throws Exception { @ParameterizedTest
void testSingleContainer(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -524,7 +533,7 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class); JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length()); assertEquals(1, json.length(), "incorrect number of elements");
JSONObject container = json.getJSONObject("container"); JSONObject container = json.getJSONObject("container");
assertEquals(containerId.toString(), container.getString("containerId")); assertEquals(containerId.toString(), container.getString("containerId"));
assertEquals("test diagnostics info", container.getString("diagnosticsInfo")); assertEquals("test diagnostics info", container.getString("diagnosticsInfo"));
@ -542,8 +551,10 @@ public class TestAHSWebServices extends JerseyTestBase {
container.getString("containerState")); container.getString("containerState"));
} }
@Test(timeout = 10000) @MethodSource("rounds")
public void testContainerLogsForFinishedApps() throws Exception { @ParameterizedTest
@Timeout(10000)
void testContainerLogsForFinishedApps(int round) throws Exception {
String fileName = "syslog"; String fileName = "syslog";
String user = "user1"; String user = "user1";
NodeId nodeId = NodeId.newInstance("test host", 100); NodeId nodeId = NodeId.newInstance("test host", 100);
@ -553,7 +564,6 @@ public class TestAHSWebServices extends JerseyTestBase {
appId, 1); appId, 1);
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1); ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1);
ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100); ContainerId containerId100 = ContainerId.newContainerId(appAttemptId, 100);
TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs,
rootLogDir, appId, Collections.singletonMap(containerId1, rootLogDir, appId, Collections.singletonMap(containerId1,
"Hello." + containerId1), "Hello." + containerId1),
@ -573,7 +583,6 @@ public class TestAHSWebServices extends JerseyTestBase {
.get(ClientResponse.class); .get(ClientResponse.class);
String responseText = response.getEntity(String.class); String responseText = response.getEntity(String.class);
assertTrue(responseText.contains("Hello." + containerId1)); assertTrue(responseText.contains("Hello." + containerId1));
// Do the same test with new API // Do the same test with new API
r = resource(); r = resource();
response = r.path("ws").path("v1") response = r.path("ws").path("v1")
@ -584,7 +593,6 @@ public class TestAHSWebServices extends JerseyTestBase {
.get(ClientResponse.class); .get(ClientResponse.class);
responseText = response.getEntity(String.class); responseText = response.getEntity(String.class);
assertTrue(responseText.contains("Hello." + containerId1)); assertTrue(responseText.contains("Hello." + containerId1));
// test whether we can find container log from remote diretory if // test whether we can find container log from remote diretory if
// the containerInfo for this container could not be fetched from AHS. // the containerInfo for this container could not be fetched from AHS.
r = resource(); r = resource();
@ -596,7 +604,6 @@ public class TestAHSWebServices extends JerseyTestBase {
.get(ClientResponse.class); .get(ClientResponse.class);
responseText = response.getEntity(String.class); responseText = response.getEntity(String.class);
assertTrue(responseText.contains("Hello." + containerId100)); assertTrue(responseText.contains("Hello." + containerId100));
// Do the same test with new API // Do the same test with new API
r = resource(); r = resource();
response = r.path("ws").path("v1") response = r.path("ws").path("v1")
@ -607,14 +614,12 @@ public class TestAHSWebServices extends JerseyTestBase {
.get(ClientResponse.class); .get(ClientResponse.class);
responseText = response.getEntity(String.class); responseText = response.getEntity(String.class);
assertTrue(responseText.contains("Hello." + containerId100)); assertTrue(responseText.contains("Hello." + containerId100));
// create an application which can not be found from AHS // create an application which can not be found from AHS
ApplicationId appId100 = ApplicationId.newInstance(0, 100); ApplicationId appId100 = ApplicationId.newInstance(0, 100);
ApplicationAttemptId appAttemptId100 = ApplicationAttemptId.newInstance( ApplicationAttemptId appAttemptId100 = ApplicationAttemptId.newInstance(
appId100, 1); appId100, 1);
ContainerId containerId1ForApp100 = ContainerId.newContainerId( ContainerId containerId1ForApp100 = ContainerId.newContainerId(
appAttemptId100, 1); appAttemptId100, 1);
TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs, TestContainerLogsUtils.createContainerLogFileInRemoteFS(conf, fs,
rootLogDir, appId100, rootLogDir, appId100,
Collections.singletonMap(containerId1ForApp100, Collections.singletonMap(containerId1ForApp100,
@ -634,7 +639,6 @@ public class TestAHSWebServices extends JerseyTestBase {
"End of LogType:syslog".length() + 50) + "\n\n"; "End of LogType:syslog".length() + 50) + "\n\n";
int tailTextSize = "\nEnd of LogType:syslog\n".getBytes().length int tailTextSize = "\nEnd of LogType:syslog\n".getBytes().length
+ tailEndSeparator.getBytes().length; + tailEndSeparator.getBytes().length;
String logMessage = "Hello." + containerId1ForApp100; String logMessage = "Hello." + containerId1ForApp100;
int fileContentSize = logMessage.getBytes().length; int fileContentSize = logMessage.getBytes().length;
// specify how many bytes we should get from logs // specify how many bytes we should get from logs
@ -655,7 +659,6 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(new String(responseText.getBytes(), assertEquals(new String(responseText.getBytes(),
(fullTextSize - fileContentSize - tailTextSize), 5), (fullTextSize - fileContentSize - tailTextSize), 5),
new String(logMessage.getBytes(), 0, 5)); new String(logMessage.getBytes(), 0, 5));
// specify how many bytes we should get from logs // specify how many bytes we should get from logs
// if we specify a negative number, it would get the last n bytes from // if we specify a negative number, it would get the last n bytes from
// container log // container log
@ -674,7 +677,6 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals(new String(responseText.getBytes(), assertEquals(new String(responseText.getBytes(),
(fullTextSize - fileContentSize - tailTextSize), 5), (fullTextSize - fileContentSize - tailTextSize), 5),
new String(logMessage.getBytes(), fileContentSize - 5, 5)); new String(logMessage.getBytes(), fileContentSize - 5, 5));
// specify the bytes which is larger than the actual file size, // specify the bytes which is larger than the actual file size,
// we would get the full logs // we would get the full logs
r = resource(); r = resource();
@ -700,8 +702,10 @@ public class TestAHSWebServices extends JerseyTestBase {
assertThat(responseText.getBytes()).hasSize(fullTextSize); assertThat(responseText.getBytes()).hasSize(fullTextSize);
} }
@Test(timeout = 10000) @MethodSource("rounds")
public void testContainerLogsForRunningApps() throws Exception { @ParameterizedTest
@Timeout(10000)
void testContainerLogsForRunningApps(int round) throws Exception {
String fileName = "syslog"; String fileName = "syslog";
String user = "user1"; String user = "user1";
ApplicationId appId = ApplicationId.newInstance( ApplicationId appId = ApplicationId.newInstance(
@ -826,8 +830,10 @@ public class TestAHSWebServices extends JerseyTestBase {
+ ContainerLogAggregationType.AGGREGATED)); + ContainerLogAggregationType.AGGREGATED));
} }
@Test(timeout = 10000) @MethodSource("rounds")
public void testContainerLogsMetaForRunningApps() throws Exception { @ParameterizedTest
@Timeout(10000)
void testContainerLogsMetaForRunningApps(int round) throws Exception {
String user = "user1"; String user = "user1";
ApplicationId appId = ApplicationId.newInstance( ApplicationId appId = ApplicationId.newInstance(
1234, 1); 1234, 1);
@ -883,7 +889,8 @@ public class TestAHSWebServices extends JerseyTestBase {
.get(ClientResponse.class); .get(ClientResponse.class);
List<ContainerLogsInfo> responseText = response.getEntity(new GenericType< List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<
List<ContainerLogsInfo>>(){}); List<ContainerLogsInfo>>(){
});
assertTrue(responseText.size() == 2); assertTrue(responseText.size() == 2);
for (ContainerLogsInfo logInfo : responseText) { for (ContainerLogsInfo logInfo : responseText) {
if (logInfo.getLogType().equals( if (logInfo.getLogType().equals(
@ -911,7 +918,8 @@ public class TestAHSWebServices extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class); .get(ClientResponse.class);
responseText = response.getEntity(new GenericType< responseText = response.getEntity(new GenericType<
List<ContainerLogsInfo>>(){}); List<ContainerLogsInfo>>(){
});
assertTrue(responseText.size() == 2); assertTrue(responseText.size() == 2);
for (ContainerLogsInfo logInfo : responseText) { for (ContainerLogsInfo logInfo : responseText) {
if (logInfo.getLogType().equals( if (logInfo.getLogType().equals(
@ -929,8 +937,10 @@ public class TestAHSWebServices extends JerseyTestBase {
} }
} }
@Test(timeout = 10000) @MethodSource("rounds")
public void testContainerLogsMetaForFinishedApps() throws Exception { @ParameterizedTest
@Timeout(10000)
void testContainerLogsMetaForFinishedApps(int round) throws Exception {
ApplicationId appId = ApplicationId.newInstance(0, 1); ApplicationId appId = ApplicationId.newInstance(0, 1);
ApplicationAttemptId appAttemptId = ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1); ApplicationAttemptId.newInstance(appId, 1);
@ -951,7 +961,8 @@ public class TestAHSWebServices extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class); .get(ClientResponse.class);
List<ContainerLogsInfo> responseText = response.getEntity(new GenericType< List<ContainerLogsInfo> responseText = response.getEntity(new GenericType<
List<ContainerLogsInfo>>(){}); List<ContainerLogsInfo>>(){
});
assertTrue(responseText.size() == 1); assertTrue(responseText.size() == 1);
assertEquals(responseText.get(0).getLogType(), assertEquals(responseText.get(0).getLogType(),
ContainerLogAggregationType.AGGREGATED.toString()); ContainerLogAggregationType.AGGREGATED.toString());

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestGenericObjectMapper.java
View File

@ -17,52 +17,56 @@
*/ */
package org.apache.hadoop.yarn.server.timeline; package org.apache.hadoop.yarn.server.timeline;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.WritableComparator;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import static org.junit.Assert.assertEquals; import org.junit.jupiter.api.Test;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.WritableComparator;
import static org.junit.jupiter.api.Assertions.assertEquals;
@InterfaceAudience.Private @InterfaceAudience.Private
@InterfaceStability.Unstable @InterfaceStability.Unstable
public class TestGenericObjectMapper { public class TestGenericObjectMapper {
@Test @Test
public void testEncoding() { void testEncoding() {
testEncoding(Long.MAX_VALUE); testEncoding(Long.MAX_VALUE);
testEncoding(Long.MIN_VALUE); testEncoding(Long.MIN_VALUE);
testEncoding(0l); testEncoding(0L);
testEncoding(128l); testEncoding(128L);
testEncoding(256l); testEncoding(256L);
testEncoding(512l); testEncoding(512L);
testEncoding(-256l); testEncoding(-256L);
} }
private static void testEncoding(long l) { private static void testEncoding(long l) {
byte[] b = GenericObjectMapper.writeReverseOrderedLong(l); byte[] b = GenericObjectMapper.writeReverseOrderedLong(l);
assertEquals("error decoding", l, assertEquals(l,
GenericObjectMapper.readReverseOrderedLong(b, 0)); GenericObjectMapper.readReverseOrderedLong(b, 0),
"error decoding");
byte[] buf = new byte[16]; byte[] buf = new byte[16];
System.arraycopy(b, 0, buf, 5, 8); System.arraycopy(b, 0, buf, 5, 8);
assertEquals("error decoding at offset", l, assertEquals(l,
GenericObjectMapper.readReverseOrderedLong(buf, 5)); GenericObjectMapper.readReverseOrderedLong(buf, 5),
"error decoding at offset");
if (l > Long.MIN_VALUE) { if (l > Long.MIN_VALUE) {
byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1); byte[] a = GenericObjectMapper.writeReverseOrderedLong(l-1);
assertEquals("error preserving ordering", 1, assertEquals(1,
WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length)); WritableComparator.compareBytes(a, 0, a.length, b, 0, b.length),
"error preserving ordering");
} }
if (l < Long.MAX_VALUE) { if (l < Long.MAX_VALUE) {
byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1); byte[] c = GenericObjectMapper.writeReverseOrderedLong(l+1);
assertEquals("error preserving ordering", 1, assertEquals(1,
WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length)); WritableComparator.compareBytes(b, 0, b.length, c, 0, c.length),
"error preserving ordering");
} }
} }
@ -71,20 +75,20 @@ public class TestGenericObjectMapper {
} }
@Test @Test
public void testValueTypes() throws IOException { void testValueTypes() throws IOException {
verify(Integer.MAX_VALUE); verify(Integer.MAX_VALUE);
verify(Integer.MIN_VALUE); verify(Integer.MIN_VALUE);
assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read( assertEquals(Integer.MAX_VALUE, GenericObjectMapper.read(
GenericObjectMapper.write((long) Integer.MAX_VALUE))); GenericObjectMapper.write((long) Integer.MAX_VALUE)));
assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read( assertEquals(Integer.MIN_VALUE, GenericObjectMapper.read(
GenericObjectMapper.write((long) Integer.MIN_VALUE))); GenericObjectMapper.write((long) Integer.MIN_VALUE)));
verify((long)Integer.MAX_VALUE + 1l); verify((long) Integer.MAX_VALUE + 1L);
verify((long)Integer.MIN_VALUE - 1l); verify((long) Integer.MIN_VALUE - 1L);
verify(Long.MAX_VALUE); verify(Long.MAX_VALUE);
verify(Long.MIN_VALUE); verify(Long.MIN_VALUE);
assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42l))); assertEquals(42, GenericObjectMapper.read(GenericObjectMapper.write(42L)));
verify(42); verify(42);
verify(1.23); verify(1.23);
verify("abc"); verify("abc");

119
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
View File

@ -17,10 +17,6 @@
*/ */
package org.apache.hadoop.yarn.server.timeline; package org.apache.hadoop.yarn.server.timeline;
import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.io.File; import java.io.File;
import java.io.FileFilter; import java.io.FileFilter;
import java.io.IOException; import java.io.IOException;
@ -29,6 +25,14 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DBException;
import org.iq80.leveldb.Options;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
@ -46,14 +50,13 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelineP
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.utils.LeveldbIterator; import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DBException; import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
import org.iq80.leveldb.Options; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.After; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.Before; import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.fail;
import org.mockito.Mockito;
@InterfaceAudience.Private @InterfaceAudience.Private
@InterfaceStability.Unstable @InterfaceStability.Unstable
@ -62,7 +65,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
private File fsPath; private File fsPath;
private Configuration config = new YarnConfiguration(); private Configuration config = new YarnConfiguration();
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
fsContext = FileContext.getLocalFSFileContext(); fsContext = FileContext.getLocalFSFileContext();
fsPath = new File("target", this.getClass().getSimpleName() + fsPath = new File("target", this.getClass().getSimpleName() +
@ -79,14 +82,14 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
loadTestDomainData(); loadTestDomainData();
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
store.stop(); store.stop();
fsContext.delete(new Path(fsPath.getAbsolutePath()), true); fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
} }
@Test @Test
public void testRootDirPermission() throws IOException { void testRootDirPermission() throws IOException {
FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); FileSystem fs = FileSystem.getLocal(new YarnConfiguration());
FileStatus file = fs.getFileStatus( FileStatus file = fs.getFileStatus(
new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME)); new Path(fsPath.getAbsolutePath(), LeveldbTimelineStore.FILENAME));
@ -133,7 +136,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testCacheSizes() { void testCacheSizes() {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
assertEquals(10000, LeveldbTimelineStore.getStartTimeReadCacheSize(conf)); assertEquals(10000, LeveldbTimelineStore.getStartTimeReadCacheSize(conf));
assertEquals(10000, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf)); assertEquals(10000, LeveldbTimelineStore.getStartTimeWriteCacheSize(conf));
@ -165,7 +168,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testGetEntityTypes() throws IOException { void testGetEntityTypes() throws IOException {
List<String> entityTypes = ((LeveldbTimelineStore) store).getEntityTypes(); List<String> entityTypes = ((LeveldbTimelineStore) store).getEntityTypes();
assertEquals(7, entityTypes.size()); assertEquals(7, entityTypes.size());
assertEquals("ACL_ENTITY_TYPE_1", entityTypes.get(0)); assertEquals("ACL_ENTITY_TYPE_1", entityTypes.get(0));
@ -177,7 +180,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testDeleteEntities() throws IOException, InterruptedException { void testDeleteEntities() throws IOException, InterruptedException {
assertEquals(3, getEntities("type_1").size()); assertEquals(3, getEntities("type_1").size());
assertEquals(1, getEntities("type_2").size()); assertEquals(1, getEntities("type_2").size());
@ -214,7 +217,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testDeleteEntitiesPrimaryFilters() void testDeleteEntitiesPrimaryFilters()
throws IOException, InterruptedException { throws IOException, InterruptedException {
Map<String, Set<Object>> primaryFilter = Map<String, Set<Object>> primaryFilter =
Collections.singletonMap("user", Collections.singleton( Collections.singletonMap("user", Collections.singleton(
@ -257,7 +260,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testFromTsWithDeletion() void testFromTsWithDeletion()
throws IOException, InterruptedException { throws IOException, InterruptedException {
long l = System.currentTimeMillis(); long l = System.currentTimeMillis();
assertEquals(3, getEntitiesFromTs("type_1", l).size()); assertEquals(3, getEntitiesFromTs("type_1", l).size());
@ -284,22 +287,22 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testCheckVersion() throws IOException { void testCheckVersion() throws IOException {
LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store; LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store;
// default version // default version
Version defaultVersion = dbStore.getCurrentVersion(); Version defaultVersion = dbStore.getCurrentVersion();
Assert.assertEquals(defaultVersion, dbStore.loadVersion()); assertEquals(defaultVersion, dbStore.loadVersion());
// compatible version // compatible version
Version compatibleVersion = Version compatibleVersion =
Version.newInstance(defaultVersion.getMajorVersion(), Version.newInstance(defaultVersion.getMajorVersion(),
defaultVersion.getMinorVersion() + 2); defaultVersion.getMinorVersion() + 2);
dbStore.storeVersion(compatibleVersion); dbStore.storeVersion(compatibleVersion);
Assert.assertEquals(compatibleVersion, dbStore.loadVersion()); assertEquals(compatibleVersion, dbStore.loadVersion());
restartTimelineStore(); restartTimelineStore();
dbStore = (LeveldbTimelineStore) store; dbStore = (LeveldbTimelineStore) store;
// overwrite the compatible version // overwrite the compatible version
Assert.assertEquals(defaultVersion, dbStore.loadVersion()); assertEquals(defaultVersion, dbStore.loadVersion());
// incompatible version // incompatible version
Version incompatibleVersion = Version.newInstance( Version incompatibleVersion = Version.newInstance(
@ -307,24 +310,24 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
dbStore.storeVersion(incompatibleVersion); dbStore.storeVersion(incompatibleVersion);
try { try {
restartTimelineStore(); restartTimelineStore();
Assert.fail("Incompatible version, should expect fail here."); fail("Incompatible version, should expect fail here.");
} catch (ServiceStateException e) { } catch (ServiceStateException e) {
Assert.assertTrue("Exception message mismatch", assertTrue(e.getMessage().contains("Incompatible version for timeline store"),
e.getMessage().contains("Incompatible version for timeline store")); "Exception message mismatch");
} }
} }
@Test @Test
public void testValidateConfig() throws IOException { void testValidateConfig() throws IOException {
Configuration copyConfig = new YarnConfiguration(config); Configuration copyConfig = new YarnConfiguration(config);
try { try {
Configuration newConfig = new YarnConfiguration(copyConfig); Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0); newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
} }
try { try {
@ -333,9 +336,9 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
} }
try { try {
@ -344,9 +347,9 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
} }
try { try {
@ -357,10 +360,9 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
0); 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert assertTrue(e
.assertTrue(e
.getMessage().contains( .getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
} }
@ -372,10 +374,9 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
0); 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert assertTrue(e
.assertTrue(e
.getMessage() .getMessage()
.contains( .contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE)); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
@ -405,7 +406,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testRelatingToNonExistingEntity() throws IOException { void testRelatingToNonExistingEntity() throws IOException {
TimelineEntity entityToStore = new TimelineEntity(); TimelineEntity entityToStore = new TimelineEntity();
entityToStore.setEntityType("TEST_ENTITY_TYPE_1"); entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
entityToStore.setEntityId("TEST_ENTITY_ID_1"); entityToStore.setEntityId("TEST_ENTITY_ID_1");
@ -416,17 +417,17 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
store.put(entities); store.put(entities);
TimelineEntity entityToGet = TimelineEntity entityToGet =
store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null); store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); assertEquals("DEFAULT", entityToGet.getDomainId());
Assert.assertEquals("TEST_ENTITY_TYPE_1", assertEquals("TEST_ENTITY_TYPE_1",
entityToGet.getRelatedEntities().keySet().iterator().next()); entityToGet.getRelatedEntities().keySet().iterator().next());
Assert.assertEquals("TEST_ENTITY_ID_1", assertEquals("TEST_ENTITY_ID_1",
entityToGet.getRelatedEntities().values().iterator().next() entityToGet.getRelatedEntities().values().iterator().next()
.iterator().next()); .iterator().next());
} }
@Test @Test
public void testRelatingToOldEntityWithoutDomainId() throws IOException { void testRelatingToOldEntityWithoutDomainId() throws IOException {
// New entity is put in the default domain // New entity is put in the default domain
TimelineEntity entityToStore = new TimelineEntity(); TimelineEntity entityToStore = new TimelineEntity();
entityToStore.setEntityType("NEW_ENTITY_TYPE_1"); entityToStore.setEntityType("NEW_ENTITY_TYPE_1");
@ -439,11 +440,11 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
TimelineEntity entityToGet = TimelineEntity entityToGet =
store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertNull(entityToGet.getDomainId()); assertNull(entityToGet.getDomainId());
Assert.assertEquals("NEW_ENTITY_TYPE_1", assertEquals("NEW_ENTITY_TYPE_1",
entityToGet.getRelatedEntities().keySet().iterator().next()); entityToGet.getRelatedEntities().keySet().iterator().next());
Assert.assertEquals("NEW_ENTITY_ID_1", assertEquals("NEW_ENTITY_ID_1",
entityToGet.getRelatedEntities().values().iterator().next() entityToGet.getRelatedEntities().values().iterator().next()
.iterator().next()); .iterator().next());
@ -456,26 +457,26 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
entities = new TimelineEntities(); entities = new TimelineEntities();
entities.addEntity(entityToStore); entities.addEntity(entityToStore);
TimelinePutResponse response = store.put(entities); TimelinePutResponse response = store.put(entities);
Assert.assertEquals(1, response.getErrors().size()); assertEquals(1, response.getErrors().size());
Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, assertEquals(TimelinePutError.FORBIDDEN_RELATION,
response.getErrors().get(0).getErrorCode()); response.getErrors().get(0).getErrorCode());
entityToGet = entityToGet =
store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertNull(entityToGet.getDomainId()); assertNull(entityToGet.getDomainId());
// Still have one related entity // Still have one related entity
Assert.assertEquals(1, entityToGet.getRelatedEntities().keySet().size()); assertEquals(1, entityToGet.getRelatedEntities().keySet().size());
Assert.assertEquals(1, entityToGet.getRelatedEntities().values() assertEquals(1, entityToGet.getRelatedEntities().values()
.iterator().next().size()); .iterator().next().size());
} }
@Test
/** /**
* Test that LevelDb repair is attempted at least once during * Test that LevelDb repair is attempted at least once during
* serviceInit for LeveldbTimelineStore in case open fails the * serviceInit for LeveldbTimelineStore in case open fails the
* first time. * first time.
*/ */
public void testLevelDbRepair() throws IOException { @Test
void testLevelDbRepair() throws IOException {
LeveldbTimelineStore store = new LeveldbTimelineStore(); LeveldbTimelineStore store = new LeveldbTimelineStore();
JniDBFactory factory = Mockito.mock(JniDBFactory.class); JniDBFactory factory = Mockito.mock(JniDBFactory.class);
@ -497,7 +498,7 @@ public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
.repair(Mockito.any(File.class), Mockito.any(Options.class)); .repair(Mockito.any(File.class), Mockito.any(Options.class));
FileFilter fileFilter = new WildcardFileFilter( FileFilter fileFilter = new WildcardFileFilter(
"*" + LeveldbTimelineStore.BACKUP_EXT + "*"); "*" + LeveldbTimelineStore.BACKUP_EXT + "*");
Assert.assertTrue(path.listFiles(fileFilter).length > 0); assertTrue(path.listFiles(fileFilter).length > 0);
} finally { } finally {
store.close(); store.close();
fsContext.delete(new Path(path.getAbsolutePath()), true); fsContext.delete(new Path(path.getAbsolutePath()), true);

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestMemoryTimelineStore.java
View File

@ -18,18 +18,17 @@
package org.apache.hadoop.yarn.server.timeline; package org.apache.hadoop.yarn.server.timeline;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
public class TestMemoryTimelineStore extends TimelineStoreTestUtils { public class TestMemoryTimelineStore extends TimelineStoreTestUtils {
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
store = new MemoryTimelineStore(); store = new MemoryTimelineStore();
store.init(new YarnConfiguration()); store.init(new YarnConfiguration());
@ -39,7 +38,7 @@ public class TestMemoryTimelineStore extends TimelineStoreTestUtils {
loadTestDomainData(); loadTestDomainData();
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
store.stop(); store.stop();
} }

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDB.java
View File

@ -19,14 +19,16 @@ package org.apache.hadoop.yarn.server.timeline;
import java.io.File; import java.io.File;
import org.iq80.leveldb.DB;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.iq80.leveldb.DB;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.Before;
import org.junit.Test;
/** Test class for verification of RollingLevelDB. */ /** Test class for verification of RollingLevelDB. */
public class TestRollingLevelDB { public class TestRollingLevelDB {
@ -53,7 +55,7 @@ public class TestRollingLevelDB {
} }
}; };
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
lfs = FileSystem.getLocal(conf); lfs = FileSystem.getLocal(conf);
File fsPath = new File("target", this.getClass().getSimpleName() + File fsPath = new File("target", this.getClass().getSimpleName() +
@ -65,26 +67,26 @@ public class TestRollingLevelDB {
} }
@Test @Test
public void testInsertAfterRollPeriodRollsDB() throws Exception { void testInsertAfterRollPeriodRollsDB() throws Exception {
rollingLevelDB.init(conf); rollingLevelDB.init(conf);
long now = rollingLevelDB.currentTimeMillis(); long now = rollingLevelDB.currentTimeMillis();
DB db = rollingLevelDB.getDBForStartTime(now); DB db = rollingLevelDB.getDBForStartTime(now);
long startTime = rollingLevelDB.getStartTimeFor(db); long startTime = rollingLevelDB.getStartTimeFor(db);
Assert.assertEquals("Received level db for incorrect start time", assertEquals(rollingLevelDB.computeCurrentCheckMillis(now),
rollingLevelDB.computeCurrentCheckMillis(now), startTime,
startTime); "Received level db for incorrect start time");
now = rollingLevelDB.getNextRollingTimeMillis(); now = rollingLevelDB.getNextRollingTimeMillis();
rollingLevelDB.setCurrentTimeMillis(now); rollingLevelDB.setCurrentTimeMillis(now);
db = rollingLevelDB.getDBForStartTime(now); db = rollingLevelDB.getDBForStartTime(now);
startTime = rollingLevelDB.getStartTimeFor(db); startTime = rollingLevelDB.getStartTimeFor(db);
Assert.assertEquals("Received level db for incorrect start time", assertEquals(rollingLevelDB.computeCurrentCheckMillis(now),
rollingLevelDB.computeCurrentCheckMillis(now), startTime,
startTime); "Received level db for incorrect start time");
} }
@Test @Test
public void testInsertForPreviousPeriodAfterRollPeriodRollsDB() void testInsertForPreviousPeriodAfterRollPeriodRollsDB()
throws Exception { throws Exception {
rollingLevelDB.init(conf); rollingLevelDB.init(conf);
@ -93,8 +95,8 @@ public class TestRollingLevelDB {
rollingLevelDB.setCurrentTimeMillis(now); rollingLevelDB.setCurrentTimeMillis(now);
DB db = rollingLevelDB.getDBForStartTime(now - 1); DB db = rollingLevelDB.getDBForStartTime(now - 1);
long startTime = rollingLevelDB.getStartTimeFor(db); long startTime = rollingLevelDB.getStartTimeFor(db);
Assert.assertEquals("Received level db for incorrect start time", assertEquals(rollingLevelDB.computeCurrentCheckMillis(now - 1),
rollingLevelDB.computeCurrentCheckMillis(now - 1), startTime,
startTime); "Received level db for incorrect start time");
} }
} }

115
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestRollingLevelDBTimelineStore.java
View File

@ -17,13 +17,18 @@
*/ */
package org.apache.hadoop.yarn.server.timeline; package org.apache.hadoop.yarn.server.timeline;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import java.io.File; import java.io.File;
import java.io.FilenameFilter; import java.io.FilenameFilter;
import java.io.IOException; import java.io.IOException;
import org.eclipse.jetty.util.log.Log;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.Options;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
@ -41,14 +46,10 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelineP
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.records.Version;
import org.fusesource.leveldbjni.JniDBFactory; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.iq80.leveldb.Options; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.junit.After; import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.fail;
import org.junit.Before;
import org.junit.Test;
import org.eclipse.jetty.util.log.Log;
import org.mockito.Mockito;
/** Test class to verify RollingLevelDBTimelineStore. */ /** Test class to verify RollingLevelDBTimelineStore. */
@InterfaceAudience.Private @InterfaceAudience.Private
@ -58,7 +59,7 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
private File fsPath; private File fsPath;
private Configuration config = new YarnConfiguration(); private Configuration config = new YarnConfiguration();
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
fsContext = FileContext.getLocalFSFileContext(); fsContext = FileContext.getLocalFSFileContext();
fsPath = new File("target", this.getClass().getSimpleName() + fsPath = new File("target", this.getClass().getSimpleName() +
@ -75,14 +76,14 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
loadTestDomainData(); loadTestDomainData();
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
store.stop(); store.stop();
fsContext.delete(new Path(fsPath.getAbsolutePath()), true); fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
} }
@Test @Test
public void testRootDirPermission() throws IOException { void testRootDirPermission() throws IOException {
FileSystem fs = FileSystem.getLocal(new YarnConfiguration()); FileSystem fs = FileSystem.getLocal(new YarnConfiguration());
FileStatus file = fs.getFileStatus(new Path(fsPath.getAbsolutePath(), FileStatus file = fs.getFileStatus(new Path(fsPath.getAbsolutePath(),
RollingLevelDBTimelineStore.FILENAME)); RollingLevelDBTimelineStore.FILENAME));
@ -130,7 +131,7 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testCacheSizes() { void testCacheSizes() {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
assertEquals(10000, assertEquals(10000,
RollingLevelDBTimelineStore.getStartTimeReadCacheSize(conf)); RollingLevelDBTimelineStore.getStartTimeReadCacheSize(conf));
@ -150,22 +151,22 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testCheckVersion() throws IOException { void testCheckVersion() throws IOException {
RollingLevelDBTimelineStore dbStore = (RollingLevelDBTimelineStore) store; RollingLevelDBTimelineStore dbStore = (RollingLevelDBTimelineStore) store;
// default version // default version
Version defaultVersion = dbStore.getCurrentVersion(); Version defaultVersion = dbStore.getCurrentVersion();
Assert.assertEquals(defaultVersion, dbStore.loadVersion()); assertEquals(defaultVersion, dbStore.loadVersion());
// compatible version // compatible version
Version compatibleVersion = Version compatibleVersion =
Version.newInstance(defaultVersion.getMajorVersion(), Version.newInstance(defaultVersion.getMajorVersion(),
defaultVersion.getMinorVersion() + 2); defaultVersion.getMinorVersion() + 2);
dbStore.storeVersion(compatibleVersion); dbStore.storeVersion(compatibleVersion);
Assert.assertEquals(compatibleVersion, dbStore.loadVersion()); assertEquals(compatibleVersion, dbStore.loadVersion());
restartTimelineStore(); restartTimelineStore();
dbStore = (RollingLevelDBTimelineStore) store; dbStore = (RollingLevelDBTimelineStore) store;
// overwrite the compatible version // overwrite the compatible version
Assert.assertEquals(defaultVersion, dbStore.loadVersion()); assertEquals(defaultVersion, dbStore.loadVersion());
// incompatible version // incompatible version
Version incompatibleVersion = Version incompatibleVersion =
@ -174,24 +175,24 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
dbStore.storeVersion(incompatibleVersion); dbStore.storeVersion(incompatibleVersion);
try { try {
restartTimelineStore(); restartTimelineStore();
Assert.fail("Incompatible version, should expect fail here."); fail("Incompatible version, should expect fail here.");
} catch (ServiceStateException e) { } catch (ServiceStateException e) {
Assert.assertTrue("Exception message mismatch", assertTrue(e.getMessage().contains("Incompatible version for timeline store"),
e.getMessage().contains("Incompatible version for timeline store")); "Exception message mismatch");
} }
} }
@Test @Test
public void testValidateConfig() throws IOException { void testValidateConfig() throws IOException {
Configuration copyConfig = new YarnConfiguration(config); Configuration copyConfig = new YarnConfiguration(config);
try { try {
Configuration newConfig = new YarnConfiguration(copyConfig); Configuration newConfig = new YarnConfiguration(copyConfig);
newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0); newConfig.setLong(YarnConfiguration.TIMELINE_SERVICE_TTL_MS, 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_TTL_MS)); YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
} }
try { try {
@ -200,9 +201,9 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS, 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS)); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
} }
try { try {
@ -211,9 +212,9 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE, -1);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE)); YarnConfiguration.TIMELINE_SERVICE_LEVELDB_READ_CACHE_SIZE));
} }
try { try {
@ -223,9 +224,9 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
0); 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration YarnConfiguration
.TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE)); .TIMELINE_SERVICE_LEVELDB_START_TIME_READ_CACHE_SIZE));
} }
@ -237,9 +238,9 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
0); 0);
config = newConfig; config = newConfig;
restartTimelineStore(); restartTimelineStore();
Assert.fail(); fail();
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
YarnConfiguration YarnConfiguration
.TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE)); .TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE));
} }
@ -268,7 +269,7 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
} }
@Test @Test
public void testRelatingToNonExistingEntity() throws IOException { void testRelatingToNonExistingEntity() throws IOException {
TimelineEntity entityToStore = new TimelineEntity(); TimelineEntity entityToStore = new TimelineEntity();
entityToStore.setEntityType("TEST_ENTITY_TYPE_1"); entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
entityToStore.setEntityId("TEST_ENTITY_ID_1"); entityToStore.setEntityId("TEST_ENTITY_ID_1");
@ -279,17 +280,17 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
store.put(entities); store.put(entities);
TimelineEntity entityToGet = TimelineEntity entityToGet =
store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null); store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); assertEquals("DEFAULT", entityToGet.getDomainId());
Assert.assertEquals("TEST_ENTITY_TYPE_1", assertEquals("TEST_ENTITY_TYPE_1",
entityToGet.getRelatedEntities().keySet().iterator().next()); entityToGet.getRelatedEntities().keySet().iterator().next());
Assert.assertEquals("TEST_ENTITY_ID_1", assertEquals("TEST_ENTITY_ID_1",
entityToGet.getRelatedEntities().values().iterator().next() entityToGet.getRelatedEntities().values().iterator().next()
.iterator().next()); .iterator().next());
} }
@Test @Test
public void testRelatingToEntityInSamePut() throws IOException { void testRelatingToEntityInSamePut() throws IOException {
TimelineEntity entityToRelate = new TimelineEntity(); TimelineEntity entityToRelate = new TimelineEntity();
entityToRelate.setEntityType("TEST_ENTITY_TYPE_2"); entityToRelate.setEntityType("TEST_ENTITY_TYPE_2");
entityToRelate.setEntityId("TEST_ENTITY_ID_2"); entityToRelate.setEntityId("TEST_ENTITY_ID_2");
@ -305,17 +306,17 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
store.put(entities); store.put(entities);
TimelineEntity entityToGet = TimelineEntity entityToGet =
store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null); store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertEquals("TEST_DOMAIN", entityToGet.getDomainId()); assertEquals("TEST_DOMAIN", entityToGet.getDomainId());
Assert.assertEquals("TEST_ENTITY_TYPE_1", assertEquals("TEST_ENTITY_TYPE_1",
entityToGet.getRelatedEntities().keySet().iterator().next()); entityToGet.getRelatedEntities().keySet().iterator().next());
Assert.assertEquals("TEST_ENTITY_ID_1", assertEquals("TEST_ENTITY_ID_1",
entityToGet.getRelatedEntities().values().iterator().next() entityToGet.getRelatedEntities().values().iterator().next()
.iterator().next()); .iterator().next());
} }
@Test @Test
public void testRelatingToOldEntityWithoutDomainId() throws IOException { void testRelatingToOldEntityWithoutDomainId() throws IOException {
// New entity is put in the default domain // New entity is put in the default domain
TimelineEntity entityToStore = new TimelineEntity(); TimelineEntity entityToStore = new TimelineEntity();
entityToStore.setEntityType("NEW_ENTITY_TYPE_1"); entityToStore.setEntityType("NEW_ENTITY_TYPE_1");
@ -328,11 +329,11 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
TimelineEntity entityToGet = TimelineEntity entityToGet =
store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); assertEquals("DEFAULT", entityToGet.getDomainId());
Assert.assertEquals("NEW_ENTITY_TYPE_1", assertEquals("NEW_ENTITY_TYPE_1",
entityToGet.getRelatedEntities().keySet().iterator().next()); entityToGet.getRelatedEntities().keySet().iterator().next());
Assert.assertEquals("NEW_ENTITY_ID_1", assertEquals("NEW_ENTITY_ID_1",
entityToGet.getRelatedEntities().values().iterator().next() entityToGet.getRelatedEntities().values().iterator().next()
.iterator().next()); .iterator().next());
@ -345,16 +346,16 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
entities = new TimelineEntities(); entities = new TimelineEntities();
entities.addEntity(entityToStore); entities.addEntity(entityToStore);
TimelinePutResponse response = store.put(entities); TimelinePutResponse response = store.put(entities);
Assert.assertEquals(1, response.getErrors().size()); assertEquals(1, response.getErrors().size());
Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, assertEquals(TimelinePutError.FORBIDDEN_RELATION,
response.getErrors().get(0).getErrorCode()); response.getErrors().get(0).getErrorCode());
entityToGet = entityToGet =
store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
Assert.assertNotNull(entityToGet); assertNotNull(entityToGet);
Assert.assertEquals("DEFAULT", entityToGet.getDomainId()); assertEquals("DEFAULT", entityToGet.getDomainId());
// Still have one related entity // Still have one related entity
Assert.assertEquals(1, entityToGet.getRelatedEntities().keySet().size()); assertEquals(1, entityToGet.getRelatedEntities().keySet().size());
Assert.assertEquals(1, entityToGet.getRelatedEntities().values() assertEquals(1, entityToGet.getRelatedEntities().values()
.iterator().next().size()); .iterator().next().size());
} }
@ -423,12 +424,12 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
Log.getLog().info("Duration for " + num + ": " + duration); Log.getLog().info("Duration for " + num + ": " + duration);
} }
@Test
/** /**
* Test that RollingLevelDb repair is attempted at least once during * Test that RollingLevelDb repair is attempted at least once during
* serviceInit for RollingLeveldbTimelineStore in case open fails the * serviceInit for RollingLeveldbTimelineStore in case open fails the
* first time. * first time.
*/ public void testLevelDbRepair() throws IOException { */ @Test
void testLevelDbRepair() throws IOException {
RollingLevelDBTimelineStore store = new RollingLevelDBTimelineStore(); RollingLevelDBTimelineStore store = new RollingLevelDBTimelineStore();
JniDBFactory factory = Mockito.mock(JniDBFactory.class); JniDBFactory factory = Mockito.mock(JniDBFactory.class);
Mockito.when(factory.open(Mockito.any(File.class), Mockito.any(Options.class))) Mockito.when(factory.open(Mockito.any(File.class), Mockito.any(Options.class)))
@ -445,7 +446,7 @@ public class TestRollingLevelDBTimelineStore extends TimelineStoreTestUtils {
.repair(Mockito.any(File.class), Mockito.any(Options.class)); .repair(Mockito.any(File.class), Mockito.any(Options.class));
FilenameFilter fileFilter = FilenameFilter fileFilter =
new WildcardFileFilter("*" + RollingLevelDBTimelineStore.BACKUP_EXT + "*"); new WildcardFileFilter("*" + RollingLevelDBTimelineStore.BACKUP_EXT + "*");
Assert.assertTrue(new File(path.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME) assertTrue(new File(path.getAbsolutePath(), RollingLevelDBTimelineStore.FILENAME)
.list(fileFilter).length > 0); .list(fileFilter).length > 0);
} finally { } finally {
store.close(); store.close();

74
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestTimelineDataManager.java
View File

@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.timeline;
import java.io.File; import java.io.File;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@ -30,10 +34,10 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager; import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.junit.After;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.Before; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertNull;
public class TestTimelineDataManager extends TimelineStoreTestUtils { public class TestTimelineDataManager extends TimelineStoreTestUtils {
@ -43,7 +47,7 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
private TimelineDataManager dataManaer; private TimelineDataManager dataManaer;
private static TimelineACLsManager aclsManager; private static TimelineACLsManager aclsManager;
private static AdminACLsManager adminACLsManager; private static AdminACLsManager adminACLsManager;
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
fsPath = new File("target", this.getClass().getSimpleName() + fsPath = new File("target", this.getClass().getSimpleName() +
"-tmpDir").getAbsoluteFile(); "-tmpDir").getAbsoluteFile();
@ -70,7 +74,7 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
adminACLsManager = new AdminACLsManager(conf); adminACLsManager = new AdminACLsManager(conf);
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
if (store != null) { if (store != null) {
store.stop(); store.stop();
@ -81,27 +85,27 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
} }
@Test @Test
public void testGetOldEntityWithOutDomainId() throws Exception { void testGetOldEntityWithOutDomainId() throws Exception {
TimelineEntity entity = dataManaer.getEntity( TimelineEntity entity = dataManaer.getEntity(
"OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1", null, "OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1", null,
UserGroupInformation.getCurrentUser()); UserGroupInformation.getCurrentUser());
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("OLD_ENTITY_ID_1", entity.getEntityId()); assertEquals("OLD_ENTITY_ID_1", entity.getEntityId());
Assert.assertEquals("OLD_ENTITY_TYPE_1", entity.getEntityType()); assertEquals("OLD_ENTITY_TYPE_1", entity.getEntityType());
Assert.assertEquals( assertEquals(
TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId()); TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId());
} }
@Test @Test
public void testGetEntitiesAclEnabled() throws Exception { void testGetEntitiesAclEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
aclsManager.setAdminACLsManager(adminACLsManager); aclsManager.setAdminACLsManager(adminACLsManager);
try { try {
TimelineEntities entities = dataManaer.getEntities( TimelineEntities entities = dataManaer.getEntities(
"ACL_ENTITY_TYPE_1", null, null, null, null, null, null, 1l, null, "ACL_ENTITY_TYPE_1", null, null, null, null, null, null, 1L, null,
UserGroupInformation.createUserForTesting("owner_1", new String[]{"group1"})); UserGroupInformation.createUserForTesting("owner_1", new String[]{"group1"}));
Assert.assertEquals(1, entities.getEntities().size()); assertEquals(1, entities.getEntities().size());
Assert.assertEquals("ACL_ENTITY_ID_11", assertEquals("ACL_ENTITY_ID_11",
entities.getEntities().get(0).getEntityId()); entities.getEntities().get(0).getEntityId());
} finally { } finally {
aclsManager.setAdminACLsManager(oldAdminACLsManager); aclsManager.setAdminACLsManager(oldAdminACLsManager);
@ -109,27 +113,27 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
} }
@Test @Test
public void testGetOldEntitiesWithOutDomainId() throws Exception { void testGetOldEntitiesWithOutDomainId() throws Exception {
TimelineEntities entities = dataManaer.getEntities( TimelineEntities entities = dataManaer.getEntities(
"OLD_ENTITY_TYPE_1", null, null, null, null, null, null, null, null, "OLD_ENTITY_TYPE_1", null, null, null, null, null, null, null, null,
UserGroupInformation.getCurrentUser()); UserGroupInformation.getCurrentUser());
Assert.assertEquals(2, entities.getEntities().size()); assertEquals(2, entities.getEntities().size());
Assert.assertEquals("OLD_ENTITY_ID_2", assertEquals("OLD_ENTITY_ID_2",
entities.getEntities().get(0).getEntityId()); entities.getEntities().get(0).getEntityId());
Assert.assertEquals("OLD_ENTITY_TYPE_1", assertEquals("OLD_ENTITY_TYPE_1",
entities.getEntities().get(0).getEntityType()); entities.getEntities().get(0).getEntityType());
Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
entities.getEntities().get(0).getDomainId()); entities.getEntities().get(0).getDomainId());
Assert.assertEquals("OLD_ENTITY_ID_1", assertEquals("OLD_ENTITY_ID_1",
entities.getEntities().get(1).getEntityId()); entities.getEntities().get(1).getEntityId());
Assert.assertEquals("OLD_ENTITY_TYPE_1", assertEquals("OLD_ENTITY_TYPE_1",
entities.getEntities().get(1).getEntityType()); entities.getEntities().get(1).getEntityType());
Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
entities.getEntities().get(1).getDomainId()); entities.getEntities().get(1).getDomainId());
} }
@Test @Test
public void testUpdatingOldEntityWithoutDomainId() throws Exception { void testUpdatingOldEntityWithoutDomainId() throws Exception {
// Set the domain to the default domain when updating // Set the domain to the default domain when updating
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
entity.setEntityType("OLD_ENTITY_TYPE_1"); entity.setEntityType("OLD_ENTITY_TYPE_1");
@ -140,16 +144,16 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
entities.addEntity(entity); entities.addEntity(entity);
TimelinePutResponse response = dataManaer.postEntities( TimelinePutResponse response = dataManaer.postEntities(
entities, UserGroupInformation.getCurrentUser()); entities, UserGroupInformation.getCurrentUser());
Assert.assertEquals(0, response.getErrors().size()); assertEquals(0, response.getErrors().size());
entity = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); entity = store.getEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
Assert.assertNotNull(entity); assertNotNull(entity);
// Even in leveldb, the domain is updated to the default domain Id // Even in leveldb, the domain is updated to the default domain Id
Assert.assertEquals( assertEquals(
TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId()); TimelineDataManager.DEFAULT_DOMAIN_ID, entity.getDomainId());
Assert.assertEquals(1, entity.getOtherInfo().size()); assertEquals(1, entity.getOtherInfo().size());
Assert.assertEquals("NEW_OTHER_INFO_KEY", assertEquals("NEW_OTHER_INFO_KEY",
entity.getOtherInfo().keySet().iterator().next()); entity.getOtherInfo().keySet().iterator().next());
Assert.assertEquals("NEW_OTHER_INFO_VALUE", assertEquals("NEW_OTHER_INFO_VALUE",
entity.getOtherInfo().values().iterator().next()); entity.getOtherInfo().values().iterator().next());
// Set the domain to the non-default domain when updating // Set the domain to the non-default domain when updating
@ -162,15 +166,15 @@ public class TestTimelineDataManager extends TimelineStoreTestUtils {
entities.addEntity(entity); entities.addEntity(entity);
response = dataManaer.postEntities( response = dataManaer.postEntities(
entities, UserGroupInformation.getCurrentUser()); entities, UserGroupInformation.getCurrentUser());
Assert.assertEquals(1, response.getErrors().size()); assertEquals(1, response.getErrors().size());
Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED,
response.getErrors().get(0).getErrorCode()); response.getErrors().get(0).getErrorCode());
entity = store.getEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null); entity = store.getEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
Assert.assertNotNull(entity); assertNotNull(entity);
// In leveldb, the domain Id is still null // In leveldb, the domain Id is still null
Assert.assertNull(entity.getDomainId()); assertNull(entity.getDomainId());
// Updating is not executed // Updating is not executed
Assert.assertEquals(0, entity.getOtherInfo().size()); assertEquals(0, entity.getOtherInfo().size());
} }
} }

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
View File

@ -17,10 +17,6 @@
*/ */
package org.apache.hadoop.yarn.server.timeline; package org.apache.hadoop.yarn.server.timeline;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
@ -35,16 +31,20 @@ import java.util.Set;
import java.util.SortedSet; import java.util.SortedSet;
import java.util.TreeSet; import java.util.TreeSet;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents.EventsOfOneEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TimelineStoreTestUtils { public class TimelineStoreTestUtils {
protected static final List<TimelineEvent> EMPTY_EVENTS = protected static final List<TimelineEvent> EMPTY_EVENTS =
@ -503,18 +503,24 @@ public class TimelineStoreTestUtils {
public void testGetEntities() throws IOException { public void testGetEntities() throws IOException {
// test getting entities // test getting entities
assertEquals("nonzero entities size for nonexistent type", 0, assertEquals(0,
getEntities("type_0").size()); getEntities("type_0").size(),
assertEquals("nonzero entities size for nonexistent type", 0, "nonzero entities size for nonexistent type");
getEntities("type_3").size()); assertEquals(0,
assertEquals("nonzero entities size for nonexistent type", 0, getEntities("type_3").size(),
getEntities("type_6").size()); "nonzero entities size for nonexistent type");
assertEquals("nonzero entities size for nonexistent type", 0, assertEquals(0,
getEntitiesWithPrimaryFilter("type_0", userFilter).size()); getEntities("type_6").size(),
assertEquals("nonzero entities size for nonexistent type", 0, "nonzero entities size for nonexistent type");
getEntitiesWithPrimaryFilter("type_3", userFilter).size()); assertEquals(0,
assertEquals("nonzero entities size for nonexistent type", 0, getEntitiesWithPrimaryFilter("type_0", userFilter).size(),
getEntitiesWithPrimaryFilter("type_6", userFilter).size()); "nonzero entities size for nonexistent type");
assertEquals(0,
getEntitiesWithPrimaryFilter("type_3", userFilter).size(),
"nonzero entities size for nonexistent type");
assertEquals(0,
getEntitiesWithPrimaryFilter("type_6", userFilter).size(),
"nonzero entities size for nonexistent type");
List<TimelineEntity> entities = getEntities("type_1"); List<TimelineEntity> entities = getEntities("type_1");
assertEquals(3, entities.size()); assertEquals(3, entities.size());
@ -681,15 +687,18 @@ public class TimelineStoreTestUtils {
public void testGetEntitiesWithPrimaryFilters() throws IOException { public void testGetEntitiesWithPrimaryFilters() throws IOException {
// test using primary filter // test using primary filter
assertEquals("nonzero entities size for primary filter", 0, assertEquals(0,
getEntitiesWithPrimaryFilter("type_1", getEntitiesWithPrimaryFilter("type_1",
new NameValuePair("none", "none")).size()); new NameValuePair("none", "none")).size(),
assertEquals("nonzero entities size for primary filter", 0, "nonzero entities size for primary filter");
assertEquals(0,
getEntitiesWithPrimaryFilter("type_2", getEntitiesWithPrimaryFilter("type_2",
new NameValuePair("none", "none")).size()); new NameValuePair("none", "none")).size(),
assertEquals("nonzero entities size for primary filter", 0, "nonzero entities size for primary filter");
assertEquals(0,
getEntitiesWithPrimaryFilter("type_3", getEntitiesWithPrimaryFilter("type_3",
new NameValuePair("none", "none")).size()); new NameValuePair("none", "none")).size(),
"nonzero entities size for primary filter");
List<TimelineEntity> entities = getEntitiesWithPrimaryFilter("type_1", List<TimelineEntity> entities = getEntitiesWithPrimaryFilter("type_1",
userFilter); userFilter);
@ -927,13 +936,12 @@ public class TimelineStoreTestUtils {
if (primaryFilters == null) { if (primaryFilters == null) {
assertNull(retrievedEntityInfo.getPrimaryFilters()); assertNull(retrievedEntityInfo.getPrimaryFilters());
} else { } else {
assertTrue(primaryFilters.equals( assertEquals(primaryFilters, retrievedEntityInfo.getPrimaryFilters());
retrievedEntityInfo.getPrimaryFilters()));
} }
if (otherInfo == null) { if (otherInfo == null) {
assertNull(retrievedEntityInfo.getOtherInfo()); assertNull(retrievedEntityInfo.getOtherInfo());
} else { } else {
assertTrue(otherInfo.equals(retrievedEntityInfo.getOtherInfo())); assertEquals(otherInfo, retrievedEntityInfo.getOtherInfo());
} }
} }

112
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/recovery/TestLeveldbTimelineStateStore.java
View File

@ -18,14 +18,13 @@
package org.apache.hadoop.yarn.server.timeline.recovery; package org.apache.hadoop.yarn.server.timeline.recovery;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@ -36,10 +35,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier; import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState; import org.apache.hadoop.yarn.server.timeline.recovery.TimelineStateStore.TimelineServiceState;
import org.junit.After;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.Before; import static org.junit.jupiter.api.Assertions.assertFalse;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TestLeveldbTimelineStateStore { public class TestLeveldbTimelineStateStore {
@ -48,7 +48,7 @@ public class TestLeveldbTimelineStateStore {
private Configuration conf; private Configuration conf;
private TimelineStateStore store; private TimelineStateStore store;
@Before @BeforeEach
public void setup() throws Exception { public void setup() throws Exception {
fsPath = new File("target", getClass().getSimpleName() + fsPath = new File("target", getClass().getSimpleName() +
"-tmpDir").getAbsoluteFile(); "-tmpDir").getAbsoluteFile();
@ -63,7 +63,7 @@ public class TestLeveldbTimelineStateStore {
fsPath.getAbsolutePath()); fsPath.getAbsolutePath());
} }
@After @AfterEach
public void tearDown() throws Exception { public void tearDown() throws Exception {
if (store != null) { if (store != null) {
store.stop(); store.stop();
@ -81,11 +81,11 @@ public class TestLeveldbTimelineStateStore {
} }
@Test @Test
public void testTokenStore() throws Exception { void testTokenStore() throws Exception {
initAndStartTimelineServiceStateStoreService(); initAndStartTimelineServiceStateStoreService();
TimelineServiceState state = store.loadState(); TimelineServiceState state = store.loadState();
assertTrue("token state not empty", state.tokenState.isEmpty()); assertTrue(state.tokenState.isEmpty(), "token state not empty");
assertTrue("key state not empty", state.tokenMasterKeyState.isEmpty()); assertTrue(state.tokenMasterKeyState.isEmpty(), "key state not empty");
final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes()); final DelegationKey key1 = new DelegationKey(1, 2, "keyData1".getBytes());
final TimelineDelegationTokenIdentifier token1 = final TimelineDelegationTokenIdentifier token1 =
@ -120,19 +120,23 @@ public class TestLeveldbTimelineStateStore {
initAndStartTimelineServiceStateStoreService(); initAndStartTimelineServiceStateStoreService();
state = store.loadState(); state = store.loadState();
assertEquals("incorrect loaded token count", 2, state.tokenState.size()); assertEquals(2, state.tokenState.size(), "incorrect loaded token count");
assertTrue("missing token 1", state.tokenState.containsKey(token1)); assertTrue(state.tokenState.containsKey(token1), "missing token 1");
assertEquals("incorrect token 1 date", tokenDate1, assertEquals(tokenDate1,
state.tokenState.get(token1)); state.tokenState.get(token1),
assertTrue("missing token 2", state.tokenState.containsKey(token2)); "incorrect token 1 date");
assertEquals("incorrect token 2 date", tokenDate2, assertTrue(state.tokenState.containsKey(token2), "missing token 2");
state.tokenState.get(token2)); assertEquals(tokenDate2,
assertEquals("incorrect master key count", 1, state.tokenState.get(token2),
state.tokenMasterKeyState.size()); "incorrect token 2 date");
assertTrue("missing master key 1", assertEquals(1,
state.tokenMasterKeyState.contains(key1)); state.tokenMasterKeyState.size(),
assertEquals("incorrect latest sequence number", 12345678, "incorrect master key count");
state.getLatestSequenceNumber()); assertTrue(state.tokenMasterKeyState.contains(key1),
"missing master key 1");
assertEquals(12345678,
state.getLatestSequenceNumber(),
"incorrect latest sequence number");
final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes()); final DelegationKey key2 = new DelegationKey(3, 4, "keyData2".getBytes());
final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes()); final DelegationKey key3 = new DelegationKey(5, 6, "keyData3".getBytes());
@ -154,46 +158,50 @@ public class TestLeveldbTimelineStateStore {
initAndStartTimelineServiceStateStoreService(); initAndStartTimelineServiceStateStoreService();
state = store.loadState(); state = store.loadState();
assertEquals("incorrect loaded token count", 2, state.tokenState.size()); assertEquals(2, state.tokenState.size(), "incorrect loaded token count");
assertFalse("token 1 not removed", state.tokenState.containsKey(token1)); assertFalse(state.tokenState.containsKey(token1), "token 1 not removed");
assertTrue("missing token 2", state.tokenState.containsKey(token2)); assertTrue(state.tokenState.containsKey(token2), "missing token 2");
assertEquals("incorrect token 2 date", newTokenDate2, assertEquals(newTokenDate2,
state.tokenState.get(token2)); state.tokenState.get(token2),
assertTrue("missing token 3", state.tokenState.containsKey(token3)); "incorrect token 2 date");
assertEquals("incorrect token 3 date", tokenDate3, assertTrue(state.tokenState.containsKey(token3), "missing token 3");
state.tokenState.get(token3)); assertEquals(tokenDate3,
assertEquals("incorrect master key count", 2, state.tokenState.get(token3),
state.tokenMasterKeyState.size()); "incorrect token 3 date");
assertFalse("master key 1 not removed", assertEquals(2,
state.tokenMasterKeyState.contains(key1)); state.tokenMasterKeyState.size(),
assertTrue("missing master key 2", "incorrect master key count");
state.tokenMasterKeyState.contains(key2)); assertFalse(state.tokenMasterKeyState.contains(key1),
assertTrue("missing master key 3", "master key 1 not removed");
state.tokenMasterKeyState.contains(key3)); assertTrue(state.tokenMasterKeyState.contains(key2),
assertEquals("incorrect latest sequence number", 12345679, "missing master key 2");
state.getLatestSequenceNumber()); assertTrue(state.tokenMasterKeyState.contains(key3),
"missing master key 3");
assertEquals(12345679,
state.getLatestSequenceNumber(),
"incorrect latest sequence number");
store.close(); store.close();
} }
@Test @Test
public void testCheckVersion() throws IOException { void testCheckVersion() throws IOException {
LeveldbTimelineStateStore store = LeveldbTimelineStateStore store =
initAndStartTimelineServiceStateStoreService(); initAndStartTimelineServiceStateStoreService();
// default version // default version
Version defaultVersion = store.getCurrentVersion(); Version defaultVersion = store.getCurrentVersion();
Assert.assertEquals(defaultVersion, store.loadVersion()); assertEquals(defaultVersion, store.loadVersion());
// compatible version // compatible version
Version compatibleVersion = Version compatibleVersion =
Version.newInstance(defaultVersion.getMajorVersion(), Version.newInstance(defaultVersion.getMajorVersion(),
defaultVersion.getMinorVersion() + 2); defaultVersion.getMinorVersion() + 2);
store.storeVersion(compatibleVersion); store.storeVersion(compatibleVersion);
Assert.assertEquals(compatibleVersion, store.loadVersion()); assertEquals(compatibleVersion, store.loadVersion());
store.stop(); store.stop();
// overwrite the compatible version // overwrite the compatible version
store = initAndStartTimelineServiceStateStoreService(); store = initAndStartTimelineServiceStateStoreService();
Assert.assertEquals(defaultVersion, store.loadVersion()); assertEquals(defaultVersion, store.loadVersion());
// incompatible version // incompatible version
Version incompatibleVersion = Version incompatibleVersion =
@ -204,10 +212,10 @@ public class TestLeveldbTimelineStateStore {
try { try {
initAndStartTimelineServiceStateStoreService(); initAndStartTimelineServiceStateStoreService();
Assert.fail("Incompatible version, should expect fail here."); fail("Incompatible version, should expect fail here.");
} catch (ServiceStateException e) { } catch (ServiceStateException e) {
Assert.assertTrue("Exception message mismatch", assertTrue(e.getMessage().contains("Incompatible version for timeline state store"),
e.getMessage().contains("Incompatible version for timeline state store")); "Exception message mismatch");
} }
} }
} }

116
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineACLsManager.java
View File

@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.timeline.security;
import java.io.IOException; import java.io.IOException;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@ -29,8 +31,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.junit.Assert;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class TestTimelineACLsManager { public class TestTimelineACLsManager {
@ -45,7 +49,7 @@ public class TestTimelineACLsManager {
} }
@Test @Test
public void testYarnACLsNotEnabledForEntity() throws Exception { void testYarnACLsNotEnabledForEntity() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
TimelineACLsManager timelineACLsManager = TimelineACLsManager timelineACLsManager =
@ -56,20 +60,20 @@ public class TestTimelineACLsManager {
TimelineStore.SystemFilter.ENTITY_OWNER TimelineStore.SystemFilter.ENTITY_OWNER
.toString(), "owner"); .toString(), "owner");
entity.setDomainId("domain_id_1"); entity.setDomainId("domain_id_1");
Assert.assertTrue( assertTrue(
"Always true when ACLs are not enabled",
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("user"), UserGroupInformation.createRemoteUser("user"),
ApplicationAccessType.VIEW_APP, entity)); ApplicationAccessType.VIEW_APP, entity),
Assert.assertTrue( "Always true when ACLs are not enabled");
"Always true when ACLs are not enabled", assertTrue(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("user"), UserGroupInformation.createRemoteUser("user"),
ApplicationAccessType.MODIFY_APP, entity)); ApplicationAccessType.MODIFY_APP, entity),
"Always true when ACLs are not enabled");
} }
@Test @Test
public void testYarnACLsEnabledForEntity() throws Exception { void testYarnACLsEnabledForEntity() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
@ -81,51 +85,51 @@ public class TestTimelineACLsManager {
TimelineStore.SystemFilter.ENTITY_OWNER TimelineStore.SystemFilter.ENTITY_OWNER
.toString(), "owner"); .toString(), "owner");
entity.setDomainId("domain_id_1"); entity.setDomainId("domain_id_1");
Assert.assertTrue( assertTrue(
"Owner should be allowed to view",
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("owner"), UserGroupInformation.createRemoteUser("owner"),
ApplicationAccessType.VIEW_APP, entity)); ApplicationAccessType.VIEW_APP, entity),
Assert.assertTrue( "Owner should be allowed to view");
"Reader should be allowed to view", assertTrue(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("reader"), UserGroupInformation.createRemoteUser("reader"),
ApplicationAccessType.VIEW_APP, entity)); ApplicationAccessType.VIEW_APP, entity),
Assert.assertFalse( "Reader should be allowed to view");
"Other shouldn't be allowed to view", assertFalse(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("other"), UserGroupInformation.createRemoteUser("other"),
ApplicationAccessType.VIEW_APP, entity)); ApplicationAccessType.VIEW_APP, entity),
Assert.assertTrue( "Other shouldn't be allowed to view");
"Admin should be allowed to view", assertTrue(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("admin"), UserGroupInformation.createRemoteUser("admin"),
ApplicationAccessType.VIEW_APP, entity)); ApplicationAccessType.VIEW_APP, entity),
"Admin should be allowed to view");
Assert.assertTrue( assertTrue(
"Owner should be allowed to modify",
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("owner"), UserGroupInformation.createRemoteUser("owner"),
ApplicationAccessType.MODIFY_APP, entity)); ApplicationAccessType.MODIFY_APP, entity),
Assert.assertTrue( "Owner should be allowed to modify");
"Writer should be allowed to modify", assertTrue(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("writer"), UserGroupInformation.createRemoteUser("writer"),
ApplicationAccessType.MODIFY_APP, entity)); ApplicationAccessType.MODIFY_APP, entity),
Assert.assertFalse( "Writer should be allowed to modify");
"Other shouldn't be allowed to modify", assertFalse(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("other"), UserGroupInformation.createRemoteUser("other"),
ApplicationAccessType.MODIFY_APP, entity)); ApplicationAccessType.MODIFY_APP, entity),
Assert.assertTrue( "Other shouldn't be allowed to modify");
"Admin should be allowed to modify", assertTrue(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("admin"), UserGroupInformation.createRemoteUser("admin"),
ApplicationAccessType.MODIFY_APP, entity)); ApplicationAccessType.MODIFY_APP, entity),
"Admin should be allowed to modify");
} }
@Test @Test
public void testCorruptedOwnerInfoForEntity() throws Exception { void testCorruptedOwnerInfoForEntity() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
@ -137,29 +141,29 @@ public class TestTimelineACLsManager {
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("owner"), UserGroupInformation.createRemoteUser("owner"),
ApplicationAccessType.VIEW_APP, entity); ApplicationAccessType.VIEW_APP, entity);
Assert.fail("Exception is expected"); fail("Exception is expected");
} catch (YarnException e) { } catch (YarnException e) {
Assert.assertTrue("It's not the exact expected exception", e.getMessage() assertTrue(e.getMessage()
.contains("doesn't exist.")); .contains("doesn't exist."), "It's not the exact expected exception");
} }
} }
@Test @Test
public void testYarnACLsNotEnabledForDomain() throws Exception { void testYarnACLsNotEnabledForDomain() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
TimelineACLsManager timelineACLsManager = TimelineACLsManager timelineACLsManager =
new TimelineACLsManager(conf); new TimelineACLsManager(conf);
TimelineDomain domain = new TimelineDomain(); TimelineDomain domain = new TimelineDomain();
domain.setOwner("owner"); domain.setOwner("owner");
Assert.assertTrue( assertTrue(
"Always true when ACLs are not enabled",
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("user"), domain)); UserGroupInformation.createRemoteUser("user"), domain),
"Always true when ACLs are not enabled");
} }
@Test @Test
public void testYarnACLsEnabledForDomain() throws Exception { void testYarnACLsEnabledForDomain() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
@ -167,22 +171,22 @@ public class TestTimelineACLsManager {
new TimelineACLsManager(conf); new TimelineACLsManager(conf);
TimelineDomain domain = new TimelineDomain(); TimelineDomain domain = new TimelineDomain();
domain.setOwner("owner"); domain.setOwner("owner");
Assert.assertTrue( assertTrue(
"Owner should be allowed to access",
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("owner"), domain)); UserGroupInformation.createRemoteUser("owner"), domain),
Assert.assertFalse( "Owner should be allowed to access");
"Other shouldn't be allowed to access", assertFalse(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("other"), domain)); UserGroupInformation.createRemoteUser("other"), domain),
Assert.assertTrue( "Other shouldn't be allowed to access");
"Admin should be allowed to access", assertTrue(
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("admin"), domain)); UserGroupInformation.createRemoteUser("admin"), domain),
"Admin should be allowed to access");
} }
@Test @Test
public void testCorruptedOwnerInfoForDomain() throws Exception { void testCorruptedOwnerInfoForDomain() throws Exception {
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner"); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
@ -192,10 +196,10 @@ public class TestTimelineACLsManager {
try { try {
timelineACLsManager.checkAccess( timelineACLsManager.checkAccess(
UserGroupInformation.createRemoteUser("owner"), domain); UserGroupInformation.createRemoteUser("owner"), domain);
Assert.fail("Exception is expected"); fail("Exception is expected");
} catch (YarnException e) { } catch (YarnException e) {
Assert.assertTrue("It's not the exact expected exception", e.getMessage() assertTrue(e.getMessage()
.contains("is corrupted.")); .contains("is corrupted."), "It's not the exact expected exception");
} }
} }

123
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilterForV1.java
View File

@ -24,6 +24,13 @@ import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil;
@ -46,23 +53,18 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
import static org.junit.Assert.fail;
import org.junit.AfterClass; import static org.apache.hadoop.yarn.conf.YarnConfiguration.TIMELINE_HTTP_AUTH_PREFIX;
import org.junit.Assert; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.BeforeClass; import static org.junit.jupiter.api.Assertions.assertFalse;
import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.junit.runner.RunWith; import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.runners.Parameterized; import static org.junit.jupiter.api.Assertions.fail;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** /**
* Test cases for authentication via TimelineAuthenticationFilter while * Test cases for authentication via TimelineAuthenticationFilter while
* publishing entities for ATSv1. * publishing entities for ATSv1.
*/ */
@RunWith(Parameterized.class)
public class TestTimelineAuthenticationFilterForV1 { public class TestTimelineAuthenticationFilterForV1 {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(TestTimelineAuthenticationFilterForV1.class); LoggerFactory.getLogger(TestTimelineAuthenticationFilterForV1.class);
@ -83,7 +85,6 @@ public class TestTimelineAuthenticationFilterForV1 {
System.getProperty("test.build.dir", "target/test-dir") + "/" System.getProperty("test.build.dir", "target/test-dir") + "/"
+ TestTimelineAuthenticationFilterForV1.class.getSimpleName(); + TestTimelineAuthenticationFilterForV1.class.getSimpleName();
@Parameterized.Parameters
public static Collection<Object[]> withSsl() { public static Collection<Object[]> withSsl() {
return Arrays.asList(new Object[][]{{false}, {true}}); return Arrays.asList(new Object[][]{{false}, {true}});
} }
@ -95,11 +96,11 @@ public class TestTimelineAuthenticationFilterForV1 {
private static Configuration conf; private static Configuration conf;
private static boolean withSsl; private static boolean withSsl;
public TestTimelineAuthenticationFilterForV1(boolean withSsl) { public void initTestTimelineAuthenticationFilterForV1(boolean isSslEnabled) {
TestTimelineAuthenticationFilterForV1.withSsl = withSsl; TestTimelineAuthenticationFilterForV1.withSsl = isSslEnabled;
} }
@BeforeClass @BeforeAll
public static void setup() { public static void setup() {
try { try {
testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR); testMiniKDC = new MiniKdc(MiniKdc.createConf(), TEST_ROOT_DIR);
@ -167,7 +168,7 @@ public class TestTimelineAuthenticationFilterForV1 {
return client; return client;
} }
@AfterClass @AfterAll
public static void tearDown() throws Exception { public static void tearDown() throws Exception {
if (testMiniKDC != null) { if (testMiniKDC != null) {
testMiniKDC.stop(); testMiniKDC.stop();
@ -184,8 +185,10 @@ public class TestTimelineAuthenticationFilterForV1 {
} }
} }
@Test @MethodSource("withSsl")
public void testPutTimelineEntities() throws Exception { @ParameterizedTest
void testPutTimelineEntities(boolean isSslEnabled) throws Exception {
initTestTimelineAuthenticationFilterForV1(isSslEnabled);
KerberosTestUtils.doAs(PRINCIPAL, new Callable<Void>() { KerberosTestUtils.doAs(PRINCIPAL, new Callable<Void>() {
@Override @Override
public Void call() throws Exception { public Void call() throws Exception {
@ -199,20 +202,22 @@ public class TestTimelineAuthenticationFilterForV1 {
if (putResponse.getErrors().size() > 0) { if (putResponse.getErrors().size() > 0) {
LOG.error("putResponse errors: {}", putResponse.getErrors()); LOG.error("putResponse errors: {}", putResponse.getErrors());
} }
Assert.assertTrue("There were some errors in the putResponse", assertTrue(putResponse.getErrors().isEmpty(),
putResponse.getErrors().isEmpty()); "There were some errors in the putResponse");
TimelineEntity entityToRead = TimelineEntity entityToRead =
testTimelineServer.getTimelineStore().getEntity("entity1", testTimelineServer.getTimelineStore().getEntity("entity1",
TestTimelineAuthenticationFilterForV1.class.getName(), null); TestTimelineAuthenticationFilterForV1.class.getName(), null);
Assert.assertNotNull("Timeline entity should not be null", assertNotNull(entityToRead,
entityToRead); "Timeline entity should not be null");
return null; return null;
} }
}); });
} }
@Test @MethodSource("withSsl")
public void testPutDomains() throws Exception { @ParameterizedTest
void testPutDomains(boolean isSslEnabled) throws Exception {
initTestTimelineAuthenticationFilterForV1(isSslEnabled);
KerberosTestUtils.doAs(PRINCIPAL, new Callable<Void>() { KerberosTestUtils.doAs(PRINCIPAL, new Callable<Void>() {
@Override @Override
public Void call() throws Exception { public Void call() throws Exception {
@ -226,15 +231,17 @@ public class TestTimelineAuthenticationFilterForV1 {
TimelineDomain domainToRead = TimelineDomain domainToRead =
testTimelineServer.getTimelineStore().getDomain( testTimelineServer.getTimelineStore().getDomain(
TestTimelineAuthenticationFilterForV1.class.getName()); TestTimelineAuthenticationFilterForV1.class.getName());
Assert.assertNotNull("Timeline domain should not be null", assertNotNull(domainToRead,
domainToRead); "Timeline domain should not be null");
return null; return null;
} }
}); });
} }
@Test @MethodSource("withSsl")
public void testDelegationTokenOperations() throws Exception { @ParameterizedTest
void testDelegationTokenOperations(boolean isSslEnabled) throws Exception {
initTestTimelineAuthenticationFilterForV1(isSslEnabled);
TimelineClient httpUserClient = TimelineClient httpUserClient =
KerberosTestUtils.doAs(PRINCIPAL, KerberosTestUtils.doAs(PRINCIPAL,
new Callable<TimelineClient>() { new Callable<TimelineClient>() {
@ -255,41 +262,41 @@ public class TestTimelineAuthenticationFilterForV1 {
// Let HTTP user to get the delegation for itself // Let HTTP user to get the delegation for itself
Token<TimelineDelegationTokenIdentifier> token = Token<TimelineDelegationTokenIdentifier> token =
httpUserClient.getDelegationToken(httpUser.getShortUserName()); httpUserClient.getDelegationToken(httpUser.getShortUserName());
Assert.assertNotNull("Delegation token should not be null", token); assertNotNull(token, "Delegation token should not be null");
TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier(); TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
Assert.assertNotNull("Delegation token identifier should not be null", assertNotNull(tDT,
tDT); "Delegation token identifier should not be null");
Assert.assertEquals("Owner of delegation token identifier does not match", assertEquals(new Text(HTTP_USER), tDT.getOwner(),
new Text(HTTP_USER), tDT.getOwner()); "Owner of delegation token identifier does not match");
// Renew token // Renew token
Assert.assertFalse("Service field of token should not be empty", assertFalse(token.getService().toString().isEmpty(),
token.getService().toString().isEmpty()); "Service field of token should not be empty");
// Renew the token from the token service address // Renew the token from the token service address
long renewTime1 = httpUserClient.renewDelegationToken(token); long renewTime1 = httpUserClient.renewDelegationToken(token);
Thread.sleep(100); Thread.sleep(100);
token.setService(new Text()); token.setService(new Text());
Assert.assertTrue("Service field of token should be empty", assertTrue(token.getService().toString().isEmpty(),
token.getService().toString().isEmpty()); "Service field of token should be empty");
// If the token service address is not available, it still can be renewed // If the token service address is not available, it still can be renewed
// from the configured address // from the configured address
long renewTime2 = httpUserClient.renewDelegationToken(token); long renewTime2 = httpUserClient.renewDelegationToken(token);
Assert.assertTrue("renewTime2 should be later than renewTime1", assertTrue(renewTime1 < renewTime2,
renewTime1 < renewTime2); "renewTime2 should be later than renewTime1");
// Cancel token // Cancel token
Assert.assertTrue("Service field of token should be empty", assertTrue(token.getService().toString().isEmpty(),
token.getService().toString().isEmpty()); "Service field of token should be empty");
// If the token service address is not available, it still can be canceled // If the token service address is not available, it still can be canceled
// from the configured address // from the configured address
httpUserClient.cancelDelegationToken(token); httpUserClient.cancelDelegationToken(token);
// Renew should not be successful because the token is canceled // Renew should not be successful because the token is canceled
try { try {
httpUserClient.renewDelegationToken(token); httpUserClient.renewDelegationToken(token);
Assert.fail("Renew of delegation token should not be successful"); fail("Renew of delegation token should not be successful");
} catch (Exception e) { } catch (Exception e) {
LOG.info("Exception while renewing delegation token", e); LOG.info("Exception while renewing delegation token", e);
Assert.assertTrue(e.getMessage().contains( assertTrue(e.getMessage().contains(
"Renewal request for unknown token")); "Renewal request for unknown token"));
} }
@ -304,35 +311,35 @@ public class TestTimelineAuthenticationFilterForV1 {
} }
}); });
token = fooUserClient.getDelegationToken(httpUser.getShortUserName()); token = fooUserClient.getDelegationToken(httpUser.getShortUserName());
Assert.assertNotNull("Delegation token should not be null", token); assertNotNull(token, "Delegation token should not be null");
tDT = token.decodeIdentifier(); tDT = token.decodeIdentifier();
Assert.assertNotNull("Delegation token identifier should not be null", assertNotNull(tDT,
tDT); "Delegation token identifier should not be null");
Assert.assertEquals("Owner of delegation token is not the expected", assertEquals(new Text(FOO_USER), tDT.getOwner(),
new Text(FOO_USER), tDT.getOwner()); "Owner of delegation token is not the expected");
Assert.assertEquals("Real user of delegation token is not the expected", assertEquals(new Text(HTTP_USER), tDT.getRealUser(),
new Text(HTTP_USER), tDT.getRealUser()); "Real user of delegation token is not the expected");
// Renew token as the renewer // Renew token as the renewer
final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token; final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew); renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew);
renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew); renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew);
Assert.assertTrue("renewTime2 should be later than renewTime1", assertTrue(renewTime1 < renewTime2,
renewTime1 < renewTime2); "renewTime2 should be later than renewTime1");
// Cancel token // Cancel token
Assert.assertFalse("Service field of token should not be empty", assertFalse(tokenToRenew.getService().toString().isEmpty(),
tokenToRenew.getService().toString().isEmpty()); "Service field of token should not be empty");
// Cancel the token from the token service address // Cancel the token from the token service address
fooUserClient.cancelDelegationToken(tokenToRenew); fooUserClient.cancelDelegationToken(tokenToRenew);
// Renew should not be successful because the token is canceled // Renew should not be successful because the token is canceled
try { try {
httpUserClient.renewDelegationToken(tokenToRenew); httpUserClient.renewDelegationToken(tokenToRenew);
Assert.fail("Renew of delegation token should not be successful"); fail("Renew of delegation token should not be successful");
} catch (Exception e) { } catch (Exception e) {
LOG.info("Exception while renewing delegation token", e); LOG.info("Exception while renewing delegation token", e);
Assert.assertTrue( assertTrue(
e.getMessage().contains("Renewal request for unknown token")); e.getMessage().contains("Renewal request for unknown token"));
} }
@ -349,10 +356,10 @@ public class TestTimelineAuthenticationFilterForV1 {
try { try {
barUserClient.getDelegationToken(httpUser.getShortUserName()); barUserClient.getDelegationToken(httpUser.getShortUserName());
Assert.fail("Retrieval of delegation token should not be successful"); fail("Retrieval of delegation token should not be successful");
} catch (Exception e) { } catch (Exception e) {
LOG.info("Exception while retrieving delegation token", e); LOG.info("Exception while retrieving delegation token", e);
Assert.assertTrue(e.getCause() instanceof AuthorizationException || assertTrue(e.getCause() instanceof AuthorizationException ||
e.getCause() instanceof AuthenticationException); e.getCause() instanceof AuthenticationException);
} }
} }

335
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
View File

@ -18,22 +18,12 @@
package org.apache.hadoop.yarn.server.timeline.webapp; package org.apache.hadoop.yarn.server.timeline.webapp;
import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.util.Collections; import java.util.Collections;
import java.util.Enumeration; import java.util.Enumeration;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import javax.servlet.FilterConfig; import javax.servlet.FilterConfig;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
import javax.servlet.ServletException; import javax.servlet.ServletException;
@ -42,17 +32,28 @@ import javax.ws.rs.core.Response.Status;
import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException; import javax.xml.bind.JAXBException;
import com.google.inject.Guice;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.JettyUtils; import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomains;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError; import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
@ -64,23 +65,23 @@ import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.google.inject.Guice; import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
import com.google.inject.servlet.ServletModule; import static org.assertj.core.api.Assertions.assertThat;
import com.sun.jersey.api.client.ClientResponse; import static org.junit.jupiter.api.Assertions.assertEquals;
import com.sun.jersey.api.client.WebResource; import static org.junit.jupiter.api.Assertions.assertNotNull;
import com.sun.jersey.api.client.config.DefaultClientConfig; import static org.junit.jupiter.api.Assertions.assertNull;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; import static org.junit.jupiter.api.Assertions.fail;
import com.sun.jersey.test.framework.WebAppDescriptor; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
public class TestTimelineWebServices extends JerseyTestBase { public class TestTimelineWebServices extends JerseyTestBase {
@ -99,7 +100,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
try { try {
store = mockTimelineStore(); store = mockTimelineStore();
} catch (Exception e) { } catch (Exception e) {
Assert.fail(); fail();
} }
Configuration conf = new YarnConfiguration(); Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
@ -138,7 +139,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
try { try {
taFilter.init(filterConfig); taFilter.init(filterConfig);
} catch (ServletException e) { } catch (ServletException e) {
Assert.fail("Unable to initialize TimelineAuthenticationFilter: " + fail("Unable to initialize TimelineAuthenticationFilter: " +
e.getMessage()); e.getMessage());
} }
@ -146,7 +147,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
try { try {
doNothing().when(taFilter).init(any(FilterConfig.class)); doNothing().when(taFilter).init(any(FilterConfig.class));
} catch (ServletException e) { } catch (ServletException e) {
Assert.fail("Unable to initialize TimelineAuthenticationFilter: " + fail("Unable to initialize TimelineAuthenticationFilter: " +
e.getMessage()); e.getMessage());
} }
filter("/*").through(taFilter); filter("/*").through(taFilter);
@ -158,7 +159,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
Guice.createInjector(new WebServletModule())); Guice.createInjector(new WebServletModule()));
} }
@Before @BeforeEach
public void setUp() throws Exception { public void setUp() throws Exception {
super.setUp(); super.setUp();
GuiceServletConfig.setInjector( GuiceServletConfig.setInjector(
@ -187,7 +188,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testAbout() throws Exception { void testAbout() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.accept(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON)
@ -197,54 +198,54 @@ public class TestTimelineWebServices extends JerseyTestBase {
TimelineAbout actualAbout = response.getEntity(TimelineAbout.class); TimelineAbout actualAbout = response.getEntity(TimelineAbout.class);
TimelineAbout expectedAbout = TimelineAbout expectedAbout =
TimelineUtils.createTimelineAbout("Timeline API"); TimelineUtils.createTimelineAbout("Timeline API");
Assert.assertNotNull( assertNotNull(
"Timeline service about response is null", actualAbout); actualAbout, "Timeline service about response is null");
Assert.assertEquals(expectedAbout.getAbout(), actualAbout.getAbout()); assertEquals(expectedAbout.getAbout(), actualAbout.getAbout());
Assert.assertEquals(expectedAbout.getTimelineServiceVersion(), assertEquals(expectedAbout.getTimelineServiceVersion(),
actualAbout.getTimelineServiceVersion()); actualAbout.getTimelineServiceVersion());
Assert.assertEquals(expectedAbout.getTimelineServiceBuildVersion(), assertEquals(expectedAbout.getTimelineServiceBuildVersion(),
actualAbout.getTimelineServiceBuildVersion()); actualAbout.getTimelineServiceBuildVersion());
Assert.assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(), assertEquals(expectedAbout.getTimelineServiceVersionBuiltOn(),
actualAbout.getTimelineServiceVersionBuiltOn()); actualAbout.getTimelineServiceVersionBuiltOn());
Assert.assertEquals(expectedAbout.getHadoopVersion(), assertEquals(expectedAbout.getHadoopVersion(),
actualAbout.getHadoopVersion()); actualAbout.getHadoopVersion());
Assert.assertEquals(expectedAbout.getHadoopBuildVersion(), assertEquals(expectedAbout.getHadoopBuildVersion(),
actualAbout.getHadoopBuildVersion()); actualAbout.getHadoopBuildVersion());
Assert.assertEquals(expectedAbout.getHadoopVersionBuiltOn(), assertEquals(expectedAbout.getHadoopVersionBuiltOn(),
actualAbout.getHadoopVersionBuiltOn()); actualAbout.getHadoopVersionBuiltOn());
} }
private static void verifyEntities(TimelineEntities entities) { private static void verifyEntities(TimelineEntities entities) {
Assert.assertNotNull(entities); assertNotNull(entities);
Assert.assertEquals(3, entities.getEntities().size()); assertEquals(3, entities.getEntities().size());
TimelineEntity entity1 = entities.getEntities().get(0); TimelineEntity entity1 = entities.getEntities().get(0);
Assert.assertNotNull(entity1); assertNotNull(entity1);
Assert.assertEquals("id_1", entity1.getEntityId()); assertEquals("id_1", entity1.getEntityId());
Assert.assertEquals("type_1", entity1.getEntityType()); assertEquals("type_1", entity1.getEntityType());
Assert.assertEquals(123l, entity1.getStartTime().longValue()); assertEquals(123L, entity1.getStartTime().longValue());
Assert.assertEquals(2, entity1.getEvents().size()); assertEquals(2, entity1.getEvents().size());
Assert.assertEquals(4, entity1.getPrimaryFilters().size()); assertEquals(4, entity1.getPrimaryFilters().size());
Assert.assertEquals(4, entity1.getOtherInfo().size()); assertEquals(4, entity1.getOtherInfo().size());
TimelineEntity entity2 = entities.getEntities().get(1); TimelineEntity entity2 = entities.getEntities().get(1);
Assert.assertNotNull(entity2); assertNotNull(entity2);
Assert.assertEquals("id_2", entity2.getEntityId()); assertEquals("id_2", entity2.getEntityId());
Assert.assertEquals("type_1", entity2.getEntityType()); assertEquals("type_1", entity2.getEntityType());
Assert.assertEquals(123l, entity2.getStartTime().longValue()); assertEquals(123L, entity2.getStartTime().longValue());
Assert.assertEquals(2, entity2.getEvents().size()); assertEquals(2, entity2.getEvents().size());
Assert.assertEquals(4, entity2.getPrimaryFilters().size()); assertEquals(4, entity2.getPrimaryFilters().size());
Assert.assertEquals(4, entity2.getOtherInfo().size()); assertEquals(4, entity2.getOtherInfo().size());
TimelineEntity entity3 = entities.getEntities().get(2); TimelineEntity entity3 = entities.getEntities().get(2);
Assert.assertNotNull(entity2); assertNotNull(entity2);
Assert.assertEquals("id_6", entity3.getEntityId()); assertEquals("id_6", entity3.getEntityId());
Assert.assertEquals("type_1", entity3.getEntityType()); assertEquals("type_1", entity3.getEntityType());
Assert.assertEquals(61l, entity3.getStartTime().longValue()); assertEquals(61L, entity3.getStartTime().longValue());
Assert.assertEquals(0, entity3.getEvents().size()); assertEquals(0, entity3.getEvents().size());
Assert.assertEquals(4, entity3.getPrimaryFilters().size()); assertEquals(4, entity3.getPrimaryFilters().size());
Assert.assertEquals(4, entity3.getOtherInfo().size()); assertEquals(4, entity3.getOtherInfo().size());
} }
@Test @Test
public void testGetEntities() throws Exception { void testGetEntities() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1") .path("type_1")
@ -256,7 +257,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testFromId() throws Exception { void testFromId() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").queryParam("fromId", "id_2") .path("type_1").queryParam("fromId", "id_2")
@ -278,7 +279,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testFromTs() throws Exception { void testFromTs() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").queryParam("fromTs", Long.toString(beforeTime)) .path("type_1").queryParam("fromTs", Long.toString(beforeTime))
@ -301,7 +302,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testPrimaryFilterString() { void testPrimaryFilterString() {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").queryParam("primaryFilter", "user:username") .path("type_1").queryParam("primaryFilter", "user:username")
@ -313,7 +314,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testPrimaryFilterInteger() { void testPrimaryFilterInteger() {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").queryParam("primaryFilter", .path("type_1").queryParam("primaryFilter",
@ -326,11 +327,11 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testPrimaryFilterLong() { void testPrimaryFilterLong() {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").queryParam("primaryFilter", .path("type_1").queryParam("primaryFilter",
"long:" + Long.toString((long) Integer.MAX_VALUE + 1l)) "long:" + Long.toString((long) Integer.MAX_VALUE + 1L))
.accept(MediaType.APPLICATION_JSON) .accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class); .get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
@ -339,7 +340,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testSecondaryFilters() { void testSecondaryFilters() {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1") .path("type_1")
@ -353,7 +354,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetEntity() throws Exception { void testGetEntity() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").path("id_1") .path("type_1").path("id_1")
@ -362,17 +363,17 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
TimelineEntity entity = response.getEntity(TimelineEntity.class); TimelineEntity entity = response.getEntity(TimelineEntity.class);
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("id_1", entity.getEntityId()); assertEquals("id_1", entity.getEntityId());
Assert.assertEquals("type_1", entity.getEntityType()); assertEquals("type_1", entity.getEntityType());
Assert.assertEquals(123l, entity.getStartTime().longValue()); assertEquals(123L, entity.getStartTime().longValue());
Assert.assertEquals(2, entity.getEvents().size()); assertEquals(2, entity.getEvents().size());
Assert.assertEquals(4, entity.getPrimaryFilters().size()); assertEquals(4, entity.getPrimaryFilters().size());
Assert.assertEquals(4, entity.getOtherInfo().size()); assertEquals(4, entity.getOtherInfo().size());
} }
@Test @Test
public void testGetEntityFields1() throws Exception { void testGetEntityFields1() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").path("id_1").queryParam("fields", "events,otherinfo") .path("type_1").path("id_1").queryParam("fields", "events,otherinfo")
@ -381,17 +382,17 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
TimelineEntity entity = response.getEntity(TimelineEntity.class); TimelineEntity entity = response.getEntity(TimelineEntity.class);
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("id_1", entity.getEntityId()); assertEquals("id_1", entity.getEntityId());
Assert.assertEquals("type_1", entity.getEntityType()); assertEquals("type_1", entity.getEntityType());
Assert.assertEquals(123l, entity.getStartTime().longValue()); assertEquals(123L, entity.getStartTime().longValue());
Assert.assertEquals(2, entity.getEvents().size()); assertEquals(2, entity.getEvents().size());
Assert.assertEquals(0, entity.getPrimaryFilters().size()); assertEquals(0, entity.getPrimaryFilters().size());
Assert.assertEquals(4, entity.getOtherInfo().size()); assertEquals(4, entity.getOtherInfo().size());
} }
@Test @Test
public void testGetEntityFields2() throws Exception { void testGetEntityFields2() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").path("id_1").queryParam("fields", "lasteventonly," + .path("type_1").path("id_1").queryParam("fields", "lasteventonly," +
@ -401,17 +402,17 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
TimelineEntity entity = response.getEntity(TimelineEntity.class); TimelineEntity entity = response.getEntity(TimelineEntity.class);
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("id_1", entity.getEntityId()); assertEquals("id_1", entity.getEntityId());
Assert.assertEquals("type_1", entity.getEntityType()); assertEquals("type_1", entity.getEntityType());
Assert.assertEquals(123l, entity.getStartTime().longValue()); assertEquals(123L, entity.getStartTime().longValue());
Assert.assertEquals(1, entity.getEvents().size()); assertEquals(1, entity.getEvents().size());
Assert.assertEquals(4, entity.getPrimaryFilters().size()); assertEquals(4, entity.getPrimaryFilters().size());
Assert.assertEquals(0, entity.getOtherInfo().size()); assertEquals(0, entity.getOtherInfo().size());
} }
@Test @Test
public void testGetEvents() throws Exception { void testGetEvents() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("type_1").path("events") .path("type_1").path("events")
@ -421,22 +422,22 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
TimelineEvents events = response.getEntity(TimelineEvents.class); TimelineEvents events = response.getEntity(TimelineEvents.class);
Assert.assertNotNull(events); assertNotNull(events);
Assert.assertEquals(1, events.getAllEvents().size()); assertEquals(1, events.getAllEvents().size());
TimelineEvents.EventsOfOneEntity partEvents = events.getAllEvents().get(0); TimelineEvents.EventsOfOneEntity partEvents = events.getAllEvents().get(0);
Assert.assertEquals(2, partEvents.getEvents().size()); assertEquals(2, partEvents.getEvents().size());
TimelineEvent event1 = partEvents.getEvents().get(0); TimelineEvent event1 = partEvents.getEvents().get(0);
Assert.assertEquals(456l, event1.getTimestamp()); assertEquals(456L, event1.getTimestamp());
Assert.assertEquals("end_event", event1.getEventType()); assertEquals("end_event", event1.getEventType());
Assert.assertEquals(1, event1.getEventInfo().size()); assertEquals(1, event1.getEventInfo().size());
TimelineEvent event2 = partEvents.getEvents().get(1); TimelineEvent event2 = partEvents.getEvents().get(1);
Assert.assertEquals(123l, event2.getTimestamp()); assertEquals(123L, event2.getTimestamp());
Assert.assertEquals("start_event", event2.getEventType()); assertEquals("start_event", event2.getEventType());
Assert.assertEquals(0, event2.getEventInfo().size()); assertEquals(0, event2.getEventInfo().size());
} }
@Test @Test
public void testPostEntitiesWithPrimaryFilter() throws Exception { void testPostEntitiesWithPrimaryFilter() throws Exception {
TimelineEntities entities = new TimelineEntities(); TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
Map<String, Set<Object>> filters = new HashMap<String, Set<Object>>(); Map<String, Set<Object>> filters = new HashMap<String, Set<Object>>();
@ -455,11 +456,11 @@ public class TestTimelineWebServices extends JerseyTestBase {
.post(ClientResponse.class, entities); .post(ClientResponse.class, entities);
TimelinePutResponse putResposne = TimelinePutResponse putResposne =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(0, putResposne.getErrors().size()); assertEquals(0, putResposne.getErrors().size());
} }
@Test @Test
public void testPostEntities() throws Exception { void testPostEntities() throws Exception {
TimelineEntities entities = new TimelineEntities(); TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity(); TimelineEntity entity = new TimelineEntity();
entity.setEntityId("test id 1"); entity.setEntityId("test id 1");
@ -486,8 +487,8 @@ public class TestTimelineWebServices extends JerseyTestBase {
response.getType().toString()); response.getType().toString());
TimelinePutResponse putResposne = TimelinePutResponse putResposne =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResposne); assertNotNull(putResposne);
Assert.assertEquals(0, putResposne.getErrors().size()); assertEquals(0, putResposne.getErrors().size());
// verify the entity exists in the store // verify the entity exists in the store
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
.path("test type 1").path("test id 1") .path("test type 1").path("test id 1")
@ -496,13 +497,13 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
entity = response.getEntity(TimelineEntity.class); entity = response.getEntity(TimelineEntity.class);
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("test id 1", entity.getEntityId()); assertEquals("test id 1", entity.getEntityId());
Assert.assertEquals("test type 1", entity.getEntityType()); assertEquals("test type 1", entity.getEntityType());
} }
@Test @Test
public void testPostIncompleteEntities() throws Exception { void testPostIncompleteEntities() throws Exception {
TimelineEntities entities = new TimelineEntities(); TimelineEntities entities = new TimelineEntities();
TimelineEntity entity1 = new TimelineEntity(); TimelineEntity entity1 = new TimelineEntity();
entity1.setEntityId("test id 1"); entity1.setEntityId("test id 1");
@ -523,7 +524,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testPostEntitiesWithYarnACLsEnabled() throws Exception { void testPostEntitiesWithYarnACLsEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -544,8 +545,8 @@ public class TestTimelineWebServices extends JerseyTestBase {
response.getType().toString()); response.getType().toString());
TimelinePutResponse putResponse = TimelinePutResponse putResponse =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResponse); assertNotNull(putResponse);
Assert.assertEquals(0, putResponse.getErrors().size()); assertEquals(0, putResponse.getErrors().size());
// override/append timeline data in the same entity with different user // override/append timeline data in the same entity with different user
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
@ -556,9 +557,9 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
putResponse = response.getEntity(TimelinePutResponse.class); putResponse = response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResponse); assertNotNull(putResponse);
Assert.assertEquals(1, putResponse.getErrors().size()); assertEquals(1, putResponse.getErrors().size());
Assert.assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED, assertEquals(TimelinePutResponse.TimelinePutError.ACCESS_DENIED,
putResponse.getErrors().get(0).getErrorCode()); putResponse.getErrors().get(0).getErrorCode());
// Cross domain relationship will be rejected // Cross domain relationship will be rejected
@ -580,9 +581,9 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
putResponse = response.getEntity(TimelinePutResponse.class); putResponse = response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResponse); assertNotNull(putResponse);
Assert.assertEquals(1, putResponse.getErrors().size()); assertEquals(1, putResponse.getErrors().size());
Assert.assertEquals(TimelinePutError.FORBIDDEN_RELATION, assertEquals(TimelinePutError.FORBIDDEN_RELATION,
putResponse.getErrors().get(0).getErrorCode()); putResponse.getErrors().get(0).getErrorCode());
// Make sure the entity has been added anyway even though the // Make sure the entity has been added anyway even though the
@ -595,16 +596,16 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
entity = response.getEntity(TimelineEntity.class); entity = response.getEntity(TimelineEntity.class);
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("test id 3", entity.getEntityId()); assertEquals("test id 3", entity.getEntityId());
Assert.assertEquals("test type 2", entity.getEntityType()); assertEquals("test type 2", entity.getEntityType());
} finally { } finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
} }
} }
@Test @Test
public void testPostEntitiesToDefaultDomain() throws Exception { void testPostEntitiesToDefaultDomain() throws Exception {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -624,8 +625,8 @@ public class TestTimelineWebServices extends JerseyTestBase {
response.getType().toString()); response.getType().toString());
TimelinePutResponse putResposne = TimelinePutResponse putResposne =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResposne); assertNotNull(putResposne);
Assert.assertEquals(0, putResposne.getErrors().size()); assertEquals(0, putResposne.getErrors().size());
// verify the entity exists in the store // verify the entity exists in the store
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
.path("test type 7").path("test id 7") .path("test type 7").path("test id 7")
@ -635,10 +636,10 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
entity = response.getEntity(TimelineEntity.class); entity = response.getEntity(TimelineEntity.class);
Assert.assertNotNull(entity); assertNotNull(entity);
Assert.assertEquals("test id 7", entity.getEntityId()); assertEquals("test id 7", entity.getEntityId());
Assert.assertEquals("test type 7", entity.getEntityType()); assertEquals("test type 7", entity.getEntityType());
Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID, assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
entity.getDomainId()); entity.getDomainId());
} finally { } finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
@ -646,7 +647,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetEntityWithYarnACLsEnabled() throws Exception { void testGetEntityWithYarnACLsEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -667,7 +668,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
response.getType().toString()); response.getType().toString());
TimelinePutResponse putResponse = TimelinePutResponse putResponse =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(0, putResponse.getErrors().size()); assertEquals(0, putResponse.getErrors().size());
// verify the system data will not be exposed // verify the system data will not be exposed
// 1. No field specification // 1. No field specification
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
@ -678,7 +679,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
entity = response.getEntity(TimelineEntity.class); entity = response.getEntity(TimelineEntity.class);
Assert.assertNull(entity.getPrimaryFilters().get( assertNull(entity.getPrimaryFilters().get(
TimelineStore.SystemFilter.ENTITY_OWNER.toString())); TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
// 2. other field // 2. other field
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
@ -690,7 +691,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
entity = response.getEntity(TimelineEntity.class); entity = response.getEntity(TimelineEntity.class);
Assert.assertNull(entity.getPrimaryFilters().get( assertNull(entity.getPrimaryFilters().get(
TimelineStore.SystemFilter.ENTITY_OWNER.toString())); TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
// 3. primaryfilters field // 3. primaryfilters field
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
@ -702,7 +703,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
entity = response.getEntity(TimelineEntity.class); entity = response.getEntity(TimelineEntity.class);
Assert.assertNull(entity.getPrimaryFilters().get( assertNull(entity.getPrimaryFilters().get(
TimelineStore.SystemFilter.ENTITY_OWNER.toString())); TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
// get entity with other user // get entity with other user
@ -720,7 +721,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetEntitiesWithYarnACLsEnabled() { void testGetEntitiesWithYarnACLsEnabled() {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -742,7 +743,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
response.getType().toString()); response.getType().toString());
TimelinePutResponse putResponse = TimelinePutResponse putResponse =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(0, putResponse.getErrors().size()); assertEquals(0, putResponse.getErrors().size());
// Put entity [4, 5] in domain 2 // Put entity [4, 5] in domain 2
entities = new TimelineEntities(); entities = new TimelineEntities();
@ -761,7 +762,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
putResponse = response.getEntity(TimelinePutResponse.class); putResponse = response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(0, putResponse.getErrors().size()); assertEquals(0, putResponse.getErrors().size());
// Query entities of type 4 // Query entities of type 4
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
@ -782,7 +783,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetEventsWithYarnACLsEnabled() { void testGetEventsWithYarnACLsEnabled() {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -808,7 +809,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
response.getType().toString()); response.getType().toString());
TimelinePutResponse putResponse = TimelinePutResponse putResponse =
response.getEntity(TimelinePutResponse.class); response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(0, putResponse.getErrors().size()); assertEquals(0, putResponse.getErrors().size());
// Put entity [5, 6] in domain 2 // Put entity [5, 6] in domain 2
entities = new TimelineEntities(); entities = new TimelineEntities();
@ -831,7 +832,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
putResponse = response.getEntity(TimelinePutResponse.class); putResponse = response.getEntity(TimelinePutResponse.class);
Assert.assertEquals(0, putResponse.getErrors().size()); assertEquals(0, putResponse.getErrors().size());
// Query events belonging to the entities of type 4 // Query events belonging to the entities of type 4
response = r.path("ws").path("v1").path("timeline") response = r.path("ws").path("v1").path("timeline")
@ -852,7 +853,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetDomain() throws Exception { void testGetDomain() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("domain").path("domain_id_1") .path("domain").path("domain_id_1")
@ -865,7 +866,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetDomainYarnACLsEnabled() { void testGetDomainYarnACLsEnabled() {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -894,7 +895,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetDomains() throws Exception { void testGetDomains() throws Exception {
WebResource r = resource(); WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline") ClientResponse response = r.path("ws").path("v1").path("timeline")
.path("domain") .path("domain")
@ -904,7 +905,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
TimelineDomains domains = response.getEntity(TimelineDomains.class); TimelineDomains domains = response.getEntity(TimelineDomains.class);
Assert.assertEquals(2, domains.getDomains().size()); assertEquals(2, domains.getDomains().size());
for (int i = 0; i < domains.getDomains().size(); ++i) { for (int i = 0; i < domains.getDomains().size(); ++i) {
verifyDomain(domains.getDomains().get(i), verifyDomain(domains.getDomains().get(i),
i == 0 ? "domain_id_4" : "domain_id_1"); i == 0 ? "domain_id_4" : "domain_id_1");
@ -912,7 +913,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testGetDomainsYarnACLsEnabled() throws Exception { void testGetDomainsYarnACLsEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -925,7 +926,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
TimelineDomains domains = response.getEntity(TimelineDomains.class); TimelineDomains domains = response.getEntity(TimelineDomains.class);
Assert.assertEquals(2, domains.getDomains().size()); assertEquals(2, domains.getDomains().size());
for (int i = 0; i < domains.getDomains().size(); ++i) { for (int i = 0; i < domains.getDomains().size(); ++i) {
verifyDomain(domains.getDomains().get(i), verifyDomain(domains.getDomains().get(i),
i == 0 ? "domain_id_4" : "domain_id_1"); i == 0 ? "domain_id_4" : "domain_id_1");
@ -940,14 +941,14 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
domains = response.getEntity(TimelineDomains.class); domains = response.getEntity(TimelineDomains.class);
Assert.assertEquals(0, domains.getDomains().size()); assertEquals(0, domains.getDomains().size());
} finally { } finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager); timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
} }
} }
@Test @Test
public void testPutDomain() throws Exception { void testPutDomain() throws Exception {
TimelineDomain domain = new TimelineDomain(); TimelineDomain domain = new TimelineDomain();
domain.setId("test_domain_id"); domain.setId("test_domain_id");
WebResource r = resource(); WebResource r = resource();
@ -977,10 +978,10 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
domain = response.getEntity(TimelineDomain.class); domain = response.getEntity(TimelineDomain.class);
Assert.assertNotNull(domain); assertNotNull(domain);
Assert.assertEquals("test_domain_id", domain.getId()); assertEquals("test_domain_id", domain.getId());
Assert.assertEquals("tester", domain.getOwner()); assertEquals("tester", domain.getOwner());
Assert.assertEquals(null, domain.getDescription()); assertNull(domain.getDescription());
// Update the domain // Update the domain
domain.setDescription("test_description"); domain.setDescription("test_description");
@ -1000,13 +1001,13 @@ public class TestTimelineWebServices extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
response.getType().toString()); response.getType().toString());
domain = response.getEntity(TimelineDomain.class); domain = response.getEntity(TimelineDomain.class);
Assert.assertNotNull(domain); assertNotNull(domain);
Assert.assertEquals("test_domain_id", domain.getId()); assertEquals("test_domain_id", domain.getId());
Assert.assertEquals("test_description", domain.getDescription()); assertEquals("test_description", domain.getDescription());
} }
@Test @Test
public void testPutDomainYarnACLsEnabled() throws Exception { void testPutDomainYarnACLsEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager = AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager); timelineACLsManager.setAdminACLsManager(adminACLsManager);
try { try {
@ -1035,7 +1036,7 @@ public class TestTimelineWebServices extends JerseyTestBase {
} }
@Test @Test
public void testContextFactory() throws Exception { void testContextFactory() throws Exception {
JAXBContext jaxbContext1 = ContextFactory.createContext( JAXBContext jaxbContext1 = ContextFactory.createContext(
new Class[]{TimelineDomain.class}, Collections.EMPTY_MAP); new Class[]{TimelineDomain.class}, Collections.EMPTY_MAP);
JAXBContext jaxbContext2 = ContextFactory.createContext( JAXBContext jaxbContext2 = ContextFactory.createContext(
@ -1045,21 +1046,21 @@ public class TestTimelineWebServices extends JerseyTestBase {
try { try {
ContextFactory.createContext(new Class[]{TimelineEntity.class}, ContextFactory.createContext(new Class[]{TimelineEntity.class},
Collections.EMPTY_MAP); Collections.EMPTY_MAP);
Assert.fail("Expected JAXBException"); fail("Expected JAXBException");
} catch (Exception e) { } catch (Exception e) {
assertThat(e).isExactlyInstanceOf(JAXBException.class); assertThat(e).isExactlyInstanceOf(JAXBException.class);
} }
} }
private static void verifyDomain(TimelineDomain domain, String domainId) { private static void verifyDomain(TimelineDomain domain, String domainId) {
Assert.assertNotNull(domain); assertNotNull(domain);
Assert.assertEquals(domainId, domain.getId()); assertEquals(domainId, domain.getId());
// The specific values have been verified in TestMemoryTimelineStore // The specific values have been verified in TestMemoryTimelineStore
Assert.assertNotNull(domain.getDescription()); assertNotNull(domain.getDescription());
Assert.assertNotNull(domain.getOwner()); assertNotNull(domain.getOwner());
Assert.assertNotNull(domain.getReaders()); assertNotNull(domain.getReaders());
Assert.assertNotNull(domain.getWriters()); assertNotNull(domain.getWriters());
Assert.assertNotNull(domain.getCreatedTime()); assertNotNull(domain.getCreatedTime());
Assert.assertNotNull(domain.getModifiedTime()); assertNotNull(domain.getModifiedTime());
} }
} }

31
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServicesWithSSL.java
View File

@ -23,6 +23,12 @@ import java.io.IOException;
import java.net.URI; import java.net.URI;
import java.util.EnumSet; import java.util.EnumSet;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation;
@ -38,13 +44,10 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore; import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field; import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timeline.TimelineStore; import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
import com.sun.jersey.api.client.Client; import static org.junit.jupiter.api.Assertions.assertEquals;
import com.sun.jersey.api.client.ClientResponse; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestTimelineWebServicesWithSSL { public class TestTimelineWebServicesWithSSL {
@ -58,7 +61,7 @@ public class TestTimelineWebServicesWithSSL {
private static TimelineStore store; private static TimelineStore store;
private static Configuration conf; private static Configuration conf;
@BeforeClass @BeforeAll
public static void setupServer() throws Exception { public static void setupServer() throws Exception {
conf = new YarnConfiguration(); conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@ -84,7 +87,7 @@ public class TestTimelineWebServicesWithSSL {
store = timelineServer.getTimelineStore(); store = timelineServer.getTimelineStore();
} }
@AfterClass @AfterAll
public static void tearDownServer() throws Exception { public static void tearDownServer() throws Exception {
if (timelineServer != null) { if (timelineServer != null) {
timelineServer.stop(); timelineServer.stop();
@ -92,7 +95,7 @@ public class TestTimelineWebServicesWithSSL {
} }
@Test @Test
public void testPutEntities() throws Exception { void testPutEntities() throws Exception {
TestTimelineClient client = new TestTimelineClient(); TestTimelineClient client = new TestTimelineClient();
try { try {
client.init(conf); client.init(conf);
@ -107,16 +110,16 @@ public class TestTimelineWebServicesWithSSL {
expectedEntity.addEvent(event); expectedEntity.addEvent(event);
TimelinePutResponse response = client.putEntities(expectedEntity); TimelinePutResponse response = client.putEntities(expectedEntity);
Assert.assertEquals(0, response.getErrors().size()); assertEquals(0, response.getErrors().size());
Assert.assertTrue(client.resp.toString().contains("https")); assertTrue(client.resp.toString().contains("https"));
TimelineEntity actualEntity = store.getEntity( TimelineEntity actualEntity = store.getEntity(
expectedEntity.getEntityId(), expectedEntity.getEntityType(), expectedEntity.getEntityId(), expectedEntity.getEntityType(),
EnumSet.allOf(Field.class)); EnumSet.allOf(Field.class));
Assert.assertNotNull(actualEntity); assertNotNull(actualEntity);
Assert.assertEquals( assertEquals(
expectedEntity.getEntityId(), actualEntity.getEntityId()); expectedEntity.getEntityId(), actualEntity.getEntityId());
Assert.assertEquals( assertEquals(
expectedEntity.getEntityType(), actualEntity.getEntityType()); expectedEntity.getEntityType(), actualEntity.getEntityType());
} finally { } finally {
client.stop(); client.stop();