From 9f5667fe27581d731f21e9f960730bf00da27743 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Fri, 12 Jul 2013 04:53:17 +0000 Subject: [PATCH] HDFS-4373. Merging change r1502328 from trunk to branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1502423 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../server/namenode/NameNodeHttpServer.java | 2 + .../namenode/StartupProgressServlet.java | 135 ++++++++++ .../namenode/TestStartupProgressServlet.java | 247 ++++++++++++++++++ 4 files changed, 386 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StartupProgressServlet.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartupProgressServlet.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d1e980f851b..ca143008d7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -95,6 +95,8 @@ Release 2.1.0-beta - 2013-07-02 HDFS-4762 Provide HDFS based NFSv3 and Mountd implementation (brandonli) HDFS-4372. Track NameNode startup progress. (cnauroth) + + HDFS-4373. Add HTTP API for querying NameNode startup progress. (cnauroth) IMPROVEMENTS diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 7c27a6bf4d8..053464730df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -194,6 +194,8 @@ public class NameNodeHttpServer { } private static void setupServlets(HttpServer httpServer, Configuration conf) { + httpServer.addInternalServlet("startupProgress", + StartupProgressServlet.PATH_SPEC, StartupProgressServlet.class); httpServer.addInternalServlet("getDelegationToken", GetDelegationTokenServlet.PATH_SPEC, GetDelegationTokenServlet.class, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StartupProgressServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StartupProgressServlet.java new file mode 100644 index 00000000000..a6b9afbd1f1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/StartupProgressServlet.java @@ -0,0 +1,135 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Phase; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressView; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.Step; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; +import org.apache.hadoop.io.IOUtils; +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonGenerator; + +import org.apache.hadoop.classification.InterfaceAudience; + +/** + * Servlet that provides a JSON representation of the namenode's current startup + * progress. + */ +@InterfaceAudience.Private +@SuppressWarnings("serial") +public class StartupProgressServlet extends DfsServlet { + + private static final String COUNT = "count"; + private static final String ELAPSED_TIME = "elapsedTime"; + private static final String FILE = "file"; + private static final String NAME = "name"; + private static final String PERCENT_COMPLETE = "percentComplete"; + private static final String PHASES = "phases"; + private static final String SIZE = "size"; + private static final String STATUS = "status"; + private static final String STEPS = "steps"; + private static final String TOTAL = "total"; + + public static final String PATH_SPEC = "/startupProgress"; + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException { + resp.setContentType("application/json; charset=UTF-8"); + StartupProgress prog = NameNodeHttpServer.getStartupProgressFromContext( + getServletContext()); + StartupProgressView view = prog.createView(); + JsonGenerator json = new JsonFactory().createJsonGenerator(resp.getWriter()); + try { + json.writeStartObject(); + json.writeNumberField(ELAPSED_TIME, view.getElapsedTime()); + json.writeNumberField(PERCENT_COMPLETE, view.getPercentComplete()); + json.writeArrayFieldStart(PHASES); + + for (Phase phase: view.getPhases()) { + json.writeStartObject(); + json.writeStringField(NAME, phase.getName()); + json.writeStringField(STATUS, view.getStatus(phase).toString()); + json.writeNumberField(PERCENT_COMPLETE, view.getPercentComplete(phase)); + json.writeNumberField(ELAPSED_TIME, view.getElapsedTime(phase)); + writeStringFieldIfNotNull(json, FILE, view.getFile(phase)); + writeNumberFieldIfDefined(json, SIZE, view.getSize(phase)); + json.writeArrayFieldStart(STEPS); + + for (Step step: view.getSteps(phase)) { + json.writeStartObject(); + StepType type = step.getType(); + String name = type != null ? type.getName() : null; + writeStringFieldIfNotNull(json, NAME, name); + json.writeNumberField(COUNT, view.getCount(phase, step)); + writeStringFieldIfNotNull(json, FILE, step.getFile()); + writeNumberFieldIfDefined(json, SIZE, step.getSize()); + json.writeNumberField(TOTAL, view.getTotal(phase, step)); + json.writeNumberField(PERCENT_COMPLETE, view.getPercentComplete(phase, + step)); + json.writeNumberField(ELAPSED_TIME, view.getElapsedTime(phase, step)); + json.writeEndObject(); + } + + json.writeEndArray(); + json.writeEndObject(); + } + + json.writeEndArray(); + json.writeEndObject(); + } finally { + IOUtils.cleanup(LOG, json); + } + } + + /** + * Writes a JSON number field only if the value is defined. + * + * @param json JsonGenerator to receive output + * @param key String key to put + * @param value long value to put + * @throws IOException if there is an I/O error + */ + private static void writeNumberFieldIfDefined(JsonGenerator json, String key, + long value) throws IOException { + if (value != Long.MIN_VALUE) { + json.writeNumberField(key, value); + } + } + + /** + * Writes a JSON string field only if the value is non-null. + * + * @param json JsonGenerator to receive output + * @param key String key to put + * @param value String value to put + * @throws IOException if there is an I/O error + */ + private static void writeStringFieldIfNotNull(JsonGenerator json, String key, + String value) throws IOException { + if (value != null) { + json.writeStringField(key, value); + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartupProgressServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartupProgressServlet.java new file mode 100644 index 00000000000..544f44f12f4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartupProgressServlet.java @@ -0,0 +1,247 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import static org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgressTestHelper.*; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +import javax.servlet.ServletContext; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import com.google.common.collect.ImmutableMap; +import org.apache.hadoop.hdfs.server.namenode.startupprogress.StartupProgress; +import org.junit.Before; +import org.junit.Test; +import org.mortbay.util.ajax.JSON; + +public class TestStartupProgressServlet { + + private HttpServletRequest req; + private HttpServletResponse resp; + private ByteArrayOutputStream respOut; + private StartupProgress startupProgress; + private StartupProgressServlet servlet; + + @Before + public void setUp() throws Exception { + startupProgress = new StartupProgress(); + ServletContext context = mock(ServletContext.class); + when(context.getAttribute(NameNodeHttpServer.STARTUP_PROGRESS_ATTRIBUTE_KEY)) + .thenReturn(startupProgress); + servlet = mock(StartupProgressServlet.class); + when(servlet.getServletContext()).thenReturn(context); + doCallRealMethod().when(servlet).doGet(any(HttpServletRequest.class), + any(HttpServletResponse.class)); + req = mock(HttpServletRequest.class); + respOut = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(respOut); + resp = mock(HttpServletResponse.class); + when(resp.getWriter()).thenReturn(writer); + } + + @Test + public void testInitialState() throws Exception { + String respBody = doGetAndReturnResponseBody(); + assertNotNull(respBody); + + Map expected = ImmutableMap.builder() + .put("percentComplete", 0.0f) + .put("phases", Arrays.asList( + ImmutableMap.builder() + .put("name", "LoadingFsImage") + .put("status", "PENDING") + .put("percentComplete", 0.0f) + .put("steps", Collections.emptyList()) + .build(), + ImmutableMap.builder() + .put("name", "LoadingEdits") + .put("status", "PENDING") + .put("percentComplete", 0.0f) + .put("steps", Collections.emptyList()) + .build(), + ImmutableMap.builder() + .put("name", "SavingCheckpoint") + .put("status", "PENDING") + .put("percentComplete", 0.0f) + .put("steps", Collections.emptyList()) + .build(), + ImmutableMap.builder() + .put("name", "SafeMode") + .put("status", "PENDING") + .put("percentComplete", 0.0f) + .put("steps", Collections.emptyList()) + .build())) + .build(); + + assertEquals(JSON.toString(expected), filterJson(respBody)); + } + + @Test + public void testRunningState() throws Exception { + setStartupProgressForRunningState(startupProgress); + String respBody = doGetAndReturnResponseBody(); + assertNotNull(respBody); + + Map expected = ImmutableMap.builder() + .put("percentComplete", 0.375f) + .put("phases", Arrays.asList( + ImmutableMap.builder() + .put("name", "LoadingFsImage") + .put("status", "COMPLETE") + .put("percentComplete", 1.0f) + .put("steps", Collections.singletonList( + ImmutableMap.builder() + .put("name", "Inodes") + .put("count", 100L) + .put("total", 100L) + .put("percentComplete", 1.0f) + .build() + )) + .build(), + ImmutableMap.builder() + .put("name", "LoadingEdits") + .put("status", "RUNNING") + .put("percentComplete", 0.5f) + .put("steps", Collections.singletonList( + ImmutableMap.builder() + .put("count", 100L) + .put("file", "file") + .put("size", 1000L) + .put("total", 200L) + .put("percentComplete", 0.5f) + .build() + )) + .build(), + ImmutableMap.builder() + .put("name", "SavingCheckpoint") + .put("status", "PENDING") + .put("percentComplete", 0.0f) + .put("steps", Collections.emptyList()) + .build(), + ImmutableMap.builder() + .put("name", "SafeMode") + .put("status", "PENDING") + .put("percentComplete", 0.0f) + .put("steps", Collections.emptyList()) + .build())) + .build(); + + assertEquals(JSON.toString(expected), filterJson(respBody)); + } + + @Test + public void testFinalState() throws Exception { + setStartupProgressForFinalState(startupProgress); + String respBody = doGetAndReturnResponseBody(); + assertNotNull(respBody); + + Map expected = ImmutableMap.builder() + .put("percentComplete", 1.0f) + .put("phases", Arrays.asList( + ImmutableMap.builder() + .put("name", "LoadingFsImage") + .put("status", "COMPLETE") + .put("percentComplete", 1.0f) + .put("steps", Collections.singletonList( + ImmutableMap.builder() + .put("name", "Inodes") + .put("count", 100L) + .put("total", 100L) + .put("percentComplete", 1.0f) + .build() + )) + .build(), + ImmutableMap.builder() + .put("name", "LoadingEdits") + .put("status", "COMPLETE") + .put("percentComplete", 1.0f) + .put("steps", Collections.singletonList( + ImmutableMap.builder() + .put("count", 200L) + .put("file", "file") + .put("size", 1000L) + .put("total", 200L) + .put("percentComplete", 1.0f) + .build() + )) + .build(), + ImmutableMap.builder() + .put("name", "SavingCheckpoint") + .put("status", "COMPLETE") + .put("percentComplete", 1.0f) + .put("steps", Collections.singletonList( + ImmutableMap.builder() + .put("name", "Inodes") + .put("count", 300L) + .put("total", 300L) + .put("percentComplete", 1.0f) + .build() + )) + .build(), + ImmutableMap.builder() + .put("name", "SafeMode") + .put("status", "COMPLETE") + .put("percentComplete", 1.0f) + .put("steps", Collections.singletonList( + ImmutableMap.builder() + .put("name", "AwaitingReportedBlocks") + .put("count", 400L) + .put("total", 400L) + .put("percentComplete", 1.0f) + .build() + )) + .build())) + .build(); + + assertEquals(JSON.toString(expected), filterJson(respBody)); + } + + /** + * Calls doGet on the servlet, captures the response body as a string, and + * returns it to the caller. + * + * @return String response body + * @throws IOException thrown if there is an I/O error + */ + private String doGetAndReturnResponseBody() throws IOException { + servlet.doGet(req, resp); + return new String(respOut.toByteArray(), "UTF-8"); + } + + /** + * Filters the given JSON response body, removing elements that would impede + * testing. Specifically, it removes elapsedTime fields, because we cannot + * predict the exact values. + * + * @param str String to filter + * @return String filtered value + */ + private String filterJson(String str) { + return str.replaceAll("\"elapsedTime\":\\d+\\,", "") + .replaceAll("\\,\"elapsedTime\":\\d+", ""); + } +}