From babd19de331c875a1dffee908617c07c3e1eb31b Mon Sep 17 00:00:00 2001 From: Christopher Douglas Date: Fri, 3 Jun 2011 23:40:53 +0000 Subject: [PATCH] HADOOP-7144. Expose JMX metrics via JSON servlet. Contributed by Robert Joseph Evans git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1131289 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 3 + .../org/apache/hadoop/http/HttpServer.java | 2 + .../org/apache/hadoop/jmx/JMXJsonServlet.java | 327 ++++++++++++++++++ .../org/apache/hadoop/jmx/package-info.java | 22 ++ .../apache/hadoop/jmx/TestJMXJsonServlet.java | 69 ++++ 5 files changed, 423 insertions(+) create mode 100644 src/java/org/apache/hadoop/jmx/JMXJsonServlet.java create mode 100644 src/java/org/apache/hadoop/jmx/package-info.java create mode 100644 src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java diff --git a/CHANGES.txt b/CHANGES.txt index bacc3715820..83fdd657534 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -38,6 +38,9 @@ Trunk (unreleased changes) HADOOP-6832. Add an authentication plugin using a configurable static user for the web UI. (Owen O'Malley and Todd Lipcon via cdouglas) + HADOOP-7144. Expose JMX metrics via JSON servlet. (Robert Joseph Evans via + cdouglas) + IMPROVEMENTS HADOOP-7042. Updates to test-patch.sh to include failed test names and diff --git a/src/java/org/apache/hadoop/http/HttpServer.java b/src/java/org/apache/hadoop/http/HttpServer.java index e482a1faabe..75b3bbac34f 100644 --- a/src/java/org/apache/hadoop/http/HttpServer.java +++ b/src/java/org/apache/hadoop/http/HttpServer.java @@ -52,6 +52,7 @@ import org.apache.hadoop.http.FilterContainer; import org.apache.hadoop.http.FilterInitializer; import org.apache.hadoop.http.HtmlQuoting; +import org.apache.hadoop.jmx.JMXJsonServlet; import org.apache.hadoop.log.LogLevel; import org.apache.hadoop.metrics.MetricsServlet; import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector; @@ -287,6 +288,7 @@ protected void addDefaultServlets() { addServlet("stacks", "/stacks", StackServlet.class); addServlet("logLevel", "/logLevel", LogLevel.Servlet.class); addServlet("metrics", "/metrics", MetricsServlet.class); + addServlet("jmx", "/jmx", JMXJsonServlet.class); addServlet("conf", "/conf", ConfServlet.class); } diff --git a/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java b/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java new file mode 100644 index 00000000000..be59f5a3013 --- /dev/null +++ b/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.jmx; + +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.management.ManagementFactory; +import java.lang.reflect.Array; +import java.util.Iterator; +import java.util.Set; + +import javax.management.AttributeNotFoundException; +import javax.management.InstanceNotFoundException; +import javax.management.IntrospectionException; +import javax.management.MBeanAttributeInfo; +import javax.management.MBeanException; +import javax.management.MBeanInfo; +import javax.management.MBeanServer; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.ReflectionException; +import javax.management.openmbean.CompositeData; +import javax.management.openmbean.CompositeType; +import javax.management.openmbean.TabularData; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.http.HttpServer; +import org.codehaus.jackson.JsonFactory; +import org.codehaus.jackson.JsonGenerator; + +/* + * This servlet is based off of the JMXProxyServlet from Tomcat 7.0.14. It has + * been rewritten to be read only and to output in a JSON format so it is not + * really that close to the original. + */ +/** + * Provides Read only web access to JMX. + *

+ * This servlet generally will be placed under the /jmx URL for each + * HttpServer. It provides read only + * access to JMX metrics. The optional qry parameter + * may be used to query only a subset of the JMX Beans. This query + * functionality is provided through the + * {@link MBeanServer#queryNames(ObjectName, javax.management.QueryExp)} + * method. + *

+ * For example http://.../jmx?qry=Hadoop:* will return + * all hadoop metrics exposed through JMX. + *

+ * If the qry parameter is not formatted correctly then a + * 400 BAD REQUEST http response code will be returned. + *

+ * The return format is JSON and in the form + *

+ * 

+ *  {
+ *    "beans" : [
+ *      {
+ *        "name":"bean-name"
+ *        ...
+ *      }
+ *    ]
+ *  }
+ *
+ *

+ * The servlet attempts to convert the the JMXBeans into JSON. Each + * bean's attributes will be converted to a JSON object member. + * + * If the attribute is a boolean, a number, a string, or an array + * it will be converted to the JSON equivalent. + * + * If the value is a {@link CompositeData} then it will be converted + * to a JSON object with the keys as the name of the JSON member and + * the value is converted following these same rules. + * + * If the value is a {@link TabularData} then it will be converted + * to an array of the {@link CompositeData} elements that it contains. + * + * All other objects will be converted to a string and output as such. + * + * The bean's name and modelerType will be returned for all beans. + */ +public class JMXJsonServlet extends HttpServlet { + private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class); + + private static final long serialVersionUID = 1L; + + // ----------------------------------------------------- Instance Variables + /** + * MBean server. + */ + protected transient MBeanServer mBeanServer = null; + + // --------------------------------------------------------- Public Methods + /** + * Initialize this servlet. + */ + @Override + public void init() throws ServletException { + // Retrieve the MBean server + mBeanServer = ManagementFactory.getPlatformMBeanServer(); + } + + /** + * Process a GET request for the specified resource. + * + * @param request + * The servlet request we are processing + * @param response + * The servlet response we are creating + */ + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) { + try { + // Do the authorization + if (!HttpServer.hasAdministratorAccess(getServletContext(), request, + response)) { + return; + } + + response.setContentType("application/json; charset=utf8"); + + PrintWriter writer = response.getWriter(); + + JsonFactory jsonFactory = new JsonFactory(); + JsonGenerator jg = jsonFactory.createJsonGenerator(writer); + jg.useDefaultPrettyPrinter(); + jg.writeStartObject(); + if (mBeanServer == null) { + jg.writeStringField("result", "ERROR"); + jg.writeStringField("message", "No MBeanServer could be found"); + jg.close(); + return; + } + String qry = request.getParameter("qry"); + if (qry == null) { + qry = "*:*"; + } + listBeans(jg, new ObjectName(qry)); + jg.close(); + } catch (IOException e) { + LOG.error("Caught an exception while processing JMX request", e); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } catch (MalformedObjectNameException e) { + LOG.error("Caught an exception while processing JMX request", e); + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + } + } + + // --------------------------------------------------------- Private Methods + private void listBeans(JsonGenerator jg, ObjectName qry) throws IOException { + LOG.debug("Listing beans for "+qry); + Set names = null; + names = mBeanServer.queryNames(qry, null); + + jg.writeArrayFieldStart("beans"); + Iterator it = names.iterator(); + while (it.hasNext()) { + ObjectName oname = it.next(); + MBeanInfo minfo; + String code; + try { + minfo = mBeanServer.getMBeanInfo(oname); + code = minfo.getClassName(); + try { + if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) { + code = (String) mBeanServer.getAttribute(oname, "modelerType"); + } + } catch (AttributeNotFoundException e) { + //Ignored the modelerType attribute was not found, so use the class name instead. + } catch (MBeanException e) { + //The code inside the attribute getter threw an exception so log it, and + // fall back on the class name + LOG.error("getting attribute modelerType of "+oname+" threw an exception", e); + } catch (RuntimeException e) { + //For some reason even with an MBeanException available to them Runtime exceptions + //can still find their way through, so treat them the same as MBeanException + LOG.error("getting attribute modelerType of "+oname+" threw an exception", e); + } catch (ReflectionException e) { + //This happens when the code inside the JMX bean (setter?? from the java docs) + //threw an exception, so log it and fall back on the class name + LOG.error("getting attribute modelerType of "+oname+" threw an exception", e); + } + } catch (InstanceNotFoundException e) { + //Ignored for some reason the bean was not found so don't output it + continue; + } catch (IntrospectionException e) { + //This is an internal error, something odd happened with reflection so log it and + //don't output the bean. + LOG.error("Problem while trying to process JMX query: "+qry+" with MBean "+oname, e); + continue; + } catch (ReflectionException e) { + //This happens when the code inside the JMX bean threw an exception, so log it and + //don't output the bean. + LOG.error("Problem while trying to process JMX query: "+qry+" with MBean "+oname, e); + continue; + } + + jg.writeStartObject(); + jg.writeStringField("name", oname.toString()); + // can't be null - I think + + jg.writeStringField("modelerType", code); + + MBeanAttributeInfo attrs[] = minfo.getAttributes(); + for (int i = 0; i < attrs.length; i++) { + writeAttribute(jg, oname, attrs[i]); + } + // LOG.error("Caught Error writing value ",t); + // ExceptionUtils.handleThrowable(t); + //} + jg.writeEndObject(); + } + jg.writeEndArray(); + } + + private void writeAttribute(JsonGenerator jg, ObjectName oname, MBeanAttributeInfo attr) throws IOException { + if (!attr.isReadable()) { + return; + } + String attName = attr.getName(); + if ("modelerType".equals(attName)) { + return; + } + if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 + || attName.indexOf(" ") >= 0) { + return; + } + Object value = null; + try { + value = mBeanServer.getAttribute(oname, attName); + } catch (AttributeNotFoundException e) { + //Ignored the attribute was not found, which should never happen because the bean + //just told us that it has this attribute, but if this happens just don't output + //the attribute. + return; + } catch (MBeanException e) { + //The code inside the attribute getter threw an exception so log it, and + // skip outputting the attribute + LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e); + return; + } catch (RuntimeException e) { + //For some reason even with an MBeanException available to them Runtime exceptions + //can still find their way through, so treat them the same as MBeanException + LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e); + return; + } catch (ReflectionException e) { + //This happens when the code inside the JMX bean (setter?? from the java docs) + //threw an exception, so log it and skip outputting the attribute + LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e); + return; + } catch (InstanceNotFoundException e) { + //Ignored the mbean itself was not found, which should never happen because we + //just accessed it (perhaps something unregistered in-between) but if this + //happens just don't output the attribute. + return; + } + + writeAttribute(jg, attName, value); + } + + private void writeAttribute(JsonGenerator jg, String attName, Object value) throws IOException { + jg.writeFieldName(attName); + writeObject(jg, value); + } + + private void writeObject(JsonGenerator jg, Object value) throws IOException { + if(value == null) { + jg.writeNull(); + } else { + Class c = value.getClass(); + if (c.isArray()) { + jg.writeStartArray(); + int len = Array.getLength(value); + for (int j = 0; j < len; j++) { + Object item = Array.get(value, j); + writeObject(jg, item); + } + jg.writeEndArray(); + } else if(value instanceof Number) { + Number n = (Number)value; + jg.writeNumber(n.toString()); + } else if(value instanceof Boolean) { + Boolean b = (Boolean)value; + jg.writeBoolean(b); + } else if(value instanceof CompositeData) { + CompositeData cds = (CompositeData)value; + CompositeType comp = cds.getCompositeType(); + Set keys = comp.keySet(); + jg.writeStartObject(); + for(String key: keys) { + writeAttribute(jg, key, cds.get(key)); + } + jg.writeEndObject(); + } else if(value instanceof TabularData) { + TabularData tds = (TabularData)value; + jg.writeStartArray(); + for(Object entry : tds.values()) { + writeObject(jg, entry); + } + jg.writeEndArray(); + } else { + jg.writeString(value.toString()); + } + } + } +} diff --git a/src/java/org/apache/hadoop/jmx/package-info.java b/src/java/org/apache/hadoop/jmx/package-info.java new file mode 100644 index 00000000000..e09d9938cd5 --- /dev/null +++ b/src/java/org/apache/hadoop/jmx/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package provides access to JMX primarily through the + * {@link org.apache.hadoop.jmx.JMXJsonServlet} class. + */ +package org.apache.hadoop.jmx; \ No newline at end of file diff --git a/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java b/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java new file mode 100644 index 00000000000..b1feaf3ae86 --- /dev/null +++ b/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.jmx; + + +import java.net.URL; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.http.HttpServer; +import org.apache.hadoop.http.HttpServerFunctionalTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestJMXJsonServlet extends HttpServerFunctionalTest { + private static final Log LOG = LogFactory.getLog(TestJMXJsonServlet.class); + private static HttpServer server; + private static URL baseUrl; + + @BeforeClass public static void setup() throws Exception { + server = createTestServer(); + server.start(); + baseUrl = getServerURL(server); + } + + @AfterClass public static void cleanup() throws Exception { + server.stop(); + } + + public static void assertReFind(String re, String value) { + Pattern p = Pattern.compile(re); + Matcher m = p.matcher(value); + assertTrue("'"+p+"' does not match "+value, m.find()); + } + + @Test public void testQury() throws Exception { + String result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Runtime")); + LOG.info("/jmx?qry=java.lang:type=Runtime RESULT: "+result); + assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Runtime\"", result); + assertReFind("\"modelerType\"", result); + + result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory")); + LOG.info("/jmx?qry=java.lang:type=Memory RESULT: "+result); + assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result); + assertReFind("\"modelerType\"", result); + + result = readOutput(new URL(baseUrl, "/jmx")); + LOG.info("/jmx RESULT: "+result); + assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result); + } +}