From 32207cd478c7caa9056c49d0c53e85b415fe310e Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Thu, 15 Apr 2021 09:07:13 -0700 Subject: [PATCH] HBASE-25770 Http InfoServers should honor gzip encoding when requested (#3159) Signed-off-by: Duo Zhang Signed-off-by: Josh Elser --- hbase-http/pom.xml | 5 ++ .../apache/hadoop/hbase/http/HttpServer.java | 20 +++++- .../hadoop/hbase/http/TestHttpServer.java | 69 ++++++++++++++++++- .../apache/hadoop/hbase/master/HMaster.java | 6 +- 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/hbase-http/pom.xml b/hbase-http/pom.xml index ad3bb9c9564..7fe7b77d192 100644 --- a/hbase-http/pom.xml +++ b/hbase-http/pom.xml @@ -220,6 +220,11 @@ junit test + + org.hamcrest + hamcrest-library + test + org.mockito mockito-core diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java index 41844e75543..1d516945ad6 100644 --- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java +++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/HttpServer.java @@ -66,7 +66,6 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.org.eclipse.jetty.http.HttpVersion; @@ -81,6 +80,7 @@ import org.apache.hbase.thirdparty.org.eclipse.jetty.server.SslConnectionFactory import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.ContextHandlerCollection; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.HandlerCollection; import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.RequestLogHandler; +import org.apache.hbase.thirdparty.org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.DefaultServlet; import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterHolder; import org.apache.hbase.thirdparty.org.eclipse.jetty.servlet.FilterMapping; @@ -575,6 +575,7 @@ public class HttpServer implements FilterContainer { this.findPort = b.findPort; this.authenticationEnabled = b.securityEnabled; initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs, b); + this.webServer.setHandler(buildGzipHandler(this.webServer.getHandler())); } private void initializeWebServer(String name, String hostName, @@ -662,6 +663,23 @@ public class HttpServer implements FilterContainer { return ctx; } + /** + * Construct and configure an instance of {@link GzipHandler}. With complex + * multi-{@link WebAppContext} configurations, it's easiest to apply this handler directly to the + * instance of {@link Server} near the end of its configuration, something like + * 
+   *    Server server = new Server();
+   *    //...
+   *    server.setHandler(buildGzipHandler(server.getHandler()));
+   *    server.start();
+   *
+ */ + public static GzipHandler buildGzipHandler(final Handler wrapped) { + final GzipHandler gzipHandler = new GzipHandler(); + gzipHandler.setHandler(wrapped); + return gzipHandler; + } + private static void addNoCacheFilter(WebAppContext ctxt) { defineFilter(ctxt, NO_CACHE_FILTER, NoCacheFilter.class.getName(), Collections. emptyMap(), new String[] { "/*" }); diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java index acd2735d6be..0d51870e0a4 100644 --- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java +++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/TestHttpServer.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,11 +17,16 @@ */ package org.apache.hadoop.hbase.http; +import static org.hamcrest.Matchers.greaterThan; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; +import java.nio.CharBuffer; import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; @@ -56,6 +61,13 @@ import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AccessControlList; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.hamcrest.MatcherAssert; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -66,7 +78,6 @@ import org.junit.experimental.categories.Category; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.apache.hbase.thirdparty.org.eclipse.jetty.server.ServerConnector; import org.apache.hbase.thirdparty.org.eclipse.jetty.util.ajax.JSON; @@ -269,6 +280,60 @@ public class TestHttpServer extends HttpServerFunctionalTest { // assertEquals("text/html; charset=utf-8", conn.getContentType()); } + @Test + public void testNegotiatesEncodingGzip() throws IOException { + final InputStream stream = ClassLoader.getSystemResourceAsStream("webapps/static/test.css"); + assertNotNull(stream); + final String sourceContent = readFully(stream); + + try (final CloseableHttpClient client = HttpClients.createMinimal()) { + final HttpGet request = new HttpGet(new URL(baseUrl, "/static/test.css").toString()); + + request.setHeader(HttpHeaders.ACCEPT_ENCODING, null); + final long unencodedContentLength; + try (final CloseableHttpResponse response = client.execute(request)) { + final HttpEntity entity = response.getEntity(); + assertNotNull(entity); + assertNull(entity.getContentEncoding()); + unencodedContentLength = entity.getContentLength(); + MatcherAssert.assertThat(unencodedContentLength, greaterThan(0L)); + final String unencodedEntityBody = readFully(entity.getContent()); + assertEquals(sourceContent, unencodedEntityBody); + } + + request.setHeader(HttpHeaders.ACCEPT_ENCODING, "gzip"); + final long encodedContentLength; + try (final CloseableHttpResponse response = client.execute(request)) { + final HttpEntity entity = response.getEntity(); + assertNotNull(entity); + assertNotNull(entity.getContentEncoding()); + assertEquals("gzip", entity.getContentEncoding().getValue()); + encodedContentLength = entity.getContentLength(); + MatcherAssert.assertThat(encodedContentLength, greaterThan(0L)); + final String encodedEntityBody = readFully(entity.getContent()); + // the encoding/decoding process, as implemented in this specific combination of dependency + // versions, does not perfectly preserve trailing whitespace. thus, `trim()`. + assertEquals(sourceContent.trim(), encodedEntityBody.trim()); + } + MatcherAssert.assertThat(unencodedContentLength, greaterThan(encodedContentLength)); + } + } + + private static String readFully(final InputStream input) throws IOException { + // TODO: when the time comes, delete me and replace with a JDK11 IO helper API. + try (final BufferedReader reader = new BufferedReader(new InputStreamReader(input))) { + final StringBuilder sb = new StringBuilder(); + final CharBuffer buffer = CharBuffer.allocate(1024 * 2); + while (reader.read(buffer) > 0) { + sb.append(buffer); + buffer.clear(); + } + return sb.toString(); + } finally { + input.close(); + } + } + /** * Dummy filter that mimics as an authentication filter. Obtains user identity * from the request parameter user.name. Wraps around the request so that diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index ac3f7698005..1089adfbd8a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -89,10 +89,10 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hadoop.hbase.exceptions.MasterStoppedException; -import org.apache.hadoop.hbase.executor.ExecutorService.ExecutorConfig; import org.apache.hadoop.hbase.executor.ExecutorType; import org.apache.hadoop.hbase.favored.FavoredNodesManager; import org.apache.hadoop.hbase.favored.FavoredNodesPromoter; +import org.apache.hadoop.hbase.http.HttpServer; import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; @@ -543,7 +543,8 @@ public class HMaster extends HRegionServer implements MasterServices { if (infoPort < 0 || infoServer == null) { return -1; } - if(infoPort == infoServer.getPort()) { + if (infoPort == infoServer.getPort()) { + // server is already running return infoPort; } final String addr = conf.get("hbase.master.info.bindAddress", "0.0.0.0"); @@ -565,6 +566,7 @@ public class HMaster extends HRegionServer implements MasterServices { connector.setPort(infoPort); masterJettyServer.addConnector(connector); masterJettyServer.setStopAtShutdown(true); + masterJettyServer.setHandler(HttpServer.buildGzipHandler(masterJettyServer.getHandler())); final String redirectHostname = StringUtils.isBlank(useThisHostnameInstead) ? null : useThisHostnameInstead;