add curl format trace logging for requests and responses

client/src/main/java/org/elasticsearch/client/RequestLogger.java

@@ -20,31 +20,111 @@
 package org.elasticsearch.client;
 
 import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.http.Header;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpEntityEnclosingRequest;
 import org.apache.http.HttpHost;
-import org.apache.http.RequestLine;
+import org.apache.http.HttpResponse;
-import org.apache.http.StatusLine;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.entity.BufferedHttpEntity;
+import org.apache.http.util.EntityUtils;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 
 /**
- * Helper class that exposes static methods to unify the way requests are logged
+ * Helper class that exposes static methods to unify the way requests are logged.
+ * Includes trace logging to log complete requests and responses in curl format.
  */
 public final class RequestLogger {
 
+    private static final Log tracer = LogFactory.getLog("tracer");
+
     private RequestLogger() {
     }
 
     /**
      * Logs a request that yielded a response
      */
-    public static void log(Log logger, String message, RequestLine requestLine, HttpHost host, StatusLine statusLine) {
+    public static void log(Log logger, String message, HttpUriRequest request, HttpHost host, HttpResponse httpResponse) {
-        logger.debug(message + " [" + requestLine.getMethod() + " " + host + requestLine.getUri() + "] [" + statusLine + "]");
+        logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() +
+                "] [" + httpResponse.getStatusLine() + "]");
+
+        if (tracer.isTraceEnabled()) {
+            String requestLine;
+            try {
+                requestLine = buildTraceRequest(request, host);
+            } catch(IOException e) {
+                requestLine = "";
+                tracer.trace("error while reading request for trace purposes", e);
+            }
+            String responseLine;
+            try {
+                responseLine = buildTraceResponse(httpResponse);
+            } catch(IOException e) {
+                responseLine = "";
+                tracer.trace("error while reading response for trace purposes", e);
+            }
+            tracer.trace(requestLine + '\n' + responseLine);
+        }
     }
 
     /**
      * Logs a request that failed
      */
-    public static void log(Log logger, String message, RequestLine requestLine, HttpHost host, IOException e) {
+    public static void log(Log logger, String message, HttpUriRequest request, HttpHost host, IOException e) {
-        logger.debug(message + " [" + requestLine.getMethod() + " " + host + requestLine.getUri() + "]", e);
+        logger.debug(message + " [" + request.getMethod() + " " + host + request.getRequestLine().getUri() + "]", e);
+        if (logger.isTraceEnabled()) {
+            String traceRequest;
+            try {
+                traceRequest = buildTraceRequest(request, host);
+            } catch (IOException e1) {
+                tracer.trace("error while reading request for trace purposes", e);
+                traceRequest = "";
+            }
+            tracer.trace(traceRequest);
+        }
+    }
+
+    /**
+     * Creates curl output for given request
+     */
+    static String buildTraceRequest(HttpUriRequest request, HttpHost host) throws IOException {
+        String requestLine = "curl -iX " + request.getMethod() + " '" + host + request.getRequestLine().getUri() + "'";
+        if (request instanceof  HttpEntityEnclosingRequest) {
+            HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
+            if (enclosingRequest.getEntity() != null) {
+                requestLine += " -d '";
+                HttpEntity entity = new BufferedHttpEntity(enclosingRequest.getEntity());
+                enclosingRequest.setEntity(entity);
+                requestLine += EntityUtils.toString(entity) + "'";
+            }
+        }
+        return requestLine;
+    }
+
+    /**
+     * Creates curl output for given response
+     */
+    static String buildTraceResponse(HttpResponse httpResponse) throws IOException {
+        String responseLine = "# " + httpResponse.getStatusLine().toString();
+        for (Header header : httpResponse.getAllHeaders()) {
+            responseLine += "\n# " + header.getName() + ": " + header.getValue();
+        }
+        responseLine += "\n#";
+        HttpEntity entity = httpResponse.getEntity();
+        if (entity != null) {
+            entity = new BufferedHttpEntity(entity);
+            httpResponse.setEntity(entity);
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent()))) {
+                String line;
+                while( (line = reader.readLine()) != null) {
+                    responseLine += "\n# " + line;
+                }
+            }
+        }
+        return responseLine;
     }
 }

client/src/main/java/org/elasticsearch/client/RestClient.java

@@ -98,7 +98,7 @@ public final class RestClient implements Closeable {
             try {
                 response = client.execute(connection.getHost(), request);
             } catch(IOException e) {
-                RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), e);
+                RequestLogger.log(logger, "request failed", request, connection.getHost(), e);
                 connectionPool.onFailure(connection);
                 lastSeenException = addSuppressedException(lastSeenException, e);
                 continue;
@@ -108,11 +108,11 @@ public final class RestClient implements Closeable {
             int statusCode = response.getStatusLine().getStatusCode();
             //TODO make ignore status code configurable. rest-spec and tests support that parameter (ignore_missing)
             if (statusCode < 300 || (request.getMethod().equals(HttpHead.METHOD_NAME) && statusCode == 404) ) {
-                RequestLogger.log(logger, "request succeeded", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+                RequestLogger.log(logger, "request succeeded", request, connection.getHost(), response);
                 connectionPool.onSuccess(connection);
                 return new ElasticsearchResponse(request.getRequestLine(), connection.getHost(), response);
             } else {
-                RequestLogger.log(logger, "request failed", request.getRequestLine(), connection.getHost(), response.getStatusLine());
+                RequestLogger.log(logger, "request failed", request, connection.getHost(), response);
                 String responseBody = null;
                 if (response.getEntity() != null) {
                     responseBody = EntityUtils.toString(response.getEntity());

client/src/main/java/org/elasticsearch/client/sniff/Sniffer.java

@@ -77,7 +77,7 @@ final class Sniffer {
         try (CloseableHttpResponse response = client.execute(host, httpGet)) {
             StatusLine statusLine = response.getStatusLine();
             if (statusLine.getStatusCode() >= 300) {
-                RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, statusLine);
+                RequestLogger.log(logger, "sniff failed", httpGet, host, response);
                 String responseBody = null;
                 if (response.getEntity() != null) {
                     responseBody = EntityUtils.toString(response.getEntity());
@@ -85,11 +85,11 @@ final class Sniffer {
                 throw new ElasticsearchResponseException(httpGet.getRequestLine(), host, response.getStatusLine(), responseBody);
             } else {
                 List<HttpHost> nodes = readHosts(response.getEntity());
-                RequestLogger.log(logger, "sniff succeeded", httpGet.getRequestLine(), host, statusLine);
+                RequestLogger.log(logger, "sniff succeeded", httpGet, host, response);
                 return nodes;
             }
         } catch(IOException e) {
-            RequestLogger.log(logger, "sniff failed", httpGet.getRequestLine(), host, e);
+            RequestLogger.log(logger, "sniff failed", httpGet, host, e);
             throw e;
         }
     }

client/src/test/java/org/elasticsearch/client/RequestLoggerTests.java

@@ -0,0 +1,107 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import com.carrotsearch.randomizedtesting.generators.RandomInts;
+import org.apache.commons.codec.Charsets;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpHost;
+import org.apache.http.ProtocolVersion;
+import org.apache.http.client.methods.HttpHead;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHttpResponse;
+import org.apache.http.message.BasicStatusLine;
+import org.apache.lucene.util.LuceneTestCase;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+
+public class RequestLoggerTests extends LuceneTestCase {
+
+    public void testTraceRequest() throws IOException, URISyntaxException {
+        HttpHost host = new HttpHost("localhost", 9200, random().nextBoolean() ? "http" : "https");
+        URI uri = new URI("/index/type/_api");
+
+        HttpRequestBase request;
+        int requestType = RandomInts.randomIntBetween(random(), 0, 4);
+        switch(requestType) {
+            case 0:
+                request = new HttpGetWithEntity(uri);
+                break;
+            case 1:
+                request = new HttpPost(uri);
+                break;
+            case 2:
+                request = new HttpPut(uri);
+                break;
+            case 3:
+                request = new HttpDeleteWithEntity(uri);
+                break;
+            case 4:
+                request = new HttpHead(uri);
+                break;
+            default:
+                throw new UnsupportedOperationException();
+        }
+
+        String expected = "curl -iX " + request.getMethod() + " '" + host + uri + "'";
+
+        if (request instanceof HttpEntityEnclosingRequest && random().nextBoolean()) {
+            HttpEntityEnclosingRequest enclosingRequest = (HttpEntityEnclosingRequest) request;
+            String requestBody = "{ \"field\": \"value\" }";
+            enclosingRequest.setEntity(new StringEntity(requestBody, Charsets.UTF_8));
+            expected += " -d '" + requestBody + "'";
+        }
+
+        String traceRequest = RequestLogger.buildTraceRequest(request, host);
+        assertThat(traceRequest, equalTo(expected));
+    }
+
+    public void testTraceResponse() throws IOException {
+        ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 1);
+        int statusCode = RandomInts.randomIntBetween(random(), 200, 599);
+        String reasonPhrase = "REASON";
+        BasicStatusLine statusLine = new BasicStatusLine(protocolVersion, statusCode, reasonPhrase);
+        String expected = "# " + statusLine.toString();
+        BasicHttpResponse httpResponse = new BasicHttpResponse(statusLine);
+        int numHeaders = RandomInts.randomIntBetween(random(), 0, 3);
+        for (int i = 0; i < numHeaders; i++) {
+            httpResponse.setHeader("header" + i, "value");
+            expected += "\n# header" + i + ": value";
+        }
+        expected += "\n#";
+        if (random().nextBoolean()) {
+            String responseBody = "{\n  \"field\": \"value\"\n}";
+            httpResponse.setEntity(new StringEntity(responseBody, Charsets.UTF_8));
+            expected += "\n# {";
+            expected += "\n#   \"field\": \"value\"";
+            expected += "\n# }";
+        }
+
+        String traceResponse = RequestLogger.buildTraceResponse(httpResponse);
+        assertThat(traceResponse, equalTo(expected));
+    }
+}