SOLR-462: Changes to CommonsHttpSolrServer.java

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@613304 13f79535-47bb-0310-9956-ffa450edef68

client/java/solrj/CHANGES.txt

@@ -30,6 +30,13 @@ Changes in runtime behavior
     The API to build documents has changed -- you need to pass a boost 
     (or null) with every field.  (ryan)
 
+ 3. SOLR-462: Changes to CommonsHttpSolrServer.java to add soTimeout (read 
+    timeout), connection pool timeout, directive to not follow HTTP redirects, 
+    configurable retries on NoHttpResponseException, compression, and not 
+    creating a new HttpClient on each request.  If your existing code overrides 
+    getHttpConnection(), you will now need to override createHttpClient()
+    (Sean Timm via ryan)
+
 Bug Fixes
 
 Other Changes

client/java/solrj/src/org/apache/solr/client/solrj/impl/CommonsHttpSolrServer.java

@@ -18,6 +18,7 @@
 package org.apache.solr.client.solrj.impl;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.Reader;
@@ -25,13 +26,17 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
 
+import org.apache.commons.httpclient.Header;
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpMethod;
 import org.apache.commons.httpclient.HttpMethodBase;
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.NoHttpResponseException;
 import org.apache.commons.httpclient.methods.GetMethod;
 import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
 import org.apache.commons.httpclient.methods.PostMethod;
@@ -66,7 +71,11 @@ public class CommonsHttpSolrServer extends BaseSolrServer
   protected String _baseURL;
   protected ModifiableSolrParams _invariantParams;
   protected ResponseParser _processor;
-  MultiThreadedHttpConnectionManager _connectionManager = new MultiThreadedHttpConnectionManager();
+  
+  private final HttpClient _httpClient;
+  private boolean _followRedirects = false;
+  private boolean _allowCompression = false;
+  private int _maxRetries = 0;
   
   /**  
    * @param solrServerUrl The URL of the Solr server.  For 
@@ -91,6 +100,8 @@ public class CommonsHttpSolrServer extends BaseSolrServer
       this._baseURL = this._baseURL.substring( 0, this._baseURL.length()-1 );
     }
     
+    this._httpClient = createHttpClient();
+    
     // increase the default connections
     this.setDefaultMaxConnectionsPerHost( 32 );  // 2
     this.setMaxTotalConnections( 128 ); // 20
@@ -104,6 +115,14 @@ public class CommonsHttpSolrServer extends BaseSolrServer
     _invariantParams.set( CommonParams.VERSION, "2.2" );
   }
   
+  /**
+   * This can be overridden to add certificates etc
+   */
+  protected HttpClient createHttpClient()
+  {
+    return new HttpClient( new MultiThreadedHttpConnectionManager() );
+  }
+
   //------------------------------------------------------------------------
   //------------------------------------------------------------------------
 
@@ -136,6 +155,13 @@ public class CommonsHttpSolrServer extends BaseSolrServer
       params = new DefaultSolrParams( _invariantParams, params );
     }
     
+    int tries = _maxRetries + 1;        
+    try {
+      while( tries-- > 0 ) {
+        // Note: since we aren't do intermittent time keeping
+        // ourselves, the potential non-timeout latency could be as
+        // much as tries-times (plus scheduling effects) the given
+        // timeAllowed.
         try {
           if( SolrRequest.METHOD.GET == request.getMethod() ) {
             if( streams != null ) {
@@ -191,11 +217,7 @@ public class CommonsHttpSolrServer extends BaseSolrServer
                 }
                 
                 // Set the multi-part request
-              post.setRequestEntity(
-                  new MultipartRequestEntity(
-                    parts,
-                    post.getParams() )
-              );
+                post.setRequestEntity( new MultipartRequestEntity( parts, post.getParams() ) );
                 method = post;
               }
 
@@ -210,9 +232,8 @@ public class CommonsHttpSolrServer extends BaseSolrServer
               // Using a loop just to get the first one
               for( ContentStream content : streams ) {
                 post.setRequestEntity( 
-              new InputStreamRequestEntity(
-                content.getStream(),
-                content.getContentType()) );
+                    new InputStreamRequestEntity( content.getStream(), content.getContentType())
+                );
                 break;
               }
               method = post;
@@ -222,17 +243,33 @@ public class CommonsHttpSolrServer extends BaseSolrServer
             throw new SolrServerException("Unsupported method: "+request.getMethod() );
           }
         }
+        catch( NoHttpResponseException r ) {
+          // This is generally safe to retry on
+          method.releaseConnection();
+          method = null;
+          // If out of tries then just rethrow (as normal error).
+          if( ( tries < 1 ) ) {
+            throw r;
+          }
+          //log.warn( "Caught: " + r + ". Retrying..." );
+        }
+      }
+    }
     catch( IOException ex ) {
       throw new SolrServerException("error reading streams", ex );
     }
     
+    method.setFollowRedirects( _followRedirects );
     method.addRequestHeader( "User-Agent", AGENT );
+    if( _allowCompression ) {
+      method.setRequestHeader( new Header( "Accept-Encoding", "gzip,deflate" ) );
+    }
     
     try {
       // Execute the method.
       //System.out.println( "EXECUTE:"+method.getURI() );
     
-      int statusCode = getHttpConnection().executeMethod(method);
+      int statusCode = _httpClient.executeMethod(method);
       if (statusCode != HttpStatus.SC_OK) {
         StringBuilder msg = new StringBuilder();
         msg.append( method.getStatusLine().getReasonPhrase() );
@@ -248,7 +285,41 @@ public class CommonsHttpSolrServer extends BaseSolrServer
       if( method instanceof HttpMethodBase ) {
         charset = ((HttpMethodBase)method).getResponseCharSet();
       }
-      Reader reader = new InputStreamReader( method.getResponseBodyAsStream(), charset ); 
+      InputStream respBody = method.getResponseBodyAsStream();
+      // Jakarta Commons HTTPClient doesn't handle any
+      // compression natively.  Handle gzip or deflate
+      // here if applicable.
+      if( _allowCompression ) {
+        Header contentEncodingHeader = method.getResponseHeader( "Content-Encoding" );
+        if( contentEncodingHeader != null ) {
+          String contentEncoding = contentEncodingHeader.getValue();
+          if( contentEncoding.contains( "gzip" ) ) {
+            //log.debug( "wrapping response in GZIPInputStream" );
+            respBody = new GZIPInputStream( respBody );
+          }
+          else if( contentEncoding.contains( "deflate" ) ) {
+            //log.debug( "wrapping response in InflaterInputStream" );
+            respBody = new InflaterInputStream(respBody);
+          }
+        }
+        else {
+          Header contentTypeHeader = method.getResponseHeader( "Content-Type" );
+          if( contentTypeHeader != null ) {
+            String contentType = contentTypeHeader.getValue();
+            if( contentType != null ) {
+              if( contentType.startsWith( "application/x-gzip-compressed" ) ) {
+                //log.debug( "wrapping response in GZIPInputStream" );
+                respBody = new GZIPInputStream( respBody );
+              }
+              else if ( contentType.startsWith("application/x-deflate") ) {
+                //log.debug( "wrapping response in InflaterInputStream" );
+                respBody = new InflaterInputStream(respBody);
+              }
+            }
+          }
+        }
+      }
+      Reader reader = new InputStreamReader( respBody, charset ); 
       return _processor.processResponse( reader );
     } 
     catch (HttpException e) {
@@ -290,27 +361,58 @@ public class CommonsHttpSolrServer extends BaseSolrServer
     _processor = processor;
   }
 
-  protected HttpClient getHttpConnection() {
-    return new HttpClient(_connectionManager);
-  }
-
   public MultiThreadedHttpConnectionManager getConnectionManager() {
-    return _connectionManager;
+    return (MultiThreadedHttpConnectionManager)_httpClient.getHttpConnectionManager();
   }
   
   /** set connectionTimeout on the underlying MultiThreadedHttpConnectionManager */
   public void setConnectionTimeout(int timeout) {
-    _connectionManager.getParams().setConnectionTimeout(timeout);
+    getConnectionManager().getParams().setConnectionTimeout(timeout);
+  }
+  
+  /** set connectionManagerTimeout on the HttpClient.**/
+  public void setConnectionManagerTimeout(int timeout) {
+    _httpClient.getParams().setConnectionManagerTimeout(timeout);
+  }
+  
+  /** set soTimeout (read timeout) on the underlying MultiThreadedHttpConnectionManager.  This is desirable for queries, but probably not for indexing. */
+  public void setSoTimeout(int timeout) {
+    getConnectionManager().getParams().setSoTimeout(timeout);
   }
   
   /** set maxConnectionsPerHost on the underlying MultiThreadedHttpConnectionManager */
   public void setDefaultMaxConnectionsPerHost(int connections) {
-    _connectionManager.getParams().setDefaultMaxConnectionsPerHost(connections);
+    getConnectionManager().getParams().setDefaultMaxConnectionsPerHost(connections);
   }
   
   /** set maxTotalConnection on the underlying MultiThreadedHttpConnectionManager */
   public void setMaxTotalConnections(int connections) {
-    _connectionManager.getParams().setMaxTotalConnections(connections);
+    getConnectionManager().getParams().setMaxTotalConnections(connections);
   }
 
+  /**
+   * set followRedirects.  This defaults to false under the
+   * assumption that if you are following a redirect to get to a Solr
+   * installation, something is misconfigured somewhere.
+   */
+  public void setFollowRedirects( boolean followRedirects ) {
+    _followRedirects = followRedirects;
+  }
+
+  /**
+   * set allowCompression.  If compression is enabled, both gzip and
+   * deflate compression will be accepted in the HTTP response.
+   */
+  public void setAllowCompression( boolean allowCompression ) {
+    _allowCompression = allowCompression;
+  }
+
+  /**
+   *  set maximum number of retries to attempt in the event of
+   *  transient errors.  Default: 0 (no) retries. No more than 1
+   *  recommended.
+   */
+  public void setMaxRetries( int maxRetries ) {
+    _maxRetries = maxRetries;
+  }
 }