Issue 107: new Apache HttpComponents HttpClient 4.0 plugin

core/src/main/java/org/jclouds/http/HttpUtils.java

@@ -57,6 +57,19 @@ import com.google.common.io.Closeables;
  */
 public class HttpUtils {
 
+   /**
+    * keys to the map are only used for socket information, not path. In this case, you should
+    * remove any path or query details from the URI.
+    */
+   public static URI createBaseEndpointFor(URI endpoint) {
+      if (endpoint.getPort() == -1) {
+         return URI.create(String.format("%s://%s", endpoint.getScheme(), endpoint.getHost()));
+      } else {
+         return URI.create(String.format("%s://%s:%d", endpoint.getScheme(), endpoint.getHost(),
+                  endpoint.getPort()));
+      }
+   }
+
    /**
     * Web browsers do not always handle '+' characters well, use the well-supported '%20' instead.
     */

extensions/apachehc/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+
+    Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+
+    ====================================================================
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+    ====================================================================
+
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <artifactId>jclouds-extensions-project</artifactId>
+        <groupId>org.jclouds</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <artifactId>jclouds-apachehc</artifactId>
+    <name>jclouds Apache Http Components Client</name>
+    <packaging>jar</packaging>
+    <description>Apache HttpComponents client</description>
+
+    <scm>
+        <connection>scm:svn:http://jclouds.googlecode.com/svn/trunk</connection>
+        <developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk</developerConnection>
+        <url>http://jclouds.googlecode.com/svn/trunk</url>
+    </scm>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.0.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.mortbay.jetty</groupId>
+            <artifactId>jetty</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

extensions/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorService.java

@@ -0,0 +1,133 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+
+package org.jclouds.http.apachehc;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+
+import javax.inject.Named;
+
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpHost;
+import org.apache.http.HttpVersion;
+import org.apache.http.client.HttpClient;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.params.ConnManagerParams;
+import org.apache.http.conn.params.ConnPerRoute;
+import org.apache.http.conn.params.ConnPerRouteBean;
+import org.apache.http.conn.scheme.PlainSocketFactory;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.scheme.SchemeRegistry;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
+import org.apache.http.params.BasicHttpParams;
+import org.apache.http.params.HttpParams;
+import org.apache.http.params.HttpProtocolParams;
+import org.jclouds.Constants;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.HttpUtils;
+import org.jclouds.http.handlers.DelegatingErrorHandler;
+import org.jclouds.http.handlers.DelegatingRetryHandler;
+import org.jclouds.http.internal.BaseHttpCommandExecutorService;
+import org.jclouds.http.internal.HttpWire;
+
+import com.google.common.base.Function;
+import com.google.common.collect.MapMaker;
+import com.google.inject.Inject;
+
+/**
+ * Simple implementation of a {@link HttpFutureCommandClient}, Apache Components HttpClient 4.x.
+ * 
+ * @author Sam Tunnicliffe
+ * @author Adrian Cole
+ */
+public class ApacheHCHttpCommandExecutorService extends
+         BaseHttpCommandExecutorService<HttpEntityEnclosingRequest> {
+
+   private final ConcurrentMap<URI, HttpClient> poolMap;
+
+   @Inject
+   ApacheHCHttpCommandExecutorService(
+            @Named(Constants.PROPERTY_IO_WORKER_THREADS) ExecutorService ioWorkerExecutor,
+            DelegatingRetryHandler retryHandler,
+            DelegatingErrorHandler errorHandler,
+            HttpWire wire,
+            @Named(Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT) final int globalMaxConnections,
+            @Named(Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST) final int globalMaxConnectionsPerHost) {
+      super(ioWorkerExecutor, retryHandler, errorHandler, wire);
+      checkArgument(globalMaxConnections > 0, Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT
+               + " must be greater than zero");
+      checkArgument(globalMaxConnectionsPerHost > 0, Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST
+               + " must be greater than zero");
+      poolMap = new MapMaker().makeComputingMap(new Function<URI, HttpClient>() {
+         public HttpClient apply(URI endPoint) {
+            checkArgument(endPoint.getHost() != null, String.format(
+                     "endPoint.getHost() is null for %s", endPoint));
+            HttpParams params = new BasicHttpParams();
+            try {
+               // TODO: have this use our executor service
+               // TODO: implement wire logging
+               ConnManagerParams.setMaxTotalConnections(params, globalMaxConnections);
+               ConnPerRoute connectionsPerRoute = new ConnPerRouteBean(globalMaxConnectionsPerHost);
+               ConnManagerParams.setMaxConnectionsPerRoute(params, connectionsPerRoute);
+               HttpProtocolParams.setVersion(params, HttpVersion.HTTP_1_1);
+               SchemeRegistry schemeRegistry = new SchemeRegistry();
+               if (endPoint.getScheme().equals("http"))
+                  schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(),
+                           80));
+               else
+                  schemeRegistry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(),
+                           443));
+               ClientConnectionManager cm = new ThreadSafeClientConnManager(params, schemeRegistry);
+               return new DefaultHttpClient(cm, params);
+            } catch (RuntimeException e) {
+               logger.error(e, "error creating entry for %s", endPoint);
+               throw e;
+            }
+         }
+      });
+   }
+
+   @Override
+   protected HttpEntityEnclosingRequest convert(HttpRequest request) throws IOException {
+      return ApacheHCUtils.convertToApacheRequest(request);
+   }
+
+   @Override
+   protected HttpResponse invoke(HttpEntityEnclosingRequest nativeRequest) throws IOException {
+      URI endpoint = URI.create(nativeRequest.getRequestLine().getUri());
+      HttpClient client = poolMap.get(HttpUtils.createBaseEndpointFor(endpoint));
+      assert (client != null) : "pool for endpoint null " + endpoint;
+      HttpHost host = new HttpHost(endpoint.getHost(), endpoint.getPort(), endpoint.getScheme());
+      org.apache.http.HttpResponse nativeResponse = client.execute(host, nativeRequest);
+      return ApacheHCUtils.convertToJCloudsResponse(nativeResponse);
+   }
+
+   @Override
+   protected void cleanup(HttpEntityEnclosingRequest nativeResponse) {
+      // TODO
+   }
+}

extensions/apachehc/src/main/java/org/jclouds/http/apachehc/ApacheHCUtils.java

@@ -0,0 +1,125 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.http.apachehc;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+
+import javax.ws.rs.core.HttpHeaders;
+
+import org.apache.http.Header;
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpVersion;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.FileEntity;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.message.BasicHttpEntityEnclosingRequest;
+import org.jclouds.http.HttpRequest;
+import org.jclouds.http.HttpResponse;
+import org.jclouds.http.Payload;
+
+/**
+ * 
+ * @author Adrian Cole
+ */
+public class ApacheHCUtils {
+   public static final String USER_AGENT = "jclouds/1.0 httpclient/4.0.1";
+
+   public static HttpEntityEnclosingRequest convertToApacheRequest(HttpRequest request) {
+
+      String uri = request.getEndpoint().toASCIIString();
+      if (request.getEndpoint().getQuery() != null)
+         uri += "?" + request.getEndpoint().getQuery();
+      BasicHttpEntityEnclosingRequest apacheRequest = new BasicHttpEntityEnclosingRequest(request
+               .getMethod(), uri, HttpVersion.HTTP_1_1);
+
+      Payload payload = request.getPayload();
+
+      // Since we may remove headers, ensure they are added to the apache
+      // request after this block
+      if (payload != null) {
+         String lengthString = request.getFirstHeaderOrNull(HttpHeaders.CONTENT_LENGTH);
+         if (lengthString == null) {
+            throw new IllegalStateException("no Content-Length header on request: " + apacheRequest);
+         }
+         long contentLength = Long.parseLong(lengthString);
+         String contentType = request.getFirstHeaderOrNull(HttpHeaders.CONTENT_TYPE);
+         addEntityForContent(apacheRequest, payload.getRawContent(), contentType, contentLength);
+      }
+
+      for (String header : request.getHeaders().keySet()) {
+         for (String value : request.getHeaders().get(header))
+            // apache automatically tries to add content length header
+            if (!header.equals(HttpHeaders.CONTENT_LENGTH))
+               apacheRequest.addHeader(header, value);
+      }
+      apacheRequest.addHeader(HttpHeaders.USER_AGENT, USER_AGENT);
+
+      return apacheRequest;
+   }
+
+   public static void addEntityForContent(HttpEntityEnclosingRequest apacheRequest, Object content,
+            String contentType, long length) {
+      if (content instanceof InputStream) {
+         InputStream inputStream = (InputStream) content;
+         if (length == -1)
+            throw new IllegalArgumentException(
+                     "you must specify size when content is an InputStream");
+         InputStreamEntity Entity = new InputStreamEntity(inputStream, length);
+         Entity.setContentType(contentType);
+         apacheRequest.setEntity(Entity);
+      } else if (content instanceof String) {
+         StringEntity nStringEntity = null;
+         try {
+            nStringEntity = new StringEntity((String) content);
+         } catch (UnsupportedEncodingException e) {
+            throw new UnsupportedOperationException("Encoding not supported", e);
+         }
+         nStringEntity.setContentType(contentType);
+         apacheRequest.setEntity(nStringEntity);
+      } else if (content instanceof File) {
+         apacheRequest.setEntity(new FileEntity((File) content, contentType));
+      } else if (content instanceof byte[]) {
+         ByteArrayEntity Entity = new ByteArrayEntity((byte[]) content);
+         Entity.setContentType(contentType);
+         apacheRequest.setEntity(Entity);
+      } else {
+         throw new UnsupportedOperationException("Content class not supported: "
+                  + content.getClass().getName());
+      }
+      assert (apacheRequest.getEntity() != null);
+   }
+
+   public static HttpResponse convertToJCloudsResponse(org.apache.http.HttpResponse apacheResponse)
+            throws IOException {
+      HttpResponse response = new HttpResponse();
+      if (apacheResponse.getEntity() != null) {
+         response.setContent(apacheResponse.getEntity().getContent());
+      }
+      for (Header header : apacheResponse.getAllHeaders()) {
+         response.getHeaders().put(header.getName(), header.getValue());
+      }
+      response.setStatusCode(apacheResponse.getStatusLine().getStatusCode());
+      response.setMessage(apacheResponse.getStatusLine().getReasonPhrase());
+      return response;
+   }
+}

extensions/apachehc/src/main/java/org/jclouds/http/apachehc/config/ApacheHCHttpCommandExecutorServiceModule.java

@@ -0,0 +1,54 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.http.apachehc.config;
+
+import org.jclouds.http.HttpCommandExecutorService;
+import org.jclouds.http.TransformingHttpCommandExecutorService;
+import org.jclouds.http.TransformingHttpCommandExecutorServiceImpl;
+import org.jclouds.http.apachehc.ApacheHCHttpCommandExecutorService;
+import org.jclouds.http.config.ConfiguresHttpCommandExecutorService;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Scopes;
+
+/**
+ * Configures {@link ApacheHCHttpCommandExecutorService}.
+ * 
+ * Note that this uses threads
+ * 
+ * @author Sam Tunnicliffe
+ * @author Adrian Cole
+ */
+@ConfiguresHttpCommandExecutorService
+public class ApacheHCHttpCommandExecutorServiceModule extends AbstractModule {
+
+   @Override
+   protected void configure() {
+      bindClient();
+   }
+
+   protected void bindClient() {
+      bind(HttpCommandExecutorService.class).to(ApacheHCHttpCommandExecutorService.class).in(
+               Scopes.SINGLETON);
+
+      bind(TransformingHttpCommandExecutorService.class).to(
+               TransformingHttpCommandExecutorServiceImpl.class).in(Scopes.SINGLETON);
+   }
+
+}

extensions/apachehc/src/test/java/org/jclouds/http/apachehc/ApacheHCHttpCommandExecutorServiceTest.java

@@ -0,0 +1,54 @@
+/**
+ *
+ * Copyright (C) 2009 Cloud Conscious, LLC. <info@cloudconscious.com>
+ *
+ * ====================================================================
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ====================================================================
+ */
+package org.jclouds.http.apachehc;
+
+import static org.jclouds.Constants.PROPERTY_IO_WORKER_THREADS;
+import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT;
+import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST;
+import static org.jclouds.Constants.PROPERTY_USER_THREADS;
+
+import java.util.Properties;
+
+import org.jclouds.http.BaseHttpCommandExecutorServiceTest;
+import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
+import org.testng.annotations.Test;
+
+import com.google.inject.Module;
+
+/**
+ * Tests the functionality of the {@link ApacheHCHttpCommandExecutorService}
+ * 
+ * @author Adrian Cole
+ */
+@Test
+public class ApacheHCHttpCommandExecutorServiceTest extends BaseHttpCommandExecutorServiceTest {
+
+   protected Module createConnectionModule() {
+      return new ApacheHCHttpCommandExecutorServiceModule();
+   }
+
+   protected void addConnectionProperties(Properties props) {
+      props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + "");
+      props.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 50 + "");
+      // IO workers not used in this executor
+      props.setProperty(PROPERTY_IO_WORKER_THREADS, 0 + "");
+      props.setProperty(PROPERTY_USER_THREADS, 5 + "");
+   }
+
+}

extensions/pom.xml

@@ -25,7 +25,7 @@
     <parent>
         <artifactId>jclouds-project</artifactId>
         <groupId>org.jclouds</groupId>
-        <version>1.0-beta-3</version>
+        <version>1.0-SNAPSHOT</version>
         <relativePath>../project/pom.xml</relativePath>
     </parent>
     <artifactId>jclouds-extensions-project</artifactId>
@@ -34,6 +34,7 @@
     <modules>
         <module>gae</module>
         <module>httpnio</module>
+        <module>apachehc</module>
         <module>joda</module>
         <module>bouncycastle</module>
         <module>log4j</module>