added larger file upload into perftest

This commit is contained in:
Adrian Cole 2011-04-07 12:52:13 -07:00
parent 25eb1581ea
commit f39514a861
9 changed files with 198 additions and 181 deletions

View File

@ -1,168 +1,181 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<!-- <!--
Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com> Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
==================================================================== ====================================================================
Licensed under the Apache License, Version 2.0 (the "License"); you Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You may may not use this file except in compliance with the License. You may
obtain a copy of the License at obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 Unless required by http://www.apache.org/licenses/LICENSE-2.0 Unless required by
applicable law or agreed to in writing, software distributed under the applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the the specific language governing permissions and limitations under the
License. License.
==================================================================== ====================================================================
--> -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion>
<modelVersion>4.0.0</modelVersion> <parent>
<parent> <groupId>org.jclouds</groupId>
<groupId>org.jclouds</groupId> <artifactId>jclouds-project</artifactId>
<artifactId>aws-common-project</artifactId> <version>1.0-SNAPSHOT</version>
<version>1.0-SNAPSHOT</version> <relativePath>../../project/pom.xml</relativePath>
</parent> </parent>
<artifactId>aws-common-perftest</artifactId> <artifactId>s3-perftest</artifactId>
<name>jclouds Performance test verses Amazon SDK implementation</name> <name>jclouds Performance test verses Amazon SDK implementation</name>
<description>Performance test verses Amazon SDK implementation</description> <description>Performance test verses Amazon SDK implementation</description>
<properties> <properties>
<!-- <test.aws-s3.loopcount>100</test.aws-s3.loopcount>
note you must set the property ${appengine.home} to a valid <test.aws-s3.identity>${test.aws.identity}</test.aws-s3.identity>
extraction of appengine-java-sdk <test.aws-s3.credential>${test.aws.credential}</test.aws-s3.credential>
--> <test.initializer>org.jclouds.aws.s3.blobstore.integration.AWSS3TestInitializer</test.initializer>
<appengine.home>YOUR_APPENGINE_HOME</appengine.home> <jclouds.version>1.0-SNAPSHOT</jclouds.version>
<test.s3.loopcount>100</test.s3.loopcount> <gae.version>1.4.0</gae.version>
<test.s3.identity>${test.aws.identity}</test.s3.identity> </properties>
<test.s3.credential>${test.aws.credential}</test.s3.credential> <!-- bootstrapping: need to fetch the project POM -->
<test.initializer>org.jclouds.s3.blobstore.integration.S3TestInitializer</test.initializer> <repositories>
</properties> <repository>
<id>jclouds-sona-snapshots-nexus</id>
<dependencies> <url>https://oss.sonatype.org/content/repositories/snapshots</url>
<dependency> <releases>
<groupId>${project.groupId}</groupId> <enabled>false</enabled>
<artifactId>jclouds-apachehc</artifactId> </releases>
<version>${project.version}</version> <snapshots>
</dependency> <enabled>true</enabled>
<dependency> </snapshots>
<groupId>org.jclouds.api</groupId> </repository>
<artifactId>s3</artifactId> </repositories>
<version>${project.version}</version> <dependencies>
</dependency> <dependency>
<dependency> <groupId>org.jclouds.provider</groupId>
<groupId>com.amazonaws</groupId> <artifactId>aws-s3</artifactId>
<artifactId>aws-java-sdk</artifactId> <version>${jclouds.version}</version>
<version>1.1.1</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>com.amazonaws</groupId>
<groupId>net.java.dev.jets3t</groupId> <artifactId>aws-java-sdk</artifactId>
<artifactId>jets3t</artifactId> <version>1.1.9</version>
<version>0.8.0</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>net.java.dev.jets3t</groupId>
<groupId>org.jclouds.driver</groupId> <artifactId>jets3t</artifactId>
<artifactId>jclouds-enterprise</artifactId> <version>0.8.0</version>
<version>${project.version}</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>org.jclouds.driver</groupId>
<groupId>org.jclouds.driver</groupId> <artifactId>jclouds-enterprise</artifactId>
<artifactId>jclouds-apachehc</artifactId> <version>${jclouds.version}</version>
<version>${project.version}</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>org.jclouds.driver</groupId>
<groupId>org.jclouds.driver</groupId> <artifactId>jclouds-log4j</artifactId>
<artifactId>jclouds-gae</artifactId> <version>${jclouds.version}</version>
<version>${project.version}</version> </dependency>
</dependency> <dependency>
<dependency> <groupId>org.jclouds.driver</groupId>
<groupId>org.jclouds.api</groupId> <artifactId>jclouds-apachehc</artifactId>
<artifactId>s3</artifactId> <version>${jclouds.version}</version>
<version>${project.version}</version> </dependency>
<type>test-jar</type> <dependency>
<scope>test</scope> <groupId>org.jclouds.driver</groupId>
</dependency> <artifactId>jclouds-gae</artifactId>
<!-- start gae stuff --> <version>${jclouds.version}</version>
<dependency> </dependency>
<groupId>com.google.appengine</groupId> <dependency>
<artifactId>appengine-api-stubs</artifactId> <groupId>com.google.appengine</groupId>
<version>1.3.5</version> <artifactId>appengine-tools-sdk</artifactId>
<scope>test</scope> <version>${gae.version}</version>
</dependency> <scope>system</scope>
<dependency> <systemPath>${env.APPENGINE_HOME}/lib/appengine-tools-api.jar</systemPath>
<groupId>com.google.appengine</groupId> </dependency>
<artifactId>appengine-testing</artifactId> <dependency>
<version>1.3.5</version> <groupId>org.jclouds</groupId>
<scope>test</scope> <artifactId>jclouds-core</artifactId>
</dependency> <version>${jclouds.version}</version>
<dependency> <type>test-jar</type>
<groupId>com.google.appengine</groupId> <scope>test</scope>
<artifactId>appengine-local-runtime</artifactId> </dependency>
<version>1.3.5</version> <dependency>
<scope>test</scope> <groupId>org.jclouds</groupId>
</dependency> <artifactId>jclouds-blobstore</artifactId>
<dependency> <version>${jclouds.version}</version>
<groupId>${project.groupId}</groupId> <type>test-jar</type>
<artifactId>jclouds-core</artifactId> <scope>test</scope>
<version>${project.version}</version> </dependency>
<type>test-jar</type> <dependency>
<scope>test</scope> <groupId>org.jclouds.api</groupId>
<!-- <artifactId>s3</artifactId>
<exclusions> <exclusion> <groupId>org.mortbay.jetty</groupId> <version>${jclouds.version}</version>
<artifactId>jetty</artifactId> </exclusion> </exclusions> <type>test-jar</type>
--> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.geronimo.specs</groupId> <groupId>org.jclouds.provider</groupId>
<artifactId>geronimo-servlet_2.5_spec</artifactId> <artifactId>aws-s3</artifactId>
<version>1.2</version> <version>${jclouds.version}</version>
<scope>test</scope> <type>test-jar</type>
</dependency> <scope>test</scope>
</dependency>
</dependencies> <!-- start gae stuff -->
<dependency>
<scm> <groupId>com.google.appengine</groupId>
<connection>scm:svn:http://jclouds.googlecode.com/svn/trunk</connection> <artifactId>appengine-tools-sdk</artifactId>
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk</developerConnection> <version>${gae.version}</version>
<url>http://jclouds.googlecode.com/svn/trunk</url> <scope>system</scope>
</scm> <systemPath>${env.APPENGINE_HOME}/lib/appengine-tools-api.jar</systemPath>
</dependency>
<build> <dependency>
<plugins> <groupId>com.google.appengine</groupId>
<plugin> <artifactId>appengine-api-stubs</artifactId>
<artifactId>maven-surefire-plugin</artifactId> <version>${gae.version}</version>
<executions> <scope>test</scope>
<execution> </dependency>
<id>integration</id> <dependency>
<phase>integration-test</phase> <groupId>com.google.appengine</groupId>
<goals> <artifactId>appengine-testing</artifactId>
<goal>test</goal> <version>${gae.version}</version>
</goals> <scope>test</scope>
<configuration> </dependency>
<threadCount>1</threadCount> </dependencies>
<systemProperties> <build>
<property> <plugins>
<name>test.s3.identity</name> <plugin>
<value>${test.s3.identity}</value> <artifactId>maven-surefire-plugin</artifactId>
</property> <executions>
<property> <execution>
<name>test.s3.credential</name> <id>integration</id>
<value>${test.s3.credential}</value> <phase>integration-test</phase>
</property> <goals>
<property> <goal>test</goal>
<name>test.initializer</name> </goals>
<value>${test.initializer}</value> <configuration>
</property> <threadCount>1</threadCount>
<property> <systemProperties>
<name>test.s3.loopcount</name> <property>
<value>${test.s3.loopcount}</value> <name>test.aws-s3.identity</name>
</property> <value>${test.aws-s3.identity}</value>
</systemProperties> </property>
</configuration> <property>
</execution> <name>test.aws-s3.credential</name>
</executions> <value>${test.aws-s3.credential}</value>
</plugin> </property>
</plugins> <property>
</build> <name>test.initializer</name>
<value>${test.initializer}</value>
</property>
<property>
<name>test.aws-s3.loopcount</name>
<value>${test.aws-s3.loopcount}</value>
</property>
</systemProperties>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project> </project>

View File

@ -50,8 +50,8 @@ public class AmazonPerformanceLiveTest extends BasePerformanceLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext); super.setUpResourcesOnThisThread(testContext);
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
s3 = new AmazonS3Client(new BasicAWSCredentials(System.getProperty("test.s3.identity"), s3 = new AmazonS3Client(new BasicAWSCredentials(System.getProperty("test.aws-s3.identity"),
System.getProperty("test.s3.credential"))); System.getProperty("test.aws-s3.credential")));
} }
@Override @Override

View File

@ -30,7 +30,8 @@ import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.S3Object;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;
@ -59,7 +60,7 @@ public abstract class BaseJCloudsPerformanceLiveTest extends BasePerformanceLive
// } // }
protected void overrideWithSysPropertiesAndPrint(Properties overrides, String contextName) { protected void overrideWithSysPropertiesAndPrint(Properties overrides, String contextName) {
overrides.putAll(System.getProperties()); overrides.putAll(System.getProperties());
System.out.printf("%s: loopCount(%s), perContext(%s), perHost(%s),ioWorkers(%s), userThreads(%s)%n", contextName, System.out.printf("%s: loopCount(%s), perContext(%s), perHost(%s), ioWorkers(%s), userThreads(%s)%n", contextName,
loopCount, overrides.getProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT), overrides loopCount, overrides.getProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT), overrides
.getProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST), overrides.getProperty(PROPERTY_IO_WORKER_THREADS), .getProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST), overrides.getProperty(PROPERTY_IO_WORKER_THREADS),
overrides.getProperty(PROPERTY_USER_THREADS)); overrides.getProperty(PROPERTY_USER_THREADS));

View File

@ -51,7 +51,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
containerCount = 1; containerCount = 1;
} }
protected int timeoutSeconds = 15; protected int timeoutSeconds = 15;
protected int loopCount = Integer.parseInt(System.getProperty("test.s3.loopcount", "1000")); protected int loopCount = Integer.parseInt(System.getProperty("test.aws-s3.loopcount", "1000"));
protected ExecutorService exec; protected ExecutorService exec;
protected Logger logger = Logger.NULL;; protected Logger logger = Logger.NULL;;
@ -141,13 +141,13 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
} }
private void doParallel(Provider<ListenableFuture<?>> provider, int loopCount, String containerName) private void doParallel(Provider<ListenableFuture<?>> provider, int loopCount, String containerName)
throws InterruptedException, ExecutionException, TimeoutException { throws InterruptedException, ExecutionException, TimeoutException {
Map<Integer, ListenableFuture<?>> responses = Maps.newHashMap(); Map<Integer, ListenableFuture<?>> responses = Maps.newHashMap();
for (int i = 0; i < loopCount; i++) for (int i = 0; i < loopCount; i++)
responses.put(i, provider.get()); responses.put(i, provider.get());
Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, null, Logger.NULL, String.format( Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, null, Logger.NULL, String.format(
"putting into containerName: %s", containerName)); "putting into containerName: %s", containerName));
assert exceptions.size() == 0 : exceptions; assert exceptions.size() == 0 : exceptions;
@ -164,13 +164,13 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
public ListenableFuture<?> get() { public ListenableFuture<?> get() {
return Futures.makeListenable(putByteArray(bucketName, key.getAndIncrement() + "", test, return Futures.makeListenable(putByteArray(bucketName, key.getAndIncrement() + "", test,
"application/octetstring")); "application/octetstring"));
} }
} }
class PutFileFuture implements Provider<ListenableFuture<?>> { class PutFileFuture implements Provider<ListenableFuture<?>> {
final AtomicInteger key = new AtomicInteger(0); final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml"); protected File file = new File(getClass().getResource("/testimg.png").getFile());
private final String bucketName; private final String bucketName;
public PutFileFuture(String bucketName) { public PutFileFuture(String bucketName) {
@ -195,7 +195,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
public ListenableFuture<?> get() { public ListenableFuture<?> get() {
return Futures.makeListenable(putInputStream(bucketName, key.getAndIncrement() + "", new ByteArrayInputStream( return Futures.makeListenable(putInputStream(bucketName, key.getAndIncrement() + "", new ByteArrayInputStream(
test), "application/octetstring")); test), "application/octetstring"));
} }
} }

View File

@ -33,6 +33,7 @@ import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule; import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule; import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
import org.jclouds.logging.config.NullLoggingModule; import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.ITestContext; import org.testng.ITestContext;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -46,8 +47,8 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi
@BeforeClass(groups = { "integration", "live" }) @BeforeClass(groups = { "integration", "live" })
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
String accesskeyid = System.getProperty("test.s3.identity"); String accesskeyid = System.getProperty("test.aws-s3.identity");
String secretkey = System.getProperty("test.s3.credential"); String secretkey = System.getProperty("test.aws-s3.credential");
Properties overrides = new Properties(); Properties overrides = new Properties();
overrides.setProperty(PROPERTY_SO_TIMEOUT, 5000 + ""); overrides.setProperty(PROPERTY_SO_TIMEOUT, 5000 + "");
overrides.setProperty(PROPERTY_CONNECTION_TIMEOUT, 5000 + ""); overrides.setProperty(PROPERTY_CONNECTION_TIMEOUT, 5000 + "");
@ -57,7 +58,7 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi
overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); overrides.setProperty(PROPERTY_USER_THREADS, 0 + "");
String contextName = "enterprise"; String contextName = "enterprise";
overrideWithSysPropertiesAndPrint(overrides, contextName); overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(), new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(),
new EnterpriseConfigurationModule()), overrides); new EnterpriseConfigurationModule()), overrides);
} }

View File

@ -30,6 +30,7 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.gae.config.GoogleAppEngineConfigurationModule; import org.jclouds.gae.config.GoogleAppEngineConfigurationModule;
import org.jclouds.logging.config.NullLoggingModule; import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
@ -143,7 +144,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes
Properties overrides = new Properties(); Properties overrides = new Properties();
String contextName = "gae"; String contextName = "gae";
overrideWithSysPropertiesAndPrint(overrides, contextName); overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new GoogleAppEngineConfigurationModule()), overrides); new NullLoggingModule(), new GoogleAppEngineConfigurationModule()), overrides);
} }

View File

@ -30,6 +30,7 @@ import java.util.concurrent.Executors;
import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule; import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.logging.config.NullLoggingModule; import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.ITestContext; import org.testng.ITestContext;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -49,8 +50,8 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest {
@BeforeClass(groups = { "integration", "live" }) @BeforeClass(groups = { "integration", "live" })
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
String accesskeyid = System.getProperty("test.s3.identity"); String accesskeyid = System.getProperty("test.aws-s3.identity");
String secretkey = System.getProperty("test.s3.credential"); String secretkey = System.getProperty("test.aws-s3.credential");
Properties overrides = new Properties(); Properties overrides = new Properties();
overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + ""); overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + "");
overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 30 + ""); overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 30 + "");
@ -58,7 +59,7 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest {
overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); overrides.setProperty(PROPERTY_USER_THREADS, 0 + "");
String contextName = "standard"; String contextName = "standard";
overrideWithSysPropertiesAndPrint(overrides, contextName); overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new EnterpriseConfigurationModule()), overrides); new NullLoggingModule(), new EnterpriseConfigurationModule()), overrides);
} }

View File

@ -49,8 +49,8 @@ public class Jets3tPerformanceLiveTest extends BasePerformanceLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext); super.setUpResourcesOnThisThread(testContext);
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
jetClient = new RestS3Service(new AWSCredentials(System.getProperty("test.s3.identity"), jetClient = new RestS3Service(new AWSCredentials(System.getProperty("test.aws-s3.identity"),
System.getProperty("test.s3.credential"))); System.getProperty("test.aws-s3.credential")));
} }
@Override @Override

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 MiB