added larger file upload into perftest

This commit is contained in:
Adrian Cole 2011-04-07 12:52:13 -07:00
parent 25eb1581ea
commit f39514a861
9 changed files with 198 additions and 181 deletions

View File

@ -1,168 +1,181 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
<!--
Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
Copyright (C) 2010 Cloud Conscious, LLC. <info@cloudconscious.com>
====================================================================
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You may
obtain a copy of the License at
====================================================================
Licensed under the Apache License, Version 2.0 (the "License"); you
may not use this file except in compliance with the License. You may
obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 Unless required by
applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the
License.
====================================================================
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jclouds</groupId>
<artifactId>aws-common-project</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>
<artifactId>aws-common-perftest</artifactId>
<name>jclouds Performance test verses Amazon SDK implementation</name>
<description>Performance test verses Amazon SDK implementation</description>
<properties>
<!--
note you must set the property ${appengine.home} to a valid
extraction of appengine-java-sdk
-->
<appengine.home>YOUR_APPENGINE_HOME</appengine.home>
<test.s3.loopcount>100</test.s3.loopcount>
<test.s3.identity>${test.aws.identity}</test.s3.identity>
<test.s3.credential>${test.aws.credential}</test.s3.credential>
<test.initializer>org.jclouds.s3.blobstore.integration.S3TestInitializer</test.initializer>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-apachehc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.api</groupId>
<artifactId>s3</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.1.1</version>
</dependency>
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
<version>0.8.0</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-enterprise</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-apachehc</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-gae</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.api</groupId>
<artifactId>s3</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<!-- start gae stuff -->
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-api-stubs</artifactId>
<version>1.3.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-testing</artifactId>
<version>1.3.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-local-runtime</artifactId>
<version>1.3.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<!--
<exclusions> <exclusion> <groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId> </exclusion> </exclusions>
-->
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-servlet_2.5_spec</artifactId>
<version>1.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<scm>
<connection>scm:svn:http://jclouds.googlecode.com/svn/trunk</connection>
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk</developerConnection>
<url>http://jclouds.googlecode.com/svn/trunk</url>
</scm>
<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<executions>
<execution>
<id>integration</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<threadCount>1</threadCount>
<systemProperties>
<property>
<name>test.s3.identity</name>
<value>${test.s3.identity}</value>
</property>
<property>
<name>test.s3.credential</name>
<value>${test.s3.credential}</value>
</property>
<property>
<name>test.initializer</name>
<value>${test.initializer}</value>
</property>
<property>
<name>test.s3.loopcount</name>
<value>${test.s3.loopcount}</value>
</property>
</systemProperties>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
http://www.apache.org/licenses/LICENSE-2.0 Unless required by
applicable law or agreed to in writing, software distributed under the
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
CONDITIONS OF ANY KIND, either express or implied. See the License for
the specific language governing permissions and limitations under the
License.
====================================================================
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-project</artifactId>
<version>1.0-SNAPSHOT</version>
<relativePath>../../project/pom.xml</relativePath>
</parent>
<artifactId>s3-perftest</artifactId>
<name>jclouds Performance test verses Amazon SDK implementation</name>
<description>Performance test verses Amazon SDK implementation</description>
<properties>
<test.aws-s3.loopcount>100</test.aws-s3.loopcount>
<test.aws-s3.identity>${test.aws.identity}</test.aws-s3.identity>
<test.aws-s3.credential>${test.aws.credential}</test.aws-s3.credential>
<test.initializer>org.jclouds.aws.s3.blobstore.integration.AWSS3TestInitializer</test.initializer>
<jclouds.version>1.0-SNAPSHOT</jclouds.version>
<gae.version>1.4.0</gae.version>
</properties>
<!-- bootstrapping: need to fetch the project POM -->
<repositories>
<repository>
<id>jclouds-sona-snapshots-nexus</id>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.jclouds.provider</groupId>
<artifactId>aws-s3</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.1.9</version>
</dependency>
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
<version>0.8.0</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-enterprise</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-log4j</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-apachehc</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-gae</artifactId>
<version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-tools-sdk</artifactId>
<version>${gae.version}</version>
<scope>system</scope>
<systemPath>${env.APPENGINE_HOME}/lib/appengine-tools-api.jar</systemPath>
</dependency>
<dependency>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-core</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-blobstore</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds.api</groupId>
<artifactId>s3</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds.provider</groupId>
<artifactId>aws-s3</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<!-- start gae stuff -->
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-tools-sdk</artifactId>
<version>${gae.version}</version>
<scope>system</scope>
<systemPath>${env.APPENGINE_HOME}/lib/appengine-tools-api.jar</systemPath>
</dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-api-stubs</artifactId>
<version>${gae.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-testing</artifactId>
<version>${gae.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<executions>
<execution>
<id>integration</id>
<phase>integration-test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<threadCount>1</threadCount>
<systemProperties>
<property>
<name>test.aws-s3.identity</name>
<value>${test.aws-s3.identity}</value>
</property>
<property>
<name>test.aws-s3.credential</name>
<value>${test.aws-s3.credential}</value>
</property>
<property>
<name>test.initializer</name>
<value>${test.initializer}</value>
</property>
<property>
<name>test.aws-s3.loopcount</name>
<value>${test.aws-s3.loopcount}</value>
</property>
</systemProperties>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -50,8 +50,8 @@ public class AmazonPerformanceLiveTest extends BasePerformanceLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
exec = Executors.newCachedThreadPool();
s3 = new AmazonS3Client(new BasicAWSCredentials(System.getProperty("test.s3.identity"),
System.getProperty("test.s3.credential")));
s3 = new AmazonS3Client(new BasicAWSCredentials(System.getProperty("test.aws-s3.identity"),
System.getProperty("test.aws-s3.credential")));
}
@Override

View File

@ -30,7 +30,8 @@ import java.io.InputStream;
import java.util.Properties;
import java.util.concurrent.Future;
import org.jclouds.aws.s3.domain.S3Object;
import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.S3Object;
import com.google.common.base.Throwables;
@ -59,7 +60,7 @@ public abstract class BaseJCloudsPerformanceLiveTest extends BasePerformanceLive
// }
protected void overrideWithSysPropertiesAndPrint(Properties overrides, String contextName) {
overrides.putAll(System.getProperties());
System.out.printf("%s: loopCount(%s), perContext(%s), perHost(%s),ioWorkers(%s), userThreads(%s)%n", contextName,
System.out.printf("%s: loopCount(%s), perContext(%s), perHost(%s), ioWorkers(%s), userThreads(%s)%n", contextName,
loopCount, overrides.getProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT), overrides
.getProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST), overrides.getProperty(PROPERTY_IO_WORKER_THREADS),
overrides.getProperty(PROPERTY_USER_THREADS));

View File

@ -51,7 +51,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
containerCount = 1;
}
protected int timeoutSeconds = 15;
protected int loopCount = Integer.parseInt(System.getProperty("test.s3.loopcount", "1000"));
protected int loopCount = Integer.parseInt(System.getProperty("test.aws-s3.loopcount", "1000"));
protected ExecutorService exec;
protected Logger logger = Logger.NULL;;
@ -141,13 +141,13 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
}
private void doParallel(Provider<ListenableFuture<?>> provider, int loopCount, String containerName)
throws InterruptedException, ExecutionException, TimeoutException {
throws InterruptedException, ExecutionException, TimeoutException {
Map<Integer, ListenableFuture<?>> responses = Maps.newHashMap();
for (int i = 0; i < loopCount; i++)
responses.put(i, provider.get());
Map<Integer, Exception> exceptions = awaitCompletion(responses, exec, null, Logger.NULL, String.format(
"putting into containerName: %s", containerName));
"putting into containerName: %s", containerName));
assert exceptions.size() == 0 : exceptions;
@ -164,13 +164,13 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
public ListenableFuture<?> get() {
return Futures.makeListenable(putByteArray(bucketName, key.getAndIncrement() + "", test,
"application/octetstring"));
"application/octetstring"));
}
}
class PutFileFuture implements Provider<ListenableFuture<?>> {
final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml");
protected File file = new File(getClass().getResource("/testimg.png").getFile());
private final String bucketName;
public PutFileFuture(String bucketName) {
@ -195,7 +195,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
public ListenableFuture<?> get() {
return Futures.makeListenable(putInputStream(bucketName, key.getAndIncrement() + "", new ByteArrayInputStream(
test), "application/octetstring"));
test), "application/octetstring"));
}
}

View File

@ -33,6 +33,7 @@ import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@ -46,8 +47,8 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi
@BeforeClass(groups = { "integration", "live" })
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
exec = Executors.newCachedThreadPool();
String accesskeyid = System.getProperty("test.s3.identity");
String secretkey = System.getProperty("test.s3.credential");
String accesskeyid = System.getProperty("test.aws-s3.identity");
String secretkey = System.getProperty("test.aws-s3.credential");
Properties overrides = new Properties();
overrides.setProperty(PROPERTY_SO_TIMEOUT, 5000 + "");
overrides.setProperty(PROPERTY_CONNECTION_TIMEOUT, 5000 + "");
@ -57,7 +58,7 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi
overrides.setProperty(PROPERTY_USER_THREADS, 0 + "");
String contextName = "enterprise";
overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of(
context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(),
new EnterpriseConfigurationModule()), overrides);
}

View File

@ -30,6 +30,7 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.gae.config.GoogleAppEngineConfigurationModule;
import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
@ -143,7 +144,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes
Properties overrides = new Properties();
String contextName = "gae";
overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of(
context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new GoogleAppEngineConfigurationModule()), overrides);
}

View File

@ -30,6 +30,7 @@ import java.util.concurrent.Executors;
import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.ITestContext;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@ -49,8 +50,8 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest {
@BeforeClass(groups = { "integration", "live" })
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
exec = Executors.newCachedThreadPool();
String accesskeyid = System.getProperty("test.s3.identity");
String secretkey = System.getProperty("test.s3.credential");
String accesskeyid = System.getProperty("test.aws-s3.identity");
String secretkey = System.getProperty("test.aws-s3.credential");
Properties overrides = new Properties();
overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + "");
overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 30 + "");
@ -58,7 +59,7 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest {
overrides.setProperty(PROPERTY_USER_THREADS, 0 + "");
String contextName = "standard";
overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of(
context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new EnterpriseConfigurationModule()), overrides);
}

View File

@ -49,8 +49,8 @@ public class Jets3tPerformanceLiveTest extends BasePerformanceLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext);
exec = Executors.newCachedThreadPool();
jetClient = new RestS3Service(new AWSCredentials(System.getProperty("test.s3.identity"),
System.getProperty("test.s3.credential")));
jetClient = new RestS3Service(new AWSCredentials(System.getProperty("test.aws-s3.identity"),
System.getProperty("test.aws-s3.credential")));
}
@Override

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 MiB