Apache
/
jclouds
mirror of https://github.com/apache/jclouds.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added larger file upload into perftest

This commit is contained in:
Adrian Cole 2011-04-07 12:52:13 -07:00
parent 25eb1581ea
commit f39514a861
9 changed files with 198 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
demos/perftest/pom.xml
View File

@ -17,44 +17,48 @@
License. License.
==================================================================== ====================================================================
--> -->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<groupId>org.jclouds</groupId> <groupId>org.jclouds</groupId>
<artifactId>aws-common-project</artifactId> <artifactId>jclouds-project</artifactId>
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
<relativePath>../../project/pom.xml</relativePath>
</parent> </parent>
<artifactId>aws-common-perftest</artifactId> <artifactId>s3-perftest</artifactId>
<name>jclouds Performance test verses Amazon SDK implementation</name> <name>jclouds Performance test verses Amazon SDK implementation</name>
<description>Performance test verses Amazon SDK implementation</description> <description>Performance test verses Amazon SDK implementation</description>
<properties> <properties>
<!-- <test.aws-s3.loopcount>100</test.aws-s3.loopcount>
note you must set the property ${appengine.home} to a valid <test.aws-s3.identity>${test.aws.identity}</test.aws-s3.identity>
extraction of appengine-java-sdk <test.aws-s3.credential>${test.aws.credential}</test.aws-s3.credential>
--> <test.initializer>org.jclouds.aws.s3.blobstore.integration.AWSS3TestInitializer</test.initializer>
<appengine.home>YOUR_APPENGINE_HOME</appengine.home> <jclouds.version>1.0-SNAPSHOT</jclouds.version>
<test.s3.loopcount>100</test.s3.loopcount> <gae.version>1.4.0</gae.version>
<test.s3.identity>${test.aws.identity}</test.s3.identity>
<test.s3.credential>${test.aws.credential}</test.s3.credential>
<test.initializer>org.jclouds.s3.blobstore.integration.S3TestInitializer</test.initializer>
</properties> </properties>
<!-- bootstrapping: need to fetch the project POM -->
<repositories>
<repository>
<id>jclouds-sona-snapshots-nexus</id>
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>${project.groupId}</groupId> <groupId>org.jclouds.provider</groupId>
<artifactId>jclouds-apachehc</artifactId> <artifactId>aws-s3</artifactId>
<version>${project.version}</version> <version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.api</groupId>
<artifactId>s3</artifactId>
<version>${project.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.amazonaws</groupId> <groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId> <artifactId>aws-java-sdk</artifactId>
<version>1.1.1</version> <version>1.1.9</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>net.java.dev.jets3t</groupId> <groupId>net.java.dev.jets3t</groupId>
@ -64,70 +68,79 @@
<dependency> <dependency>
<groupId>org.jclouds.driver</groupId> <groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-enterprise</artifactId> <artifactId>jclouds-enterprise</artifactId>
<version>${project.version}</version> <version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-log4j</artifactId>
<version>${jclouds.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jclouds.driver</groupId> <groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-apachehc</artifactId> <artifactId>jclouds-apachehc</artifactId>
<version>${project.version}</version> <version>${jclouds.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jclouds.driver</groupId> <groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-gae</artifactId> <artifactId>jclouds-gae</artifactId>
<version>${project.version}</version> <version>${jclouds.version}</version>
</dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-tools-sdk</artifactId>
<version>${gae.version}</version>
<scope>system</scope>
<systemPath>${env.APPENGINE_HOME}/lib/appengine-tools-api.jar</systemPath>
</dependency>
<dependency>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-core</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds</groupId>
<artifactId>jclouds-blobstore</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.jclouds.api</groupId> <groupId>org.jclouds.api</groupId>
<artifactId>s3</artifactId> <artifactId>s3</artifactId>
<version>${project.version}</version> <version>${jclouds.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jclouds.provider</groupId>
<artifactId>aws-s3</artifactId>
<version>${jclouds.version}</version>
<type>test-jar</type> <type>test-jar</type>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<!-- start gae stuff --> <!-- start gae stuff -->
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-tools-sdk</artifactId>
<version>${gae.version}</version>
<scope>system</scope>
<systemPath>${env.APPENGINE_HOME}/lib/appengine-tools-api.jar</systemPath>
</dependency>
<dependency> <dependency>
<groupId>com.google.appengine</groupId> <groupId>com.google.appengine</groupId>
<artifactId>appengine-api-stubs</artifactId> <artifactId>appengine-api-stubs</artifactId>
<version>1.3.5</version> <version>${gae.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>com.google.appengine</groupId> <groupId>com.google.appengine</groupId>
<artifactId>appengine-testing</artifactId> <artifactId>appengine-testing</artifactId>
<version>1.3.5</version> <version>${gae.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>com.google.appengine</groupId>
<artifactId>appengine-local-runtime</artifactId>
<version>1.3.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>jclouds-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
<!--
<exclusions> <exclusion> <groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId> </exclusion> </exclusions>
-->
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-servlet_2.5_spec</artifactId>
<version>1.2</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<scm>
<connection>scm:svn:http://jclouds.googlecode.com/svn/trunk</connection>
<developerConnection>scm:svn:https://jclouds.googlecode.com/svn/trunk</developerConnection>
<url>http://jclouds.googlecode.com/svn/trunk</url>
</scm>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
@ -143,20 +156,20 @@
<threadCount>1</threadCount> <threadCount>1</threadCount>
<systemProperties> <systemProperties>
<property> <property>
<name>test.s3.identity</name> <name>test.aws-s3.identity</name>
<value>${test.s3.identity}</value> <value>${test.aws-s3.identity}</value>
</property> </property>
<property> <property>
<name>test.s3.credential</name> <name>test.aws-s3.credential</name>
<value>${test.s3.credential}</value> <value>${test.aws-s3.credential}</value>
</property> </property>
<property> <property>
<name>test.initializer</name> <name>test.initializer</name>
<value>${test.initializer}</value> <value>${test.initializer}</value>
</property> </property>
<property> <property>
<name>test.s3.loopcount</name> <name>test.aws-s3.loopcount</name>
<value>${test.s3.loopcount}</value> <value>${test.aws-s3.loopcount}</value>
</property> </property>
</systemProperties> </systemProperties>
</configuration> </configuration>

4
demos/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java
View File

@ -50,8 +50,8 @@ public class AmazonPerformanceLiveTest extends BasePerformanceLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext); super.setUpResourcesOnThisThread(testContext);
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
s3 = new AmazonS3Client(new BasicAWSCredentials(System.getProperty("test.s3.identity"), s3 = new AmazonS3Client(new BasicAWSCredentials(System.getProperty("test.aws-s3.identity"),
System.getProperty("test.s3.credential"))); System.getProperty("test.aws-s3.credential")));
} }
@Override @Override

3
demos/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformanceLiveTest.java
View File

@ -30,7 +30,8 @@ import java.io.InputStream;
import java.util.Properties; import java.util.Properties;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import org.jclouds.aws.s3.domain.S3Object; import org.jclouds.s3.S3AsyncClient;
import org.jclouds.s3.domain.S3Object;
import com.google.common.base.Throwables; import com.google.common.base.Throwables;

4
demos/perftest/src/test/java/org/jclouds/aws/s3/BasePerformanceLiveTest.java
View File

@ -51,7 +51,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
containerCount = 1; containerCount = 1;
} }
protected int timeoutSeconds = 15; protected int timeoutSeconds = 15;
protected int loopCount = Integer.parseInt(System.getProperty("test.s3.loopcount", "1000")); protected int loopCount = Integer.parseInt(System.getProperty("test.aws-s3.loopcount", "1000"));
protected ExecutorService exec; protected ExecutorService exec;
protected Logger logger = Logger.NULL;; protected Logger logger = Logger.NULL;;
@ -170,7 +170,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe
class PutFileFuture implements Provider<ListenableFuture<?>> { class PutFileFuture implements Provider<ListenableFuture<?>> {
final AtomicInteger key = new AtomicInteger(0); final AtomicInteger key = new AtomicInteger(0);
protected File file = new File("pom.xml"); protected File file = new File(getClass().getResource("/testimg.png").getFile());
private final String bucketName; private final String bucketName;
public PutFileFuture(String bucketName) { public PutFileFuture(String bucketName) {

7
demos/perftest/src/test/java/org/jclouds/aws/s3/JCloudsApacheHCPerformanceLiveTest.java
View File

@ -33,6 +33,7 @@ import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule; import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule; import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule;
import org.jclouds.logging.config.NullLoggingModule; import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.ITestContext; import org.testng.ITestContext;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -46,8 +47,8 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi
@BeforeClass(groups = { "integration", "live" }) @BeforeClass(groups = { "integration", "live" })
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
String accesskeyid = System.getProperty("test.s3.identity"); String accesskeyid = System.getProperty("test.aws-s3.identity");
String secretkey = System.getProperty("test.s3.credential"); String secretkey = System.getProperty("test.aws-s3.credential");
Properties overrides = new Properties(); Properties overrides = new Properties();
overrides.setProperty(PROPERTY_SO_TIMEOUT, 5000 + ""); overrides.setProperty(PROPERTY_SO_TIMEOUT, 5000 + "");
overrides.setProperty(PROPERTY_CONNECTION_TIMEOUT, 5000 + ""); overrides.setProperty(PROPERTY_CONNECTION_TIMEOUT, 5000 + "");
@ -57,7 +58,7 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi
overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); overrides.setProperty(PROPERTY_USER_THREADS, 0 + "");
String contextName = "enterprise"; String contextName = "enterprise";
overrideWithSysPropertiesAndPrint(overrides, contextName); overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(), new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(),
new EnterpriseConfigurationModule()), overrides); new EnterpriseConfigurationModule()), overrides);
} }

3
demos/perftest/src/test/java/org/jclouds/aws/s3/JCloudsGaePerformanceLiveTest.java
View File

@ -30,6 +30,7 @@ import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.gae.config.GoogleAppEngineConfigurationModule; import org.jclouds.gae.config.GoogleAppEngineConfigurationModule;
import org.jclouds.logging.config.NullLoggingModule; import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.annotations.AfterClass; import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod; import org.testng.annotations.BeforeMethod;
@ -143,7 +144,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes
Properties overrides = new Properties(); Properties overrides = new Properties();
String contextName = "gae"; String contextName = "gae";
overrideWithSysPropertiesAndPrint(overrides, contextName); overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new GoogleAppEngineConfigurationModule()), overrides); new NullLoggingModule(), new GoogleAppEngineConfigurationModule()), overrides);
} }

7
demos/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java
View File

@ -30,6 +30,7 @@ import java.util.concurrent.Executors;
import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.blobstore.BlobStoreContextFactory;
import org.jclouds.enterprise.config.EnterpriseConfigurationModule; import org.jclouds.enterprise.config.EnterpriseConfigurationModule;
import org.jclouds.logging.config.NullLoggingModule; import org.jclouds.logging.config.NullLoggingModule;
import org.jclouds.s3.S3AsyncClient;
import org.testng.ITestContext; import org.testng.ITestContext;
import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test; import org.testng.annotations.Test;
@ -49,8 +50,8 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest {
@BeforeClass(groups = { "integration", "live" }) @BeforeClass(groups = { "integration", "live" })
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
String accesskeyid = System.getProperty("test.s3.identity"); String accesskeyid = System.getProperty("test.aws-s3.identity");
String secretkey = System.getProperty("test.s3.credential"); String secretkey = System.getProperty("test.aws-s3.credential");
Properties overrides = new Properties(); Properties overrides = new Properties();
overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + ""); overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + "");
overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 30 + ""); overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 30 + "");
@ -58,7 +59,7 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest {
overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); overrides.setProperty(PROPERTY_USER_THREADS, 0 + "");
String contextName = "standard"; String contextName = "standard";
overrideWithSysPropertiesAndPrint(overrides, contextName); overrideWithSysPropertiesAndPrint(overrides, contextName);
context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( context = new BlobStoreContextFactory().createContext("aws-s3", accesskeyid, secretkey, ImmutableSet.of(
new NullLoggingModule(), new EnterpriseConfigurationModule()), overrides); new NullLoggingModule(), new EnterpriseConfigurationModule()), overrides);
} }

4
demos/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java
View File

@ -49,8 +49,8 @@ public class Jets3tPerformanceLiveTest extends BasePerformanceLiveTest {
public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception {
super.setUpResourcesOnThisThread(testContext); super.setUpResourcesOnThisThread(testContext);
exec = Executors.newCachedThreadPool(); exec = Executors.newCachedThreadPool();
jetClient = new RestS3Service(new AWSCredentials(System.getProperty("test.s3.identity"), jetClient = new RestS3Service(new AWSCredentials(System.getProperty("test.aws-s3.identity"),
System.getProperty("test.s3.credential"))); System.getProperty("test.aws-s3.credential")));
} }
@Override @Override

BIN
demos/perftest/src/test/resources/testimg.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.7 MiB