From 5bd3eb0571cff819f2ae41acd6e3607f440bab0c Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 21 Jul 2010 11:50:06 -0700 Subject: [PATCH] updated perftest to use amazon sdk and also stop using httpnio --- aws/perftest/README.txt | 38 + aws/perftest/pom.xml | 306 ++-- aws/perftest/src/main/java/S3Driver.java | 159 -- .../java/com/amazon/s3/AWSAuthConnection.java | 633 ------- .../src/main/java/com/amazon/s3/Bucket.java | 41 - .../java/com/amazon/s3/CallingFormat.java | 103 -- .../java/com/amazon/s3/CommonPrefixEntry.java | 17 - .../main/java/com/amazon/s3/GetResponse.java | 70 - .../amazon/s3/ListAllMyBucketsResponse.java | 106 -- .../com/amazon/s3/ListBucketResponse.java | 243 --- .../main/java/com/amazon/s3/ListEntry.java | 51 - .../java/com/amazon/s3/LocationResponse.java | 89 - .../src/main/java/com/amazon/s3/Owner.java | 18 - .../amazon/s3/QueryStringAuthGenerator.java | 264 --- .../src/main/java/com/amazon/s3/Response.java | 25 - .../src/main/java/com/amazon/s3/S3Object.java | 30 - .../src/main/java/com/amazon/s3/Utils.java | 324 ---- .../java/com/amazon/thirdparty/Base64.java | 1459 ----------------- aws/perftest/src/main/resources/README | 84 - .../java/com/amazon/s3/DateServiceTest.java | 52 - .../test/java/com/amazon/s3/S3ParserTest.java | 162 -- .../aws/s3/AmazonPerformanceLiveTest.java | 84 +- .../s3/BaseJCloudsPerformanceLiveTest.java | 22 +- .../aws/s3/BasePerformanceLiveTest.java | 41 +- .../JCloudsApacheHCPerformanceLiveTest.java | 15 +- .../aws/s3/JCloudsGaePerformanceLiveTest.java | 81 +- .../aws/s3/JCloudsNioPerformanceLiveTest.java | 61 - .../aws/s3/JCloudsPerformanceLiveTest.java | 11 +- .../aws/s3/Jets3tPerformanceLiveTest.java | 6 +- 29 files changed, 303 insertions(+), 4292 deletions(-) create mode 100644 aws/perftest/README.txt delete mode 100644 aws/perftest/src/main/java/S3Driver.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/AWSAuthConnection.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/Bucket.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/CallingFormat.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/CommonPrefixEntry.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/GetResponse.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/ListAllMyBucketsResponse.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/ListBucketResponse.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/ListEntry.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/LocationResponse.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/Owner.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/QueryStringAuthGenerator.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/Response.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/S3Object.java delete mode 100644 aws/perftest/src/main/java/com/amazon/s3/Utils.java delete mode 100644 aws/perftest/src/main/java/com/amazon/thirdparty/Base64.java delete mode 100755 aws/perftest/src/main/resources/README delete mode 100755 aws/perftest/src/test/java/com/amazon/s3/DateServiceTest.java delete mode 100755 aws/perftest/src/test/java/com/amazon/s3/S3ParserTest.java delete mode 100755 aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java diff --git a/aws/perftest/README.txt b/aws/perftest/README.txt new file mode 100644 index 0000000000..83d0a812eb --- /dev/null +++ b/aws/perftest/README.txt @@ -0,0 +1,38 @@ +This sample uses the Google App Engine for Java SDK located at http://googleappengine.googlecode.com/files/appengine-java-sdk-1.3.5.zip + +Please unzip the above file and modify your maven settings.xml like below before attempting to run 'mvn -Plive install' + + + appengine + + true + + + /path/to/appengine-java-sdk-1.3.5 + + + + + aws + + true + + + YOUR_ACCESS_KEY_ID + YOUR_SECRET_KEY + + + + + + jclouds + http://jclouds.googlecode.com/svn/trunk/repo + + + jclouds-rimu-snapshots-nexus + http://jclouds.rimuhosting.com:8081/nexus/content/repositories/snapshots + + true + + + diff --git a/aws/perftest/pom.xml b/aws/perftest/pom.xml index 3cd2a1d1e4..6d7a987f49 100644 --- a/aws/perftest/pom.xml +++ b/aws/perftest/pom.xml @@ -1,146 +1,168 @@ - - - - 4.0.0 - - org.jclouds - jclouds-aws-project - 1.0-SNAPSHOT - - jclouds-aws-perftest - jclouds Performance test verses Amazon samples implementation - Performance test verses Amazon samples implementation - - - - ${project.groupId} - jclouds-apachehc - ${project.version} - - - ${project.groupId} - jclouds-aws - ${project.version} - - - net.java.dev.jets3t - jets3t - 0.7.1 - - - ${project.groupId} - jclouds-enterprise - ${project.version} - - - ${project.groupId} - jclouds-httpnio - ${project.version} - - - ${project.groupId} - jclouds-gae - ${project.version} - - - - com.google.appengine - appengine-api-stubs - 1.3.0 - test - - - com.google.appengine - appengine-local-runtime - 1.3.0 - test - - - ${project.groupId} - jclouds-core - ${project.version} - test-jar - test - - - org.mortbay.jetty - jetty - - - - - org.apache.geronimo.specs - geronimo-servlet_2.5_spec - 1.2 - test - - - - ${project.groupId} - jclouds-aws - ${project.version} - test-jar - test - - - - - scm:svn:http://jclouds.googlecode.com/svn/trunk - scm:svn:https://jclouds.googlecode.com/svn/trunk - http://jclouds.googlecode.com/svn/trunk - - - - - - maven-surefire-plugin - - - integration - integration-test - - test - - - 1 - - - - - - + + 4.0.0 + + org.jclouds + jclouds-aws-project + 1.0-SNAPSHOT + + jclouds-aws-perftest + jclouds Performance test verses Amazon SDK implementation + Performance test verses Amazon SDK implementation + + + YOUR_APPENGINE_HOME + 100 + ${jclouds.aws.accesskeyid} + ${jclouds.aws.secretaccesskey} + org.jclouds.aws.s3.blobstore.integration.S3TestInitializer + + + + + ${project.groupId} + jclouds-apachehc + ${project.version} + + + ${project.groupId} + jclouds-aws + ${project.version} + + + com.amazonaws + aws-java-sdk + 1.0.007 + + + net.java.dev.jets3t + jets3t + 0.7.4 + + + ${project.groupId} + jclouds-enterprise + ${project.version} + + + ${project.groupId} + jclouds-apachehc + ${project.version} + + + ${project.groupId} + jclouds-gae + ${project.version} + + + ${project.groupId} + jclouds-aws + ${project.version} + test-jar + test + + + + com.google.appengine + appengine-api-stubs + 1.3.5 + test + + + com.google.appengine + appengine-testing + 1.3.5 + test + + + com.google.appengine + appengine-local-runtime + 1.3.5 + test + + + ${project.groupId} + jclouds-core + ${project.version} + test-jar + test + + + + org.apache.geronimo.specs + geronimo-servlet_2.5_spec + 1.2 + test + + + + + + scm:svn:http://jclouds.googlecode.com/svn/trunk + scm:svn:https://jclouds.googlecode.com/svn/trunk + http://jclouds.googlecode.com/svn/trunk + + + + + + maven-surefire-plugin + + + integration + integration-test + + test + + + 1 + + + jclouds.test.identity + ${jclouds.test.identity} + + + jclouds.test.credential + ${jclouds.test.credential} + + + jclouds.test.initializer + ${jclouds.test.initializer} + + + jclouds.test.loopcount + ${jclouds.test.loopcount} + + + + + + + + diff --git a/aws/perftest/src/main/java/S3Driver.java b/aws/perftest/src/main/java/S3Driver.java deleted file mode 100644 index 14cf58fe6e..0000000000 --- a/aws/perftest/src/main/java/S3Driver.java +++ /dev/null @@ -1,159 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its -// affiliates. - -import java.util.Map; -import java.util.TreeMap; -import java.util.Arrays; - -import com.amazon.s3.AWSAuthConnection; -import com.amazon.s3.CallingFormat; -import com.amazon.s3.QueryStringAuthGenerator; -import com.amazon.s3.S3Object; - -public class S3Driver { - - static final String awsAccessKeyId = ""; - static final String awsSecretAccessKey = ""; - - - // convert the bucket to lowercase for vanity domains - // the bucket name must be lowercase since DNS is case-insensitive - static final String bucketName = awsAccessKeyId.toLowerCase() + "-test-bucket"; - static final String keyName = "test-key"; - static final String copiedKeyName = "copy-of-" + keyName; - - public static void main(String args[]) throws Exception { - if (awsAccessKeyId.startsWith(" "); - System.in.read(); - - System.out.println("\nNow try just the url without the query string arguments. It should fail.\n"); - System.out.println(generator.makeBareURL(bucketName, keyName)); - System.out.print("\npress enter> "); - System.in.read(); - - System.out.println("----- putting object with metadata and public read acl -----"); - - Map metadata = new TreeMap(); - metadata.put("blah", Arrays.asList(new String[] { "foo" })); - object = new S3Object("this is a publicly readable test".getBytes(), metadata); - - headers = new TreeMap(); - headers.put("x-amz-acl", Arrays.asList(new String[] { "public-read" })); - headers.put("Content-Type", Arrays.asList(new String[] { "text/plain" })); - - System.out.println( - conn.put(bucketName, keyName + "-public", object, headers).connection.getResponseMessage() - ); - - System.out.println("----- anonymous read test -----"); - System.out.println("\nYou should be able to try this in your browser\n"); - System.out.println(generator.makeBareURL(bucketName, keyName + "-public")); - System.out.print("\npress enter> "); - System.in.read(); - - System.out.println("----- path style url example -----"); - System.out.println("\nNon-location-constrained buckets can also be specified as part of the url path. (This was the original url style supported by S3.)"); - System.out.println("\nTry this url out in your browser (it will only be valid for 60 seconds)\n"); - generator.setCallingFormat(CallingFormat.getPathCallingFormat()); - // could also have been done like this: - // generator = new QueryStringAuthGenerator(awsAccessKeyId, awsSecretAccessKey, true, Utils.DEFAULT_HOST, CallingFormat.getPathCallingFormat()); - generator.setExpiresIn(60 * 1000); - System.out.println(generator.get(bucketName, keyName, null)); - System.out.print("\npress enter> "); - System.in.read(); - - System.out.println("----- getting object's acl -----"); - System.out.println(new String(conn.getACL(bucketName, keyName, null).object.data)); - - System.out.println("----- deleting objects -----"); - System.out.println( - conn.delete(bucketName, copiedKeyName, null).connection.getResponseMessage() - ); - System.out.println( - conn.delete(bucketName, keyName, null).connection.getResponseMessage() - ); - System.out.println( - conn.delete(bucketName, keyName + "-public", null).connection.getResponseMessage() - ); - - System.out.println("----- listing bucket -----"); - System.out.println(conn.listBucket(bucketName, null, null, null, null).entries); - - System.out.println("----- listing all my buckets -----"); - System.out.println(conn.listAllMyBuckets(null).entries); - - System.out.println("----- deleting bucket -----"); - System.out.println( - conn.deleteBucket(bucketName, null).connection.getResponseMessage() - ); - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/AWSAuthConnection.java b/aws/perftest/src/main/java/com/amazon/s3/AWSAuthConnection.java deleted file mode 100644 index 674f93085e..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/AWSAuthConnection.java +++ /dev/null @@ -1,633 +0,0 @@ -/** - * - * Copyright (C) 2009 Adrian Cole - * - * ==================================================================== - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * ==================================================================== - */ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.net.HttpURLConnection; -import java.net.MalformedURLException; -import java.net.URL; -import java.text.SimpleDateFormat; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.TimeZone; - -/** - * An interface into the S3 system. It is initially configured with - * authentication and connection parameters and exposes methods to access and - * manipulate S3 data. - */ -public class AWSAuthConnection { - public static final String LOCATION_DEFAULT = null; - public static final String LOCATION_EU = "EU"; - - private String awsAccessKeyId; - private String awsSecretAccessKey; - private boolean isSecure; - private String server; - private int port; - private CallingFormat callingFormat; - - public AWSAuthConnection(String awsAccessKeyId, String awsSecretAccessKey) { - this(awsAccessKeyId, awsSecretAccessKey, true); - } - - public AWSAuthConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure) { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, Utils.DEFAULT_HOST); - } - - public AWSAuthConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, - String server) - { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, - isSecure ? Utils.SECURE_PORT : Utils.INSECURE_PORT); - } - - public AWSAuthConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, - String server, int port) { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, port, CallingFormat.getSubdomainCallingFormat()); - - } - - public AWSAuthConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, - String server, CallingFormat format) { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, - isSecure ? Utils.SECURE_PORT : Utils.INSECURE_PORT, - format); - } - - /** - * Create a new interface to interact with S3 with the given credential and connection - * parameters - * - * @param awsAccessKeyId Your user key into AWS - * @param awsSecretAccessKey The secret string used to generate signatures for authentication. - * @param isSecure use SSL encryption - * @param server Which host to connect to. Usually, this will be s3.amazonaws.com - * @param port Which port to use. - * @param callingFormat Type of request Regular/Vanity or Pure Vanity domain - */ - public AWSAuthConnection(String awsAccessKeyId, String awsSecretAccessKey, boolean isSecure, - String server, int port, CallingFormat format) - { - this.awsAccessKeyId = awsAccessKeyId; - this.awsSecretAccessKey = awsSecretAccessKey; - this.isSecure = isSecure; - this.server = server; - this.port = port; - this.callingFormat = format; - } - - /** - * Creates a new bucket. - * @param bucket The name of the bucket to create. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - * @param metadata A Map of String to List of Strings representing the s3 - * metadata for this bucket (can be null). - * @deprecated use version that specifies location - */ - public Response createBucket(String bucket, Map headers) - throws MalformedURLException, IOException - { - return createBucket(bucket, null, headers); - } - - /** - * Creates a new bucket. - * @param bucket The name of the bucket to create. - * @param location Desired location ("EU") (or null for default). - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - * @param metadata A Map of String to List of Strings representing the s3 - * metadata for this bucket (can be null). - * @throws IllegalArgumentException on invalid location - */ - public Response createBucket(String bucket, String location, Map headers) - throws MalformedURLException, IOException - { - String body; - if (location == null) { - body = null; - } else if (LOCATION_EU.equals(location)) { - if (!callingFormat.supportsLocatedBuckets()) - throw new IllegalArgumentException("Creating location-constrained bucket with unsupported calling-format"); - body = "" + location + ""; - } else - throw new IllegalArgumentException("Invalid Location: "+location); - - // validate bucket name - if (!Utils.validateBucketName(bucket, callingFormat, location != null)) - throw new IllegalArgumentException("Invalid S3Bucket Name: "+bucket); - - HttpURLConnection request = makeRequest("PUT", bucket, "", null, headers); - if (body != null) - { - request.setDoOutput(true); - request.getOutputStream().write(body.getBytes("UTF-8")); - } - return new Response(request); - } - - /** - * Check if the specified bucket exists (via a HEAD request) - * @param bucket The name of the bucket to check - * @return true if HEAD access returned success - */ - public boolean checkBucketExists(String bucket) throws MalformedURLException, IOException - { - HttpURLConnection response = makeRequest("HEAD", bucket, "", null, null); - int httpCode = response.getResponseCode(); - return httpCode >= 200 && httpCode < 300; - } - - /** - * Lists the contents of a bucket. - * @param bucket The name of the bucket to create. - * @param prefix All returned keys will start with this string (can be null). - * @param marker All returned keys will be lexographically greater than - * this string (can be null). - * @param maxKeys The maximum number of keys to return (can be null). - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public ListBucketResponse listBucket(String bucket, String prefix, String marker, - Integer maxKeys, Map headers) - throws MalformedURLException, IOException - { - return listBucket(bucket, prefix, marker, maxKeys, null, headers); - } - - /** - * Lists the contents of a bucket. - * @param bucket The name of the bucket to list. - * @param prefix All returned keys will start with this string (can be null). - * @param marker All returned keys will be lexographically greater than - * this string (can be null). - * @param maxKeys The maximum number of keys to return (can be null). - * @param delimiter Keys that contain a string between the prefix and the first - * occurrence of the delimiter will be rolled up into a single element. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public ListBucketResponse listBucket(String bucket, String prefix, String marker, - Integer maxKeys, String delimiter, Map headers) - throws MalformedURLException, IOException - { - - Map pathArgs = Utils.paramsForListOptions(prefix, marker, maxKeys, delimiter); - return new ListBucketResponse(makeRequest("GET", bucket, "", pathArgs, headers)); - } - - /** - * Deletes a bucket. - * @param bucket The name of the bucket to delete. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response deleteBucket(String bucket, Map headers) - throws MalformedURLException, IOException - { - return new Response(makeRequest("DELETE", bucket, "", null, headers)); - } - - /** - * Writes an object to S3. - * @param bucket The name of the bucket to which the object will be added. - * @param key The name of the key to use. - * @param object An S3Object containing the data to write. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response put(String bucket, String key, S3Object object, Map headers) - throws MalformedURLException, IOException - { - HttpURLConnection request = - makeRequest("PUT", bucket, Utils.urlencode(key), null, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.data == null ? new byte[] {} : object.data); - - return new Response(request); - } - - /** - * Creates a copy of an existing S3 Object. In this signature, we will copy the - * existing metadata. The default access control policy is private; if you want - * to override it, please use x-amz-acl in the headers. - * @param sourceBucket The name of the bucket where the source object lives. - * @param sourceKey The name of the key to copy. - * @param destinationBucket The name of the bucket to which the object will be added. - * @param destinationKey The name of the key to use. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). You may wish to set the x-amz-acl header appropriately. - */ - public Response copy( String sourceBucket, String sourceKey, String destinationBucket, String destinationKey, Map headers ) - throws MalformedURLException, IOException - { - S3Object object = new S3Object(new byte[] {}, new HashMap()); - headers = headers == null ? new HashMap() : new HashMap(headers); - headers.put("x-amz-copy-source", Arrays.asList( new String[] { sourceBucket + "/" + sourceKey } ) ); - headers.put("x-amz-metadata-directive", Arrays.asList( new String[] { "COPY" } ) ); - return verifyCopy( put( destinationBucket, destinationKey, object, headers ) ); - } - - /** - * Creates a copy of an existing S3 Object. In this signature, we will replace the - * existing metadata. The default access control policy is private; if you want - * to override it, please use x-amz-acl in the headers. - * @param sourceBucket The name of the bucket where the source object lives. - * @param sourceKey The name of the key to copy. - * @param destinationBucket The name of the bucket to which the object will be added. - * @param destinationKey The name of the key to use. - * @param metadata A Map of String to List of Strings representing the S3 metadata - * for the new object. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). You may wish to set the x-amz-acl header appropriately. - */ - public Response copy( String sourceBucket, String sourceKey, String destinationBucket, String destinationKey, Map metadata, Map headers ) - throws MalformedURLException, IOException - { - S3Object object = new S3Object(new byte[] {}, metadata); - headers = headers == null ? new HashMap() : new HashMap(headers); - headers.put("x-amz-copy-source", Arrays.asList( new String[] { sourceBucket + "/" + sourceKey } ) ); - headers.put("x-amz-metadata-directive", Arrays.asList( new String[] { "REPLACE" } ) ); - return verifyCopy( put( destinationBucket, destinationKey, object, headers ) ); - } - - /** - * Copy sometimes returns a successful response and starts to send whitespace - * characters to us. This method processes those whitespace characters and - * will throw an exception if the response is either unknown or an error. - * @param response Response object from the PUT request. - * @return The response with the input stream drained. - * @throws IOException If anything goes wrong. - */ - private Response verifyCopy( Response response ) throws IOException { - if (response.connection.getResponseCode() < 400) { - byte[] body = GetResponse.slurpInputStream(response.connection.getInputStream()); - String message = new String( body ); - if ( message.indexOf( "" ) != -1 ) { - // It worked! - } else { - throw new IOException( "Unexpected response: " + message ); - } - } - return response; - } - - /** - * Reads an object from S3. - * @param bucket The name of the bucket where the object lives. - * @param key The name of the key to use. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public GetResponse get(String bucket, String key, Map headers) - throws MalformedURLException, IOException - { - return new GetResponse(makeRequest("GET", bucket, Utils.urlencode(key), null, headers)); - } - - /** - * Deletes an object from S3. - * @param bucket The name of the bucket where the object lives. - * @param key The name of the key to use. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response delete(String bucket, String key, Map headers) - throws MalformedURLException, IOException - { - return new Response(makeRequest("DELETE", bucket, Utils.urlencode(key), null, headers)); - } - - /** - * Get the requestPayment xml document for a given bucket - * @param bucket The name of the bucket - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public GetResponse getBucketRequestPayment(String bucket, Map headers) - throws MalformedURLException, IOException - { - Map pathArgs = new HashMap(); - pathArgs.put("requestPayment", null); - return new GetResponse(makeRequest("GET", bucket, "", pathArgs, headers)); - } - - /** - * Write a new requestPayment xml document for a given bucket - * @param loggingXMLDoc The xml representation of the requestPayment configuration as a String - * @param bucket The name of the bucket - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response putBucketRequestPayment(String bucket, String requestPaymentXMLDoc, Map headers) - throws MalformedURLException, IOException - { - Map pathArgs = new HashMap(); - pathArgs.put("requestPayment", null); - S3Object object = new S3Object(requestPaymentXMLDoc.getBytes(), null); - HttpURLConnection request = makeRequest("PUT", bucket, "", pathArgs, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.data == null ? new byte[] {} : object.data); - - return new Response(request); - } - - /** - * Get the logging xml document for a given bucket - * @param bucket The name of the bucket - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public GetResponse getBucketLogging(String bucket, Map headers) - throws MalformedURLException, IOException - { - Map pathArgs = new HashMap(); - pathArgs.put("logging", null); - return new GetResponse(makeRequest("GET", bucket, "", pathArgs, headers)); - } - - /** - * Write a new logging xml document for a given bucket - * @param loggingXMLDoc The xml representation of the logging configuration as a String - * @param bucket The name of the bucket - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response putBucketLogging(String bucket, String loggingXMLDoc, Map headers) - throws MalformedURLException, IOException - { - Map pathArgs = new HashMap(); - pathArgs.put("logging", null); - S3Object object = new S3Object(loggingXMLDoc.getBytes(), null); - HttpURLConnection request = makeRequest("PUT", bucket, "", pathArgs, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.data == null ? new byte[] {} : object.data); - - return new Response(request); - } - - /** - * Get the ACL for a given bucket - * @param bucket The name of the bucket where the object lives. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public GetResponse getBucketACL(String bucket, Map headers) - throws MalformedURLException, IOException - { - return getACL(bucket, "", headers); - } - - /** - * Get the ACL for a given object (or bucket, if key is null). - * @param bucket The name of the bucket where the object lives. - * @param key The name of the key to use. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public GetResponse getACL(String bucket, String key, Map headers) - throws MalformedURLException, IOException - { - if (key == null) key = ""; - - Map pathArgs = new HashMap(); - pathArgs.put("acl", null); - - return new GetResponse( - makeRequest("GET", bucket, Utils.urlencode(key), pathArgs, headers) - ); - } - - /** - * Write a new ACL for a given bucket - * @param aclXMLDoc The xml representation of the ACL as a String - * @param bucket The name of the bucket where the object lives. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response putBucketACL(String bucket, String aclXMLDoc, Map headers) - throws MalformedURLException, IOException - { - return putACL(bucket, "", aclXMLDoc, headers); - } - - /** - * Write a new ACL for a given object - * @param aclXMLDoc The xml representation of the ACL as a String - * @param bucket The name of the bucket where the object lives. - * @param key The name of the key to use. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public Response putACL(String bucket, String key, String aclXMLDoc, Map headers) - throws MalformedURLException, IOException - { - S3Object object = new S3Object(aclXMLDoc.getBytes(), null); - - Map pathArgs = new HashMap(); - pathArgs.put("acl", null); - - HttpURLConnection request = - makeRequest("PUT", bucket, Utils.urlencode(key), pathArgs, headers, object); - - request.setDoOutput(true); - request.getOutputStream().write(object.data == null ? new byte[] {} : object.data); - - return new Response(request); - } - - public LocationResponse getBucketLocation(String bucket) - throws MalformedURLException, IOException - { - Map pathArgs = new HashMap(); - pathArgs.put("location", null); - return new LocationResponse(makeRequest("GET", bucket, "", pathArgs, null)); - } - - - /** - * List all the buckets created by this account. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - public ListAllMyBucketsResponse listAllMyBuckets(Map headers) - throws MalformedURLException, IOException - { - return new ListAllMyBucketsResponse(makeRequest("GET", "", "", null, headers)); - } - - - - /** - * Make a new HttpURLConnection without passing an S3Object parameter. - * Use this method for key operations that do require arguments - * @param method The method to invoke - * @param bucketName the bucket this request is for - * @param key the key this request is for - * @param pathArgs the - * @param headers - * @return - * @throws MalformedURLException - * @throws IOException - */ - private HttpURLConnection makeRequest(String method, String bucketName, String key, Map pathArgs, Map headers) - throws MalformedURLException, IOException - { - return makeRequest(method, bucketName, key, pathArgs, headers, null); - } - - - - /** - * Make a new HttpURLConnection. - * @param method The HTTP method to use (GET, PUT, DELETE) - * @param bucketName The bucket name this request affects - * @param key The key this request is for - * @param pathArgs parameters if any to be sent along this request - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - * @param object The S3Object that is to be written (can be null). - */ - private HttpURLConnection makeRequest(String method, String bucket, String key, Map pathArgs, Map headers, - S3Object object) - throws MalformedURLException, IOException - { - CallingFormat callingFormat = Utils.getCallingFormatForBucket( this.callingFormat, bucket ); - if ( isSecure && callingFormat != CallingFormat.getPathCallingFormat() && bucket.indexOf( "." ) != -1 ) { - System.err.println( "You are making an SSL connection, however, the bucket contains periods and the wildcard certificate will not match by default. Please consider using HTTP." ); - } - - // build the domain based on the calling format - URL url = callingFormat.getURL(isSecure, server, this.port, bucket, key, pathArgs); - - HttpURLConnection connection = (HttpURLConnection)url.openConnection(); - connection.setRequestMethod(method); - - // subdomain-style urls may encounter http redirects. - // Ensure that redirects are supported. - if (!connection.getInstanceFollowRedirects() - && callingFormat.supportsLocatedBuckets()) - throw new RuntimeException("HTTP redirect support required."); - - addHeaders(connection, headers); - if (object != null) addMetadataHeaders(connection, object.metadata); - addAuthHeader(connection, method, bucket, key, pathArgs); - - return connection; - } - - /** - * Add the given headers to the HttpURLConnection. - * @param connection The HttpURLConnection to which the headers will be added. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - */ - private void addHeaders(HttpURLConnection connection, Map headers) { - addHeaders(connection, headers, ""); - } - - /** - * Add the given metadata fields to the HttpURLConnection. - * @param connection The HttpURLConnection to which the headers will be added. - * @param metadata A Map of String to List of Strings representing the s3 - * metadata for this resource. - */ - private void addMetadataHeaders(HttpURLConnection connection, Map metadata) { - addHeaders(connection, metadata, Utils.METADATA_PREFIX); - } - - /** - * Add the given headers to the HttpURLConnection with a prefix before the keys. - * @param connection The HttpURLConnection to which the headers will be added. - * @param headers A Map of String to List of Strings representing the http - * headers to pass (can be null). - * @param prefix The string to prepend to each key before adding it to the connection. - */ - private void addHeaders(HttpURLConnection connection, Map headers, String prefix) { - if (headers != null) { - for (Iterator i = headers.keySet().iterator(); i.hasNext(); ) { - String key = (String)i.next(); - for (Iterator j = ((List)headers.get(key)).iterator(); j.hasNext(); ) { - String value = (String)j.next(); - connection.addRequestProperty(prefix + key, value); - } - } - } - } - - /** - * Add the appropriate Authorization header to the HttpURLConnection. - * @param connection The HttpURLConnection to which the header will be added. - * @param method The HTTP method to use (GET, PUT, DELETE) - * @param bucket the bucket name this request is for - * @param key the key this request is for - * @param pathArgs path arguments which are part of this request - */ - private void addAuthHeader(HttpURLConnection connection, String method, String bucket, String key, Map pathArgs) { - if (connection.getRequestProperty("Date") == null) { - connection.setRequestProperty("Date", httpDate()); - } - if (connection.getRequestProperty("Content-Type") == null) { - connection.setRequestProperty("Content-Type", ""); - } - - String canonicalString = - Utils.makeCanonicalString(method, bucket, key, pathArgs, connection.getRequestProperties()); - String encodedCanonical = Utils.encode(this.awsSecretAccessKey, canonicalString, false); - connection.setRequestProperty("Authorization", - "AWS " + this.awsAccessKeyId + ":" + encodedCanonical); - } - - - /** - * Generate an rfc822 date for use in the Date HTTP header. - */ - public static String httpDate() { - final String DateFormat = "EEE, dd MMM yyyy HH:mm:ss "; - SimpleDateFormat format = new SimpleDateFormat( DateFormat, Locale.US ); - format.setTimeZone( TimeZone.getTimeZone( "GMT" ) ); - return format.format( new Date() ) + "GMT"; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/Bucket.java b/aws/perftest/src/main/java/com/amazon/s3/Bucket.java deleted file mode 100644 index aab89a6ea2..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/Bucket.java +++ /dev/null @@ -1,41 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.util.Date; - -/** - * A class representing a single bucket. Returned as a component of ListAllMyBucketsResponse. - */ -public class Bucket { - /** - * The name of the bucket. - */ - public String name; - - /** - * The bucket's creation date. - */ - public Date creationDate; - - public Bucket() { - this.name = null; - this.creationDate = null; - } - - public Bucket(String name, Date creationDate) { - this.name = name; - this.creationDate = creationDate; - } - - public String toString() { - return this.name; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/CallingFormat.java b/aws/perftest/src/main/java/com/amazon/s3/CallingFormat.java deleted file mode 100644 index 5be4cd70e9..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/CallingFormat.java +++ /dev/null @@ -1,103 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.Map; - -public abstract class CallingFormat { - - protected static CallingFormat pathCallingFormat = new PathCallingFormat(); - protected static CallingFormat subdomainCallingFormat = new SubdomainCallingFormat(); - protected static CallingFormat vanityCallingFormat = new VanityCallingFormat(); - - public abstract boolean supportsLocatedBuckets(); - public abstract String getEndpoint(String server, int port, String bucket); - public abstract String getPathBase(String bucket, String key); - public abstract URL getURL (boolean isSecure, String server, int port, String bucket, String key, Map pathArgs) - throws MalformedURLException; - - public static CallingFormat getPathCallingFormat() { - return pathCallingFormat; - } - - public static CallingFormat getSubdomainCallingFormat() { - return subdomainCallingFormat; - } - - public static CallingFormat getVanityCallingFormat() { - return vanityCallingFormat; - } - - static private class PathCallingFormat extends CallingFormat { - public boolean supportsLocatedBuckets() { - return false; - } - - public String getPathBase(String bucket, String key) { - return isBucketSpecified(bucket) ? "/" + bucket + "/" + key : "/"; - } - - public String getEndpoint(String server, int port, String bucket) { - return server + ":" + port; - } - - public URL getURL(boolean isSecure, String server, int port, String bucket, String key, Map pathArgs) - throws MalformedURLException { - String pathBase = isBucketSpecified(bucket) ? "/" + bucket + "/" + key : "/"; - String pathArguments = Utils.convertPathArgsHashToString(pathArgs); - return new URL(isSecure ? "https": "http", server, port, pathBase + pathArguments); - } - - private boolean isBucketSpecified(String bucket) { - if(bucket == null) return false; - if(bucket.length() == 0) return false; - return true; - } - } - - static private class SubdomainCallingFormat extends CallingFormat { - public boolean supportsLocatedBuckets() { - return true; - } - - public String getServer(String server, String bucket) { - return bucket + "." + server; - } - public String getEndpoint(String server, int port, String bucket) { - return getServer(server, bucket) + ":" + port ; - } - public String getPathBase(String bucket, String key) { - return "/" + key; - } - - public URL getURL(boolean isSecure, String server, int port, String bucket, String key, Map pathArgs) - throws MalformedURLException { - if (bucket == null || bucket.length() == 0) - { - //The bucket is null, this is listAllBuckets request - String pathArguments = Utils.convertPathArgsHashToString(pathArgs); - return new URL(isSecure ? "https": "http", server, port, "/" + pathArguments); - } else { - String serverToUse = getServer(server, bucket); - String pathBase = getPathBase(bucket, key); - String pathArguments = Utils.convertPathArgsHashToString(pathArgs); - return new URL(isSecure ? "https": "http", serverToUse, port, pathBase + pathArguments); - } - } - } - - static private class VanityCallingFormat extends SubdomainCallingFormat { - public String getServer(String server, String bucket) { - return bucket; - } - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/CommonPrefixEntry.java b/aws/perftest/src/main/java/com/amazon/s3/CommonPrefixEntry.java deleted file mode 100644 index 80198bcad0..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/CommonPrefixEntry.java +++ /dev/null @@ -1,17 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -public class CommonPrefixEntry { - /** - * The prefix common to the delimited keys it represents - */ - public String prefix; -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/GetResponse.java b/aws/perftest/src/main/java/com/amazon/s3/GetResponse.java deleted file mode 100644 index 76b767346c..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/GetResponse.java +++ /dev/null @@ -1,70 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.net.HttpURLConnection; -import java.io.IOException; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; - -/** - * A Response object returned from AWSAuthConnection.get(). Exposes the attribute object, which - * represents the retrieved object. - */ -public class GetResponse extends Response { - public S3Object object; - - /** - * Pulls a representation of an S3Object out of the HttpURLConnection response. - */ - public GetResponse(HttpURLConnection connection) throws IOException { - super(connection); - if (connection.getResponseCode() < 400) { - Map metadata = extractMetadata(connection); - byte[] body = slurpInputStream(connection.getInputStream()); - this.object = new S3Object(body, metadata); - } - } - - /** - * Examines the response's header fields and returns a Map from String to List of Strings - * representing the object's metadata. - */ - private Map extractMetadata(HttpURLConnection connection) { - TreeMap metadata = new TreeMap(); - Map headers = connection.getHeaderFields(); - for (Iterator i = headers.keySet().iterator(); i.hasNext(); ) { - String key = (String)i.next(); - if (key == null) continue; - if (key.startsWith(Utils.METADATA_PREFIX)) { - metadata.put(key.substring(Utils.METADATA_PREFIX.length()), headers.get(key)); - } - } - - return metadata; - } - - /** - * Read the input stream and dump it all into a big byte array - */ - static byte[] slurpInputStream(InputStream stream) throws IOException { - final int chunkSize = 2048; - byte[] buf = new byte[chunkSize]; - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(chunkSize); - int count; - - while ((count=stream.read(buf)) != -1) byteStream.write(buf, 0, count); - - return byteStream.toByteArray(); - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/ListAllMyBucketsResponse.java b/aws/perftest/src/main/java/com/amazon/s3/ListAllMyBucketsResponse.java deleted file mode 100644 index f4d6c1e9c3..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/ListAllMyBucketsResponse.java +++ /dev/null @@ -1,106 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.SimpleTimeZone; - -import org.xml.sax.Attributes; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; - -/** - * Returned by AWSAuthConnection.listAllMyBuckets(). - */ -public class ListAllMyBucketsResponse extends Response { - /** - * A list of Bucket objects, one for each of this account's buckets. Will be null if - * the request fails. - */ - public List entries; - - public ListAllMyBucketsResponse(HttpURLConnection connection) throws IOException { - super(connection); - if (connection.getResponseCode() < 400) { - try { - XMLReader xr = Utils.createXMLReader();; - ListAllMyBucketsHandler handler = new ListAllMyBucketsHandler(); - xr.setContentHandler(handler); - xr.setErrorHandler(handler); - - xr.parse(new InputSource(connection.getInputStream())); - this.entries = handler.getEntries(); - } catch (SAXException e) { - throw new RuntimeException("Unexpected error parsing ListAllMyBuckets xml", e); - } - } - } - - static class ListAllMyBucketsHandler extends DefaultHandler { - - private List entries = null; - private Bucket currBucket = null; - private StringBuffer currText = null; - private SimpleDateFormat iso8601Parser = null; - - public ListAllMyBucketsHandler() { - super(); - entries = new ArrayList(); - this.iso8601Parser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - this.iso8601Parser.setTimeZone(new SimpleTimeZone(0, "GMT")); - this.currText = new StringBuffer(); - } - - public void startDocument() { - // ignore - } - - public void endDocument() { - // ignore - } - - public void startElement(String uri, String name, String qName, Attributes attrs) { - if (name.equals("Bucket")) { - this.currBucket = new Bucket(); - } - } - - public void endElement(String uri, String name, String qName) { - if (name.equals("Bucket")) { - this.entries.add(this.currBucket); - } else if (name.equals("Name")) { - this.currBucket.name = this.currText.toString(); - } else if (name.equals("CreationDate")) { - try { - this.currBucket.creationDate = this.iso8601Parser.parse(this.currText.toString()); - } catch (ParseException e) { - throw new RuntimeException("Unexpected date format in list bucket output", e); - } - } - this.currText = new StringBuffer(); - } - - public void characters(char ch[], int start, int length) { - this.currText.append(ch, start, length); - } - - public List getEntries() { - return this.entries; - } - } -} - diff --git a/aws/perftest/src/main/java/com/amazon/s3/ListBucketResponse.java b/aws/perftest/src/main/java/com/amazon/s3/ListBucketResponse.java deleted file mode 100644 index 56bbf4fb5f..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/ListBucketResponse.java +++ /dev/null @@ -1,243 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.SimpleTimeZone; - -import org.xml.sax.Attributes; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.InputSource; -import org.xml.sax.XMLReader; -import org.xml.sax.SAXException; - - -/** - * Returned by AWSAuthConnection.listBucket() - */ -public class ListBucketResponse extends Response { - - /** - * The name of the bucket being listed. Null if request fails. - */ - public String name = null; - - /** - * The prefix echoed back from the request. Null if request fails. - */ - public String prefix = null; - - /** - * The marker echoed back from the request. Null if request fails. - */ - public String marker = null; - - /** - * The delimiter echoed back from the request. Null if not specified in - * the request, or if it fails. - */ - public String delimiter = null; - - /** - * The maxKeys echoed back from the request if specified. 0 if request fails. - */ - public int maxKeys = 0; - - /** - * Indicates if there are more results to the list. True if the current - * list results have been truncated. false if request fails. - */ - public boolean isTruncated = false; - - /** - * Indicates what to use as a marker for subsequent list requests in the event - * that the results are truncated. Present only when a delimiter is specified. - * Null if request fails. - */ - public String nextMarker = null; - - /** - * A List of ListEntry objects representing the objects in the given bucket. - * Null if the request fails. - */ - public List entries = null; - - /** - * A List of CommonPrefixEntry objects representing the common prefixes of the - * keys that matched up to the delimiter. Null if the request fails. - */ - public List commonPrefixEntries = null; - - public ListBucketResponse(HttpURLConnection connection) throws IOException { - super(connection); - if (connection.getResponseCode() < 400) { - try { - XMLReader xr = Utils.createXMLReader(); - ListBucketHandler handler = new ListBucketHandler(); - xr.setContentHandler(handler); - xr.setErrorHandler(handler); - - xr.parse(new InputSource(connection.getInputStream())); - - this.name = handler.getName(); - this.prefix = handler.getPrefix(); - this.marker = handler.getMarker(); - this.delimiter = handler.getDelimiter(); - this.maxKeys = handler.getMaxKeys(); - this.isTruncated = handler.getIsTruncated(); - this.nextMarker = handler.getNextMarker(); - this.entries = handler.getKeyEntries(); - this.commonPrefixEntries = handler.getCommonPrefixEntries(); - - } catch (SAXException e) { - throw new RuntimeException("Unexpected error parsing ListBucket xml", e); - } - } - } - - class ListBucketHandler extends DefaultHandler { - - private String name = null; - private String prefix = null; - private String marker = null; - private String delimiter = null; - private int maxKeys = 0; - private boolean isTruncated = false; - private String nextMarker = null; - private boolean isEchoedPrefix = false; - private List keyEntries = null; - private ListEntry keyEntry = null; - private List commonPrefixEntries = null; - private CommonPrefixEntry commonPrefixEntry = null; - private StringBuffer currText = null; - private SimpleDateFormat iso8601Parser = null; - - public ListBucketHandler() { - super(); - keyEntries = new ArrayList(); - commonPrefixEntries = new ArrayList(); - this.iso8601Parser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - this.iso8601Parser.setTimeZone(new SimpleTimeZone(0, "GMT")); - this.currText = new StringBuffer(); - } - - public void startDocument() { - this.isEchoedPrefix = true; - } - - public void endDocument() { - // ignore - } - - public void startElement(String uri, String name, String qName, Attributes attrs) { - if (name.equals("Contents")) { - this.keyEntry = new ListEntry(); - } else if (name.equals("Owner")) { - this.keyEntry.owner = new Owner(); - } else if (name.equals("CommonPrefixes")){ - this.commonPrefixEntry = new CommonPrefixEntry(); - } - } - - public void endElement(String uri, String name, String qName) { - if (name.equals("Name")) { - this.name = this.currText.toString(); - } - // this prefix is the one we echo back from the request - else if (name.equals("Prefix") && this.isEchoedPrefix) { - this.prefix = this.currText.toString(); - this.isEchoedPrefix = false; - } else if (name.equals("Marker")) { - this.marker = this.currText.toString(); - } else if (name.equals("MaxKeys")) { - this.maxKeys = Integer.parseInt(this.currText.toString()); - } else if (name.equals("Delimiter")) { - this.delimiter = this.currText.toString(); - } else if (name.equals("IsTruncated")) { - this.isTruncated = Boolean.valueOf(this.currText.toString()); - } else if (name.equals("NextMarker")) { - this.nextMarker = this.currText.toString(); - } else if (name.equals("Contents")) { - this.keyEntries.add(this.keyEntry); - } else if (name.equals("Key")) { - this.keyEntry.key = this.currText.toString(); - } else if (name.equals("LastModified")) { - try { - this.keyEntry.lastModified = this.iso8601Parser.parse(this.currText.toString()); - } catch (ParseException e) { - throw new RuntimeException("Unexpected date format in list bucket output", e); - } - } else if (name.equals("ETag")) { - this.keyEntry.eTag = this.currText.toString(); - } else if (name.equals("Size")) { - this.keyEntry.size = Long.parseLong(this.currText.toString()); - } else if (name.equals("StorageClass")) { - this.keyEntry.storageClass = this.currText.toString(); - } else if (name.equals("ID")) { - this.keyEntry.owner.id = this.currText.toString(); - } else if (name.equals("DisplayName")) { - this.keyEntry.owner.displayName = this.currText.toString(); - } else if (name.equals("CommonPrefixes")) { - this.commonPrefixEntries.add(this.commonPrefixEntry); - } - // this is the common prefix for keys that match up to the delimiter - else if (name.equals("Prefix")) { - this.commonPrefixEntry.prefix = this.currText.toString(); - } - if(this.currText.length() != 0) - this.currText = new StringBuffer(); - } - - public void characters(char ch[], int start, int length) { - this.currText.append(ch, start, length); - } - - public String getName() { - return this.name; - } - - public String getPrefix() { - return this.prefix; - } - - public String getMarker() { - return this.marker; - } - - public String getDelimiter() { - return this.delimiter; - } - - public int getMaxKeys(){ - return this.maxKeys; - } - - public boolean getIsTruncated() { - return this.isTruncated; - } - - public String getNextMarker() { - return this.nextMarker; - } - - public List getKeyEntries() { - return this.keyEntries; - } - - public List getCommonPrefixEntries() { - return this.commonPrefixEntries; - } - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/ListEntry.java b/aws/perftest/src/main/java/com/amazon/s3/ListEntry.java deleted file mode 100644 index 5be8810c18..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/ListEntry.java +++ /dev/null @@ -1,51 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.util.Date; - -/** - * A structure representing a single object stored in S3. Returned as a part of ListBucketResponse. - */ -public class ListEntry { - /** - * The name of the object - */ - public String key; - - /** - * The date at which the object was last modified. - */ - public Date lastModified; - - /** - * The object's ETag, which can be used for conditional GETs. - */ - public String eTag; - - /** - * The size of the object in bytes. - */ - public long size; - - /** - * The object's storage class - */ - public String storageClass; - - /** - * The object's owner - */ - public Owner owner; - - public String toString() { - return key; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/LocationResponse.java b/aws/perftest/src/main/java/com/amazon/s3/LocationResponse.java deleted file mode 100644 index 07661514f7..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/LocationResponse.java +++ /dev/null @@ -1,89 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.io.IOException; -import java.net.HttpURLConnection; - -import org.xml.sax.Attributes; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.DefaultHandler; - -/** - * A Response object returned from AWSAuthConnection.getBucketLocation(). - * Parses the response XML and exposes the location constraint - * via the geteLocation() method. - */ -public class LocationResponse extends Response { - String location; - - /** - * Parse the response to a ?location query. - */ - public LocationResponse(HttpURLConnection connection) throws IOException { - super(connection); - if (connection.getResponseCode() < 400) { - try { - XMLReader xr = Utils.createXMLReader();; - LocationResponseHandler handler = new LocationResponseHandler(); - xr.setContentHandler(handler); - xr.setErrorHandler(handler); - - xr.parse(new InputSource(connection.getInputStream())); - this.location = handler.location; - } catch (SAXException e) { - throw new RuntimeException("Unexpected error parsing ListAllMyBuckets xml", e); - } - } else { - this.location = ""; - } - } - - /** - * Report the location-constraint for a bucket. - * A value of null indicates an error; - * the empty string indicates no constraint; - * and any other value is an actual location constraint value. - */ - public String getLocation() { - return location; - } - - /** - * Helper class to parse LocationConstraint response XML - */ - static class LocationResponseHandler extends DefaultHandler { - String location = null; - private StringBuffer currText = null; - - public void startDocument() { - } - - public void startElement(String uri, String name, String qName, Attributes attrs) { - if (name.equals("LocationConstraint")) { - this.currText = new StringBuffer(); - } - } - - public void endElement(String uri, String name, String qName) { - if (name.equals("LocationConstraint")) { - location = this.currText.toString(); - this.currText = null; - } - } - - public void characters(char ch[], int start, int length) { - if (currText != null) - this.currText.append(ch, start, length); - } - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/Owner.java b/aws/perftest/src/main/java/com/amazon/s3/Owner.java deleted file mode 100644 index 4693bf740c..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/Owner.java +++ /dev/null @@ -1,18 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -/** - * A structure representing the owner of an object, used as a part of ListEntry. - */ -public class Owner { - public String id; - public String displayName; -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/QueryStringAuthGenerator.java b/aws/perftest/src/main/java/com/amazon/s3/QueryStringAuthGenerator.java deleted file mode 100644 index 4f2ee4baa6..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/QueryStringAuthGenerator.java +++ /dev/null @@ -1,264 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -/** - * This class mimics the behavior of AWSAuthConnection, except instead of actually performing - * the operation, QueryStringAuthGenerator will return URLs with query string parameters that - * can be used to do the same thing. These parameters include an expiration date, so that - * if you hand them off to someone else, they will only work for a limited amount of time. - */ -public class QueryStringAuthGenerator { - - private String awsAccessKeyId; - private String awsSecretAccessKey; - private boolean isSecure; - private String server; - private int port; - private CallingFormat callingFormat; - - private Long expiresIn = null; - private Long expires = null; - - // by default, expire in 1 minute. - private static final Long DEFAULT_EXPIRES_IN = new Long(60 * 1000); - - public QueryStringAuthGenerator(String awsAccessKeyId, String awsSecretAccessKey) { - this(awsAccessKeyId, awsSecretAccessKey, true); - } - - public QueryStringAuthGenerator(String awsAccessKeyId, String awsSecretAccessKey, - boolean isSecure) - { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, Utils.DEFAULT_HOST); - } - - public QueryStringAuthGenerator(String awsAccessKeyId, String awsSecretAccessKey, - boolean isSecure, String server) - { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, - isSecure ? Utils.SECURE_PORT : Utils.INSECURE_PORT); - } - - public QueryStringAuthGenerator(String awsAccessKeyId, String awsSecretAccessKey, - boolean isSecure, String server, int port) - { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, - port, CallingFormat.getSubdomainCallingFormat()); - } - - public QueryStringAuthGenerator(String awsAccessKeyId, String awsSecretAccessKey, - boolean isSecure, String server, CallingFormat callingFormat) - { - this(awsAccessKeyId, awsSecretAccessKey, isSecure, server, - isSecure ? Utils.SECURE_PORT : Utils.INSECURE_PORT, - callingFormat); - } - - public QueryStringAuthGenerator(String awsAccessKeyId, String awsSecretAccessKey, - boolean isSecure, String server, int port, CallingFormat callingFormat) - { - this.awsAccessKeyId = awsAccessKeyId; - this.awsSecretAccessKey = awsSecretAccessKey; - this.isSecure = isSecure; - this.server = server; - this.port = port; - this.callingFormat = callingFormat; - - this.expiresIn = DEFAULT_EXPIRES_IN; - this.expires = null; - } - - - public void setCallingFormat(CallingFormat format) { - this.callingFormat = format; - } - - public void setExpires(long millisSinceEpoch) { - expires = new Long(millisSinceEpoch); - expiresIn = null; - } - - public void setExpiresIn(long millis) { - expiresIn = new Long(millis); - expires = null; - } - - public String createBucket(String bucket, Map headers) - { - // validate bucket name - if (!Utils.validateBucketName(bucket, callingFormat, false)) - throw new IllegalArgumentException("Invalid S3Bucket Name: "+bucket); - - Map pathArgs = new HashMap(); - return generateURL("PUT", bucket, "", pathArgs, headers); - } - - public String listBucket(String bucket, String prefix, String marker, - Integer maxKeys, Map headers){ - return listBucket(bucket, prefix, marker, maxKeys, null, headers); - } - - public String listBucket(String bucket, String prefix, String marker, - Integer maxKeys, String delimiter, Map headers) - { - Map pathArgs = Utils.paramsForListOptions(prefix, marker, maxKeys, delimiter); - return generateURL("GET", bucket, "", pathArgs, headers); - } - - public String deleteBucket(String bucket, Map headers) - { - Map pathArgs = new HashMap(); - return generateURL("DELETE", bucket, "", pathArgs, headers); - } - - public String put(String bucket, String key, S3Object object, Map headers) { - Map metadata = null; - Map pathArgs = new HashMap(); - if (object != null) { - metadata = object.metadata; - } - - - return generateURL("PUT", bucket, Utils.urlencode(key), pathArgs, mergeMeta(headers, metadata)); - } - - public String get(String bucket, String key, Map headers) - { - Map pathArgs = new HashMap(); - return generateURL("GET", bucket, Utils.urlencode(key), pathArgs, headers); - } - - public String delete(String bucket, String key, Map headers) - { - Map pathArgs = new HashMap(); - return generateURL("DELETE", bucket, Utils.urlencode(key), pathArgs, headers); - } - - public String getBucketLogging(String bucket, Map headers) { - Map pathArgs = new HashMap(); - pathArgs.put("logging", null); - return generateURL("GET", bucket, "", pathArgs, headers); - } - - public String putBucketLogging(String bucket, String loggingXMLDoc, Map headers) { - Map pathArgs = new HashMap(); - pathArgs.put("logging", null); - return generateURL("PUT", bucket, "", pathArgs, headers); - } - - public String getBucketACL(String bucket, Map headers) { - return getACL(bucket, "", headers); - } - - public String getACL(String bucket, String key, Map headers) - { - Map pathArgs = new HashMap(); - pathArgs.put("acl", null); - return generateURL("GET", bucket, Utils.urlencode(key), pathArgs, headers); - } - - public String putBucketACL(String bucket, String aclXMLDoc, Map headers) { - return putACL(bucket, "", aclXMLDoc, headers); - } - - public String putACL(String bucket, String key, String aclXMLDoc, Map headers) - { - Map pathArgs = new HashMap(); - pathArgs.put("acl", null); - return generateURL("PUT", bucket, Utils.urlencode(key), pathArgs, headers); - } - - public String listAllMyBuckets(Map headers) - { - Map pathArgs = new HashMap(); - return generateURL("GET", "", "", pathArgs, headers); - } - - public String makeBareURL(String bucket, String key) { - StringBuffer buffer = new StringBuffer(); - if (this.isSecure) { - buffer.append("https://"); - } else { - buffer.append("http://"); - } - buffer.append(this.server).append(":").append(this.port).append("/").append(bucket); - buffer.append("/").append(Utils.urlencode(key)); - - return buffer.toString(); - } - - private String generateURL(String method, String bucketName, String key, Map pathArgs, Map headers) - { - long expires = 0L; - if (this.expiresIn != null) { - expires = System.currentTimeMillis() + this.expiresIn.longValue(); - } else if (this.expires != null) { - expires = this.expires.longValue(); - } else { - throw new RuntimeException("Illegal expires state"); - } - - // convert to seconds - expires /= 1000; - - String canonicalString = Utils.makeCanonicalString(method, bucketName, key, pathArgs, headers, ""+expires); - String encodedCanonical = Utils.encode(this.awsSecretAccessKey, canonicalString, true); - - pathArgs.put("Signature", encodedCanonical); - pathArgs.put("Expires", Long.toString(expires)); - pathArgs.put("AWSAccessKeyId", this.awsAccessKeyId); - - CallingFormat callingFormat = Utils.getCallingFormatForBucket( this.callingFormat, bucketName ); - if ( isSecure && callingFormat != CallingFormat.getPathCallingFormat() && bucketName.indexOf( "." ) != -1 ) { - System.err.println( "You are making an SSL connection, however, the bucket contains periods and the wildcard certificate will not match by default. Please consider using HTTP." ); - } - - String returnString; - try { - URL url = callingFormat.getURL(isSecure, server, port, bucketName, key, pathArgs); - returnString = url.toString(); - } catch (MalformedURLException e) { - returnString = "Exception generating url " + e; - } - - return returnString; - } - - private Map mergeMeta(Map headers, Map metadata) { - Map merged = new TreeMap(); - if (headers != null) { - for (Iterator i = headers.keySet().iterator(); i.hasNext(); ) { - String key = (String)i.next(); - merged.put(key, headers.get(key)); - } - } - if (metadata != null) { - for (Iterator i = metadata.keySet().iterator(); i.hasNext(); ) { - String key = (String)i.next(); - String metadataKey = Utils.METADATA_PREFIX + key; - if (merged.containsKey(metadataKey)) { - ((List)merged.get(metadataKey)).addAll((List)metadata.get(key)); - } else { - merged.put(metadataKey, metadata.get(key)); - } - } - } - return merged; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/Response.java b/aws/perftest/src/main/java/com/amazon/s3/Response.java deleted file mode 100644 index b6b5f01c58..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/Response.java +++ /dev/null @@ -1,25 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.net.HttpURLConnection; -import java.io.IOException; - -/** - * The parent class of all other Responses. This class keeps track of the - * HttpURLConnection response. - */ -public class Response { - public HttpURLConnection connection; - - public Response(HttpURLConnection connection) throws IOException { - this.connection = connection; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/S3Object.java b/aws/perftest/src/main/java/com/amazon/s3/S3Object.java deleted file mode 100644 index d43a3bca9f..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/S3Object.java +++ /dev/null @@ -1,30 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.util.Map; - -/** - * A representation of a single object stored in S3. - */ -public class S3Object { - - public byte[] data; - - /** - * A Map from String to List of Strings representing the object's metadata - */ - public Map metadata; - - public S3Object(byte[] data, Map metadata) { - this.data = data; - this.metadata = metadata; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/s3/Utils.java b/aws/perftest/src/main/java/com/amazon/s3/Utils.java deleted file mode 100644 index b2b5937ffb..0000000000 --- a/aws/perftest/src/main/java/com/amazon/s3/Utils.java +++ /dev/null @@ -1,324 +0,0 @@ -// This software code is made available "AS IS" without warranties of any -// kind. You may copy, display, modify and redistribute the software -// code either by itself or as incorporated into your code; provided that -// you do not remove any proprietary notices. Your use of this software -// code is at your own risk and you waive any claim against Amazon -// Digital Services, Inc. or its affiliates with respect to your use of -// this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its -// affiliates. - -package com.amazon.s3; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.TreeMap; - -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.XMLReaderFactory; -import org.xml.sax.SAXException; - -import com.amazon.thirdparty.Base64; - -public class Utils { - static final String METADATA_PREFIX = "x-amz-meta-"; - static final String AMAZON_HEADER_PREFIX = "x-amz-"; - static final String ALTERNATIVE_DATE_HEADER = "x-amz-date"; - public static final String DEFAULT_HOST = "s3.amazonaws.com"; - - public static final int SECURE_PORT = 443; - public static final int INSECURE_PORT = 80; - - - /** - * HMAC/SHA1 Algorithm per RFC 2104. - */ - private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1"; - - static String makeCanonicalString(String method, String bucket, String key, Map pathArgs, Map headers) { - return makeCanonicalString(method, bucket, key, pathArgs, headers, null); - } - - /** - * Calculate the canonical string. When expires is non-null, it will be - * used instead of the Date header. - */ - static String makeCanonicalString(String method, String bucketName, String key, Map pathArgs, - Map headers, String expires) - { - StringBuffer buf = new StringBuffer(); - buf.append(method + "\n"); - - // Add all interesting headers to a list, then sort them. "Interesting" - // is defined as Content-MD5, Content-Type, Date, and x-amz- - SortedMap interestingHeaders = new TreeMap(); - if (headers != null) { - for (Iterator i = headers.keySet().iterator(); i.hasNext(); ) { - String hashKey = (String)i.next(); - if (hashKey == null) continue; - String lk = hashKey.toLowerCase(); - - // Ignore any headers that are not particularly interesting. - if (lk.equals("content-type") || lk.equals("content-md5") || lk.equals("date") || - lk.startsWith(AMAZON_HEADER_PREFIX)) - { - List s = (List)headers.get(hashKey); - interestingHeaders.put(lk, concatenateList(s)); - } - } - } - - if (interestingHeaders.containsKey(ALTERNATIVE_DATE_HEADER)) { - interestingHeaders.put("date", ""); - } - - // if the expires is non-null, use that for the date field. this - // trumps the x-amz-date behavior. - if (expires != null) { - interestingHeaders.put("date", expires); - } - - // these headers require that we still put a new line in after them, - // even if they don't exist. - if (! interestingHeaders.containsKey("content-type")) { - interestingHeaders.put("content-type", ""); - } - if (! interestingHeaders.containsKey("content-eTag")) { - interestingHeaders.put("content-eTag", ""); - } - - // Finally, add all the interesting headers (i.e.: all that startwith x-amz- ;-)) - for (Iterator i = interestingHeaders.keySet().iterator(); i.hasNext(); ) { - String headerKey = (String)i.next(); - if (headerKey.startsWith(AMAZON_HEADER_PREFIX)) { - buf.append(headerKey).append(':').append(interestingHeaders.get(headerKey)); - } else { - buf.append(interestingHeaders.get(headerKey)); - } - buf.append("\n"); - } - - // build the path using the bucket and key - if (bucketName != null && !bucketName.equals("")) { - buf.append("/" + bucketName ); - } - - // append the key (it might be an empty string) - // append a slash regardless - buf.append("/"); - if(key != null) { - buf.append(key); - } - - // if there is an acl, logging or torrent parameter - // add them to the string - if (pathArgs != null ) { - if (pathArgs.containsKey("acl")) { - buf.append("?acl"); - } else if (pathArgs.containsKey("torrent")) { - buf.append("?torrent"); - } else if (pathArgs.containsKey("logging")) { - buf.append("?logging"); - } else if (pathArgs.containsKey("location")) { - buf.append("?location"); - } - } - - return buf.toString(); - - } - - /** - * Calculate the HMAC/SHA1 on a string. - * @param data Data to sign - * @param passcode Passcode to sign it with - * @return Signature - * @throws NoSuchAlgorithmException If the algorithm does not exist. Unlikely - * @throws InvalidKeyException If the key is invalid. - */ - static String encode(String awsSecretAccessKey, String canonicalString, - boolean urlencode) - { - // The following HMAC/SHA1 code for the signature is taken from the - // AWS Platform's implementation of RFC2104 (amazon.webservices.common.Signature) - // - // Acquire an HMAC/SHA1 from the raw key bytes. - SecretKeySpec signingKey = - new SecretKeySpec(awsSecretAccessKey.getBytes(), HMAC_SHA1_ALGORITHM); - - // Acquire the MAC instance and initialize with the signing key. - Mac mac = null; - try { - mac = Mac.getInstance(HMAC_SHA1_ALGORITHM); - } catch (NoSuchAlgorithmException e) { - // should not happen - throw new RuntimeException("Could not find sha1 algorithm", e); - } - try { - mac.init(signingKey); - } catch (InvalidKeyException e) { - // also should not happen - throw new RuntimeException("Could not initialize the MAC algorithm", e); - } - - // Compute the HMAC on the digest, and set it. - String b64 = Base64.encodeBytes(mac.doFinal(canonicalString.getBytes())); - - if (urlencode) { - return urlencode(b64); - } else { - return b64; - } - } - - static Map paramsForListOptions(String prefix, String marker, Integer maxKeys) { - return paramsForListOptions(prefix, marker, maxKeys, null); - } - - static Map paramsForListOptions(String prefix, String marker, Integer maxKeys, String delimiter) { - - Map argParams = new HashMap(); - // these three params must be url encoded - if (prefix != null) - argParams.put("prefix", urlencode(prefix)); - if (marker != null) - argParams.put("marker", urlencode(marker)); - if (delimiter != null) - argParams.put("delimiter", urlencode(delimiter)); - - if (maxKeys != null) - argParams.put("max-keys", Integer.toString(maxKeys.intValue())); - - return argParams; - - } - - /** - * Converts the Path Arguments from a map to String which can be used in url construction - * @param pathArgs a map of arguments - * @return a string representation of pathArgs - */ - public static String convertPathArgsHashToString(Map pathArgs) { - StringBuffer pathArgsString = new StringBuffer(); - String argumentValue; - boolean firstRun = true; - if (pathArgs != null) { - for (Iterator argumentIterator = pathArgs.keySet().iterator(); argumentIterator.hasNext(); ) { - String argument = (String)argumentIterator.next(); - if (firstRun) { - firstRun = false; - pathArgsString.append("?"); - } else { - pathArgsString.append("&"); - } - - argumentValue = (String)pathArgs.get(argument); - pathArgsString.append(argument); - if (argumentValue != null) { - pathArgsString.append("="); - pathArgsString.append(argumentValue); - } - } - } - - return pathArgsString.toString(); - } - - - - - static String urlencode(String unencoded) { - try { - return URLEncoder.encode(unencoded, "UTF-8"); - } catch (UnsupportedEncodingException e) { - // should never happen - throw new RuntimeException("Could not url encode to UTF-8", e); - } - } - - static XMLReader createXMLReader() { - try { - return XMLReaderFactory.createXMLReader(); - } catch (SAXException e) { - // oops, lets try doing this (needed in 1.4) - System.setProperty("org.xml.sax.driver", "org.apache.crimson.parser.XMLReaderImpl"); - } - try { - // try once more - return XMLReaderFactory.createXMLReader(); - } catch (SAXException e) { - throw new RuntimeException("Couldn't initialize a sax driver for the XMLReader"); - } - } - - /** - * Concatenates a bunch of header values, seperating them with a comma. - * @param values List of header values. - * @return String of all headers, with commas. - */ - private static String concatenateList(List values) { - StringBuffer buf = new StringBuffer(); - for (int i = 0, size = values.size(); i < size; ++ i) { - buf.append(((String)values.get(i)).replaceAll("\n", "").trim()); - if (i != (size - 1)) { - buf.append(","); - } - } - return buf.toString(); - } - - /** - * Validate bucket-name - */ - static boolean validateBucketName(String bucketName, CallingFormat callingFormat, boolean located) { - if (callingFormat == CallingFormat.getPathCallingFormat()) - { - final int MIN_BUCKET_LENGTH = 3; - final int MAX_BUCKET_LENGTH = 255; - final String BUCKET_NAME_REGEX = "^[0-9A-Za-z\\.\\-_]*$"; - - return null != bucketName && - bucketName.length() >= MIN_BUCKET_LENGTH && - bucketName.length() <= MAX_BUCKET_LENGTH && - bucketName.matches(BUCKET_NAME_REGEX); - } else { - return isValidSubdomainBucketName( bucketName ); - } - } - - static boolean isValidSubdomainBucketName( String bucketName ) { - final int MIN_BUCKET_LENGTH = 3; - final int MAX_BUCKET_LENGTH = 63; - // don't allow names that look like 127.0.0.1 - final String IPv4_REGEX = "^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$"; - // dns sub-name restrictions - final String BUCKET_NAME_REGEX = "^[a-z0-9]([a-z0-9\\-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9\\-]*[a-z0-9])?)*$"; - - // If there wasn't a location-constraint, then the current actual - // restriction is just that no 'part' of the name (i.e. sequence - // of characters between any 2 '.'s has to be 63) but the recommendation - // is to keep the entire bucket name under 63. - return null != bucketName && - bucketName.length() >= MIN_BUCKET_LENGTH && - bucketName.length() <= MAX_BUCKET_LENGTH && - !bucketName.matches(IPv4_REGEX) && - bucketName.matches(BUCKET_NAME_REGEX); - } - - static CallingFormat getCallingFormatForBucket( CallingFormat desiredFormat, String bucketName ) { - CallingFormat callingFormat = desiredFormat; - if ( callingFormat == CallingFormat.getSubdomainCallingFormat() && ! Utils.isValidSubdomainBucketName( bucketName ) ) { - callingFormat = CallingFormat.getPathCallingFormat(); - } - return callingFormat; - } -} diff --git a/aws/perftest/src/main/java/com/amazon/thirdparty/Base64.java b/aws/perftest/src/main/java/com/amazon/thirdparty/Base64.java deleted file mode 100644 index bda181c9c1..0000000000 --- a/aws/perftest/src/main/java/com/amazon/thirdparty/Base64.java +++ /dev/null @@ -1,1459 +0,0 @@ -// -// NOTE: The following source code is the iHarder.net public domain -// Base64 library and is provided here as a convenience. For updates, -// problems, questions, etc. regarding this code, please visit: -// http://iharder.sourceforge.net/current/java/base64/ -// - -package com.amazon.thirdparty; - - -/** - * Encodes and decodes to and from Base64 notation. - * - *

- * Change Log: - *

- *
    - *
  • v2.1 - Cleaned up javadoc comments and unused variables and methods. Added - * some convenience methods for reading and writing to and from files.
    • - *
  • v2.0.2 - Now specifies UTF-8 encoding in places where the code fails on systems - * with other encodings (like EBCDIC).
    • - *
  • v2.0.1 - Fixed an error when decoding a single byte, that is, when the - * encoded data was a single byte.
    • - *
  • v2.0 - I got rid of methods that used booleans to set options. - * Now everything is more consolidated and cleaner. The code now detects - * when data that's being decoded is gzip-compressed and will decompress it - * automatically. Generally things are cleaner. You'll probably have to - * change some method calls that you were making to support the new - * options format (ints that you "OR" together).
    • - *
  • v1.5.1 - Fixed bug when decompressing and decoding to a - * byte[] using decode( String s, boolean gzipCompressed ). - * Added the ability to "suspend" encoding in the Output Stream so - * you can turn on and off the encoding if you need to embed base64 - * data in an otherwise "normal" stream (like an XML file).
    • - *
  • v1.5 - Output stream pases on flush() command but doesn't do anything itself. - * This helps when using GZIP streams. - * Added the ability to GZip-compress objects before encoding them.
    • - *
  • v1.4 - Added helper methods to read/write files.
    • - *
  • v1.3.6 - Fixed OutputStream.flush() so that 'position' is reset.
    • - *
  • v1.3.5 - Added flag to turn on and off line breaks. Fixed bug in input stream - * where last buffer being read, if not completely full, was not returned.
    • - *
  • v1.3.4 - Fixed when "improperly padded stream" error was thrown at the wrong time.
    • - *
  • v1.3.3 - Fixed I/O streams which were totally messed up.
    • - *
- * - *

- * I am placing this code in the Public Domain. Do with it as you will. - * This software comes with no guarantees or warranties but with - * plenty of well-wishing instead! - * Please visit http://iharder.net/base64 - * periodically to check for updates or to contribute improvements. - *

- * - * @author Robert Harder - * @author rob@iharder.net - * @version 2.1 - */ -public class Base64 -{ - -/* ******** P U B L I C F I E L D S ******** */ - - - /** No options specified. Value is zero. */ - public final static int NO_OPTIONS = 0; - - /** Specify encoding. */ - public final static int ENCODE = 1; - - - /** Specify decoding. */ - public final static int DECODE = 0; - - - /** Specify that data should be gzip-compressed. */ - public final static int GZIP = 2; - - - /** Don't break lines when encoding (violates strict Base64 specification) */ - public final static int DONT_BREAK_LINES = 8; - - -/* ******** P R I V A T E F I E L D S ******** */ - - - /** Maximum line length (76) of Base64 output. */ - private final static int MAX_LINE_LENGTH = 76; - - - /** The equals sign (=) as a byte. */ - private final static byte EQUALS_SIGN = (byte)'='; - - - /** The new line character (\n) as a byte. */ - private final static byte NEW_LINE = (byte)'\n'; - - - /** Preferred encoding. */ - private final static String PREFERRED_ENCODING = "UTF-8"; - - - /** The 64 valid Base64 values. */ - private final static byte[] ALPHABET; - private final static byte[] _NATIVE_ALPHABET = /* May be something funny like EBCDIC */ - { - (byte)'A', (byte)'B', (byte)'C', (byte)'D', (byte)'E', (byte)'F', (byte)'G', - (byte)'H', (byte)'I', (byte)'J', (byte)'K', (byte)'L', (byte)'M', (byte)'N', - (byte)'O', (byte)'P', (byte)'Q', (byte)'R', (byte)'S', (byte)'T', (byte)'U', - (byte)'V', (byte)'W', (byte)'X', (byte)'Y', (byte)'Z', - (byte)'a', (byte)'b', (byte)'c', (byte)'d', (byte)'e', (byte)'f', (byte)'g', - (byte)'h', (byte)'i', (byte)'j', (byte)'k', (byte)'l', (byte)'m', (byte)'n', - (byte)'o', (byte)'p', (byte)'q', (byte)'r', (byte)'s', (byte)'t', (byte)'u', - (byte)'v', (byte)'w', (byte)'x', (byte)'y', (byte)'z', - (byte)'0', (byte)'1', (byte)'2', (byte)'3', (byte)'4', (byte)'5', - (byte)'6', (byte)'7', (byte)'8', (byte)'9', (byte)'+', (byte)'/' - }; - - /** Determine which ALPHABET to use. */ - static - { - byte[] __bytes; - try - { - __bytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".getBytes( PREFERRED_ENCODING ); - } // end try - catch (java.io.UnsupportedEncodingException use) - { - __bytes = _NATIVE_ALPHABET; // Fall back to native encoding - } // end catch - ALPHABET = __bytes; - } // end static - - - /** - * Translates a Base64 value to either its 6-bit reconstruction value - * or a negative number indicating some other meaning. - **/ - private final static byte[] DECODABET = - { - -9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 0 - 8 - -5,-5, // Whitespace: Tab and Linefeed - -9,-9, // Decimal 11 - 12 - -5, // Whitespace: Carriage Return - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 14 - 26 - -9,-9,-9,-9,-9, // Decimal 27 - 31 - -5, // Whitespace: Space - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 33 - 42 - 62, // Plus sign at decimal 43 - -9,-9,-9, // Decimal 44 - 46 - 63, // Slash at decimal 47 - 52,53,54,55,56,57,58,59,60,61, // Numbers zero through nine - -9,-9,-9, // Decimal 58 - 60 - -1, // Equals sign at decimal 61 - -9,-9,-9, // Decimal 62 - 64 - 0,1,2,3,4,5,6,7,8,9,10,11,12,13, // Letters 'A' through 'N' - 14,15,16,17,18,19,20,21,22,23,24,25, // Letters 'O' through 'Z' - -9,-9,-9,-9,-9,-9, // Decimal 91 - 96 - 26,27,28,29,30,31,32,33,34,35,36,37,38, // Letters 'a' through 'm' - 39,40,41,42,43,44,45,46,47,48,49,50,51, // Letters 'n' through 'z' - -9,-9,-9,-9 // Decimal 123 - 126 - /*,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 127 - 139 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 140 - 152 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 153 - 165 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 166 - 178 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 179 - 191 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 192 - 204 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 205 - 217 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 218 - 230 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9, // Decimal 231 - 243 - -9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9,-9 // Decimal 244 - 255 */ - }; - - // I think I end up not using the BAD_ENCODING indicator. - //private final static byte BAD_ENCODING = -9; // Indicates error in encoding - private final static byte WHITE_SPACE_ENC = -5; // Indicates white space in encoding - private final static byte EQUALS_SIGN_ENC = -1; // Indicates equals sign in encoding - - - /** Defeats instantiation. */ - private Base64(){} - - - -/* ******** E N C O D I N G M E T H O D S ******** */ - - - /** - * Encodes up to the first three bytes of array threeBytes - * and returns a four-byte array in Base64 notation. - * The actual number of significant bytes in your array is - * given by numSigBytes. - * The array threeBytes needs only be as big as - * numSigBytes. - * Code can reuse a byte array by passing a four-byte array as b4. - * - * @param b4 A reusable byte array to reduce array instantiation - * @param threeBytes the array to convert - * @param numSigBytes the number of significant bytes in your array - * @return four byte array in Base64 notation. - * @since 1.5.1 - */ - private static byte[] encode3to4( byte[] b4, byte[] threeBytes, int numSigBytes ) - { - encode3to4( threeBytes, 0, numSigBytes, b4, 0 ); - return b4; - } // end encode3to4 - - - /** - * Encodes up to three bytes of the array source - * and writes the resulting four Base64 bytes to destination. - * The source and destination arrays can be manipulated - * anywhere along their length by specifying - * srcOffset and destOffset. - * This method does not check to make sure your arrays - * are large enough to accomodate srcOffset + 3 for - * the source array or destOffset + 4 for - * the destination array. - * The actual number of significant bytes in your array is - * given by numSigBytes. - * - * @param source the array to convert - * @param srcOffset the index where conversion begins - * @param numSigBytes the number of significant bytes in your array - * @param destination the array to hold the conversion - * @param destOffset the index where output will be put - * @return the destination array - * @since 1.3 - */ - private static byte[] encode3to4( - byte[] source, int srcOffset, int numSigBytes, - byte[] destination, int destOffset ) - { - // 1 2 3 - // 01234567890123456789012345678901 Bit position - // --------000000001111111122222222 Array position from threeBytes - // --------| || || || | Six bit groups to index ALPHABET - // >>18 >>12 >> 6 >> 0 Right shift necessary - // 0x3f 0x3f 0x3f Additional AND - - // Create buffer with zero-padding if there are only one or two - // significant bytes passed in the array. - // We have to shift left 24 in order to flush out the 1's that appear - // when Java treats a value as negative that is cast from a byte to an int. - int inBuff = ( numSigBytes > 0 ? ((source[ srcOffset ] << 24) >>> 8) : 0 ) - | ( numSigBytes > 1 ? ((source[ srcOffset + 1 ] << 24) >>> 16) : 0 ) - | ( numSigBytes > 2 ? ((source[ srcOffset + 2 ] << 24) >>> 24) : 0 ); - - switch( numSigBytes ) - { - case 3: - destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; - destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; - destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>> 6) & 0x3f ]; - destination[ destOffset + 3 ] = ALPHABET[ (inBuff ) & 0x3f ]; - return destination; - - case 2: - destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; - destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; - destination[ destOffset + 2 ] = ALPHABET[ (inBuff >>> 6) & 0x3f ]; - destination[ destOffset + 3 ] = EQUALS_SIGN; - return destination; - - case 1: - destination[ destOffset ] = ALPHABET[ (inBuff >>> 18) ]; - destination[ destOffset + 1 ] = ALPHABET[ (inBuff >>> 12) & 0x3f ]; - destination[ destOffset + 2 ] = EQUALS_SIGN; - destination[ destOffset + 3 ] = EQUALS_SIGN; - return destination; - - default: - return destination; - } // end switch - } // end encode3to4 - - - - /** - * Serializes an object and returns the Base64-encoded - * version of that serialized object. If the object - * cannot be serialized or there is another error, - * the method will return null. - * The object is not GZip-compressed before being encoded. - * - * @param serializableObject The object to encode - * @return The Base64-encoded object - * @since 1.4 - */ - public static String encodeObject( java.io.Serializable serializableObject ) - { - return encodeObject( serializableObject, NO_OPTIONS ); - } // end encodeObject - - - - /** - * Serializes an object and returns the Base64-encoded - * version of that serialized object. If the object - * cannot be serialized or there is another error, - * the method will return null. - *

- * Valid options:

-     *   GZIP: gzip-compresses object before encoding it.
-     *   DONT_BREAK_LINES: don't break lines at 76 characters
-     *     Note: Technically, this makes your encoding non-compliant.
-     *
- *

- * Example: encodeObject( myObj, Base64.GZIP ) or - *

- * Example: encodeObject( myObj, Base64.GZIP | Base64.DONT_BREAK_LINES ) - * - * @param serializableObject The object to encode - * @param options Specified options - * @return The Base64-encoded object - * @see Base64#GZIP - * @see Base64#DONT_BREAK_LINES - * @since 2.0 - */ - public static String encodeObject( java.io.Serializable serializableObject, int options ) - { - // Streams - java.io.ByteArrayOutputStream baos = null; - java.io.OutputStream b64os = null; - java.io.ObjectOutputStream oos = null; - java.util.zip.GZIPOutputStream gzos = null; - - // Isolate options - int gzip = (options & GZIP); - int dontBreakLines = (options & DONT_BREAK_LINES); - - try - { - // ObjectOutputStream -> (GZIP) -> Base64 -> ByteArrayOutputStream - baos = new java.io.ByteArrayOutputStream(); - b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines ); - - // GZip? - if( gzip == GZIP ) - { - gzos = new java.util.zip.GZIPOutputStream( b64os ); - oos = new java.io.ObjectOutputStream( gzos ); - } // end if: gzip - else - oos = new java.io.ObjectOutputStream( b64os ); - - oos.writeObject( serializableObject ); - } // end try - catch( java.io.IOException e ) - { - e.printStackTrace(); - return null; - } // end catch - finally - { - try{ oos.close(); } catch( Exception e ){} - try{ gzos.close(); } catch( Exception e ){} - try{ b64os.close(); } catch( Exception e ){} - try{ baos.close(); } catch( Exception e ){} - } // end finally - - // Return value according to relevant encoding. - try - { - return new String( baos.toByteArray(), PREFERRED_ENCODING ); - } // end try - catch (java.io.UnsupportedEncodingException uue) - { - return new String( baos.toByteArray() ); - } // end catch - - } // end encode - - - - /** - * Encodes a byte array into Base64 notation. - * Does not GZip-compress data. - * - * @param source The data to convert - * @since 1.4 - */ - public static String encodeBytes( byte[] source ) - { - return encodeBytes( source, 0, source.length, NO_OPTIONS ); - } // end encodeBytes - - - - /** - * Encodes a byte array into Base64 notation. - *

- * Valid options:

-     *   GZIP: gzip-compresses object before encoding it.
-     *   DONT_BREAK_LINES: don't break lines at 76 characters
-     *     Note: Technically, this makes your encoding non-compliant.
-     *
- *

- * Example: encodeBytes( myData, Base64.GZIP ) or - *

- * Example: encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES ) - * - * - * @param source The data to convert - * @param options Specified options - * @see Base64#GZIP - * @see Base64#DONT_BREAK_LINES - * @since 2.0 - */ - public static String encodeBytes( byte[] source, int options ) - { - return encodeBytes( source, 0, source.length, options ); - } // end encodeBytes - - - /** - * Encodes a byte array into Base64 notation. - * Does not GZip-compress data. - * - * @param source The data to convert - * @param off Offset in array where conversion should begin - * @param len Length of data to convert - * @since 1.4 - */ - public static String encodeBytes( byte[] source, int off, int len ) - { - return encodeBytes( source, off, len, NO_OPTIONS ); - } // end encodeBytes - - - - /** - * Encodes a byte array into Base64 notation. - *

- * Valid options:

-     *   GZIP: gzip-compresses object before encoding it.
-     *   DONT_BREAK_LINES: don't break lines at 76 characters
-     *     Note: Technically, this makes your encoding non-compliant.
-     *
- *

- * Example: encodeBytes( myData, Base64.GZIP ) or - *

- * Example: encodeBytes( myData, Base64.GZIP | Base64.DONT_BREAK_LINES ) - * - * - * @param source The data to convert - * @param off Offset in array where conversion should begin - * @param len Length of data to convert - * @param options Specified options - * @see Base64#GZIP - * @see Base64#DONT_BREAK_LINES - * @since 2.0 - */ - public static String encodeBytes( byte[] source, int off, int len, int options ) - { - // Isolate options - int dontBreakLines = ( options & DONT_BREAK_LINES ); - int gzip = ( options & GZIP ); - - // Compress? - if( gzip == GZIP ) - { - java.io.ByteArrayOutputStream baos = null; - java.util.zip.GZIPOutputStream gzos = null; - Base64.OutputStream b64os = null; - - - try - { - // GZip -> Base64 -> ByteArray - baos = new java.io.ByteArrayOutputStream(); - b64os = new Base64.OutputStream( baos, ENCODE | dontBreakLines ); - gzos = new java.util.zip.GZIPOutputStream( b64os ); - - gzos.write( source, off, len ); - gzos.close(); - } // end try - catch( java.io.IOException e ) - { - e.printStackTrace(); - return null; - } // end catch - finally - { - try{ gzos.close(); } catch( Exception e ){} - try{ b64os.close(); } catch( Exception e ){} - try{ baos.close(); } catch( Exception e ){} - } // end finally - - // Return value according to relevant encoding. - try - { - return new String( baos.toByteArray(), PREFERRED_ENCODING ); - } // end try - catch (java.io.UnsupportedEncodingException uue) - { - return new String( baos.toByteArray() ); - } // end catch - } // end if: compress - - // Else, don't compress. Better not to use streams at all then. - else - { - // Convert option to boolean in way that code likes it. - boolean breakLines = dontBreakLines == 0; - - int len43 = len * 4 / 3; - byte[] outBuff = new byte[ ( len43 ) // Main 4:3 - + ( (len % 3) > 0 ? 4 : 0 ) // Account for padding - + (breakLines ? ( len43 / MAX_LINE_LENGTH ) : 0) ]; // New lines - int d = 0; - int e = 0; - int len2 = len - 2; - int lineLength = 0; - for( ; d < len2; d+=3, e+=4 ) - { - encode3to4( source, d+off, 3, outBuff, e ); - - lineLength += 4; - if( breakLines && lineLength == MAX_LINE_LENGTH ) - { - outBuff[e+4] = NEW_LINE; - e++; - lineLength = 0; - } // end if: end of line - } // en dfor: each piece of array - - if( d < len ) - { - encode3to4( source, d+off, len - d, outBuff, e ); - e += 4; - } // end if: some padding needed - - - // Return value according to relevant encoding. - try - { - return new String( outBuff, 0, e, PREFERRED_ENCODING ); - } // end try - catch (java.io.UnsupportedEncodingException uue) - { - return new String( outBuff, 0, e ); - } // end catch - - } // end else: don't compress - - } // end encodeBytes - - - - - -/* ******** D E C O D I N G M E T H O D S ******** */ - - - /** - * Decodes four bytes from array source - * and writes the resulting bytes (up to three of them) - * to destination. - * The source and destination arrays can be manipulated - * anywhere along their length by specifying - * srcOffset and destOffset. - * This method does not check to make sure your arrays - * are large enough to accomodate srcOffset + 4 for - * the source array or destOffset + 3 for - * the destination array. - * This method returns the actual number of bytes that - * were converted from the Base64 encoding. - * - * - * @param source the array to convert - * @param srcOffset the index where conversion begins - * @param destination the array to hold the conversion - * @param destOffset the index where output will be put - * @return the number of decoded bytes converted - * @since 1.3 - */ - private static int decode4to3( byte[] source, int srcOffset, byte[] destination, int destOffset ) - { - // Example: Dk== - if( source[ srcOffset + 2] == EQUALS_SIGN ) - { - // Two ways to do the same thing. Don't know which way I like best. - //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) - // | ( ( DECODABET[ source[ srcOffset + 1] ] << 24 ) >>> 12 ); - int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) - | ( ( DECODABET[ source[ srcOffset + 1] ] & 0xFF ) << 12 ); - - destination[ destOffset ] = (byte)( outBuff >>> 16 ); - return 1; - } - - // Example: DkL= - else if( source[ srcOffset + 3 ] == EQUALS_SIGN ) - { - // Two ways to do the same thing. Don't know which way I like best. - //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) - // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) - // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ); - int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) - | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 ) - | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) << 6 ); - - destination[ destOffset ] = (byte)( outBuff >>> 16 ); - destination[ destOffset + 1 ] = (byte)( outBuff >>> 8 ); - return 2; - } - - // Example: DkLE - else - { - try{ - // Two ways to do the same thing. Don't know which way I like best. - //int outBuff = ( ( DECODABET[ source[ srcOffset ] ] << 24 ) >>> 6 ) - // | ( ( DECODABET[ source[ srcOffset + 1 ] ] << 24 ) >>> 12 ) - // | ( ( DECODABET[ source[ srcOffset + 2 ] ] << 24 ) >>> 18 ) - // | ( ( DECODABET[ source[ srcOffset + 3 ] ] << 24 ) >>> 24 ); - int outBuff = ( ( DECODABET[ source[ srcOffset ] ] & 0xFF ) << 18 ) - | ( ( DECODABET[ source[ srcOffset + 1 ] ] & 0xFF ) << 12 ) - | ( ( DECODABET[ source[ srcOffset + 2 ] ] & 0xFF ) << 6) - | ( ( DECODABET[ source[ srcOffset + 3 ] ] & 0xFF ) ); - - - destination[ destOffset ] = (byte)( outBuff >> 16 ); - destination[ destOffset + 1 ] = (byte)( outBuff >> 8 ); - destination[ destOffset + 2 ] = (byte)( outBuff ); - - return 3; - }catch( Exception e){ - System.out.println(""+source[srcOffset]+ ": " + ( DECODABET[ source[ srcOffset ] ] ) ); - System.out.println(""+source[srcOffset+1]+ ": " + ( DECODABET[ source[ srcOffset + 1 ] ] ) ); - System.out.println(""+source[srcOffset+2]+ ": " + ( DECODABET[ source[ srcOffset + 2 ] ] ) ); - System.out.println(""+source[srcOffset+3]+ ": " + ( DECODABET[ source[ srcOffset + 3 ] ] ) ); - return -1; - } //e nd catch - } - } // end decodeToBytes - - - - - /** - * Very low-level access to decoding ASCII characters in - * the form of a byte array. Does not support automatically - * gunzipping or any other "fancy" features. - * - * @param source The Base64 encoded data - * @param off The offset of where to begin decoding - * @param len The length of characters to decode - * @return decoded data - * @since 1.3 - */ - public static byte[] decode( byte[] source, int off, int len ) - { - int len34 = len * 3 / 4; - byte[] outBuff = new byte[ len34 ]; // Upper limit on size of output - int outBuffPosn = 0; - - byte[] b4 = new byte[4]; - int b4Posn = 0; - int i = 0; - byte sbiCrop = 0; - byte sbiDecode = 0; - for( i = off; i < off+len; i++ ) - { - sbiCrop = (byte)(source[i] & 0x7f); // Only the low seven bits - sbiDecode = DECODABET[ sbiCrop ]; - - if( sbiDecode >= WHITE_SPACE_ENC ) // White space, Equals sign or better - { - if( sbiDecode >= EQUALS_SIGN_ENC ) - { - b4[ b4Posn++ ] = sbiCrop; - if( b4Posn > 3 ) - { - outBuffPosn += decode4to3( b4, 0, outBuff, outBuffPosn ); - b4Posn = 0; - - // If that was the equals sign, break out of 'for' loop - if( sbiCrop == EQUALS_SIGN ) - break; - } // end if: quartet built - - } // end if: equals sign or better - - } // end if: white space, equals sign or better - else - { - System.err.println( "Bad Base64 input character at " + i + ": " + source[i] + "(decimal)" ); - return null; - } // end else: - } // each input character - - byte[] out = new byte[ outBuffPosn ]; - System.arraycopy( outBuff, 0, out, 0, outBuffPosn ); - return out; - } // end decode - - - - - /** - * Decodes data from Base64 notation, automatically - * detecting gzip-compressed data and decompressing it. - * - * @param s the string to decode - * @return the decoded data - * @since 1.4 - */ - public static byte[] decode( String s ) - { - byte[] bytes; - try - { - bytes = s.getBytes( PREFERRED_ENCODING ); - } // end try - catch( java.io.UnsupportedEncodingException uee ) - { - bytes = s.getBytes(); - } // end catch - // - - // Decode - bytes = decode( bytes, 0, bytes.length ); - - - // Check to see if it's gzip-compressed - // GZIP Magic Two-Byte Number: 0x8b1f (35615) - if( bytes != null && bytes.length >= 4 ) - { - - int head = ((int)bytes[0] & 0xff) | ((bytes[1] << 8) & 0xff00); - if( java.util.zip.GZIPInputStream.GZIP_MAGIC == head ) - { - java.io.ByteArrayInputStream bais = null; - java.util.zip.GZIPInputStream gzis = null; - java.io.ByteArrayOutputStream baos = null; - byte[] buffer = new byte[2048]; - int length = 0; - - try - { - baos = new java.io.ByteArrayOutputStream(); - bais = new java.io.ByteArrayInputStream( bytes ); - gzis = new java.util.zip.GZIPInputStream( bais ); - - while( ( length = gzis.read( buffer ) ) >= 0 ) - { - baos.write(buffer,0,length); - } // end while: reading input - - // No error? Get new bytes. - bytes = baos.toByteArray(); - - } // end try - catch( java.io.IOException e ) - { - // Just return originally-decoded bytes - } // end catch - finally - { - try{ baos.close(); } catch( Exception e ){} - try{ gzis.close(); } catch( Exception e ){} - try{ bais.close(); } catch( Exception e ){} - } // end finally - - } // end if: gzipped - } // end if: bytes.length >= 2 - - return bytes; - } // end decode - - - - - /** - * Attempts to decode Base64 data and deserialize a Java - * Object within. Returns null if there was an error. - * - * @param encodedObject The Base64 data to decode - * @return The decoded and deserialized object - * @since 1.5 - */ - public static Object decodeToObject( String encodedObject ) - { - // Decode and gunzip if necessary - byte[] objBytes = decode( encodedObject ); - - java.io.ByteArrayInputStream bais = null; - java.io.ObjectInputStream ois = null; - Object obj = null; - - try - { - bais = new java.io.ByteArrayInputStream( objBytes ); - ois = new java.io.ObjectInputStream( bais ); - - obj = ois.readObject(); - } // end try - catch( java.io.IOException e ) - { - e.printStackTrace(); - obj = null; - } // end catch - catch( java.lang.ClassNotFoundException e ) - { - e.printStackTrace(); - obj = null; - } // end catch - finally - { - try{ bais.close(); } catch( Exception e ){} - try{ ois.close(); } catch( Exception e ){} - } // end finally - - return obj; - } // end decodeObject - - - - /** - * Convenience method for encoding data to a file. - * - * @param dataToEncode byte array of data to encode in base64 form - * @param filename Filename for saving encoded data - * @return true if successful, false otherwise - * - * @since 2.1 - */ - public static boolean encodeToFile( byte[] dataToEncode, String filename ) - { - boolean success = false; - Base64.OutputStream bos = null; - try - { - bos = new Base64.OutputStream( - new java.io.FileOutputStream( filename ), Base64.ENCODE ); - bos.write( dataToEncode ); - success = true; - } // end try - catch( java.io.IOException e ) - { - - success = false; - } // end catch: IOException - finally - { - try{ bos.close(); } catch( Exception e ){} - } // end finally - - return success; - } // end encodeToFile - - - /** - * Convenience method for decoding data to a file. - * - * @param dataToDecode Base64-encoded data as a string - * @param filename Filename for saving decoded data - * @return true if successful, false otherwise - * - * @since 2.1 - */ - public static boolean decodeToFile( String dataToDecode, String filename ) - { - boolean success = false; - Base64.OutputStream bos = null; - try - { - bos = new Base64.OutputStream( - new java.io.FileOutputStream( filename ), Base64.DECODE ); - bos.write( dataToDecode.getBytes( PREFERRED_ENCODING ) ); - success = true; - } // end try - catch( java.io.IOException e ) - { - success = false; - } // end catch: IOException - finally - { - try{ bos.close(); } catch( Exception e ){} - } // end finally - - return success; - } // end decodeToFile - - - - - /** - * Convenience method for reading a base64-encoded - * file and decoding it. - * - * @param filename Filename for reading encoded data - * @return decoded byte array or null if unsuccessful - * - * @since 2.1 - */ - public static byte[] decodeFromFile( String filename ) - { - byte[] decodedData = null; - Base64.InputStream bis = null; - try - { - // Set up some useful variables - java.io.File file = new java.io.File( filename ); - byte[] buffer = null; - int length = 0; - int numBytes = 0; - - // Check for size of file - if( file.length() > Integer.MAX_VALUE ) - { - System.err.println( "File is too big for this convenience method (" + file.length() + " bytes)." ); - return null; - } // end if: file too big for int index - buffer = new byte[ (int)file.length() ]; - - // Open a stream - bis = new Base64.InputStream( - new java.io.BufferedInputStream( - new java.io.FileInputStream( file ) ), Base64.DECODE ); - - // Read until done - while( ( numBytes = bis.read( buffer, length, 4096 ) ) >= 0 ) - length += numBytes; - - // Save in a variable to return - decodedData = new byte[ length ]; - System.arraycopy( buffer, 0, decodedData, 0, length ); - - } // end try - catch( java.io.IOException e ) - { - System.err.println( "Error decoding from file " + filename ); - } // end catch: IOException - finally - { - try{ bis.close(); } catch( Exception e) {} - } // end finally - - return decodedData; - } // end decodeFromFile - - - - /** - * Convenience method for reading a binary file - * and base64-encoding it. - * - * @param filename Filename for reading binary data - * @return base64-encoded string or null if unsuccessful - * - * @since 2.1 - */ - public static String encodeFromFile( String filename ) - { - String encodedData = null; - Base64.InputStream bis = null; - try - { - // Set up some useful variables - java.io.File file = new java.io.File( filename ); - byte[] buffer = new byte[ (int)(file.length() * 1.4) ]; - int length = 0; - int numBytes = 0; - - // Open a stream - bis = new Base64.InputStream( - new java.io.BufferedInputStream( - new java.io.FileInputStream( file ) ), Base64.ENCODE ); - - // Read until done - while( ( numBytes = bis.read( buffer, length, 4096 ) ) >= 0 ) - length += numBytes; - - // Save in a variable to return - encodedData = new String( buffer, 0, length, Base64.PREFERRED_ENCODING ); - - } // end try - catch( java.io.IOException e ) - { - System.err.println( "Error encoding from file " + filename ); - } // end catch: IOException - finally - { - try{ bis.close(); } catch( Exception e) {} - } // end finally - - return encodedData; - } // end encodeFromFile - - - - - /* ******** I N N E R C L A S S I N P U T S T R E A M ******** */ - - - - /** - * A {@link Base64.InputStream} will read data from another - * java.io.InputStream, given in the constructor, - * and encode/decode to/from Base64 notation on the fly. - * - * @see Base64 - * @since 1.3 - */ - public static class InputStream extends java.io.FilterInputStream - { - private boolean encode; // Encoding or decoding - private int position; // Current position in the buffer - private byte[] buffer; // Small buffer holding converted data - private int bufferLength; // Length of buffer (3 or 4) - private int numSigBytes; // Number of meaningful bytes in the buffer - private int lineLength; - private boolean breakLines; // Break lines at less than 80 characters - - - /** - * Constructs a {@link Base64.InputStream} in DECODE mode. - * - * @param in the java.io.InputStream from which to read data. - * @since 1.3 - */ - public InputStream( java.io.InputStream in ) - { - this( in, DECODE ); - } // end constructor - - - /** - * Constructs a {@link Base64.InputStream} in - * either ENCODE or DECODE mode. - *

- * Valid options:

-         *   ENCODE or DECODE: Encode or Decode as data is read.
-         *   DONT_BREAK_LINES: don't break lines at 76 characters
-         *     (only meaningful when encoding)
-         *     Note: Technically, this makes your encoding non-compliant.
-         *
- *

- * Example: new Base64.InputStream( in, Base64.DECODE ) - * - * - * @param in the java.io.InputStream from which to read data. - * @param options Specified options - * @see Base64#ENCODE - * @see Base64#DECODE - * @see Base64#DONT_BREAK_LINES - * @since 2.0 - */ - public InputStream( java.io.InputStream in, int options ) - { - super( in ); - this.breakLines = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES; - this.encode = (options & ENCODE) == ENCODE; - this.bufferLength = encode ? 4 : 3; - this.buffer = new byte[ bufferLength ]; - this.position = -1; - this.lineLength = 0; - } // end constructor - - /** - * Reads enough of the input stream to convert - * to/from Base64 and returns the next byte. - * - * @return next byte - * @since 1.3 - */ - public int read() throws java.io.IOException - { - // Do we need to get data? - if( position < 0 ) - { - if( encode ) - { - byte[] b3 = new byte[3]; - int numBinaryBytes = 0; - for( int i = 0; i < 3; i++ ) - { - try - { - int b = in.read(); - - // If end of stream, b is -1. - if( b >= 0 ) - { - b3[i] = (byte)b; - numBinaryBytes++; - } // end if: not end of stream - - } // end try: read - catch( java.io.IOException e ) - { - // Only a problem if we got no data at all. - if( i == 0 ) - throw e; - - } // end catch - } // end for: each needed input byte - - if( numBinaryBytes > 0 ) - { - encode3to4( b3, 0, numBinaryBytes, buffer, 0 ); - position = 0; - numSigBytes = 4; - } // end if: got data - else - { - return -1; - } // end else - } // end if: encoding - - // Else decoding - else - { - byte[] b4 = new byte[4]; - int i = 0; - for( i = 0; i < 4; i++ ) - { - // Read four "meaningful" bytes: - int b = 0; - do{ b = in.read(); } - while( b >= 0 && DECODABET[ b & 0x7f ] <= WHITE_SPACE_ENC ); - - if( b < 0 ) - break; // Reads a -1 if end of stream - - b4[i] = (byte)b; - } // end for: each needed input byte - - if( i == 4 ) - { - numSigBytes = decode4to3( b4, 0, buffer, 0 ); - position = 0; - } // end if: got four characters - else if( i == 0 ){ - return -1; - } // end else if: also padded correctly - else - { - // Must have broken out from above. - throw new java.io.IOException( "Improperly padded Base64 input." ); - } // end - - } // end else: decode - } // end else: get data - - // Got data? - if( position >= 0 ) - { - // End of relevant data? - if( /*!encode &&*/ position >= numSigBytes ) - return -1; - - if( encode && breakLines && lineLength >= MAX_LINE_LENGTH ) - { - lineLength = 0; - return '\n'; - } // end if - else - { - lineLength++; // This isn't important when decoding - // but throwing an extra "if" seems - // just as wasteful. - - int b = buffer[ position++ ]; - - if( position >= bufferLength ) - position = -1; - - return b & 0xFF; // This is how you "cast" a byte that's - // intended to be unsigned. - } // end else - } // end if: position >= 0 - - // Else error - else - { - // When JDK1.4 is more accepted, use an assertion here. - throw new java.io.IOException( "Error in Base64 code reading stream." ); - } // end else - } // end read - - - /** - * Calls {@link #read()} repeatedly until the end of stream - * is reached or len bytes are read. - * Returns number of bytes read into array or -1 if - * end of stream is encountered. - * - * @param dest array to hold values - * @param off offset for array - * @param len max number of bytes to read into array - * @return bytes read into array or -1 if end of stream is encountered. - * @since 1.3 - */ - public int read( byte[] dest, int off, int len ) throws java.io.IOException - { - int i; - int b; - for( i = 0; i < len; i++ ) - { - b = read(); - - //if( b < 0 && i == 0 ) - // return -1; - - if( b >= 0 ) - dest[off + i] = (byte)b; - else if( i == 0 ) - return -1; - else - break; // Out of 'for' loop - } // end for: each byte read - return i; - } // end read - - } // end inner class InputStream - - - - - - - /* ******** I N N E R C L A S S O U T P U T S T R E A M ******** */ - - - - /** - * A {@link Base64.OutputStream} will write data to another - * java.io.OutputStream, given in the constructor, - * and encode/decode to/from Base64 notation on the fly. - * - * @see Base64 - * @since 1.3 - */ - public static class OutputStream extends java.io.FilterOutputStream - { - private boolean encode; - private int position; - private byte[] buffer; - private int bufferLength; - private int lineLength; - private boolean breakLines; - private byte[] b4; // Scratch used in a few places - private boolean suspendEncoding; - - /** - * Constructs a {@link Base64.OutputStream} in ENCODE mode. - * - * @param out the java.io.OutputStream to which data will be written. - * @since 1.3 - */ - public OutputStream( java.io.OutputStream out ) - { - this( out, ENCODE ); - } // end constructor - - - /** - * Constructs a {@link Base64.OutputStream} in - * either ENCODE or DECODE mode. - *

- * Valid options:

-         *   ENCODE or DECODE: Encode or Decode as data is read.
-         *   DONT_BREAK_LINES: don't break lines at 76 characters
-         *     (only meaningful when encoding)
-         *     Note: Technically, this makes your encoding non-compliant.
-         *
- *

- * Example: new Base64.OutputStream( out, Base64.ENCODE ) - * - * @param out the java.io.OutputStream to which data will be written. - * @param options Specified options. - * @see Base64#ENCODE - * @see Base64#DECODE - * @see Base64#DONT_BREAK_LINES - * @since 1.3 - */ - public OutputStream( java.io.OutputStream out, int options ) - { - super( out ); - this.breakLines = (options & DONT_BREAK_LINES) != DONT_BREAK_LINES; - this.encode = (options & ENCODE) == ENCODE; - this.bufferLength = encode ? 3 : 4; - this.buffer = new byte[ bufferLength ]; - this.position = 0; - this.lineLength = 0; - this.suspendEncoding = false; - this.b4 = new byte[4]; - } // end constructor - - - /** - * Writes the byte to the output stream after - * converting to/from Base64 notation. - * When encoding, bytes are buffered three - * at a time before the output stream actually - * gets a write() call. - * When decoding, bytes are buffered four - * at a time. - * - * @param theByte the byte to write - * @since 1.3 - */ - public void write(int theByte) throws java.io.IOException - { - // Encoding suspended? - if( suspendEncoding ) - { - super.out.write( theByte ); - return; - } // end if: supsended - - // Encode? - if( encode ) - { - buffer[ position++ ] = (byte)theByte; - if( position >= bufferLength ) // Enough to encode. - { - out.write( encode3to4( b4, buffer, bufferLength ) ); - - lineLength += 4; - if( breakLines && lineLength >= MAX_LINE_LENGTH ) - { - out.write( NEW_LINE ); - lineLength = 0; - } // end if: end of line - - position = 0; - } // end if: enough to output - } // end if: encoding - - // Else, Decoding - else - { - // Meaningful Base64 character? - if( DECODABET[ theByte & 0x7f ] > WHITE_SPACE_ENC ) - { - buffer[ position++ ] = (byte)theByte; - if( position >= bufferLength ) // Enough to output. - { - int len = Base64.decode4to3( buffer, 0, b4, 0 ); - out.write( b4, 0, len ); - //out.write( Base64.decode4to3( buffer ) ); - position = 0; - } // end if: enough to output - } // end if: meaningful base64 character - else if( DECODABET[ theByte & 0x7f ] != WHITE_SPACE_ENC ) - { - throw new java.io.IOException( "Invalid character in Base64 data." ); - } // end else: not white space either - } // end else: decoding - } // end write - - - - /** - * Calls {@link #write(int)} repeatedly until len - * bytes are written. - * - * @param theBytes array from which to read bytes - * @param off offset for array - * @param len max number of bytes to read into array - * @since 1.3 - */ - public void write( byte[] theBytes, int off, int len ) throws java.io.IOException - { - // Encoding suspended? - if( suspendEncoding ) - { - super.out.write( theBytes, off, len ); - return; - } // end if: supsended - - for( int i = 0; i < len; i++ ) - { - write( theBytes[ off + i ] ); - } // end for: each byte written - - } // end write - - - - /** - * Method added by PHIL. [Thanks, PHIL. -Rob] - * This pads the buffer without closing the stream. - */ - public void flushBase64() throws java.io.IOException - { - if( position > 0 ) - { - if( encode ) - { - out.write( encode3to4( b4, buffer, position ) ); - position = 0; - } // end if: encoding - else - { - throw new java.io.IOException( "Base64 input not properly padded." ); - } // end else: decoding - } // end if: buffer partially full - - } // end flush - - - /** - * Flushes and closes (I think, in the superclass) the stream. - * - * @since 1.3 - */ - public void close() throws java.io.IOException - { - // 1. Ensure that pending characters are written - flushBase64(); - - // 2. Actually close the stream - // Base class both flushes and closes. - super.close(); - - buffer = null; - out = null; - } // end close - - - - /** - * Suspends encoding of the stream. - * May be helpful if you need to embed a piece of - * base640-encoded data in a stream. - * - * @since 1.5.1 - */ - public void suspendEncoding() throws java.io.IOException - { - flushBase64(); - this.suspendEncoding = true; - } // end suspendEncoding - - - /** - * Resumes encoding of the stream. - * May be helpful if you need to embed a piece of - * base640-encoded data in a stream. - * - * @since 1.5.1 - */ - public void resumeEncoding() - { - this.suspendEncoding = false; - } // end resumeEncoding - - - - } // end inner class OutputStream - - -} // end class Base64 diff --git a/aws/perftest/src/main/resources/README b/aws/perftest/src/main/resources/README deleted file mode 100755 index 3a8de1c414..0000000000 --- a/aws/perftest/src/main/resources/README +++ /dev/null @@ -1,84 +0,0 @@ -This is one of a collection of interface libraries that can be used to interact -with the Amazon S3 system in a number of different languages. They each expose -two main interface classes, AWSAuthConnection and QueryStringAuthGenerator. -The first actually performs all the operations using the appropriate libraries -for the language, including header signing. The second, -QueryStringAuthGenerator, has the same interface, but instead of performing -the operation, this class will return urls with the right query string -authentication parameters set. - - -Basic Operations: - -object requests: - -GetResponse get(bucketName, keyName) - retrieves an object -GetResponse getACL(bucketName, keyName) - returns the xml acl doc -Response put(bucketName, keyName, object) - writes an object -Response putACL(bucketName, keyName, aclXMLDoc) - sets the xml acl doc -Response delete(bucketName, keyName) - deletes an object - -bucket requests: - -Response createBucket(bucketName, location) - creates a bucket -ListResponse listBucket(bucketName) - lists a bucket's contents -LocationResponse getBucketLocation(bucketName) - return the location-constraint of this bucket -GetResponse getBucketACL(bucketName) - returns the xml representation of this bucket's access control list -Response putBucketAcl(bucketName, aclXMLDoc) - sets xml representation of the bucket acl -Response deleteBucket(bucketName) - delete an empty bucket -GetResponse getBucketLogging(bucketName) - returns the xml representation of this bucket's access logging configuration -Response putBucketLogging(bucketName, loggingXMLDoc) - sets the xml representation of the bucket logging configuration - -ListAllMyBucketsResponse listAllMyBuckets() - returns a list of all buckets owned by this AWS Access Key Id - - - -Dependencies: - -None, beyond the standard libraries. - - -Notes: - -Please note that this uses the public domain iHarder.net Base64 library. For updates to that library, -see http://iharder.sourceforge.net/current/java/base64/ . - -If your bucket name contains periods, you will need to use a non-HTTPS connection as the SSL certificate -presented by s3.amazonaws.com will not match if you do. - -Limitations: - -One of the main limitations of these sample AWSAuthConnection implementations -is that the interfaces are not streaming. This means that you have to pass the -data in as a string or as a byte array and the operation returns a string or a -byte array back. This is conceptually simpler, and fine for smaller objects, -but large objects, say a couple megabytes or greater, will show poor -performance, since everything is being passed around in memory. More -sophisticated libraries would pass streams in and out, and would only read the -data on-demand, rather than storing it all in memory (S3 itself would have no -problem with such streaming applications). Another upshot of this is that the -interfaces are all blocking---that is, you cannot look at the data until all of -it has downloaded. Again, this is fine for smaller objects, but unacceptable -for larger ones. - -These libraries have nearly non-existent error handling. All errors from lower -libraries are simply passed up. The response code in the connection object needs -to be checked after each request to verify whether the request succeeded. - -Only the java library has proper handling for repeated headers. The others -assume that each header will have only one value. - -It is our intention that these libraries act as a starting point for future -development. They are meant to show off the various operations and provide an -example of how to negotiate the authentication process. - - - -This software code is made available "AS IS" without warranties of any -kind. You may copy, display, modify and redistribute the software -code either by itself or as incorporated into your code; provided that -you do not remove any proprietary notices. Your use of this software -code is at your own risk and you waive any claim against Amazon -Digital Services, Inc. or its affiliates with respect to your use of -this software code. (c) 2006 Amazon Digital Services, Inc. or its -affiliates. diff --git a/aws/perftest/src/test/java/com/amazon/s3/DateServiceTest.java b/aws/perftest/src/test/java/com/amazon/s3/DateServiceTest.java deleted file mode 100755 index 37f065252b..0000000000 --- a/aws/perftest/src/test/java/com/amazon/s3/DateServiceTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * - * Copyright (C) 2009 Cloud Conscious, LLC. - * - * ==================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ==================================================================== - */ -package com.amazon.s3; - -import java.util.ArrayList; -import java.util.List; - -import org.testng.annotations.Test; - -/** - * Compares performance of date operations - * - * @author Adrian Cole - * @author James Murty - */ -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.DateTest") -public class DateServiceTest extends org.jclouds.date.DateServiceTest { - - @Test - void testAmazonParseDateSerialResponseTime() { - for (int i = 0; i < LOOP_COUNT; i++) - AWSAuthConnection.httpDate(); - } - - @Test - void testFormatAmazonDatePerformanceInParallel() throws Throwable { - List tasks = new ArrayList(testData.length); - tasks.add(new Runnable() { - public void run() { - AWSAuthConnection.httpDate(); - } - }); - executeMultiThreadedPerformanceTest("testFormatAmazonDatePerformanceInParallel", tasks); - } - -} \ No newline at end of file diff --git a/aws/perftest/src/test/java/com/amazon/s3/S3ParserTest.java b/aws/perftest/src/test/java/com/amazon/s3/S3ParserTest.java deleted file mode 100755 index 144eff8cbb..0000000000 --- a/aws/perftest/src/test/java/com/amazon/s3/S3ParserTest.java +++ /dev/null @@ -1,162 +0,0 @@ -/** - * - * Copyright (C) 2009 Cloud Conscious, LLC. - * - * ==================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ==================================================================== - */ -package com.amazon.s3; - -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.util.Date; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorCompletionService; - -import org.jclouds.date.internal.SimpleDateFormatDateService; -import org.testng.annotations.Test; - -/** - * Compares performance of xml parsing apis. - * - * @author Adrian Cole - */ -@Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "s3.S3ParserTest") -public class S3ParserTest extends org.jclouds.aws.s3.xml.S3ParserTest { - - class MockHttpURLClient extends HttpURLConnection { - private String content; - - @Override - public InputStream getInputStream() throws IOException { - return org.jclouds.util.Utils.toInputStream(content); - } - - protected MockHttpURLClient(String content) { - super(null); - this.content = content; - } - - public void disconnect() { - } - - public boolean usingProxy() { - return false; - } - - @Override - public int getResponseCode() throws IOException { - return 200; - } - - public void connect() throws IOException { - } - } - - @Test - void testAmazonParseListAllMyBucketsSerialResponseTime() throws IOException { - for (int i = 0; i < LOOP_COUNT; i++) - runAmazonParseListAllMyBuckets(); - } - - @Test - void testAmazonParseListAllMyBucketsParallelResponseTime() throws InterruptedException, - ExecutionException { - CompletionService completer = new ExecutorCompletionService(exec); - - for (int i = 0; i < LOOP_COUNT; i++) - completer.submit(new Callable() { - public Boolean call() throws IOException { - runAmazonParseListAllMyBuckets(); - return true; - } - }); - for (int i = 0; i < LOOP_COUNT; i++) - assert completer.take().get(); - } - - @SuppressWarnings("unchecked") - @Test(enabled = false) - public void testAmazonCanParseListAllMyBuckets() throws IOException { - ListAllMyBucketsResponse response = runAmazonParseListAllMyBuckets(); - List buckets = response.entries; - Bucket bucket1 = (Bucket) buckets.get(0); - assert bucket1.name.equals("adrianjbosstest"); - Date expectedDate1 = new SimpleDateFormatDateService() - .iso8601DateParse("2009-03-12T02:00:07.000Z"); - Date date1 = bucket1.creationDate; - assert date1.toString().equals(expectedDate1.toString()); - Bucket bucket2 = (Bucket) buckets.get(1); - assert bucket2.name.equals("adrianjbosstest2"); - Date expectedDate2 = new SimpleDateFormatDateService() - .iso8601DateParse("2009-03-12T02:00:09.000Z"); - Date date2 = bucket2.creationDate; - assert date2.toString().equals(expectedDate2.toString()); - assert buckets.size() == 2; - } - - private ListAllMyBucketsResponse runAmazonParseListAllMyBuckets() throws IOException { - ListAllMyBucketsResponse response = new ListAllMyBucketsResponse(new MockHttpURLClient( - listAllMyBucketsResultOn200)); - return response; - } - - @Test(enabled = false) - public void testAmazonCanParseListBucketResult() throws IOException { - ListBucketResponse response = runAmazonParseListBucketResult(); - ListEntry content = (ListEntry) response.entries.get(0); - assert content.key.equals("3366"); - assert content.lastModified.equals(new SimpleDateFormatDateService() - .iso8601DateParse("2009-03-12T02:00:13.000Z")); - assert content.eTag.equals("\"9d7bb64e8e18ee34eec06dd2cf37b766\""); - assert content.size == 136; - assert content.owner.id - .equals("e1a5f66a480ca99a4fdfe8e318c3020446c9989d7004e7778029fbcc5d990fa0"); - assert content.owner.displayName.equals("ferncam"); - assert content.storageClass.equals("STANDARD"); - } - - private ListBucketResponse runAmazonParseListBucketResult() throws IOException { - ListBucketResponse response = new ListBucketResponse(new MockHttpURLClient( - listAllMyBucketsResultOn200)); - return response; - } - - @Test(enabled = false) - void testAmazonParseListBucketResultSerialResponseTime() throws IOException { - for (int i = 0; i < LOOP_COUNT; i++) - runAmazonParseListBucketResult(); - } - - @Test(enabled = false) - void testAmazonParseListBucketResultParallelResponseTime() throws InterruptedException, - ExecutionException { - CompletionService completer = new ExecutorCompletionService(exec); - - for (int i = 0; i < LOOP_COUNT; i++) - completer.submit(new Callable() { - public Boolean call() throws IOException { - runAmazonParseListBucketResult(); - return true; - } - }); - for (int i = 0; i < LOOP_COUNT; i++) - assert completer.take().get(); - } - -} diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java index 69f500b418..8235540e3f 100755 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/AmazonPerformanceLiveTest.java @@ -18,14 +18,12 @@ */ package org.jclouds.aws.s3; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.jets3t.service.S3ServiceException; @@ -33,7 +31,11 @@ import org.testng.ITestContext; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -import com.amazon.s3.AWSAuthConnection; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; /** * Runs operations that amazon s3 sample code is capable of performing. @@ -42,42 +44,19 @@ import com.amazon.s3.AWSAuthConnection; */ @Test(sequential = true, timeOut = 2 * 60 * 1000, testName = "perftest.AmazonPerformanceLiveTest", groups = { "live" }) public class AmazonPerformanceLiveTest extends BasePerformanceLiveTest { - private AWSAuthConnection amzClient; + private AmazonS3 s3; @BeforeClass(groups = { "live" }, dependsOnMethods = "setUpResourcesOnThisThread") protected void createLiveS3Context(ITestContext testContext) throws S3ServiceException { - if (testContext.getAttribute("jclouds.test.user") != null) { - amzClient = new AWSAuthConnection((String) testContext.getAttribute("jclouds.test.user"), - (String) testContext.getAttribute("jclouds.test.key"), false); + exec = Executors.newCachedThreadPool(); + if (testContext.getAttribute("jclouds.test.identity") != null) { + s3 = new AmazonS3Client(new BasicAWSCredentials((String) testContext.getAttribute("jclouds.test.identity"), + (String) testContext.getAttribute("jclouds.test.credential"))); } else { throw new RuntimeException("not configured properly"); } } - @Override - @Test(enabled = false) - public void testPutFileSerial() throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - @Test(enabled = false) - public void testPutFileParallel() throws InterruptedException, ExecutionException { - throw new UnsupportedOperationException(); - } - - @Override - @Test(enabled = false) - public void testPutInputStreamSerial() throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - @Test(enabled = false) - public void testPutInputStreamParallel() throws InterruptedException, ExecutionException { - throw new UnsupportedOperationException(); - } - @Override @Test(enabled = false) public void testPutStringSerial() throws Exception { @@ -92,28 +71,45 @@ public class AmazonPerformanceLiveTest extends BasePerformanceLiveTest { @SuppressWarnings("unchecked") @Override - protected Future putByteArray(final String bucket, final String key, byte[] data, - String contentType) { - final com.amazon.s3.S3Object object = new com.amazon.s3.S3Object(data, null); - final Map> headers = new TreeMap>(); - headers.put("Content-Type", Arrays.asList(new String[] { contentType })); + protected Future putByteArray(final String bucket, final String key, final byte[] data, final String contentType) { + return exec.submit(new Callable() { @Override public Object call() throws Exception { - return amzClient.put(bucket, key, object, headers).connection.getResponseMessage(); + ObjectMetadata md = new ObjectMetadata(); + md.setContentType(contentType); + md.setContentLength(data.length); + return s3.putObject(new PutObjectRequest(bucket, key, new ByteArrayInputStream(data), md)).getETag(); } }); } + @SuppressWarnings("unchecked") @Override - protected Future putFile(String bucket, String key, File data, String contentType) { - throw new UnsupportedOperationException(); + protected Future putFile(final String bucket, final String key, final File data, String contentType) { + + return exec.submit(new Callable() { + @Override + public Object call() throws Exception { + return s3.putObject(new PutObjectRequest(bucket, key, data)).getETag(); + } + }); } + @SuppressWarnings("unchecked") @Override - protected Future putInputStream(String bucket, String key, InputStream data, - String contentType) { - throw new UnsupportedOperationException(); + protected Future putInputStream(final String bucket, final String key, final InputStream data, + final String contentType) { + + return exec.submit(new Callable() { + @Override + public Object call() throws Exception { + ObjectMetadata md = new ObjectMetadata(); + md.setContentType(contentType); + md.setContentLength(data.available()); + return s3.putObject(new PutObjectRequest(bucket, key, data, md)).getETag(); + } + }); } @Override diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformanceLiveTest.java index c4b121d613..4d2cdc2a80 100644 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/BaseJCloudsPerformanceLiveTest.java @@ -18,7 +18,8 @@ */ package org.jclouds.aws.s3; -import static org.jclouds.Constants.*; +import static org.jclouds.Constants.PROPERTY_IO_WORKER_THREADS; +import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT; import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST; import static org.jclouds.Constants.PROPERTY_USER_THREADS; @@ -30,7 +31,7 @@ import java.util.concurrent.Future; import org.jclouds.aws.s3.domain.S3Object; -import com.google.appengine.repackaged.com.google.common.base.Throwables; +import com.google.common.base.Throwables; /** * // TODO: Adrian: Document this! @@ -57,12 +58,10 @@ public abstract class BaseJCloudsPerformanceLiveTest extends BasePerformanceLive // } protected void overrideWithSysPropertiesAndPrint(Properties overrides, String contextName) { overrides.putAll(System.getProperties()); - System.out.printf( - "%s: loopCount(%s), perContext(%s), perHost(%s),ioWorkers(%s), userThreads(%s)%n", - contextName, loopCount, overrides.getProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT), - overrides.getProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST), overrides - .getProperty(PROPERTY_IO_WORKER_THREADS), overrides - .getProperty(PROPERTY_USER_THREADS)); + System.out.printf("%s: loopCount(%s), perContext(%s), perHost(%s),ioWorkers(%s), userThreads(%s)%n", contextName, + loopCount, overrides.getProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT), overrides + .getProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST), overrides.getProperty(PROPERTY_IO_WORKER_THREADS), + overrides.getProperty(PROPERTY_USER_THREADS)); } @Override @@ -90,16 +89,15 @@ public abstract class BaseJCloudsPerformanceLiveTest extends BasePerformanceLive } @Override - protected Future putInputStream(String bucket, String key, InputStream data, - String contentType) { + protected Future putInputStream(String bucket, String key, InputStream data, String contentType) { S3Object object = newObject(key); - object.getMetadata().setContentType(contentType); object.setPayload(data); try { - object.setContentLength(new Long(data.available())); + object.getPayload().setContentLength(new Long(data.available())); } catch (IOException e) { Throwables.propagate(e); } + object.getPayload().setContentType(contentType); return getApi().putObject(bucket, object); } diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/BasePerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/BasePerformanceLiveTest.java index e7c3076c5a..6ef7fe878e 100755 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/BasePerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/BasePerformanceLiveTest.java @@ -50,7 +50,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe containerCount = 1; } protected int timeoutSeconds = 15; - protected int loopCount = 1000; + protected int loopCount = Integer.parseInt(System.getProperty("jclouds.test.loopcount", "1000")); protected ExecutorService exec; protected Logger logger = Logger.NULL;; @@ -65,8 +65,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } @Test - public void testPutBytesParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutBytesParallel() throws InterruptedException, ExecutionException, TimeoutException { String bucketName = getContainerName(); try { doParallel(new PutBytesFuture(bucketName), loopCount, bucketName); @@ -86,8 +85,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } @Test - public void testPutFileParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutFileParallel() throws InterruptedException, ExecutionException, TimeoutException { String bucketName = getContainerName(); try { doParallel(new PutFileFuture(bucketName), loopCount, bucketName); @@ -107,8 +105,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } @Test - public void testPutInputStreamParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutInputStreamParallel() throws InterruptedException, ExecutionException, TimeoutException { String bucketName = getContainerName(); try { doParallel(new PutInputStreamFuture(bucketName), loopCount, bucketName); @@ -128,8 +125,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } @Test - public void testPutStringParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutStringParallel() throws InterruptedException, ExecutionException, TimeoutException { String bucketName = getContainerName(); try { doParallel(new PutStringFuture(bucketName), loopCount, bucketName); @@ -138,19 +134,18 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } } - private void doSerial(Provider> provider, int loopCount) throws Exception, - ExecutionException { + private void doSerial(Provider> provider, int loopCount) throws Exception, ExecutionException { for (int i = 0; i < loopCount; i++) assert provider.get().get() != null; } - private void doParallel(Provider> provider, int loopCount, - String containerName) throws InterruptedException, ExecutionException, TimeoutException { + private void doParallel(Provider> provider, int loopCount, String containerName) + throws InterruptedException, ExecutionException, TimeoutException { Map> responses = Maps.newHashMap(); for (int i = 0; i < loopCount; i++) responses.put(i, provider.get()); assert awaitCompletion(responses, exec, null, logger, - String.format("putting into containerName: %s", containerName)).size() == 0; + String.format("putting into containerName: %s", containerName)).size() == 0; } class PutBytesFuture implements Provider> { @@ -164,7 +159,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe public ListenableFuture get() { return Futures.makeListenable(putByteArray(bucketName, key.getAndIncrement() + "", test, - "application/octetstring")); + "application/octetstring")); } } @@ -178,8 +173,7 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } public ListenableFuture get() { - return Futures.makeListenable(putFile(bucketName, key.getAndIncrement() + "", file, - "text/xml")); + return Futures.makeListenable(putFile(bucketName, key.getAndIncrement() + "", file, "text/xml")); } } @@ -195,8 +189,8 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe @Override public ListenableFuture get() { - return Futures.makeListenable(putInputStream(bucketName, key.getAndIncrement() + "", - new ByteArrayInputStream(test), "application/octetstring")); + return Futures.makeListenable(putInputStream(bucketName, key.getAndIncrement() + "", new ByteArrayInputStream( + test), "application/octetstring")); } } @@ -211,18 +205,15 @@ public abstract class BasePerformanceLiveTest extends BaseBlobStoreIntegrationTe } public ListenableFuture get() { - return Futures.makeListenable(putString(bucketName, key.getAndIncrement() + "", - testString, "text/plain")); + return Futures.makeListenable(putString(bucketName, key.getAndIncrement() + "", testString, "text/plain")); } } - protected abstract Future putByteArray(String bucket, String key, byte[] data, - String contentType); + protected abstract Future putByteArray(String bucket, String key, byte[] data, String contentType); protected abstract Future putFile(String bucket, String key, File data, String contentType); - protected abstract Future putInputStream(String bucket, String key, InputStream data, - String contentType); + protected abstract Future putInputStream(String bucket, String key, InputStream data, String contentType); protected abstract Future putString(String bucket, String key, String data, String contentType); diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsApacheHCPerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsApacheHCPerformanceLiveTest.java index e249fae1f6..0b06f9e18c 100644 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsApacheHCPerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsApacheHCPerformanceLiveTest.java @@ -28,6 +28,7 @@ import static org.jclouds.Constants.PROPERTY_USER_THREADS; import java.util.Properties; import java.util.concurrent.Executors; +import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.enterprise.config.EnterpriseConfigurationModule; import org.jclouds.http.apachehc.config.ApacheHCHttpCommandExecutorServiceModule; import org.jclouds.logging.config.NullLoggingModule; @@ -35,15 +36,17 @@ import org.testng.ITestContext; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -@Test(sequential = true, testName = "perftest.JCloudsNioPerformanceLiveTest", groups = { "live" }) +import com.google.common.collect.ImmutableSet; + +@Test(sequential = true, testName = "perftest.JCloudsApacheHCPerformanceLiveTest", groups = { "live" }) public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest { @Override @BeforeClass(groups = { "integration", "live" }) public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { exec = Executors.newCachedThreadPool(); - String accesskeyid = System.getProperty("jclouds.test.user"); - String secretkey = System.getProperty("jclouds.test.key"); + String accesskeyid = System.getProperty("jclouds.test.identity"); + String secretkey = System.getProperty("jclouds.test.credential"); Properties overrides = new Properties(); overrides.setProperty(PROPERTY_SO_TIMEOUT, 5000 + ""); overrides.setProperty(PROPERTY_CONNECTION_TIMEOUT, 5000 + ""); @@ -53,9 +56,9 @@ public class JCloudsApacheHCPerformanceLiveTest extends BaseJCloudsPerformanceLi overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); String contextName = "enterprise"; overrideWithSysPropertiesAndPrint(overrides, contextName); - context = S3ContextFactory.createContext(overrides, accesskeyid, secretkey, - new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(), - new EnterpriseConfigurationModule()); + context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( + new NullLoggingModule(), new ApacheHCHttpCommandExecutorServiceModule(), + new EnterpriseConfigurationModule()), overrides); } @Override diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsGaePerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsGaePerformanceLiveTest.java index 97afa433c3..909863bd29 100755 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsGaePerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsGaePerformanceLiveTest.java @@ -20,24 +20,23 @@ package org.jclouds.aws.s3; import java.io.File; import java.io.InputStream; -import java.util.Map; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.enterprise.config.EnterpriseConfigurationModule; +import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.gae.config.GoogleAppEngineConfigurationModule; import org.jclouds.logging.config.NullLoggingModule; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import org.testng.v6.Maps; -import com.google.appengine.tools.development.ApiProxyLocalImpl; -import com.google.apphosting.api.ApiProxy; +import com.google.appengine.tools.development.testing.LocalServiceTestHelper; +import com.google.appengine.tools.development.testing.LocalURLFetchServiceTestConfig; +import com.google.common.collect.ImmutableSet; /** * @@ -50,8 +49,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes @Override @Test(enabled = false) - public void testPutBytesParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutBytesParallel() throws InterruptedException, ExecutionException, TimeoutException { throw new UnsupportedOperationException(); } @@ -63,8 +61,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes @Override @Test(enabled = false) - public void testPutFileParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutFileParallel() throws InterruptedException, ExecutionException, TimeoutException { throw new UnsupportedOperationException(); } @@ -76,8 +73,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes @Override @Test(enabled = false) - public void testPutInputStreamParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutInputStreamParallel() throws InterruptedException, ExecutionException, TimeoutException { throw new UnsupportedOperationException(); } @@ -89,8 +85,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes @Override @Test(enabled = false) - public void testPutStringParallel() throws InterruptedException, ExecutionException, - TimeoutException { + public void testPutStringParallel() throws InterruptedException, ExecutionException, TimeoutException { throw new UnsupportedOperationException(); } @@ -103,7 +98,8 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes public JCloudsGaePerformanceLiveTest() { super(); // otherwise, we'll get timeout errors - // TODO sdk 1.2.3 should give the ability to set a higher timeout then 5 seconds allowing this + // TODO sdk 1.2.3 should give the ability to set a higher timeout then 5 + // seconds allowing this // to be removed loopCount = 5; } @@ -121,8 +117,7 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes } @Override - protected Future putInputStream(String bucket, String key, InputStream data, - String contentType) { + protected Future putInputStream(String bucket, String key, InputStream data, String contentType) { setupApiProxy(); return super.putInputStream(bucket, key, data, contentType); } @@ -135,64 +130,20 @@ public class JCloudsGaePerformanceLiveTest extends BaseJCloudsPerformanceLiveTes @BeforeMethod void setupApiProxy() { - ApiProxy.setEnvironmentForCurrentThread(new TestEnvironment()); - ApiProxy.setDelegate(new ApiProxyLocalImpl(new File(".")) { - }); - } - - class TestEnvironment implements ApiProxy.Environment { - public String getAppId() { - return "Unit Tests"; - } - - public String getVersionId() { - return "1.0"; - } - - public void setDefaultNamespace(String s) { - } - - public String getRequestNamespace() { - return null; - } - - public String getDefaultNamespace() { - return null; - } - - public String getAuthDomain() { - return null; - } - - public boolean isLoggedIn() { - return false; - } - - public String getEmail() { - return null; - } - - public boolean isAdmin() { - return false; - } - - public Map getAttributes() { - return Maps.newHashMap(); - } + new LocalServiceTestHelper(new LocalURLFetchServiceTestConfig()).setUp(); } private BlobStoreContext perfContext; @BeforeClass(groups = { "live" }) void setup() { - String accesskeyid = System.getProperty("jclouds.test.user"); - String secretkey = System.getProperty("jclouds.test.key"); + String accesskeyid = System.getProperty("jclouds.test.identity"); + String secretkey = System.getProperty("jclouds.test.credential"); Properties overrides = new Properties(); String contextName = "gae"; overrideWithSysPropertiesAndPrint(overrides, contextName); - this.perfContext = S3ContextFactory.createContext(overrides, accesskeyid, secretkey, - new NullLoggingModule(), new EnterpriseConfigurationModule(), - new GoogleAppEngineConfigurationModule()); + context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( + new NullLoggingModule(), new GoogleAppEngineConfigurationModule()), overrides); } @AfterClass(groups = { "live" }) diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java deleted file mode 100755 index da20c3b03e..0000000000 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsNioPerformanceLiveTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * - * Copyright (C) 2009 Cloud Conscious, LLC. - * - * ==================================================================== - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ==================================================================== - */ -package org.jclouds.aws.s3; - -import static org.jclouds.Constants.PROPERTY_IO_WORKER_THREADS; -import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_CONTEXT; -import static org.jclouds.Constants.PROPERTY_MAX_CONNECTIONS_PER_HOST; -import static org.jclouds.Constants.PROPERTY_USER_THREADS; - -import java.util.Properties; -import java.util.concurrent.Executors; - -import org.jclouds.enterprise.config.EnterpriseConfigurationModule; -import org.jclouds.http.httpnio.config.NioTransformingHttpCommandExecutorServiceModule; -import org.jclouds.logging.config.NullLoggingModule; -import org.testng.ITestContext; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -@Test(sequential = true, testName = "perftest.JCloudsNioPerformanceLiveTest", groups = { "live" }) -public class JCloudsNioPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest { - - @Override - @BeforeClass(groups = { "integration", "live" }) - public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { - exec = Executors.newCachedThreadPool(); - String accesskeyid = System.getProperty("jclouds.test.user"); - String secretkey = System.getProperty("jclouds.test.key"); - Properties overrides = new Properties(); - overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 20 + ""); - overrides.setProperty(PROPERTY_IO_WORKER_THREADS, 3 + ""); - overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 12 + ""); - overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); - String contextName = "nio"; - overrideWithSysPropertiesAndPrint(overrides, contextName); - context = S3ContextFactory.createContext(overrides, accesskeyid, secretkey, - new NullLoggingModule(), new EnterpriseConfigurationModule(), - new NioTransformingHttpCommandExecutorServiceModule()); - } - - @Override - public S3AsyncClient getApi() { - return (S3AsyncClient) context.getProviderSpecificContext().getAsyncApi(); - } -} diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java index 7ac10bb545..5e26a66a4e 100755 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/JCloudsPerformanceLiveTest.java @@ -26,12 +26,15 @@ import static org.jclouds.Constants.PROPERTY_USER_THREADS; import java.util.Properties; import java.util.concurrent.Executors; +import org.jclouds.blobstore.BlobStoreContextFactory; import org.jclouds.enterprise.config.EnterpriseConfigurationModule; import org.jclouds.logging.config.NullLoggingModule; import org.testng.ITestContext; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import com.google.common.collect.ImmutableSet; + /** * Tests the default JClouds client. * @@ -45,8 +48,8 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest { @BeforeClass(groups = { "integration", "live" }) public void setUpResourcesOnThisThread(ITestContext testContext) throws Exception { exec = Executors.newCachedThreadPool(); - String accesskeyid = System.getProperty("jclouds.test.user"); - String secretkey = System.getProperty("jclouds.test.key"); + String accesskeyid = System.getProperty("jclouds.test.identity"); + String secretkey = System.getProperty("jclouds.test.credential"); Properties overrides = new Properties(); overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_CONTEXT, 50 + ""); overrides.setProperty(PROPERTY_MAX_CONNECTIONS_PER_HOST, 30 + ""); @@ -54,8 +57,8 @@ public class JCloudsPerformanceLiveTest extends BaseJCloudsPerformanceLiveTest { overrides.setProperty(PROPERTY_USER_THREADS, 0 + ""); String contextName = "standard"; overrideWithSysPropertiesAndPrint(overrides, contextName); - context = S3ContextFactory.createContext(overrides, accesskeyid, secretkey, - new NullLoggingModule(), new EnterpriseConfigurationModule()); + context = new BlobStoreContextFactory().createContext("s3", accesskeyid, secretkey, ImmutableSet.of( + new NullLoggingModule(), new EnterpriseConfigurationModule()), overrides); } @Override diff --git a/aws/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java b/aws/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java index 78a510f749..42f0b51daa 100755 --- a/aws/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java +++ b/aws/perftest/src/test/java/org/jclouds/aws/s3/Jets3tPerformanceLiveTest.java @@ -69,10 +69,10 @@ public class Jets3tPerformanceLiveTest extends BasePerformanceLiveTest { @BeforeClass(groups = { "live" }, dependsOnMethods = "setUpResourcesOnThisThread") protected void createLiveS3Context(ITestContext testContext) throws S3ServiceException { - if (testContext.getAttribute("jclouds.test.user") != null) { + if (testContext.getAttribute("jclouds.test.identity") != null) { AWSCredentials credentials = new AWSCredentials((String) testContext - .getAttribute("jclouds.test.user"), (String) testContext - .getAttribute("jclouds.test.key")); + .getAttribute("jclouds.test.identity"), (String) testContext + .getAttribute("jclouds.test.credential")); jetClient = new RestS3Service(credentials); } else { throw new RuntimeException("not configured properly");