Improved support for concurrent testing in PerformanceTest. Also more fiddling measuring performance metrics for DateService, without getting any reliable results

git-svn-id: http://jclouds.googlecode.com/svn/trunk@841 3d8758e0-26b5-11de-8745-db77d3ebf521

aws/s3/core/src/main/java/org/jclouds/aws/s3/util/DateService.java

@@ -37,6 +37,8 @@ import org.joda.time.DateTimeZone;
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * Parses and formats the ISO8601 and RFC822 date formats found in 
  * XML responses and HTTP response headers.
@@ -122,4 +124,22 @@ public class DateService {
 		}
 	}
 
+    /*
+     * Alternative implementations of Format and Parse -- used to
+     * test relative speeds.
+     * TODO: Remove methods below once sufficient performance testing is complete. 
+     */
+    
+    @VisibleForTesting
+    public final DateTime jodaIso8601DateParse(String toParse) {
+    	return new DateTime(toParse);
+    }
+
+    @VisibleForTesting
+	public final String sdfIso8601DateFormat(DateTime dateTime) {
+		synchronized (iso8601SimpleDateFormat) {
+			return iso8601SimpleDateFormat.format(dateTime.toDate());
+		}
+    }
+
 }

aws/s3/core/src/test/java/org/jclouds/aws/PerformanceTest.java

@@ -23,6 +23,12 @@
  */
 package org.jclouds.aws;
 
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -38,6 +44,7 @@ import org.testng.annotations.Test;
 @Test(groups = "performance")
 public class PerformanceTest {
    protected static int LOOP_COUNT = 1000;
+   protected static int THREAD_COUNT = 1000;
    protected ExecutorService exec;
 
    @BeforeTest
@@ -50,4 +57,78 @@ public class PerformanceTest {
       exec.shutdownNow();
       exec = null;
    }
+   
+   /**
+    * Executes a list of Runnable tasks in {@link #THREAD_COUNT}
+    * simultaneous threads, and outputs the timing results. 
+    * <p>
+    * This method is careful to time only the actual task execution 
+    * time, not the overhead of creating and queuing the tasks.
+    * We also use CountDownLatches to ensure that all tasks start
+    * at the same time, so concurrency is fully tested without
+    * ramp-up or ramp-down times.
+    * <p>
+    * This code is heavily based on Listing 5.11 in 
+    * "Java Concurrency in Practice" by Brian Goetz et al,
+    * Addison-Wesley Professional.
+    * 
+    * @see {@link DateServiceTest} for example usage.
+    * 
+    * @param performanceTestName
+    * @param tasks
+    * @throws InterruptedException
+    * @throws ExecutionException
+    * @throws Throwable
+    */
+   protected void executeMultiThreadedPerformanceTest(String performanceTestName, List<Runnable> tasks) 
+   	  throws InterruptedException, ExecutionException, Throwable
+   {
+      CompletionService<Throwable> completer = new ExecutorCompletionService<Throwable>(exec);
+      final CountDownLatch startGate = new CountDownLatch(1);
+      final CountDownLatch endGate = new CountDownLatch(THREAD_COUNT);
+            
+      for (int i = 0; i < THREAD_COUNT; i++) {
+    	 final Runnable task = tasks.get(i % tasks.size());
+    	 // Wrap task so we can count down endGate.
+         completer.submit(new Callable<Throwable>() {
+             public Throwable call() {
+            	try { 
+                   startGate.await();  // Wait to start simultaneously
+            	   task.run();
+                   return null;
+            	} catch (Throwable t) {
+            	   return t;
+            	} finally {
+            	   endGate.countDown();  // Notify that I've finished
+            	}
+             }
+          });
+      }      
+      
+      // Only time the execution time for all tasks, not start/stop times.
+      long startTime = System.nanoTime();
+      startGate.countDown();  // Trigger start of all tasks
+      endGate.await();
+      long endTime = System.nanoTime() - startTime;
+
+      // Check for assertion failures
+	  Throwable t;
+      for (int i = 0; i < THREAD_COUNT; i++) {
+    	  t = completer.take().get();
+    	  if (t != null) {
+    		  throw t;
+    	  }
+      }      
+      if (performanceTestName != null) {
+	      System.out.printf("TIMING: Multi-threaded %s took %.3fms for %d threads\n", 
+		     performanceTestName, ((double)endTime / 1000000), THREAD_COUNT);
+      }
+   }
+
+   protected void executeMultiThreadedCorrectnessTest(List<Runnable> tasks) 
+	  throws InterruptedException, ExecutionException, Throwable
+   {
+	   executeMultiThreadedPerformanceTest(null, tasks);
+   }
+
 }

aws/s3/perftest/src/test/java/com/amazon/s3/DateServiceTest.java

@@ -24,21 +24,14 @@
 package com.amazon.s3;
 
 import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
 
-import java.util.Date;
+import java.util.ArrayList;
-import java.util.Locale;
+import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
 
 import org.jclouds.aws.PerformanceTest;
 import org.jclouds.aws.s3.util.DateService;
 import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
 import org.testng.annotations.Test;
 
 import com.google.inject.Guice;
@@ -66,71 +59,40 @@ public class DateServiceTest extends PerformanceTest {
    class TestData {
       public final String iso8601DateString;
       public final String rfc822DateString;
-      public final Date date;
+      public final DateTime date;
 
-      TestData(String iso8601, String rfc822, Date date) {
+      TestData(String iso8601, String rfc822, DateTime dateTime) {
          this.iso8601DateString = iso8601;
          this.rfc822DateString = rfc822;
-         this.date = date;
+         this.date = dateTime;
       }
    }
 
-   private long startTime;
-
    public DateServiceTest() {
       // Constant time test values, each TestData item must contain matching times!
       testData = new TestData[] {
-               new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", new Date(
+               new TestData("2009-03-12T02:00:07.000Z", "Thu, 12 Mar 2009 02:00:07 GMT", 
-                        1236823207000l)),
+        		   new DateTime(1236823207000l)),
-               new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", new Date(
+               new TestData("2009-03-14T04:00:07.000Z", "Sat, 14 Mar 2009 04:00:07 GMT", 
-                        1237003207000l)),
+        		   new DateTime(1237003207000l)),
-               new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", new Date(
+               new TestData("2009-03-16T06:00:07.000Z", "Mon, 16 Mar 2009 06:00:07 GMT", 
-                        1237183207000l)),
+        		   new DateTime(1237183207000l)),
-               new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", new Date(
+               new TestData("2009-03-18T08:00:07.000Z", "Wed, 18 Mar 2009 08:00:07 GMT", 
-                        1237363207000l)),
+        		   new DateTime(1237363207000l)),
-               new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", new Date(
+               new TestData("2009-03-20T10:00:07.000Z", "Fri, 20 Mar 2009 10:00:07 GMT", 
-                        1237543207000l)) };
+        		   new DateTime(1237543207000l)) };
-   }
-
-   // Joda items for performance comparisons
-   private DateTimeFormatter headerDateFormat = DateTimeFormat.forPattern(
-            "EEE, dd MMM yyyy HH:mm:ss 'GMT'").withLocale(Locale.US);
-
-   private DateTime jodaParseIso8601(String toParse) {
-      return new DateTime(toParse);
-   }
-
-   private DateTime jodaParseRfc822(String toParse) {
-      return headerDateFormat.parseDateTime(toParse);
-   }
-
-   private String timestampAsHeaderString() {
-      return toHeaderString(new DateTime());
-   }
-
-   private String toHeaderString(DateTime date) {
-      return headerDateFormat.print(date.withZone(DateTimeZone.UTC));
-   }
-
-   private void startClock() {
-      startTime = System.currentTimeMillis();
-   }
-
-   private void printElapsedClockTime(String testName) {
-      System.out.println(testName + " took " + (System.currentTimeMillis() - startTime) + "ms for "
-               + LOOP_COUNT + " loops");
    }
 
    @Test
    public void testIso8601DateParse() throws ExecutionException, InterruptedException {
       DateTime dsDate = dateService.iso8601DateParse(testData[0].iso8601DateString);
-      assertEquals(dsDate.toDate(), testData[0].date);
+      assertEquals(dsDate, testData[0].date);
    }
 
    @Test
    public void testRfc822DateParse() throws ExecutionException, InterruptedException {
       DateTime dsDate = dateService.rfc822DateParse(testData[0].rfc822DateString);
-      assertEquals(dsDate.toDate(), testData[0].date);
+      assertEquals(dsDate, testData[0].date);
    }
 
    @Test
@@ -158,69 +120,72 @@ public class DateServiceTest extends PerformanceTest {
    }
 
    @Test
-   void testFormatIso8601DateInParallel() throws InterruptedException, ExecutionException {
+   void testFormatIso8601DateCorrectnessInParallel() throws Throwable {
-      CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
-      startClock();
+      for (final TestData myData: testData) {
-      for (int i = 0; i < LOOP_COUNT; i++) {
+    	  tasks.add(new Runnable() {
-         final TestData myData = testData[i % testData.length];
+    		 public void run() {
-         completer.submit(new Callable<Boolean>() {
+                 String dsString = dateService.iso8601DateFormat(myData.date);
-            public Boolean call() throws ExecutionException, InterruptedException {
+                 assertEquals(dsString, myData.iso8601DateString);    			 
-               String dsString = dateService.iso8601DateFormat(myData.date);
+    		 }
-               /*
+    	  });
-                * Comment-in the assert below to test thread safety. Comment it out to test
-                * performance
-                */
-               assertEquals(dsString, myData.iso8601DateString);
-               return true;
-            }
-         });
       }
-      for (int i = 0; i < LOOP_COUNT; i++)
+      executeMultiThreadedCorrectnessTest(tasks);
-         assertTrue(completer.take().get());
-      printElapsedClockTime("testFormatIso8601DateInParallel");
    }
 
    @Test
-   void testFormatAmazonDateInParallel() throws InterruptedException, ExecutionException {
+   void testFormatIso8601DatePerformanceInParallel() throws Throwable {
-      CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
-      startClock();
+      for (final TestData myData: testData) {
-      for (int i = 0; i < LOOP_COUNT; i++)
+    	  tasks.add(new Runnable() {
-         completer.submit(new Callable<Boolean>() {
+    		 public void run() {
-            public Boolean call() {
+                dateService.iso8601DateFormat(myData.date);
-               AWSAuthConnection.httpDate();
+    		 }
-               return true;
+    	  });
-            }
-         });
-      for (int i = 0; i < LOOP_COUNT; i++)
-         assertTrue(completer.take().get());
-      printElapsedClockTime("testFormatAmazonDateInParallel");
-   }
-
-   @Test
-   void testFormatJodaDateInParallel() throws InterruptedException, ExecutionException {
-      CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
-      startClock();
-      for (int i = 0; i < LOOP_COUNT; i++) {
-         final TestData myData = testData[i % testData.length];
-         completer.submit(new Callable<Boolean>() {
-            public Boolean call() {
-               String jodaString = toHeaderString(new DateTime(myData.date));
-               assertEquals(jodaString, myData.rfc822DateString);
-               return true;
-            }
-         });
       }
-      for (int i = 0; i < LOOP_COUNT; i++)
+      executeMultiThreadedPerformanceTest("testFormatIso8601DatePerformanceInParallel", tasks);
-         assertTrue(completer.take().get());
-      printElapsedClockTime("testFormatJodaDateInParallel");
    }
    
    @Test
-   void testIso8601ParseDateSerialResponseTime() throws ExecutionException, InterruptedException {
+   void testFormatIso8601DatePerformanceInParallel_SdfAlternative() throws Throwable {
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
+      for (final TestData myData: testData) {
+    	  tasks.add(new Runnable() {
+    		 public void run() {
+    			dateService.sdfIso8601DateFormat(myData.date);
+    		 }
+    	  });
+      }
+      executeMultiThreadedPerformanceTest(
+         "testFormatIso8601DatePerformanceInParallel_SdfAlternative", tasks);
+   }
+   
+   @Test
+   void testFormatAmazonDatePerformanceInParallel() throws Throwable {
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
+      tasks.add(
+	     new Runnable() {
+			public void run() {
+		       AWSAuthConnection.httpDate();
+			}}
+	     );
+      executeMultiThreadedPerformanceTest("testFormatAmazonDatePerformanceInParallel", tasks);
+   }
+   
+   @Test
+   void testParseIso8601DateSerialResponseTime() throws ExecutionException, InterruptedException {
       for (int i = 0; i < LOOP_COUNT; i++)
          dateService.iso8601DateParse(testData[0].iso8601DateString);
    }
 
+   @Test
+   void testParseIso8601DateSerialResponseTime_JodaAlternative() 
+      throws ExecutionException, InterruptedException
+   {
+      for (int i = 0; i < LOOP_COUNT; i++)
+    	 dateService.jodaIso8601DateParse(testData[0].iso8601DateString);
+   }
+
    @Test
    void testAmazonParseDateSerialResponseTime() {
       for (int i = 0; i < LOOP_COUNT; i++)
@@ -228,57 +193,44 @@ public class DateServiceTest extends PerformanceTest {
    }
 
    @Test
-   void testParseIso8601DateInParallel() throws InterruptedException, ExecutionException {
+   void testParseIso8601DateCorrectnessInParallel() throws Throwable {
-      CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
-      startClock();
+      for (final TestData myData: testData) {
-      for (int i = 0; i < LOOP_COUNT; i++) {
+    	  tasks.add(new Runnable() {
-         final TestData myData = testData[i % testData.length];
+    		 public void run() {
-         completer.submit(new Callable<Boolean>() {
+                 DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString);
-            public Boolean call() throws ExecutionException, InterruptedException {
+                 assertEquals(dsDate, myData.date);
-               DateTime dsDate = dateService.iso8601DateParse(myData.iso8601DateString);
+    		 }
-               assertEquals(dsDate.toDate(), myData.date);
+    	  });
-               return true;
-            }
-         });
       }
-      for (int i = 0; i < LOOP_COUNT; i++)
+      executeMultiThreadedCorrectnessTest(tasks);
-         assertTrue(completer.take().get());
-      printElapsedClockTime("testParseIso8601DateInParallel");
    }
 
    @Test
-   void testParseAmazonDateInParallel() throws InterruptedException, ExecutionException {
+   void testParseIso8601DatePerformanceInParallel() throws Throwable {
-      CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
-      startClock();
+      for (final TestData myData: testData) {
-      for (int i = 0; i < LOOP_COUNT; i++)
+    	  tasks.add(new Runnable() {
-         completer.submit(new Callable<Boolean>() {
+    		 public void run() {
-            public Boolean call() {
+                 dateService.iso8601DateParse(myData.iso8601DateString);
-               AWSAuthConnection.httpDate();
+    		 }
-               return true;
+    	  });
-            }
+      }
-         });
+      executeMultiThreadedPerformanceTest("testParseIso8601DatePerformanceInParallel", tasks);
-      for (int i = 0; i < LOOP_COUNT; i++)
-         assertTrue(completer.take().get());
-      printElapsedClockTime("testParseAmazonDateInParallel");
    }
 
    @Test
-   void testParseJodaDateInParallel() throws InterruptedException, ExecutionException {
+   void testParseIso8601DatePerformanceInParallel_JodaAlternative() throws Throwable {
-      CompletionService<Boolean> completer = new ExecutorCompletionService<Boolean>(exec);
+	  List<Runnable> tasks = new ArrayList<Runnable>(testData.length);
-      startClock();
+      for (final TestData myData: testData) {
-      for (int i = 0; i < LOOP_COUNT; i++) {
+    	  tasks.add(new Runnable() {
-         final TestData myData = testData[i % testData.length];
+    		 public void run() {
-         completer.submit(new Callable<Boolean>() {
+                dateService.jodaIso8601DateParse(myData.iso8601DateString);
-            public Boolean call() {
+    		 }
-               Date jodaDate = jodaParseIso8601(myData.iso8601DateString).toDate();
+    	  });
-               assertEquals(jodaDate, myData.date);
-               return true;
-            }
-         });
       }
-      for (int i = 0; i < LOOP_COUNT; i++)
+      executeMultiThreadedPerformanceTest(
-         assertTrue(completer.take().get());
+		 "testParseIso8601DatePerformanceInParallel_JodaAlternative", tasks);
-      printElapsedClockTime("testParseJodaDateInParallel");
    }
 
 }