Merge pull request #2030 from rasahner/intTestRmAllSegments

int tests: in unloadAndKill, remove all segments if no date range given

integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java

@@ -38,6 +38,7 @@ import org.joda.time.Interval;
 import java.net.URL;
 import java.net.URLEncoder;
 import java.util.Map;
+import java.util.ArrayList;
 
 public class CoordinatorResourceTestClient
 {
@@ -66,11 +67,35 @@ public class CoordinatorResourceTestClient
     );
   }
 
+  private String getIntervalsURL(String dataSource)
+  {
+    return String.format("%sdatasources/%s/intervals", getCoordinatorURL(), dataSource);
+  }
+
   private String getLoadStatusURL()
   {
       return String.format("%s%s", getCoordinatorURL(), "loadstatus");
   }
 
+  // return a list of the segment dates for the specified datasource
+  public ArrayList<String> getSegmentIntervals(final String dataSource) throws Exception
+  {
+    ArrayList<String> segments = null;
+    try {
+      StatusResponseHolder response = makeRequest(HttpMethod.GET, getIntervalsURL(dataSource));
+
+      segments = jsonMapper.readValue(
+          response.getContent(), new TypeReference<ArrayList<String>>()
+          {
+          }
+      );
+    }
+    catch (Exception e) {
+      throw Throwables.propagate(e);
+    }
+    return segments;
+  }
+
   private Map<String, Integer> getLoadStatus()
   {
     Map<String, Integer> status = null;
@@ -84,7 +109,7 @@ public class CoordinatorResourceTestClient
       );
     }
     catch (Exception e) {
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
     return status;
   }

integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java

@@ -33,6 +33,8 @@ import org.joda.time.Interval;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.concurrent.Callable;
+import java.util.ArrayList;
+import java.util.Collections;
 
 public abstract class AbstractIndexerTest
 {
@@ -52,7 +54,17 @@ public abstract class AbstractIndexerTest
 
   protected void unloadAndKillData(final String dataSource) throws Exception
   {
-      unloadAndKillData (dataSource, "2013-01-01T00:00:00.000Z", "2013-12-01T00:00:00.000Z");
+      ArrayList<String> intervals = coordinator.getSegmentIntervals(dataSource);
+
+      // each element in intervals has this form:
+      //   2015-12-01T23:15:00.000Z/2015-12-01T23:16:00.000Z
+      // we'll sort the list (ISO dates have lexicographic order)
+      // then delete segments from the 1st date in the first string
+      // to the 2nd date in the last string
+      Collections.sort (intervals);
+      String first = intervals.get(0).split("/")[0];
+      String last = intervals.get(intervals.size() -1).split("/")[1];
+      unloadAndKillData (dataSource, first, last);
   }
 
   protected void unloadAndKillData(final String dataSource, String start, String end) throws Exception

integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java

@@ -66,7 +66,7 @@ public class ITIndexerTest extends AbstractIndexerTest
     }
     catch (Exception e) {
       e.printStackTrace();
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
     finally {
       unloadAndKillData(INDEX_DATASOURCE);

integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java

@@ -196,7 +196,7 @@ public class ITKafkaTest extends AbstractIndexerTest
         producer.send(message);
       }
       catch (Exception ioe) {
-        Throwables.propagate(ioe);
+        throw Throwables.propagate(ioe);
       }
 
       try {
@@ -238,7 +238,7 @@ public class ITKafkaTest extends AbstractIndexerTest
     try {
       this.queryHelper.testQueriesFromString(queryStr, 2);
     } catch (Exception e) {
-	Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
 
     // wait for segments to be handed off
@@ -259,7 +259,7 @@ public class ITKafkaTest extends AbstractIndexerTest
       );
     }
     catch (Exception e) {
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
     LOG.info("segments are present");
     segmentsExist = true;
@@ -269,7 +269,7 @@ public class ITKafkaTest extends AbstractIndexerTest
       this.queryHelper.testQueriesFromString(queryStr, 2);
     }
     catch (Exception e) {
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
   }
 
@@ -287,9 +287,7 @@ public class ITKafkaTest extends AbstractIndexerTest
     // remove segments
     if (segmentsExist) {
       try {
-        String first = DateTimeFormat.forPattern("yyyy-MM-dd'T00:00:00.000Z'").print(dtFirst);
+        unloadAndKillData(DATASOURCE);
-        String last = DateTimeFormat.forPattern("yyyy-MM-dd'T00:00:00.000Z'").print(dtFirst.plusDays(1));
-        unloadAndKillData(DATASOURCE, first, last);
       }
       catch (Exception e) {
         LOG.warn("exception while removing segments: [%s]", e.getMessage());

integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java

@@ -142,7 +142,7 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest
         this.queryHelper.testQueriesFromString(getRouterURL(), queryStr, 2);
       }
       catch (Exception e) {
-        Throwables.propagate(e);
+        throw Throwables.propagate(e);
       }
 
       // wait for the task to complete
@@ -168,12 +168,10 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest
       this.queryHelper.testQueriesFromString(getRouterURL(), queryStr, 2);
     }
     catch (Exception e) {
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
     finally {
-      String first = DateTimeFormat.forPattern("yyyy-MM-dd'T'00:00:00.000'Z'").print(dtFirst);
+      unloadAndKillData(INDEX_DATASOURCE);
-      String last = DateTimeFormat.forPattern("yyyy-MM-dd'T'00:00:00.000'Z'").print(dtFirst.plusDays(1));
-      unloadAndKillData(INDEX_DATASOURCE, first, last);
     }
   }
 
@@ -193,7 +191,7 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest
       isr = new InputStreamReader(ITRealtimeIndexTaskTest.class.getResourceAsStream(EVENT_DATA_FILE));
     }
     catch (Exception e) {
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
     try {
       reader = new BufferedReader(isr);
@@ -245,7 +243,7 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest
       }
     }
     catch (Exception e) {
-      Throwables.propagate(e);
+      throw Throwables.propagate(e);
     }
     finally {
       reader.close();