From ae4357dfabdb60367c45984a30c433cfae657066 Mon Sep 17 00:00:00 2001 From: sahner Date: Wed, 2 Dec 2015 11:40:59 -0600 Subject: [PATCH] integration tests: in unloadAndKill, remove all segments if no date range given --- .../CoordinatorResourceTestClient.java | 27 ++++++++++++++++++- .../tests/indexer/AbstractIndexerTest.java | 14 +++++++++- .../io/druid/tests/indexer/ITIndexerTest.java | 2 +- .../io/druid/tests/indexer/ITKafkaTest.java | 12 ++++----- .../indexer/ITRealtimeIndexTaskTest.java | 12 ++++----- 5 files changed, 50 insertions(+), 17 deletions(-) diff --git a/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java b/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java index 51c5b4d47de..258be38fbfa 100644 --- a/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java +++ b/integration-tests/src/main/java/io/druid/testing/clients/CoordinatorResourceTestClient.java @@ -36,6 +36,7 @@ import org.joda.time.Interval; import java.net.URL; import java.net.URLEncoder; import java.util.Map; +import java.util.ArrayList; public class CoordinatorResourceTestClient { @@ -64,11 +65,35 @@ public class CoordinatorResourceTestClient ); } + private String getIntervalsURL(String dataSource) + { + return String.format("%sdatasources/%s/intervals", getCoordinatorURL(), dataSource); + } + private String getLoadStatusURL() { return String.format("%s%s", getCoordinatorURL(), "loadstatus"); } + // return a list of the segment dates for the specified datasource + public ArrayList getSegmentIntervals(final String dataSource) throws Exception + { + ArrayList segments = null; + try { + StatusResponseHolder response = makeRequest(HttpMethod.GET, getIntervalsURL(dataSource)); + + segments = jsonMapper.readValue( + response.getContent(), new TypeReference>() + { + } + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + return segments; + } + private Map getLoadStatus() { Map status = null; @@ -82,7 +107,7 @@ public class CoordinatorResourceTestClient ); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } return status; } diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java index 5fb492bdb2e..f5a6a5d6987 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/AbstractIndexerTest.java @@ -31,6 +31,8 @@ import org.joda.time.Interval; import java.io.IOException; import java.io.InputStream; import java.util.concurrent.Callable; +import java.util.ArrayList; +import java.util.Collections; public abstract class AbstractIndexerTest { @@ -50,7 +52,17 @@ public abstract class AbstractIndexerTest protected void unloadAndKillData(final String dataSource) throws Exception { - unloadAndKillData (dataSource, "2013-01-01T00:00:00.000Z", "2013-12-01T00:00:00.000Z"); + ArrayList intervals = coordinator.getSegmentIntervals(dataSource); + + // each element in intervals has this form: + // 2015-12-01T23:15:00.000Z/2015-12-01T23:16:00.000Z + // we'll sort the list (ISO dates have lexicographic order) + // then delete segments from the 1st date in the first string + // to the 2nd date in the last string + Collections.sort (intervals); + String first = intervals.get(0).split("/")[0]; + String last = intervals.get(intervals.size() -1).split("/")[1]; + unloadAndKillData (dataSource, first, last); } protected void unloadAndKillData(final String dataSource, String start, String end) throws Exception diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java index 6dd5c0072eb..8c4d9e66efd 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITIndexerTest.java @@ -64,7 +64,7 @@ public class ITIndexerTest extends AbstractIndexerTest } catch (Exception e) { e.printStackTrace(); - Throwables.propagate(e); + throw Throwables.propagate(e); } finally { unloadAndKillData(INDEX_DATASOURCE); diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java index 6f28dc46a11..a89404dec19 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITKafkaTest.java @@ -196,7 +196,7 @@ public class ITKafkaTest extends AbstractIndexerTest producer.send(message); } catch (Exception ioe) { - Throwables.propagate(ioe); + throw Throwables.propagate(ioe); } try { @@ -238,7 +238,7 @@ public class ITKafkaTest extends AbstractIndexerTest try { this.queryHelper.testQueriesFromString(queryStr, 2); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } // wait for segments to be handed off @@ -259,7 +259,7 @@ public class ITKafkaTest extends AbstractIndexerTest ); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } LOG.info("segments are present"); segmentsExist = true; @@ -269,7 +269,7 @@ public class ITKafkaTest extends AbstractIndexerTest this.queryHelper.testQueriesFromString(queryStr, 2); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } } @@ -287,9 +287,7 @@ public class ITKafkaTest extends AbstractIndexerTest // remove segments if (segmentsExist) { try { - String first = DateTimeFormat.forPattern("yyyy-MM-dd'T00:00:00.000Z'").print(dtFirst); - String last = DateTimeFormat.forPattern("yyyy-MM-dd'T00:00:00.000Z'").print(dtFirst.plusDays(1)); - unloadAndKillData(DATASOURCE, first, last); + unloadAndKillData(DATASOURCE); } catch (Exception e) { LOG.warn("exception while removing segments: [%s]", e.getMessage()); diff --git a/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java b/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java index 59f6446082f..5c8afe5800e 100644 --- a/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java +++ b/integration-tests/src/test/java/io/druid/tests/indexer/ITRealtimeIndexTaskTest.java @@ -140,7 +140,7 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest this.queryHelper.testQueriesFromString(getRouterURL(), queryStr, 2); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } // wait for the task to complete @@ -166,12 +166,10 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest this.queryHelper.testQueriesFromString(getRouterURL(), queryStr, 2); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } finally { - String first = DateTimeFormat.forPattern("yyyy-MM-dd'T'00:00:00.000'Z'").print(dtFirst); - String last = DateTimeFormat.forPattern("yyyy-MM-dd'T'00:00:00.000'Z'").print(dtFirst.plusDays(1)); - unloadAndKillData(INDEX_DATASOURCE, first, last); + unloadAndKillData(INDEX_DATASOURCE); } } @@ -191,7 +189,7 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest isr = new InputStreamReader(ITRealtimeIndexTaskTest.class.getResourceAsStream(EVENT_DATA_FILE)); } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } try { reader = new BufferedReader(isr); @@ -243,7 +241,7 @@ public class ITRealtimeIndexTaskTest extends AbstractIndexerTest } } catch (Exception e) { - Throwables.propagate(e); + throw Throwables.propagate(e); } finally { reader.close();