From a049b426742a10b6b8545f702047d76d0453368e Mon Sep 17 00:00:00 2001 From: fjy Date: Thu, 7 Nov 2013 18:01:35 -0800 Subject: [PATCH] fix an issue with task tables not getting created automatically and prepare for next release --- build.sh | 2 +- docs/content/Booting-a-production-cluster.md | 2 +- docs/content/Download.md | 2 +- docs/content/Examples.md | 4 +- docs/content/Realtime.md | 4 +- .../Tutorial:-A-First-Look-at-Druid.md | 4 +- .../Tutorial:-Loading-Your-Data-Part-2.md | 2 +- docs/content/Tutorial:-The-Druid-Cluster.md | 6 +- docs/content/Tutorial:-Webstream.md | 4 +- docs/content/Twitter-Tutorial.textile | 2 +- examples/config/historical/runtime.properties | 2 +- examples/config/realtime/runtime.properties | 2 +- .../indexing/overlord/DbTaskStorage.java | 62 ++++++++++++------- .../overlord/IndexerDBCoordinator.java | 4 +- .../indexing/overlord/TaskLifecycleTest.java | 2 +- .../java/io/druid/cli/CliCoordinator.java | 2 +- .../java/io/druid/cli/CliHadoopIndexer.java | 2 +- .../main/java/io/druid/cli/CliHistorical.java | 2 +- .../main/java/io/druid/cli/CliOverlord.java | 2 +- 19 files changed, 65 insertions(+), 47 deletions(-) diff --git a/build.sh b/build.sh index 4695f3cf8b2..59348b08f66 100755 --- a/build.sh +++ b/build.sh @@ -30,4 +30,4 @@ echo "For examples, see: " echo " " ls -1 examples/*/*sh echo " " -echo "See also http://druid.io/docs/0.6.3" +echo "See also http://druid.io/docs/0.6.4" diff --git a/docs/content/Booting-a-production-cluster.md b/docs/content/Booting-a-production-cluster.md index 53c9540d7ad..b5476917710 100644 --- a/docs/content/Booting-a-production-cluster.md +++ b/docs/content/Booting-a-production-cluster.md @@ -3,7 +3,7 @@ layout: doc_page --- # Booting a Single Node Cluster # -[Loading Your Data](Tutorial%3A-Loading-Your-Data-Part-2.html) and [All About Queries](Tutorial%3A-All-About-Queries.html) contain recipes to boot a small druid cluster on localhost. Here we will boot a small cluster on EC2. You can checkout the code, or download a tarball from [here](http://static.druid.io/artifacts/druid-services-0.6.3-bin.tar.gz). +[Loading Your Data](Tutorial%3A-Loading-Your-Data-Part-2.html) and [All About Queries](Tutorial%3A-All-About-Queries.html) contain recipes to boot a small druid cluster on localhost. Here we will boot a small cluster on EC2. You can checkout the code, or download a tarball from [here](http://static.druid.io/artifacts/druid-services-0.6.4-bin.tar.gz). The [ec2 run script](https://github.com/metamx/druid/blob/master/examples/bin/run_ec2.sh), run_ec2.sh, is located at 'examples/bin' if you have checked out the code, or at the root of the project if you've downloaded a tarball. The scripts rely on the [Amazon EC2 API Tools](http://aws.amazon.com/developertools/351), and you will need to set three environment variables: diff --git a/docs/content/Download.md b/docs/content/Download.md index b25681d02be..a228bb0b7f9 100644 --- a/docs/content/Download.md +++ b/docs/content/Download.md @@ -6,7 +6,7 @@ A version may be declared as a release candidate if it has been deployed to a si Release Candidate ----------------- -The current release candidate is tagged at version [0.6.3](https://github.com/metamx/druid/tree/druid-0.6.3). +The current release candidate is tagged at version [0.6.4](https://github.com/metamx/druid/tree/druid-0.6.4). Stable Release -------------- diff --git a/docs/content/Examples.md b/docs/content/Examples.md index 8456e8ef616..2307dbcd6ce 100644 --- a/docs/content/Examples.md +++ b/docs/content/Examples.md @@ -19,13 +19,13 @@ Clone Druid and build it: git clone https://github.com/metamx/druid.git druid cd druid git fetch --tags -git checkout druid-0.6.3 +git checkout druid-0.6.4 ./build.sh ``` ### Downloading the DSK (Druid Standalone Kit) -[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.3-bin.tar.gz) a stand-alone tarball and run it: +[Download](http://static.druid.io/artifacts/releases/druid-services-0.6.4-bin.tar.gz) a stand-alone tarball and run it: ``` bash tar -xzf druid-services-0.X.X-bin.tar.gz diff --git a/docs/content/Realtime.md b/docs/content/Realtime.md index bd01a6f6db7..d0e0d7c29a6 100644 --- a/docs/content/Realtime.md +++ b/docs/content/Realtime.md @@ -187,8 +187,8 @@ Extending the code Realtime integration is intended to be extended in two ways: -1. Connect to data streams from varied systems ([Firehose](https://github.com/metamx/druid/blob/druid-0.6.3/realtime/src/main/java/com/metamx/druid/realtime/firehose/FirehoseFactory.java)) -2. Adjust the publishing strategy to match your needs ([Plumber](https://github.com/metamx/druid/blob/druid-0.6.3/realtime/src/main/java/com/metamx/druid/realtime/plumber/PlumberSchool.java)) +1. Connect to data streams from varied systems ([Firehose](https://github.com/metamx/druid/blob/druid-0.6.4/realtime/src/main/java/com/metamx/druid/realtime/firehose/FirehoseFactory.java)) +2. Adjust the publishing strategy to match your needs ([Plumber](https://github.com/metamx/druid/blob/druid-0.6.4/realtime/src/main/java/com/metamx/druid/realtime/plumber/PlumberSchool.java)) The expectations are that the former will be very common and something that users of Druid will do on a fairly regular basis. Most users will probably never have to deal with the latter form of customization. Indeed, we hope that all potential use cases can be packaged up as part of Druid proper without requiring proprietary customization. diff --git a/docs/content/Tutorial:-A-First-Look-at-Druid.md b/docs/content/Tutorial:-A-First-Look-at-Druid.md index 595b9d8733f..eee289816ef 100644 --- a/docs/content/Tutorial:-A-First-Look-at-Druid.md +++ b/docs/content/Tutorial:-A-First-Look-at-Druid.md @@ -47,7 +47,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu ### Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.3-bin.tar.gz). Download this file to a directory of your choosing. +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.4-bin.tar.gz). Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -58,7 +58,7 @@ tar -zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.3 +cd druid-services-0.6.4 ``` You should see a bunch of files: diff --git a/docs/content/Tutorial:-Loading-Your-Data-Part-2.md b/docs/content/Tutorial:-Loading-Your-Data-Part-2.md index 61384186500..8749f6e1541 100644 --- a/docs/content/Tutorial:-Loading-Your-Data-Part-2.md +++ b/docs/content/Tutorial:-Loading-Your-Data-Part-2.md @@ -42,7 +42,7 @@ With real-world data, we recommend having a message bus such as [Apache Kafka](h #### Setting up Kafka -[KafkaFirehoseFactory](https://github.com/metamx/druid/blob/druid-0.6.3/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java) is how druid communicates with Kafka. Using this [Firehose](Firehose.html) with the right configuration, we can import data into Druid in real-time without writing any code. To load data to a real-time node via Kafka, we'll first need to initialize Zookeeper and Kafka, and then configure and initialize a [Realtime](Realtime.html) node. +[KafkaFirehoseFactory](https://github.com/metamx/druid/blob/druid-0.6.4/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java) is how druid communicates with Kafka. Using this [Firehose](Firehose.html) with the right configuration, we can import data into Druid in real-time without writing any code. To load data to a real-time node via Kafka, we'll first need to initialize Zookeeper and Kafka, and then configure and initialize a [Realtime](Realtime.html) node. Instructions for booting a Zookeeper and then Kafka cluster are available [here](http://kafka.apache.org/07/quickstart.html). diff --git a/docs/content/Tutorial:-The-Druid-Cluster.md b/docs/content/Tutorial:-The-Druid-Cluster.md index 8b8907b3252..8c0a02349d8 100644 --- a/docs/content/Tutorial:-The-Druid-Cluster.md +++ b/docs/content/Tutorial:-The-Druid-Cluster.md @@ -11,7 +11,7 @@ In this tutorial, we will set up other types of Druid nodes as well as and exter If you followed the first tutorial, you should already have Druid downloaded. If not, let's go back and do that first. -You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.3-bin.tar.gz) +You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.6.4-bin.tar.gz) and untar the contents within by issuing: @@ -147,7 +147,7 @@ druid.port=8081 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.3"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.4"] # Dummy read only AWS account (used to download example data) druid.s3.secretKey=QyyfVZ7llSiRg6Qcrql1eEUG7buFpAK6T6engr1b @@ -237,7 +237,7 @@ druid.port=8083 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.3-SNAPSHOT"] +druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.4-SNAPSHOT"] druid.db.connector.connectURI=jdbc\:mysql\://localhost\:3306/druid druid.db.connector.user=druid diff --git a/docs/content/Tutorial:-Webstream.md b/docs/content/Tutorial:-Webstream.md index f551a5bead4..6f70cc286b8 100644 --- a/docs/content/Tutorial:-Webstream.md +++ b/docs/content/Tutorial:-Webstream.md @@ -37,7 +37,7 @@ There are two ways to setup Druid: download a tarball, or [Build From Source](Bu h3. Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.3-bin.tar.gz) +We've built a tarball that contains everything you'll need. You'll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.6.4-bin.tar.gz) Download this file to a directory of your choosing. You can extract the awesomeness within by issuing: @@ -48,7 +48,7 @@ tar zxvf druid-services-*-bin.tar.gz Not too lost so far right? That's great! If you cd into the directory: ``` -cd druid-services-0.6.3 +cd druid-services-0.6.4 ``` You should see a bunch of files: diff --git a/docs/content/Twitter-Tutorial.textile b/docs/content/Twitter-Tutorial.textile index cf4d6841f84..3336b07d4f7 100644 --- a/docs/content/Twitter-Tutorial.textile +++ b/docs/content/Twitter-Tutorial.textile @@ -9,7 +9,7 @@ There are two ways to setup Druid: download a tarball, or build it from source. h3. Download a Tarball -We've built a tarball that contains everything you'll need. You'll find it "here":http://static.druid.io/artifacts/releases/druid-services-0.6.3-bin.tar.gz. +We've built a tarball that contains everything you'll need. You'll find it "here":http://static.druid.io/artifacts/releases/druid-services-0.6.4-bin.tar.gz. Download this bad boy to a directory of your choosing. You can extract the awesomeness within by issuing: diff --git a/examples/config/historical/runtime.properties b/examples/config/historical/runtime.properties index a539c1008aa..729148dbb77 100644 --- a/examples/config/historical/runtime.properties +++ b/examples/config/historical/runtime.properties @@ -4,7 +4,7 @@ druid.port=8081 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.3"] +druid.extensions.coordinates=["io.druid.extensions:druid-s3-extensions:0.6.4"] # Dummy read only AWS account (used to download example data) druid.s3.secretKey=QyyfVZ7llSiRg6Qcrql1eEUG7buFpAK6T6engr1b diff --git a/examples/config/realtime/runtime.properties b/examples/config/realtime/runtime.properties index 612d8c36da4..2bc9d0a710d 100644 --- a/examples/config/realtime/runtime.properties +++ b/examples/config/realtime/runtime.properties @@ -4,7 +4,7 @@ druid.port=8083 druid.zk.service.host=localhost -druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.3"] +druid.extensions.coordinates=["io.druid.extensions:druid-examples:0.6.4"] druid.db.connector.connectURI=jdbc\:mysql\://localhost\:3306/druid druid.db.connector.user=druid diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/DbTaskStorage.java b/indexing-service/src/main/java/io/druid/indexing/overlord/DbTaskStorage.java index 5100f6ca613..ec2e2f99feb 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/DbTaskStorage.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/DbTaskStorage.java @@ -29,7 +29,10 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.inject.Inject; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.emitter.EmittingLogger; +import io.druid.db.DbConnector; import io.druid.db.DbTablesConfig; import io.druid.indexing.common.TaskLock; import io.druid.indexing.common.TaskStatus; @@ -47,19 +50,33 @@ import java.util.Map; public class DbTaskStorage implements TaskStorage { private final ObjectMapper jsonMapper; + private final DbConnector dbConnector; private final DbTablesConfig dbTables; private final IDBI dbi; private static final EmittingLogger log = new EmittingLogger(DbTaskStorage.class); @Inject - public DbTaskStorage(ObjectMapper jsonMapper, DbTablesConfig dbTables, IDBI dbi) + public DbTaskStorage(ObjectMapper jsonMapper, DbConnector dbConnector, DbTablesConfig dbTables, IDBI dbi) { this.jsonMapper = jsonMapper; + this.dbConnector = dbConnector; this.dbTables = dbTables; this.dbi = dbi; } + @LifecycleStart + public void start() + { + dbConnector.createTaskTables(); + } + + @LifecycleStop + public void stop() + { + // do nothing + } + @Override public void insert(final Task task, final TaskStatus status) { @@ -99,9 +116,10 @@ public class DbTaskStorage implements TaskStorage } } ); - } catch (StatementException e) { + } + catch (StatementException e) { // Might be a duplicate task ID. - if(getTask(task.getId()).isPresent()) { + if (getTask(task.getId()).isPresent()) { throw new TaskExistsException(task.getId(), e); } else { throw e; @@ -128,15 +146,15 @@ public class DbTaskStorage implements TaskStorage dbTables.getTasksTable() ) ) - .bind("id", status.getId()) - .bind("active", status.isRunnable() ? 1 : 0) - .bind("status_payload", jsonMapper.writeValueAsBytes(status)) - .execute(); + .bind("id", status.getId()) + .bind("active", status.isRunnable() ? 1 : 0) + .bind("status_payload", jsonMapper.writeValueAsBytes(status)) + .execute(); } } ); - if(updated != 1) { + if (updated != 1) { throw new IllegalStateException(String.format("Active task not found: %s", status.getId())); } } @@ -160,11 +178,11 @@ public class DbTaskStorage implements TaskStorage .bind("id", taskid) .list(); - if(dbTasks.size() == 0) { + if (dbTasks.size() == 0) { return Optional.absent(); } else { final Map dbStatus = Iterables.getOnlyElement(dbTasks); - return Optional.of(jsonMapper.readValue((byte[])dbStatus.get("payload"), Task.class)); + return Optional.of(jsonMapper.readValue((byte[]) dbStatus.get("payload"), Task.class)); } } } @@ -190,11 +208,11 @@ public class DbTaskStorage implements TaskStorage .bind("id", taskid) .list(); - if(dbStatuses.size() == 0) { + if (dbStatuses.size() == 0) { return Optional.absent(); } else { final Map dbStatus = Iterables.getOnlyElement(dbStatuses); - return Optional.of(jsonMapper.readValue((byte[])dbStatus.get("status_payload"), TaskStatus.class)); + return Optional.of(jsonMapper.readValue((byte[]) dbStatus.get("status_payload"), TaskStatus.class)); } } } @@ -224,13 +242,14 @@ public class DbTaskStorage implements TaskStorage final String id = row.get("id").toString(); try { - final Task task = jsonMapper.readValue((byte[])row.get("payload"), Task.class); - final TaskStatus status = jsonMapper.readValue((byte[])row.get("status_payload"), TaskStatus.class); + final Task task = jsonMapper.readValue((byte[]) row.get("payload"), Task.class); + final TaskStatus status = jsonMapper.readValue((byte[]) row.get("status_payload"), TaskStatus.class); if (status.isRunnable()) { tasks.add(task); } - } catch (Exception e) { + } + catch (Exception e) { log.makeAlert(e, "Failed to parse task payload").addData("task", id).emit(); } } @@ -282,11 +301,11 @@ public class DbTaskStorage implements TaskStorage final Map taskLocks = getLocksWithIds(taskid); - for(final Map.Entry taskLockWithId : taskLocks.entrySet()) { + for (final Map.Entry taskLockWithId : taskLocks.entrySet()) { final long id = taskLockWithId.getKey(); final TaskLock taskLock = taskLockWithId.getValue(); - if(taskLock.equals(taskLockToRemove)) { + if (taskLock.equals(taskLockToRemove)) { log.info("Deleting TaskLock with id[%d]: %s", id, taskLock); dbi.withHandle( @@ -380,8 +399,9 @@ public class DbTaskStorage implements TaskStorage public TaskAction apply(Map row) { try { - return jsonMapper.readValue((byte[])row.get("log_payload"), TaskAction.class); - } catch(Exception e) { + return jsonMapper.readValue((byte[]) row.get("log_payload"), TaskAction.class); + } + catch (Exception e) { throw Throwables.propagate(e); } } @@ -411,8 +431,8 @@ public class DbTaskStorage implements TaskStorage .list(); final Map retMap = Maps.newHashMap(); - for(final Map row : dbTaskLocks) { - retMap.put((Long)row.get("id"), jsonMapper.readValue((byte[])row.get("lock_payload"), TaskLock.class)); + for (final Map row : dbTaskLocks) { + retMap.put((Long) row.get("id"), jsonMapper.readValue((byte[]) row.get("lock_payload"), TaskLock.class)); } return retMap; } diff --git a/indexing-service/src/main/java/io/druid/indexing/overlord/IndexerDBCoordinator.java b/indexing-service/src/main/java/io/druid/indexing/overlord/IndexerDBCoordinator.java index b3056c274ac..2a4b5e8912d 100644 --- a/indexing-service/src/main/java/io/druid/indexing/overlord/IndexerDBCoordinator.java +++ b/indexing-service/src/main/java/io/druid/indexing/overlord/IndexerDBCoordinator.java @@ -28,6 +28,7 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.metamx.common.logger.Logger; +import io.druid.db.DbConnector; import io.druid.db.DbConnectorConfig; import io.druid.db.DbTablesConfig; import io.druid.timeline.DataSegment; @@ -58,20 +59,17 @@ public class IndexerDBCoordinator private static final Logger log = new Logger(IndexerDBCoordinator.class); private final ObjectMapper jsonMapper; - private final DbConnectorConfig dbConnectorConfig; private final DbTablesConfig dbTables; private final IDBI dbi; @Inject public IndexerDBCoordinator( ObjectMapper jsonMapper, - DbConnectorConfig dbConnectorConfig, DbTablesConfig dbTables, IDBI dbi ) { this.jsonMapper = jsonMapper; - this.dbConnectorConfig = dbConnectorConfig; this.dbTables = dbTables; this.dbi = dbi; } diff --git a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java index b155b31dd0f..0b3a4e9ed97 100644 --- a/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/overlord/TaskLifecycleTest.java @@ -433,7 +433,7 @@ public class TaskLifecycleTest private MockIndexerDBCoordinator() { - super(null, null, null, null); + super(null, null, null); } @Override diff --git a/services/src/main/java/io/druid/cli/CliCoordinator.java b/services/src/main/java/io/druid/cli/CliCoordinator.java index 0816d3992cb..3dde52888ac 100644 --- a/services/src/main/java/io/druid/cli/CliCoordinator.java +++ b/services/src/main/java/io/druid/cli/CliCoordinator.java @@ -63,7 +63,7 @@ import java.util.List; */ @Command( name = "coordinator", - description = "Runs the Coordinator, see http://druid.io/docs/0.6.3/Coordinator.html for a description." + description = "Runs the Coordinator, see http://druid.io/docs/0.6.4/Coordinator.html for a description." ) public class CliCoordinator extends ServerRunnable { diff --git a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java index 7ec8f971750..546b6cc8842 100644 --- a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java +++ b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java @@ -41,7 +41,7 @@ import java.util.List; */ @Command( name = "hadoop", - description = "Runs the batch Hadoop Druid Indexer, see http://druid.io/docs/0.6.3/Batch-ingestion.html for a description." + description = "Runs the batch Hadoop Druid Indexer, see http://druid.io/docs/0.6.4/Batch-ingestion.html for a description." ) public class CliHadoopIndexer implements Runnable { diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index d525eb38862..0dd021edeb7 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -42,7 +42,7 @@ import java.util.List; */ @Command( name = "historical", - description = "Runs a Historical node, see http://druid.io/docs/0.6.3/Historical.html for a description" + description = "Runs a Historical node, see http://druid.io/docs/0.6.4/Historical.html for a description" ) public class CliHistorical extends ServerRunnable { diff --git a/services/src/main/java/io/druid/cli/CliOverlord.java b/services/src/main/java/io/druid/cli/CliOverlord.java index 9fb298f3d59..f4ccc95964b 100644 --- a/services/src/main/java/io/druid/cli/CliOverlord.java +++ b/services/src/main/java/io/druid/cli/CliOverlord.java @@ -160,7 +160,7 @@ public class CliOverlord extends ServerRunnable storageBinder.addBinding("local").to(HeapMemoryTaskStorage.class); binder.bind(HeapMemoryTaskStorage.class).in(LazySingleton.class); - storageBinder.addBinding("db").to(DbTaskStorage.class); + storageBinder.addBinding("db").to(DbTaskStorage.class).in(ManageLifecycle.class); binder.bind(DbTaskStorage.class).in(LazySingleton.class); }