From 7557eb28004e2640f5228463f1955b971aa3f46c Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Wed, 2 Mar 2016 10:01:14 -0800 Subject: [PATCH] CliPeon: Fix deadlock on startup by eagerly creating ExecutorLifecycle, ChatHandlerResource. See stack traces here, from current master: https://gist.github.com/gianm/bd9a66c826995f97fc8f 1. The thread "qtp925672150-62" holds the lock on InternalInjectorCreator.class, used by Scopes.SINGLETON, and wants the lock on "handlers" in Lifecycle.addMaybeStartHandler called by DiscoveryModule.getServiceAnnouncer. 2. The main thread holds the lock on "handlers" in Lifecycle.addMaybeStartHandler, which it took because it's trying to add the ExecutorLifecycle to the lifecycle. main is trying to get the InternalInjectorCreator.class lock because it's running ExecutorLifecycle.start, which does some Jackson deserialization, and Jackson needs that lock in order to inject stuff into the Task it's deserializing. This patch eagerly instantiates ChatHandlerResource (which I believe is what's trying to create the ServiceAnnouncer in the qtp925672150-62 jetty thread) and the ExecutorLifecycle. --- .../server/initialization/jetty/ChatHandlerServerModule.java | 1 + services/src/main/java/io/druid/cli/CliPeon.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/io/druid/server/initialization/jetty/ChatHandlerServerModule.java b/server/src/main/java/io/druid/server/initialization/jetty/ChatHandlerServerModule.java index b7147b25f54..0acc6581339 100644 --- a/server/src/main/java/io/druid/server/initialization/jetty/ChatHandlerServerModule.java +++ b/server/src/main/java/io/druid/server/initialization/jetty/ChatHandlerServerModule.java @@ -55,6 +55,7 @@ public class ChatHandlerServerModule implements Module public void configure(Binder binder) { Jerseys.addResource(binder, ChatHandlerResource.class); + LifecycleModule.register(binder, ChatHandlerResource.class); if (properties.containsKey(MAX_CHAT_REQUESTS_PROPERTY)) { final int maxRequests = Integer.parseInt(properties.getProperty(MAX_CHAT_REQUESTS_PROPERTY)); diff --git a/services/src/main/java/io/druid/cli/CliPeon.java b/services/src/main/java/io/druid/cli/CliPeon.java index d6820f3284f..bb465b39c62 100644 --- a/services/src/main/java/io/druid/cli/CliPeon.java +++ b/services/src/main/java/io/druid/cli/CliPeon.java @@ -72,7 +72,6 @@ import io.druid.segment.loading.OmniDataSegmentMover; import io.druid.segment.loading.SegmentLoaderConfig; import io.druid.segment.loading.StorageLocationConfig; import io.druid.segment.realtime.firehose.ChatHandlerProvider; -import io.druid.segment.realtime.firehose.ChatHandlerResource; import io.druid.segment.realtime.firehose.NoopChatHandlerProvider; import io.druid.segment.realtime.firehose.ServiceAnnouncingChatHandlerProvider; import io.druid.segment.realtime.plumber.CoordinatorBasedSegmentHandoffNotifierConfig; @@ -157,6 +156,7 @@ public class CliPeon extends GuiceRunnable binder.bind(DataSegmentArchiver.class).to(OmniDataSegmentArchiver.class).in(LazySingleton.class); binder.bind(ExecutorLifecycle.class).in(ManageLifecycle.class); + LifecycleModule.register(binder, ExecutorLifecycle.class); binder.bind(ExecutorLifecycleConfig.class).toInstance( new ExecutorLifecycleConfig() .setTaskFile(new File(taskAndStatusFile.get(0)))