From 654cdc07f57f35f03d0f2f32c47271668d7c19dc Mon Sep 17 00:00:00 2001 From: Himanshu Date: Wed, 1 Nov 2017 11:59:27 -0500 Subject: [PATCH] Document HTTP based segment management and Deprecate classes to remove in future (#4997) * document http segment management * deprecated classes that shouldn't be used any further --- docs/content/configuration/broker.md | 5 +++++ docs/content/configuration/coordinator.md | 11 +++++++++++ .../io/druid/client/BatchServerInventoryView.java | 2 ++ .../io/druid/client/SingleServerInventoryView.java | 2 ++ .../curator/discovery/CuratorServiceAnnouncer.java | 3 +++ .../druid/curator/discovery/CuratorServiceUtils.java | 4 ++++ .../druid/curator/discovery/NoopServiceAnnouncer.java | 1 + .../curator/discovery/ServerDiscoveryFactory.java | 2 ++ .../curator/discovery/ServerDiscoverySelector.java | 2 ++ .../io/druid/curator/discovery/ServiceAnnouncer.java | 3 +++ .../curator/inventory/CuratorInventoryManager.java | 3 +++ .../main/java/io/druid/query/lookup/LookupModule.java | 1 + .../CuratorDataSegmentServerAnnouncer.java | 2 ++ .../coordination/DataSegmentServerAnnouncer.java | 2 ++ .../io/druid/server/coordination/ZkCoordinator.java | 2 ++ .../server/coordinator/CuratorLoadQueuePeon.java | 2 ++ .../listener/announcer/ListenerResourceAnnouncer.java | 3 ++- 17 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/content/configuration/broker.md b/docs/content/configuration/broker.md index af97d2dffa5..9989c76a80c 100644 --- a/docs/content/configuration/broker.md +++ b/docs/content/configuration/broker.md @@ -109,6 +109,11 @@ You can optionally only configure caching to be enabled on the broker by setting See [cache configuration](caching.html) for how to configure cache settings. +### Segment Discovery +|Property|Possible Values|Description|Default| +|--------|---------------|-----------|-------| +|`druid.announcer.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of zookeeper.|batch| + ### Others |Property|Possible Values|Description|Default| diff --git a/docs/content/configuration/coordinator.md b/docs/content/configuration/coordinator.md index 818d0ec95dc..2dfd97098df 100644 --- a/docs/content/configuration/coordinator.md +++ b/docs/content/configuration/coordinator.md @@ -38,6 +38,17 @@ The coordinator node uses several of the global configs in [Configuration](../co |`druid.coordinator.asOverlord.enabled`|Boolean value for whether this coordinator node should act like an overlord as well. This configuration allows users to simplify a druid cluster by not having to deploy any standalone overlord nodes. If set to true, then overlord console is available at `http://coordinator-host:port/console.html` and be sure to set `druid.coordinator.asOverlord.overlordService` also. See next.|false| |`druid.coordinator.asOverlord.overlordService`| Required, if `druid.coordinator.asOverlord.enabled` is `true`. This must be same value as `druid.service` on standalone Overlord nodes and `druid.selectors.indexing.serviceName` on Middle Managers.|NULL| +### Segment Management +|Property|Possible Values|Description|Default| +|--------|---------------|-----------|-------| +|`druid.announcer.type`|batch or http|Segment discovery method to use. "http" enables discovering segments using HTTP instead of zookeeper.|batch| +|`druid.coordinator.loadqueuepeon.type`|curator or http|Whether to use "http" or "curator" implementation to assign segment loads/drops to historical|curator| + +#### Additional config when "http" loadqueuepeon is used +|Property|Description|Default| +|--------|-----------|-------| +|`druid.coordinator.loadqueuepeon.http.batchSize`|Number of segment load/drop requests to batch in one HTTP request. Note that it must be smaller than `druid.segmentCache.numLoadingThreads` config on historical node.|1| + ### Metadata Retrieval |Property|Description|Default| diff --git a/server/src/main/java/io/druid/client/BatchServerInventoryView.java b/server/src/main/java/io/druid/client/BatchServerInventoryView.java index d0eff0549cb..abdeeaf2313 100644 --- a/server/src/main/java/io/druid/client/BatchServerInventoryView.java +++ b/server/src/main/java/io/druid/client/BatchServerInventoryView.java @@ -43,7 +43,9 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; /** + * This class is deprecated. Use {@link HttpServerInventoryView} instead. */ +@Deprecated @ManageLifecycle public class BatchServerInventoryView extends AbstractCuratorServerInventoryView> implements FilteredServerInventoryView diff --git a/server/src/main/java/io/druid/client/SingleServerInventoryView.java b/server/src/main/java/io/druid/client/SingleServerInventoryView.java index 183ff129396..ef69679be23 100644 --- a/server/src/main/java/io/druid/client/SingleServerInventoryView.java +++ b/server/src/main/java/io/druid/client/SingleServerInventoryView.java @@ -38,7 +38,9 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; /** + * This class is deprecated. Use {@link HttpServerInventoryView} instead. */ +@Deprecated @ManageLifecycle public class SingleServerInventoryView extends AbstractCuratorServerInventoryView implements FilteredServerInventoryView { diff --git a/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java index 0020e179a39..8b737125bf9 100644 --- a/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java +++ b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java @@ -30,8 +30,11 @@ import org.apache.curator.x.discovery.ServiceInstance; import java.util.Map; /** + * This class is deprecated, Add service to {@link io.druid.discovery.DruidNodeAnnouncer} node announcement instead. + * * Uses the Curator Service Discovery recipe to announce services. */ +@Deprecated public class CuratorServiceAnnouncer implements ServiceAnnouncer { private static final EmittingLogger log = new EmittingLogger(CuratorServiceAnnouncer.class); diff --git a/server/src/main/java/io/druid/curator/discovery/CuratorServiceUtils.java b/server/src/main/java/io/druid/curator/discovery/CuratorServiceUtils.java index c5acbb51a04..2dafe5c3dd0 100644 --- a/server/src/main/java/io/druid/curator/discovery/CuratorServiceUtils.java +++ b/server/src/main/java/io/druid/curator/discovery/CuratorServiceUtils.java @@ -19,6 +19,10 @@ package io.druid.curator.discovery; +/** + * This class is only used by Deprecated classes. + */ +@Deprecated public class CuratorServiceUtils { /** diff --git a/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java index b1c389c5d9a..23dff1adea8 100644 --- a/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java +++ b/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java @@ -24,6 +24,7 @@ import io.druid.server.DruidNode; /** * Does nothing. */ +@Deprecated public class NoopServiceAnnouncer implements ServiceAnnouncer { @Override diff --git a/server/src/main/java/io/druid/curator/discovery/ServerDiscoveryFactory.java b/server/src/main/java/io/druid/curator/discovery/ServerDiscoveryFactory.java index fbbbc1c4d46..4ae5fa813bb 100644 --- a/server/src/main/java/io/druid/curator/discovery/ServerDiscoveryFactory.java +++ b/server/src/main/java/io/druid/curator/discovery/ServerDiscoveryFactory.java @@ -28,7 +28,9 @@ import java.io.IOException; import java.util.Collection; /** + * Use {@link io.druid.discovery.DruidNodeDiscovery} for discovery. */ +@Deprecated public class ServerDiscoveryFactory { private final ServiceDiscovery serviceDiscovery; diff --git a/server/src/main/java/io/druid/curator/discovery/ServerDiscoverySelector.java b/server/src/main/java/io/druid/curator/discovery/ServerDiscoverySelector.java index e174b2ee14b..b5daff14653 100644 --- a/server/src/main/java/io/druid/curator/discovery/ServerDiscoverySelector.java +++ b/server/src/main/java/io/druid/curator/discovery/ServerDiscoverySelector.java @@ -36,7 +36,9 @@ import java.util.Collection; import java.util.Collections; /** + * Use {@link io.druid.discovery.DruidNodeDiscovery} for discovery. */ +@Deprecated public class ServerDiscoverySelector implements DiscoverySelector { private static final Logger log = new Logger(ServerDiscoverySelector.class); diff --git a/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java index b3b18138c9a..d1cae0283f3 100644 --- a/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java +++ b/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java @@ -22,9 +22,12 @@ package io.druid.curator.discovery; import io.druid.server.DruidNode; /** + * This class is deprecated, Add service to {@link io.druid.discovery.DruidNodeAnnouncer} node announcement instead. + * * Announces our ability to serve a particular function. Multiple users may announce the same service, in which * case they are treated as interchangeable instances of that service. */ +@Deprecated public interface ServiceAnnouncer { void announce(DruidNode node); diff --git a/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java index 20fc02e8ec5..857b5b2250e 100644 --- a/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java +++ b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java @@ -43,6 +43,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; /** + * This class is deprecated. Use {@link io.druid.client.HttpServerInventoryView} for segment discovery. + * * An InventoryManager watches updates to inventory on Zookeeper (or some other discovery-like service publishing * system). It is built up on two object types: containers and inventory objects. *

@@ -52,6 +54,7 @@ import java.util.concurrent.atomic.AtomicReference; * A Strategy is provided to the constructor of an Inventory manager, this strategy provides all of the * object-specific logic to serialize, deserialize, compose and alter the container and inventory objects. */ +@Deprecated public class CuratorInventoryManager { private static final Logger log = new Logger(CuratorInventoryManager.class); diff --git a/server/src/main/java/io/druid/query/lookup/LookupModule.java b/server/src/main/java/io/druid/query/lookup/LookupModule.java index 59c0d7f96c3..a1d387cf722 100644 --- a/server/src/main/java/io/druid/query/lookup/LookupModule.java +++ b/server/src/main/java/io/druid/query/lookup/LookupModule.java @@ -214,6 +214,7 @@ class LookupListeningResource extends ListenerResource } } +@Deprecated class LookupResourceListenerAnnouncer extends ListenerResourceAnnouncer { @Inject diff --git a/server/src/main/java/io/druid/server/coordination/CuratorDataSegmentServerAnnouncer.java b/server/src/main/java/io/druid/server/coordination/CuratorDataSegmentServerAnnouncer.java index 6b2af0f6d82..0b7ad8724e4 100644 --- a/server/src/main/java/io/druid/server/coordination/CuratorDataSegmentServerAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/CuratorDataSegmentServerAnnouncer.java @@ -29,7 +29,9 @@ import io.druid.server.initialization.ZkPathsConfig; import org.apache.curator.utils.ZKPaths; /** + * {@link DataSegmentServerAnnouncer} is deprecated. */ +@Deprecated public class CuratorDataSegmentServerAnnouncer implements DataSegmentServerAnnouncer { private static final Logger log = new Logger(CuratorDataSegmentServerAnnouncer.class); diff --git a/server/src/main/java/io/druid/server/coordination/DataSegmentServerAnnouncer.java b/server/src/main/java/io/druid/server/coordination/DataSegmentServerAnnouncer.java index 791c62c57a7..836a95d42a4 100644 --- a/server/src/main/java/io/druid/server/coordination/DataSegmentServerAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentServerAnnouncer.java @@ -20,7 +20,9 @@ package io.druid.server.coordination; /** + * Use announcement made by {@link io.druid.discovery.DruidNodeAnnouncer} */ +@Deprecated public interface DataSegmentServerAnnouncer { void announce(); diff --git a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 4978cf8e302..98d226e9944 100644 --- a/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -37,7 +37,9 @@ import org.apache.curator.utils.ZKPaths; import java.io.IOException; /** + * Use {@link io.druid.server.coordinator.HttpLoadQueuePeon} for segment load/drops. */ +@Deprecated public class ZkCoordinator { private static final EmittingLogger log = new EmittingLogger(ZkCoordinator.class); diff --git a/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java b/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java index 8ab64a466d3..99761cd43a2 100644 --- a/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/coordinator/CuratorLoadQueuePeon.java @@ -51,7 +51,9 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; /** + * Use {@link HttpLoadQueuePeon} instead. */ +@Deprecated public class CuratorLoadQueuePeon extends LoadQueuePeon { private static final EmittingLogger log = new EmittingLogger(CuratorLoadQueuePeon.class); diff --git a/server/src/main/java/io/druid/server/listener/announcer/ListenerResourceAnnouncer.java b/server/src/main/java/io/druid/server/listener/announcer/ListenerResourceAnnouncer.java index 809e4e1827c..a0203f4b4c8 100644 --- a/server/src/main/java/io/druid/server/listener/announcer/ListenerResourceAnnouncer.java +++ b/server/src/main/java/io/druid/server/listener/announcer/ListenerResourceAnnouncer.java @@ -31,8 +31,9 @@ import org.apache.curator.utils.ZKPaths; import java.nio.ByteBuffer; /** - * Announces that there is a particular ListenerResource at the listener_key. + * Starting 0.11.0 Coordinator uses announcements made by {@link io.druid.discovery.DruidNodeAnnouncer} . */ +@Deprecated public abstract class ListenerResourceAnnouncer { private static final byte[] ANNOUNCE_BYTES = ByteBuffer