mirror of https://github.com/apache/druid.git
Merge pull request #1087 from metamx/fixIndexerBaseRelativity
Fix overriding base of IndexerZkConfig to be absolute instead of relative
This commit is contained in:
Fangjin Yang
commit
cb4f894ef7
|
|
@ -25,41 +25,14 @@ Many of Druid's external dependencies can be plugged in as modules. Extensions c
|
|||
|`druid.extensions.coordinates`|The list of extensions to include.|[]|
|
||||
|
||||
### Zookeeper
|
||||
|
||||
Druid uses [Curator](http://curator.incubator.apache.org/) for all [Zookeeper](http://zookeeper.apache.org/) interactions.
|
||||
We recommend just setting the base ZK path and the ZK service host, but all ZK paths that Druid uses can be overwritten to absolute paths.
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.zk.paths.base`|Base Zookeeper path.|`/druid`|
|
||||
|`druid.zk.service.host`|The ZooKeeper hosts to connect to. This is a REQUIRED property and therefore a host address must be supplied.|none|
|
||||
|`druid.zk.service.sessionTimeoutMs`|ZooKeeper session timeout, in milliseconds.|30000|
|
||||
|`druid.curator.compress`|Boolean flag for whether or not created Znodes should be compressed.|false|
|
||||
|
||||
We recommend just setting the base ZK path, but all ZK paths that Druid uses can be overwritten.
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.zk.paths.base`|Base Zookeeper path.|druid|
|
||||
|`druid.zk.paths.propertiesPath`|Zookeeper properties path.|druid/properties|
|
||||
|`druid.zk.paths.announcementsPath`|Druid node announcement path.|druid/announcements|
|
||||
|`druid.zk.paths.liveSegmentsPath`|Current path for where Druid nodes announce their segments.|druid/segments|
|
||||
|`druid.zk.paths.loadQueuePath`|Entries here cause historical nodes to load and drop segments.|druid/loadQueue|
|
||||
|`druid.zk.paths.coordinatorPath`|Used by the coordinator for leader election.|druid/coordinator|
|
||||
|`druid.zk.paths.servedSegmentsPath`|@Deprecated. Legacy path for where Druid nodes announce their segments.|druid/servedSegments|
|
||||
|
||||
The indexing service also uses its own set of paths. These configs can be included in the common configuration.
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.zk.paths.indexer.announcementsPath`|Middle managers announce themselves here.|druid/indexer/announcements|
|
||||
|`druid.zk.paths.indexer.tasksPath`|Used to assign tasks to middle managers.|druid/indexer/tasks|
|
||||
|`druid.zk.paths.indexer.statusPath`|Parent path for announcement of task statuses.|druid/indexer/status|
|
||||
|`druid.zk.paths.indexer.leaderLatchPath`|Used for Overlord leader election.|druid/indexer/leaderLatchPath|
|
||||
|
||||
The following path is used service discovery.
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.discovery.curator.path`|Services announce themselves under this ZooKeeper path.|/druid/discovery|
|
||||
See the [Zookeeper](ZooKeeper.html) page for more information on configuration options for ZK integration.
|
||||
|
||||
### Request Logging
|
||||
|
||||
|
|
|
|||
|
|
@ -14,46 +14,44 @@ Druid uses [ZooKeeper](http://zookeeper.apache.org/) (ZK) for management of curr
|
|||
|
||||
ZooKeeper paths are set via the `runtime.properties` configuration file. Druid will automatically create paths that do not exist, so typos in config files is a very easy way to become split-brained.
|
||||
|
||||
There is a prefix path that is required and can be used as the only (well, kinda, see the note below) path-related zookeeper configuration parameter (everything else will be a default based on the prefix):
|
||||
|
||||
```
|
||||
druid.zk.paths.base
|
||||
```
|
||||
|
||||
You can also override each individual path (defaults are shown below):
|
||||
|
||||
```
|
||||
druid.zk.paths.propertiesPath=${druid.zk.paths.base}/properties
|
||||
druid.zk.paths.announcementsPath=${druid.zk.paths.base}/announcements
|
||||
druid.zk.paths.servedSegmentsPath=${druid.zk.paths.base}/servedSegments
|
||||
druid.zk.paths.loadQueuePath=${druid.zk.paths.base}/loadQueue
|
||||
druid.zk.paths.coordinatorPath=${druid.zk.paths.base}/coordinator
|
||||
druid.zk.paths.indexer.announcementsPath=${druid.zk.paths.base}/indexer/announcements
|
||||
druid.zk.paths.indexer.tasksPath=${druid.zk.paths.base}/indexer/tasks
|
||||
druid.zk.paths.indexer.statusPath=${druid.zk.paths.base}/indexer/status
|
||||
druid.zk.paths.indexer.leaderLatchPath=${druid.zk.paths.base}/indexer/leaderLatchPath
|
||||
```
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.zk.paths.base`|Base Zookeeper path.|druid|
|
||||
|`druid.zk.paths.propertiesPath`|Zookeeper properties path.|druid/properties|
|
||||
|`druid.zk.paths.announcementsPath`|Druid node announcement path.|druid/announcements|
|
||||
|`druid.zk.paths.servedSegmentsPath`|Legacy path for where Druid nodes announce their segments.|druid/servedSegments|
|
||||
|`druid.zk.paths.liveSegmentsPath`|Current path for where Druid nodes announce their segments.|druid/segments|
|
||||
|`druid.zk.paths.loadQueuePath`|Entries here cause historical nodes to load and drop segments.|druid/loadQueue|
|
||||
|`druid.zk.paths.coordinatorPath`|Used by the coordinator for leader election.|druid/coordinator|
|
||||
|`druid.zk.paths.indexer.announcementsPath`|Middle managers announce themselves here.|druid/indexer/announcements|
|
||||
|`druid.zk.paths.indexer.tasksPath`|Used to assign tasks to middle managers.|druid/indexer/tasks|
|
||||
|`druid.zk.paths.indexer.statusPath`|Parent path for announcement of task statuses.|druid/indexer/status|
|
||||
|`druid.zk.paths.indexer.leaderLatchPath`|Used for Overlord leader election.|druid/indexer/leaderLatchPath|
|
||||
|`druid.zk.service.host`|The ZooKeeper hosts to connect to. This is a REQUIRED property and therefore a host address must be supplied.|none|
|
||||
|`druid.zk.service.sessionTimeoutMs`|ZooKeeper session timeout, in milliseconds.|`30000`|
|
||||
|`druid.curator.compress`|Boolean flag for whether or not created Znodes should be compressed.|`false`|
|
||||
|
||||
### Path Configuration
|
||||
Druid interacts with ZK through a set of standard path configurations. We recommend just setting the base ZK path, but all ZK paths that Druid uses can be overwritten to absolute paths.
|
||||
|
||||
NOTE: We also use Curator’s service discovery module to expose some services via zookeeper. This also uses a zookeeper path, but this path is **not** affected by `druid.zk.paths.base` and **must** be specified separately. This property is
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.zk.paths.base`|Base Zookeeper path.|`/druid`|
|
||||
|`druid.zk.paths.propertiesPath`|Zookeeper properties path.|`${druid.zk.paths.base}/properties`|
|
||||
|`druid.zk.paths.announcementsPath`|Druid node announcement path.|`${druid.zk.paths.base}/announcements`|
|
||||
|`druid.zk.paths.liveSegmentsPath`|Current path for where Druid nodes announce their segments.|`${druid.zk.paths.base}/segments`|
|
||||
|`druid.zk.paths.loadQueuePath`|Entries here cause historical nodes to load and drop segments.|`${druid.zk.paths.base}/loadQueue`|
|
||||
|`druid.zk.paths.coordinatorPath`|Used by the coordinator for leader election.|`${druid.zk.paths.base}/coordinator`|
|
||||
|`druid.zk.paths.servedSegmentsPath`|@Deprecated. Legacy path for where Druid nodes announce their segments.|`${druid.zk.paths.base}/servedSegments`|
|
||||
|
||||
```
|
||||
druid.zk.paths.discoveryPath
|
||||
```
|
||||
The indexing service also uses its own set of paths. These configs can be included in the common configuration.
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.zk.paths.indexer.base`|Base zookeeper path for |`${druid.zk.paths.base}/indexer`|
|
||||
|`druid.zk.paths.indexer.announcementsPath`|Middle managers announce themselves here.|`${druid.zk.paths.indexer.base}/announcements`|
|
||||
|`druid.zk.paths.indexer.tasksPath`|Used to assign tasks to middle managers.|`${druid.zk.paths.indexer.base}/tasks`|
|
||||
|`druid.zk.paths.indexer.statusPath`|Parent path for announcement of task statuses.|`${druid.zk.paths.indexer.base}/status`|
|
||||
|`druid.zk.paths.indexer.leaderLatchPath`|Used for Overlord leader election.|`${druid.zk.paths.indexer.base}/leaderLatchPath`|
|
||||
|
||||
If `druid.zk.paths.base` and `druid.zk.paths.indexer.base` are both set, and none of the other `druid.zk.paths.*` or `druid.zk.paths.indexer.*` values are set, then the other properties will be evaluated relative to their respective `base`.
|
||||
For example, if `druid.zk.paths.base` is set to `/druid1` and `druid.zk.paths.indexer.base` is set to `/druid2` then `druid.zk.paths.announcementsPath` will default to `/druid1/announcements` while `druid.zk.paths.indexer.announcementsPath` will default to `/druid2/announcements`.
|
||||
|
||||
The following path is used service discovery and are **not** affected by `druid.zk.paths.base` and **must** be specified separately.
|
||||
|
||||
|Property|Description|Default|
|
||||
|--------|-----------|-------|
|
||||
|`druid.discovery.curator.path`|Services announce themselves under this ZooKeeper path.|`/druid/discovery`|
|
||||
|
||||
### Coordinator Leader Election
|
||||
|
||||
|
|
|
|||
|
|
@ -65,12 +65,12 @@ public class IndexerZkConfig
|
|||
|
||||
private String defaultIndexerPath(final String subPath)
|
||||
{
|
||||
return getZkPathsConfig().defaultPath(ZKPaths.makePath(getBase(), subPath));
|
||||
return ZKPaths.makePath(getBase(), subPath);
|
||||
}
|
||||
|
||||
public String getBase()
|
||||
{
|
||||
return base == null ? "indexer" : base;
|
||||
return base == null ? getZkPathsConfig().defaultPath("indexer") : base;
|
||||
}
|
||||
|
||||
public String getAnnouncementsPath()
|
||||
|
|
|
|||
|
|
@ -182,6 +182,44 @@ public class IndexerZkConfigTest
|
|||
Assert.assertEquals(clobberableProperties.size(), assertions);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
public void testIndexerBaseOverride()
|
||||
{
|
||||
final String overrideValue = "/foo/bar/baz";
|
||||
final String indexerPropertyKey = indexerPropertyString + ".base";
|
||||
final String priorValue = System.getProperty(indexerPropertyKey);
|
||||
System.setProperty(indexerPropertyKey, overrideValue); // Set it here so that the binding picks it up
|
||||
final Injector injector = Initialization.makeInjectorWithModules(
|
||||
GuiceInjectors.makeStartupInjector(),
|
||||
ImmutableList.<Module>of(simpleZkConfigModule)
|
||||
);
|
||||
propertyValues.clear();
|
||||
propertyValues.setProperty(indexerPropertyKey, overrideValue); // Have to set it here as well annoyingly enough
|
||||
|
||||
|
||||
JsonConfigurator configurator = injector.getBinding(JsonConfigurator.class).getProvider().get();
|
||||
|
||||
JsonConfigProvider<IndexerZkConfig> indexerPathsConfig = JsonConfigProvider.of(
|
||||
indexerPropertyString,
|
||||
IndexerZkConfig.class
|
||||
);
|
||||
indexerPathsConfig.inject(propertyValues, configurator);
|
||||
IndexerZkConfig indexerZkConfig = indexerPathsConfig.get().get();
|
||||
|
||||
|
||||
// Rewind value before we potentially fail
|
||||
if(priorValue == null){
|
||||
System.clearProperty(indexerPropertyKey);
|
||||
}else {
|
||||
System.setProperty(indexerPropertyKey, priorValue);
|
||||
}
|
||||
|
||||
Assert.assertEquals(overrideValue, indexerZkConfig.getBase());
|
||||
Assert.assertEquals(overrideValue + "/announcements", indexerZkConfig.getAnnouncementsPath());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExactConfig()
|
||||
{
|
||||
|
|
@ -205,7 +243,7 @@ public class IndexerZkConfigTest
|
|||
|
||||
IndexerZkConfig indexerZkConfig = new IndexerZkConfig(zkPathsConfig1, null, null, null, null, null);
|
||||
|
||||
Assert.assertEquals("indexer", indexerZkConfig.getBase());
|
||||
Assert.assertEquals("/druid/metrics/indexer", indexerZkConfig.getBase());
|
||||
Assert.assertEquals("/druid/metrics/indexer/announcements", indexerZkConfig.getAnnouncementsPath());
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue