Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix lookups endpoint collisions (#5058) 

* fix lookups endpoint collissions

* fix errors
This commit is contained in:
Himanshu 2017-11-09 19:39:53 -06:00 committed by Gian Merlino
parent 81f249874b
commit bbb678efd7
4 changed files with 29 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
docs/content/querying/lookups.md
View File

@ -48,12 +48,12 @@ The tiers for lookups are completely independent of historical tiers.
These configs are accessed using JSON through the following URI template These configs are accessed using JSON through the following URI template
``` ```
http://<COORDINATOR_IP>:<PORT>/druid/coordinator/v1/lookups/{tier}/{id} http://<COORDINATOR_IP>:<PORT>/druid/coordinator/v1/lookups/config/{tier}/{id}
``` ```
All URIs below are assumed to have `http://<COORDINATOR_IP>:<PORT>` prepended. All URIs below are assumed to have `http://<COORDINATOR_IP>:<PORT>` prepended.
If you have NEVER configured lookups before, you MUST post an empty json object `{}` to `/druid/coordinator/v1/lookups` to initialize the configuration. If you have NEVER configured lookups before, you MUST post an empty json object `{}` to `/druid/coordinator/v1/lookups/config` to initialize the configuration.
These endpoints will return one of the following results: These endpoints will return one of the following results:
@ -70,7 +70,7 @@ The coordinator periodically checks if any of the nodes need to load/drop lookup
# API for configuring lookups # API for configuring lookups
## Bulk update ## Bulk update
Lookups can be updated in bulk by posting a JSON object to `/druid/coordinator/v1/lookups`. The format of the json object is as follows: Lookups can be updated in bulk by posting a JSON object to `/druid/coordinator/v1/lookups/config`. The format of the json object is as follows:
```json ```json
{ {
@ -188,9 +188,9 @@ For example, a config might look something like:
All entries in the map will UPDATE existing entries. No entries will be deleted. All entries in the map will UPDATE existing entries. No entries will be deleted.
## Update Lookup ## Update Lookup
A `POST` to a particular lookup extractor factory via `/druid/coordinator/v1/lookups/{tier}/{id}` will update that specific extractor factory. A `POST` to a particular lookup extractor factory via `/druid/coordinator/v1/lookups/config/{tier}/{id}` will update that specific extractor factory.
For example, a post to `/druid/coordinator/v1/lookups/realtime_customer1/site_id_customer1` might contain the following: For example, a post to `/druid/coordinator/v1/lookups/config/realtime_customer1/site_id_customer1` might contain the following:
```json ```json
{ {
@ -209,7 +209,7 @@ This will replace the `site_id_customer1` lookup in the `realtime_customer1` wit
## Get Lookup ## Get Lookup
A `GET` to a particular lookup extractor factory is accomplished via `/druid/coordinator/v1/lookups/{tier}/{id}` A `GET` to a particular lookup extractor factory is accomplished via `/druid/coordinator/v1/lookups/{tier}/{id}`
Using the prior example, a `GET` to `/druid/coordinator/v1/lookups/realtime_customer2/site_id_customer2` should return Using the prior example, a `GET` to `/druid/coordinator/v1/lookups/config/realtime_customer2/site_id_customer2` should return
```json ```json
{ {
@ -224,14 +224,14 @@ Using the prior example, a `GET` to `/druid/coordinator/v1/lookups/realtime_cust
``` ```
## Delete Lookup ## Delete Lookup
A `DELETE` to `/druid/coordinator/v1/lookups/{tier}/{id}` will remove that lookup from the cluster. A `DELETE` to `/druid/coordinator/v1/lookups/config/{tier}/{id}` will remove that lookup from the cluster.
## List tier names ## List tier names
A `GET` to `/druid/coordinator/v1/lookups` will return a list of known tier names in the dynamic configuration. A `GET` to `/druid/coordinator/v1/lookups/config` will return a list of known tier names in the dynamic configuration.
To discover a list of tiers currently active in the cluster **instead of** ones known in the dynamic configuration, the parameter `discover=true` can be added as per `/druid/coordinator/v1/lookups?discover=true`. To discover a list of tiers currently active in the cluster **instead of** ones known in the dynamic configuration, the parameter `discover=true` can be added as per `/druid/coordinator/v1/lookups?discover=true`.
## List lookup names ## List lookup names
A `GET` to `/druid/coordinator/v1/lookups/{tier}` will return a list of known lookup names for that tier. A `GET` to `/druid/coordinator/v1/lookups/config/{tier}` will return a list of known lookup names for that tier.
# Additional API related to status of configured lookups # Additional API related to status of configured lookups
These end points can be used to get the propagation status of configured lookups to lookup nodes such as historicals. These end points can be used to get the propagation status of configured lookups to lookup nodes such as historicals.

2
server/src/main/java/io/druid/query/lookup/LookupReferencesManager.java
View File

@ -577,7 +577,7 @@ public class LookupReferencesManager
return druidLeaderClient.go( return druidLeaderClient.go(
druidLeaderClient.makeRequest( druidLeaderClient.makeRequest(
HttpMethod.GET, HttpMethod.GET,
StringUtils.format("/druid/coordinator/v1/lookups/%s?detailed=true", tier) StringUtils.format("/druid/coordinator/v1/lookups/config/%s?detailed=true", tier)
) )
); );
} }

10
server/src/main/java/io/druid/server/http/LookupCoordinatorResource.java
View File

@ -90,6 +90,7 @@ public class LookupCoordinatorResource
} }
@GET @GET
@Path("/config")
@Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
public Response getTiers( public Response getTiers(
@DefaultValue("false") @QueryParam("discover") boolean discover @DefaultValue("false") @QueryParam("discover") boolean discover
@ -113,6 +114,7 @@ public class LookupCoordinatorResource
} }
@POST @POST
@Path("/config")
@Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
@Consumes({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Consumes({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
public Response updateAllLookups( public Response updateAllLookups(
@ -148,7 +150,7 @@ public class LookupCoordinatorResource
@DELETE @DELETE
@Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
@Path("/{tier}/{lookup}") @Path("/config/{tier}/{lookup}")
public Response deleteLookup( public Response deleteLookup(
@PathParam("tier") String tier, @PathParam("tier") String tier,
@PathParam("lookup") String lookup, @PathParam("lookup") String lookup,
@ -184,7 +186,7 @@ public class LookupCoordinatorResource
@POST @POST
@Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
@Path("/{tier}/{lookup}") @Path("/config/{tier}/{lookup}")
public Response createOrUpdateLookup( public Response createOrUpdateLookup(
@PathParam("tier") String tier, @PathParam("tier") String tier,
@PathParam("lookup") String lookup, @PathParam("lookup") String lookup,
@ -234,7 +236,7 @@ public class LookupCoordinatorResource
@GET @GET
@Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
@Path("/{tier}/{lookup}") @Path("/config/{tier}/{lookup}")
public Response getSpecificLookup( public Response getSpecificLookup(
@PathParam("tier") String tier, @PathParam("tier") String tier,
@PathParam("lookup") String lookup @PathParam("lookup") String lookup
@ -267,7 +269,7 @@ public class LookupCoordinatorResource
@GET @GET
@Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE}) @Produces({MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE})
@Path("/{tier}") @Path("/config/{tier}")
public Response getSpecificTier( public Response getSpecificTier(
@PathParam("tier") String tier, @PathParam("tier") String tier,
@DefaultValue("false") @QueryParam("detailed") boolean detailed @DefaultValue("false") @QueryParam("detailed") boolean detailed

32
server/src/test/java/io/druid/query/lookup/LookupReferencesManagerTest.java
View File

@ -56,14 +56,8 @@ public class LookupReferencesManagerTest
private LookupListeningAnnouncerConfig config; private LookupListeningAnnouncerConfig config;
private static final String propertyBase = "some.property";
private static final String LOOKUP_TIER = "lookupTier"; private static final String LOOKUP_TIER = "lookupTier";
private static final int LOOKUP_THREADS = 1;
private static final boolean LOOKUP_DISABLE = false;
LookupExtractorFactory lookupExtractorFactory; LookupExtractorFactory lookupExtractorFactory;
LookupExtractorFactoryContainer container; LookupExtractorFactoryContainer container;
@ -110,7 +104,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -171,7 +165,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -209,7 +203,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -240,7 +234,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -268,7 +262,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -298,7 +292,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -332,7 +326,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -360,7 +354,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -411,7 +405,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -453,7 +447,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -529,7 +523,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request); .andReturn(request);
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
HttpResponseStatus.OK, HttpResponseStatus.OK,
@ -555,7 +549,7 @@ public class LookupReferencesManagerTest
Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx")); Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request) .andReturn(request)
.anyTimes(); .anyTimes();
FullResponseHolder responseHolder = new FullResponseHolder( FullResponseHolder responseHolder = new FullResponseHolder(
@ -579,7 +573,7 @@ public class LookupReferencesManagerTest
reset(druidLeaderClient); reset(druidLeaderClient);
expect(config.getLookupTier()).andReturn(LOOKUP_TIER); expect(config.getLookupTier()).andReturn(LOOKUP_TIER);
replay(config); replay(config);
expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/lookupTier?detailed=true")) expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true"))
.andReturn(request) .andReturn(request)
.anyTimes(); .anyTimes();
expect(druidLeaderClient.go(request)).andThrow(new IllegalStateException()).anyTimes(); expect(druidLeaderClient.go(request)).andThrow(new IllegalStateException()).anyTimes();