diff --git a/cassandra-storage/pom.xml b/cassandra-storage/pom.xml index 9e3674c0f8a..f65445ecf7e 100644 --- a/cassandra-storage/pom.xml +++ b/cassandra-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/common/pom.xml b/common/pom.xml index 0c43299e1a6..44578469660 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/docs/content/index.md b/docs/content/index.md index 89931b1f855..529a2325436 100644 --- a/docs/content/index.md +++ b/docs/content/index.md @@ -31,20 +31,20 @@ We have more details about the general design of the system and why you might wa When Druid? ---------- -* You need to do interactive, fast, exploration of large amounts of data -* You need analytics (not key value store) -* You have a lot of data (10s of Billions of events added per day, 10s of TB of data added per day) -* You want to do your analysis on data as it’s happening (realtime) -* Your store needs to be always-on, 24x7x365 and years into the future. +* You need to do interactive, fast, exploration on large amounts of data +* You need analytics (not a key-value store) +* You have a lot of data (10s of billions of events added per day, 10s of TB of data added per day) +* You want to do your analysis on data as it’s happening (in real-time) +* You need a data store that is always available, 24x7x365, and years into the future. Not Druid? ---------- -* The amount of data you have can easily be handled by MySql -* Your querying for individual entries or doing lookups (Not Analytics) -* Batch is good enough -* Canned queries is good enough +* The amount of data you have can easily be handled by MySQL +* You're querying for individual entries or doing lookups (not analytics) +* Batch ingestion is good enough +* Canned queries are good enough * Downtime is no big deal diff --git a/examples/pom.xml b/examples/pom.xml index 6d31c805814..5d7f1154814 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/hdfs-storage/pom.xml b/hdfs-storage/pom.xml index ebc7b4609fa..6f9014515d9 100644 --- a/hdfs-storage/pom.xml +++ b/hdfs-storage/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/hll/pom.xml b/hll/pom.xml index 4b66317204f..18b04898f58 100644 --- a/hll/pom.xml +++ b/hll/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index e7ea2010162..e33401f128d 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/indexing-service/pom.xml b/indexing-service/pom.xml index 81b31b48131..e1f64344c52 100644 --- a/indexing-service/pom.xml +++ b/indexing-service/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java index 878f950f0c4..ca38f90e3cc 100644 --- a/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java +++ b/indexing-service/src/main/java/io/druid/indexing/common/task/HadoopIndexTask.java @@ -70,7 +70,7 @@ public class HadoopIndexTask extends AbstractTask @JsonIgnore private final HadoopDruidIndexerSchema schema; @JsonIgnore - private final String hadoopCoordinates; + private final List hadoopDependencyCoordinates; /** * @param schema is used by the HadoopDruidIndexerJob to set up the appropriate parameters @@ -86,7 +86,8 @@ public class HadoopIndexTask extends AbstractTask public HadoopIndexTask( @JsonProperty("id") String id, @JsonProperty("config") HadoopDruidIndexerSchema schema, - @JsonProperty("hadoopCoordinates") String hadoopCoordinates + @JsonProperty("hadoopCoordinates") String hadoopCoordinates, + @JsonProperty("hadoopDependencyCoordinates") List hadoopDependencyCoordinates ) { super( @@ -100,7 +101,9 @@ public class HadoopIndexTask extends AbstractTask Preconditions.checkArgument(schema.getUpdaterJobSpec() == null, "updaterJobSpec must be absent"); this.schema = schema; - this.hadoopCoordinates = (hadoopCoordinates == null ? defaultHadoopCoordinates : hadoopCoordinates); + this.hadoopDependencyCoordinates = hadoopDependencyCoordinates == null ? Arrays.asList( + hadoopCoordinates == null ? defaultHadoopCoordinates : hadoopCoordinates + ) : hadoopDependencyCoordinates; } @Override @@ -132,20 +135,16 @@ public class HadoopIndexTask extends AbstractTask } @JsonProperty - public String getHadoopCoordinates() + public List getHadoopDependencyCoordinates() { - return hadoopCoordinates; + return hadoopDependencyCoordinates; } @SuppressWarnings("unchecked") @Override public TaskStatus run(TaskToolbox toolbox) throws Exception { - // setup Hadoop final DefaultTeslaAether aetherClient = Initialization.getAetherClient(extensionsConfig); - final ClassLoader hadoopLoader = Initialization.getClassLoaderForCoordinates( - aetherClient, hadoopCoordinates - ); final List extensionURLs = Lists.newArrayList(); for (String coordinate : extensionsConfig.getCoordinates()) { @@ -161,7 +160,12 @@ public class HadoopIndexTask extends AbstractTask final List driverURLs = Lists.newArrayList(); driverURLs.addAll(nonHadoopURLs); // put hadoop dependencies last to avoid jets3t & apache.httpcore version conflicts - driverURLs.addAll(Arrays.asList(((URLClassLoader) hadoopLoader).getURLs())); + for (String hadoopDependencyCoordinate : hadoopDependencyCoordinates) { + final ClassLoader hadoopLoader = Initialization.getClassLoaderForCoordinates( + aetherClient, hadoopDependencyCoordinate + ); + driverURLs.addAll(Arrays.asList(((URLClassLoader) hadoopLoader).getURLs())); + } final URLClassLoader loader = new URLClassLoader(driverURLs.toArray(new URL[driverURLs.size()]), null); Thread.currentThread().setContextClassLoader(loader); @@ -240,10 +244,10 @@ public class HadoopIndexTask extends AbstractTask String version = args[1]; final HadoopDruidIndexerSchema theSchema = HadoopDruidIndexerConfig.jsonMapper - .readValue( - schema, - HadoopDruidIndexerSchema.class - ); + .readValue( + schema, + HadoopDruidIndexerSchema.class + ); final HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfigBuilder().withSchema(theSchema) .withVersion(version) @@ -269,10 +273,10 @@ public class HadoopIndexTask extends AbstractTask final String segmentOutputPath = args[2]; final HadoopDruidIndexerSchema theSchema = HadoopDruidIndexerConfig.jsonMapper - .readValue( - schema, - HadoopDruidIndexerSchema.class - ); + .readValue( + schema, + HadoopDruidIndexerSchema.class + ); final HadoopDruidIndexerConfig config = new HadoopDruidIndexerConfigBuilder().withSchema(theSchema) .withWorkingPath(workingPath) diff --git a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java index ba5afb04d3b..5d88d97de48 100644 --- a/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java +++ b/indexing-service/src/test/java/io/druid/indexing/common/task/TaskSerdeTest.java @@ -401,6 +401,7 @@ public class TaskSerdeTest null, null ), + null, null ); diff --git a/kafka-eight/pom.xml b/kafka-eight/pom.xml index fc5436ef0c7..d6197cf9c55 100644 --- a/kafka-eight/pom.xml +++ b/kafka-eight/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/kafka-seven/pom.xml b/kafka-seven/pom.xml index d7f172a45a6..813e86dbe89 100644 --- a/kafka-seven/pom.xml +++ b/kafka-seven/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/pom.xml b/pom.xml index 7ec87899fad..46e0178117a 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ io.druid druid pom - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT druid druid diff --git a/processing/pom.xml b/processing/pom.xml index ac3ac2bac95..655df0ba849 100644 --- a/processing/pom.xml +++ b/processing/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/processing/src/main/java/io/druid/query/BaseQuery.java b/processing/src/main/java/io/druid/query/BaseQuery.java index ed13f9ddf39..71beaa26652 100644 --- a/processing/src/main/java/io/druid/query/BaseQuery.java +++ b/processing/src/main/java/io/druid/query/BaseQuery.java @@ -37,14 +37,14 @@ public abstract class BaseQuery implements Query { public static String QUERYID = "queryId"; private final DataSource dataSource; - private final Map context; + private final Map context; private final QuerySegmentSpec querySegmentSpec; private volatile Duration duration; public BaseQuery( DataSource dataSource, QuerySegmentSpec querySegmentSpec, - Map context + Map context ) { Preconditions.checkNotNull(dataSource, "dataSource can't be null"); @@ -102,28 +102,28 @@ public abstract class BaseQuery implements Query } @JsonProperty - public Map getContext() + public Map getContext() { return context; } @Override - public String getContextValue(String key) + public ContextType getContextValue(String key) { - return context == null ? null : context.get(key); + return context == null ? null : (ContextType) context.get(key); } @Override - public String getContextValue(String key, String defaultValue) + public ContextType getContextValue(String key, ContextType defaultValue) { - String retVal = getContextValue(key); + ContextType retVal = getContextValue(key); return retVal == null ? defaultValue : retVal; } - protected Map computeOverridenContext(Map overrides) + protected Map computeOverridenContext(Map overrides) { - Map overridden = Maps.newTreeMap(); - final Map context = getContext(); + Map overridden = Maps.newTreeMap(); + final Map context = getContext(); if (context != null) { overridden.putAll(context); } @@ -135,28 +135,41 @@ public abstract class BaseQuery implements Query @Override public String getId() { - return getContextValue(QUERYID); + return (String) getContextValue(QUERYID); } @Override public Query withId(String id) { - return withOverriddenContext(ImmutableMap.of(QUERYID, id)); + return withOverriddenContext(ImmutableMap.of(QUERYID, id)); } @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } BaseQuery baseQuery = (BaseQuery) o; - if (context != null ? !context.equals(baseQuery.context) : baseQuery.context != null) return false; - if (dataSource != null ? !dataSource.equals(baseQuery.dataSource) : baseQuery.dataSource != null) return false; - if (duration != null ? !duration.equals(baseQuery.duration) : baseQuery.duration != null) return false; - if (querySegmentSpec != null ? !querySegmentSpec.equals(baseQuery.querySegmentSpec) : baseQuery.querySegmentSpec != null) + if (context != null ? !context.equals(baseQuery.context) : baseQuery.context != null) { return false; + } + if (dataSource != null ? !dataSource.equals(baseQuery.dataSource) : baseQuery.dataSource != null) { + return false; + } + if (duration != null ? !duration.equals(baseQuery.duration) : baseQuery.duration != null) { + return false; + } + if (querySegmentSpec != null + ? !querySegmentSpec.equals(baseQuery.querySegmentSpec) + : baseQuery.querySegmentSpec != null) { + return false; + } return true; } diff --git a/processing/src/main/java/io/druid/query/BySegmentQueryRunner.java b/processing/src/main/java/io/druid/query/BySegmentQueryRunner.java index 79c6a4e03a9..d6150f63456 100644 --- a/processing/src/main/java/io/druid/query/BySegmentQueryRunner.java +++ b/processing/src/main/java/io/druid/query/BySegmentQueryRunner.java @@ -53,7 +53,7 @@ public class BySegmentQueryRunner implements QueryRunner @SuppressWarnings("unchecked") public Sequence run(final Query query) { - if (Boolean.parseBoolean(query.getContextValue("bySegment"))) { + if (Boolean.parseBoolean(query.getContextValue("bySegment"))) { final Sequence baseSequence = base.run(query); return new Sequence() { diff --git a/processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java b/processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java index 1711fb316f9..8e666c30b16 100644 --- a/processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java +++ b/processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java @@ -37,7 +37,7 @@ public abstract class BySegmentSkippingQueryRunner implements QueryRunner @Override public Sequence run(Query query) { - if (Boolean.parseBoolean(query.getContextValue("bySegment"))) { + if (Boolean.parseBoolean(query.getContextValue("bySegment"))) { return baseRunner.run(query); } diff --git a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java index 83d2ff48f98..d3600068a23 100644 --- a/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java @@ -83,7 +83,7 @@ public class ChainedExecutionQueryRunner implements QueryRunner @Override public Sequence run(final Query query) { - final int priority = Integer.parseInt(query.getContextValue("priority", "0")); + final int priority = Integer.parseInt((String) query.getContextValue("priority", "0")); return new BaseSequence>( new BaseSequence.IteratorMaker>() diff --git a/processing/src/main/java/io/druid/query/Druids.java b/processing/src/main/java/io/druid/query/Druids.java index fd2abd13ab8..3ab6b0a8ff7 100644 --- a/processing/src/main/java/io/druid/query/Druids.java +++ b/processing/src/main/java/io/druid/query/Druids.java @@ -304,7 +304,7 @@ public class Druids private QueryGranularity granularity; private List aggregatorSpecs; private List postAggregatorSpecs; - private Map context; + private Map context; private TimeseriesQueryBuilder() { @@ -384,7 +384,7 @@ public class Druids return postAggregatorSpecs; } - public Map getContext() + public Map getContext() { return context; } @@ -465,7 +465,7 @@ public class Druids return this; } - public TimeseriesQueryBuilder context(Map c) + public TimeseriesQueryBuilder context(Map c) { context = c; return this; @@ -505,7 +505,7 @@ public class Druids private QuerySegmentSpec querySegmentSpec; private List dimensions; private SearchQuerySpec querySpec; - private Map context; + private Map context; public SearchQueryBuilder() { @@ -660,7 +660,7 @@ public class Druids return this; } - public SearchQueryBuilder context(Map c) + public SearchQueryBuilder context(Map c) { context = c; return this; @@ -690,7 +690,7 @@ public class Druids { private DataSource dataSource; private QuerySegmentSpec querySegmentSpec; - private Map context; + private Map context; public TimeBoundaryQueryBuilder() { @@ -746,7 +746,7 @@ public class Druids return this; } - public TimeBoundaryQueryBuilder context(Map c) + public TimeBoundaryQueryBuilder context(Map c) { context = c; return this; diff --git a/processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java b/processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java index 77823a60730..2880332e184 100644 --- a/processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java +++ b/processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java @@ -48,7 +48,7 @@ public class FinalizeResultsQueryRunner implements QueryRunner @Override public Sequence run(final Query query) { - final boolean isBySegment = Boolean.parseBoolean(query.getContextValue("bySegment")); + final boolean isBySegment = Boolean.parseBoolean(query.getContextValue("bySegment")); final boolean shouldFinalize = Boolean.parseBoolean(query.getContextValue("finalize", "true")); if (shouldFinalize) { Function finalizerFn; @@ -100,7 +100,7 @@ public class FinalizeResultsQueryRunner implements QueryRunner } return Sequences.map( - baseRunner.run(query.withOverriddenContext(ImmutableMap.of("finalize", "false"))), + baseRunner.run(query.withOverriddenContext(ImmutableMap.of("finalize", "false"))), finalizerFn ); } diff --git a/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java b/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java index fb98968fb43..10dde9b26ea 100644 --- a/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java +++ b/processing/src/main/java/io/druid/query/GroupByParallelQueryRunner.java @@ -83,7 +83,7 @@ public class GroupByParallelQueryRunner implements QueryRunner query, configSupplier.get() ); - final int priority = Integer.parseInt(query.getContextValue("priority", "0")); + final int priority = Integer.parseInt((String) query.getContextValue("priority", "0")); if (Iterables.isEmpty(queryables)) { log.warn("No queryables found."); diff --git a/processing/src/main/java/io/druid/query/Query.java b/processing/src/main/java/io/druid/query/Query.java index d58798539ba..10a84328584 100644 --- a/processing/src/main/java/io/druid/query/Query.java +++ b/processing/src/main/java/io/druid/query/Query.java @@ -70,11 +70,11 @@ public interface Query public Duration getDuration(); - public String getContextValue(String key); + public ContextType getContextValue(String key); - public String getContextValue(String key, String defaultValue); + public ContextType getContextValue(String key, ContextType defaultValue); - public Query withOverriddenContext(Map contextOverride); + public Query withOverriddenContext(Map contextOverride); public Query withQuerySegmentSpec(QuerySegmentSpec spec); diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 80322a29531..3f04f30b8aa 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -86,7 +86,7 @@ public class GroupByQuery extends BaseQuery @JsonProperty("having") HavingSpec havingSpec, @JsonProperty("limitSpec") LimitSpec limitSpec, @JsonProperty("orderBy") LimitSpec orderBySpec, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); @@ -147,7 +147,7 @@ public class GroupByQuery extends BaseQuery HavingSpec havingSpec, LimitSpec orderBySpec, Function, Sequence> orderByLimitFn, - Map context + Map context ) { super(dataSource, querySegmentSpec, context); @@ -222,7 +222,7 @@ public class GroupByQuery extends BaseQuery } @Override - public GroupByQuery withOverriddenContext(Map contextOverride) + public GroupByQuery withOverriddenContext(Map contextOverride) { return new GroupByQuery( getDataSource(), @@ -268,7 +268,7 @@ public class GroupByQuery extends BaseQuery private List postAggregatorSpecs; private HavingSpec havingSpec; - private Map context; + private Map context; private LimitSpec limitSpec = null; private List orderByColumnSpecs = Lists.newArrayList(); @@ -443,7 +443,7 @@ public class GroupByQuery extends BaseQuery return this; } - public Builder setContext(Map context) + public Builder setContext(Map context) { this.context = context; return this; diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java index 0f032f50121..1b77f2299ba 100644 --- a/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java @@ -58,7 +58,7 @@ public class GroupByQueryQueryToolChest extends QueryToolChest NO_MERGE_CONTEXT = ImmutableMap.of(GROUP_BY_MERGE_KEY, "false"); + private static final Map NO_MERGE_CONTEXT = ImmutableMap.of(GROUP_BY_MERGE_KEY, "false"); private final Supplier configSupplier; private GroupByQueryEngine engine; // For running the outer query around a subquery @@ -80,7 +80,7 @@ public class GroupByQueryQueryToolChest extends QueryToolChest run(Query input) { - if (Boolean.valueOf(input.getContextValue(GROUP_BY_MERGE_KEY, "true"))) { + if (Boolean.valueOf((String) input.getContextValue(GROUP_BY_MERGE_KEY, "true"))) { return mergeGroupByResults(((GroupByQuery) input).withOverriddenContext(NO_MERGE_CONTEXT), runner); } else { return runner.run(input); diff --git a/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java index 73c60b840a8..098a5462d3b 100644 --- a/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -42,7 +42,7 @@ public class SegmentMetadataQuery extends BaseQuery @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, @JsonProperty("toInclude") ColumnIncluderator toInclude, @JsonProperty("merge") Boolean merge, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); @@ -77,7 +77,7 @@ public class SegmentMetadataQuery extends BaseQuery } @Override - public Query withOverriddenContext(Map contextOverride) + public Query withOverriddenContext(Map contextOverride) { return new SegmentMetadataQuery( getDataSource(), diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java index 59fe269de51..6e14ef1c1f3 100644 --- a/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java @@ -294,7 +294,7 @@ public class SearchQueryQueryToolChest extends QueryToolChest> @JsonProperty("searchDimensions") List dimensions, @JsonProperty("query") SearchQuerySpec querySpec, @JsonProperty("sort") SearchSortSpec sortSpec, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); @@ -112,7 +112,7 @@ public class SearchQuery extends BaseQuery> } @Override - public SearchQuery withOverriddenContext(Map contextOverrides) + public SearchQuery withOverriddenContext(Map contextOverrides) { return new SearchQuery( getDataSource(), diff --git a/processing/src/main/java/io/druid/query/select/SelectQuery.java b/processing/src/main/java/io/druid/query/select/SelectQuery.java index bcd29cb7f96..7556006734f 100644 --- a/processing/src/main/java/io/druid/query/select/SelectQuery.java +++ b/processing/src/main/java/io/druid/query/select/SelectQuery.java @@ -53,7 +53,7 @@ public class SelectQuery extends BaseQuery> @JsonProperty("dimensions") List dimensions, @JsonProperty("metrics") List metrics, @JsonProperty("pagingSpec") PagingSpec pagingSpec, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); @@ -120,7 +120,7 @@ public class SelectQuery extends BaseQuery> ); } - public SelectQuery withOverriddenContext(Map contextOverrides) + public SelectQuery withOverriddenContext(Map contextOverrides) { return new SelectQuery( getDataSource(), diff --git a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java index bad00f2bb87..357854f1958 100644 --- a/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java @@ -54,7 +54,7 @@ public class TimeBoundaryQuery extends BaseQuery public TimeBoundaryQuery( @JsonProperty("dataSource") DataSource dataSource, @JsonProperty("intervals") QuerySegmentSpec querySegmentSpec, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super( @@ -78,7 +78,7 @@ public class TimeBoundaryQuery extends BaseQuery } @Override - public TimeBoundaryQuery withOverriddenContext(Map contextOverrides) + public TimeBoundaryQuery withOverriddenContext(Map contextOverrides) { return new TimeBoundaryQuery( getDataSource(), diff --git a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java index a1de320f5ec..3a03018a63e 100644 --- a/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java @@ -55,7 +55,7 @@ public class TimeseriesQuery extends BaseQuery> @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("aggregations") List aggregatorSpecs, @JsonProperty("postAggregations") List postAggregatorSpecs, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); @@ -116,7 +116,7 @@ public class TimeseriesQuery extends BaseQuery> ); } - public TimeseriesQuery withOverriddenContext(Map contextOverrides) + public TimeseriesQuery withOverriddenContext(Map contextOverrides) { return new TimeseriesQuery( getDataSource(), diff --git a/processing/src/main/java/io/druid/query/topn/TopNQuery.java b/processing/src/main/java/io/druid/query/topn/TopNQuery.java index dc59a663b2c..0e7a796d045 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQuery.java @@ -62,7 +62,7 @@ public class TopNQuery extends BaseQuery> @JsonProperty("granularity") QueryGranularity granularity, @JsonProperty("aggregations") List aggregatorSpecs, @JsonProperty("postAggregations") List postAggregatorSpecs, - @JsonProperty("context") Map context + @JsonProperty("context") Map context ) { super(dataSource, querySegmentSpec, context); @@ -178,7 +178,7 @@ public class TopNQuery extends BaseQuery> ); } - public TopNQuery withOverriddenContext(Map contextOverrides) + public TopNQuery withOverriddenContext(Map contextOverrides) { return new TopNQuery( getDataSource(), diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java index 21efd3b8351..8f78ffe6191 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryBuilder.java @@ -69,7 +69,7 @@ public class TopNQueryBuilder private QueryGranularity granularity; private List aggregatorSpecs; private List postAggregatorSpecs; - private Map context; + private Map context; public TopNQueryBuilder() { @@ -130,7 +130,7 @@ public class TopNQueryBuilder return postAggregatorSpecs; } - public Map getContext() + public Map getContext() { return context; } @@ -290,7 +290,7 @@ public class TopNQueryBuilder return this; } - public TopNQueryBuilder context(Map c) + public TopNQueryBuilder context(Map c) { context = c; return this; diff --git a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java index 6f0400a1f1b..a7d77fde396 100644 --- a/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/topn/TopNQueryQueryToolChest.java @@ -339,7 +339,7 @@ public class TopNQueryQueryToolChest extends QueryToolChest dummy.ps + zip $@ $*.pdf $*.tex dummy.ps clean : - @rm -f *.aux *.bbl *.blg *.log + @rm -f *.aux *.bbl *.blg *.out *.log dummy.ps *.zip %.tex : %.bib diff --git a/publications/whitepaper/druid.pdf b/publications/whitepaper/druid.pdf index c6d7f5924f3..a7467101b38 100644 Binary files a/publications/whitepaper/druid.pdf and b/publications/whitepaper/druid.pdf differ diff --git a/publications/whitepaper/druid.tex b/publications/whitepaper/druid.tex index a032d87cc6a..4736b5a4453 100644 --- a/publications/whitepaper/druid.tex +++ b/publications/whitepaper/druid.tex @@ -1,4 +1,18 @@ -\documentclass{acm_proc_article-sp} +\documentclass{sig-alternate-2013} + +\newfont{\mycrnotice}{ptmr8t at 7pt} +\newfont{\myconfname}{ptmri8t at 7pt} +\let\crnotice\mycrnotice% +\let\confname\myconfname% +\permission{Permission to make digital or hard copies of all or part of this work for personal or classroom use is granted without fee provided that copies are not made or distributed for profit or commercial advantage and that copies bear this notice and the full citation on the first page. Copyrights for components of this work owned by others than the author(s) must be honored. Abstracting with credit is permitted. To copy otherwise, or republish, to post on servers or to redistribute to lists, requires prior specific permission and/or a fee. Request permissions from permissions@acm.org.} +\conferenceinfo{SIGMOD'14,}{June 22--27, 2014, Snowbird, UT, USA. \\ +{\mycrnotice{Copyright is held by the owner/author(s). Publication rights licensed to ACM.}}} +\copyrightetc{ACM \the\acmcopyr} +\crdata{978-1-4503-2376-5/14/06\ ...\$15.00.\\ +\href{http://dx.doi.org/10.1145/2588555.2595631}{http://dx.doi.org/10.1145/2588555.2595631}} +\clubpenalty=10000 +\widowpenalty = 10000 + \usepackage{graphicx} \usepackage{balance} \usepackage{fontspec} @@ -7,28 +21,47 @@ \graphicspath{{figures/}} \usepackage{enumitem} -\hyphenation{metamarkets nelson} +\hyphenation{metamarkets nelson cheddar} \begin{document} % ****************** TITLE **************************************** -\title{Druid: A Real-time Analytical Data Store} +\title{Druid} +\subtitle{A Real-time Analytical Data Store} % ****************** AUTHORS ************************************** \numberofauthors{6} \author{ -\alignauthor Fangjin Yang, Eric Tschetter, Xavier Léauté, Nelson Ray, Gian Merlino, Deep Ganguli\\ -\email{\{fangjin, cheddar, xavier, nelson, gian, deep\}@metamarkets.com} +\alignauthor Fangjin Yang\\ + \affaddr{Metamarkets Group, Inc.}\\ + % \affaddr{625 2nd St Suite 230}\\ + % \affaddr{San Francisco, CA 94107}\\ + \email{fangjin@metamarkets.com} +\alignauthor Eric Tschetter\\ + \email{echeddar@gmail.com} +\alignauthor Xavier Léauté\\ + \affaddr{Metamarkets Group, Inc.}\\ + \email{xavier@metamarkets.com} +\and +\alignauthor Nelson Ray\\ + \email{ncray86@gmail.com} +\alignauthor Gian Merlino\\ + \affaddr{Metamarkets Group, Inc.}\\ + \email{gian@metamarkets.com} +\alignauthor Deep Ganguli\\ + \affaddr{Metamarkets Group, Inc.}\\ + \email{deep@metamarkets.com} } \date{21 March 2013} +% ****************** AUTHORS ************************************** + \maketitle \begin{abstract} -Druid is an open -source\footnote{\href{https://github.com/metamx/druid}{https://github.com/metamx/druid}} +Druid is an open source\footnote{\href{http://druid.io/}{http://druid.io/} \href{https://github.com/metamx/druid}{https://github.com/metamx/druid}} data store designed for real-time exploratory analytics on large data sets. The system combines a column-oriented storage layout, a distributed, shared-nothing architecture, and an advanced indexing structure to allow for @@ -37,13 +70,19 @@ this paper, we describe Druid's architecture, and detail how it supports fast aggregations, flexible filters, and low latency data ingestion. \end{abstract} +% A category with the (minimum) three required fields +\category{H.2.4}{Database Management}{Systems}[Distributed databases] +% \category{D.2.8}{Software Engineering}{Metrics}[complexity measures, performance measures] +\keywords{distributed; real-time; fault-tolerant; analytics; column-oriented; OLAP} + + \section{Introduction} In recent years, the proliferation of internet technology has created a surge in machine-generated events. Individually, these events contain minimal useful information and are of low value. Given the time and resources required to extract meaning from large collections of events, many companies were willing to discard this data instead. Although -infrastructure has been built to handle event based data (e.g. IBM's +infrastructure has been built to handle event-based data (e.g. IBM's Netezza\cite{singh2011introduction}, HP's Vertica\cite{bear2012vertica}, and EMC's Greenplum\cite{miner2012unified}), they are largely sold at high price points and are only targeted towards those companies who can afford the offering. @@ -73,13 +112,12 @@ highly concurrent environment (1000+ users), Hadoop wasn't going to meet our needs. We explored different solutions in the space, and after trying both Relational Database Management Systems and NoSQL architectures, we came to the conclusion that there was nothing in the open source world that -could be fully leveraged for our requirements. - -We ended up creating Druid, an open-source, distributed, column-oriented, -real-time analytical data store. In many ways, Druid shares similarities with -other OLAP systems \cite{oehler2012ibm, schrader2009oracle, lachev2005applied}, +could be fully leveraged for our requirements. We ended up creating Druid, an +open-source, distributed, column-oriented, real-time analytical data store. In +many ways, Druid shares similarities with other OLAP systems +\cite{oehler2012ibm, schrader2009oracle, lachev2005applied}, interactive query systems \cite{melnik2010dremel}, main-memory databases -\cite{farber2012sap}, and widely-known distributed data stores +\cite{farber2012sap}, as well as widely known distributed data stores \cite{chang2008bigtable, decandia2007dynamo, lakshman2010cassandra}. The distribution and query model also borrow ideas from current generation search infrastructure \cite{linkedin2013senseidb, apache2013solr, @@ -91,7 +129,6 @@ service, and attempts to help inform anyone who faces a similar problem about a potential method of solving it. Druid is deployed in production at several technology companies\footnote{\href{http://druid.io/druid.html}{http://druid.io/druid.html}}. - The structure of the paper is as follows: we first describe the problem in Section \ref{sec:problem-definition}. Next, we detail system architecture from the point of view of how data flows through the system in Section @@ -105,6 +142,21 @@ in Section \ref{sec:related}. \section{Problem Definition} \label{sec:problem-definition} +\begin{table*} + \centering + \begin{tabular}{| l | l | l | l | l | l | l | l |} + \hline + \textbf{Timestamp} & \textbf{Page} & \textbf{Username} & \textbf{Gender} & \textbf{City} & \textbf{Characters Added} & \textbf{Characters Removed} \\ \hline + 2011-01-01T01:00:00Z & Justin Bieber & Boxer & Male & San Francisco & 1800 & 25 \\ \hline + 2011-01-01T01:00:00Z & Justin Bieber & Reach & Male & Waterloo & 2912 & 42 \\ \hline + 2011-01-01T02:00:00Z & Ke\$ha & Helz & Male & Calgary & 1953 & 17 \\ \hline + 2011-01-01T02:00:00Z & Ke\$ha & Xeno & Male & Taiyuan & 3194 & 170 \\ \hline + \end{tabular} + \caption{Sample Druid data for edits that have occurred on Wikipedia.} + \label{tab:sample_data} +\end{table*} + + Druid was originally designed to solve problems around ingesting and exploring large quantities of transactional events (log data). This form of timeseries data is commonly found in OLAP workflows and the nature of the data tends to be @@ -120,20 +172,6 @@ there are a set of metric columns that contain values (usually numeric) that can be aggregated, such as the number of characters added or removed in an edit. -\begin{table*} - \centering - \begin{tabular}{| l | l | l | l | l | l | l | l |} - \hline - \textbf{Timestamp} & \textbf{Page} & \textbf{Username} & \textbf{Gender} & \textbf{City} & \textbf{Characters Added} & \textbf{Characters Removed} \\ \hline - 2011-01-01T01:00:00Z & Justin Bieber & Boxer & Male & San Francisco & 1800 & 25 \\ \hline - 2011-01-01T01:00:00Z & Justin Bieber & Reach & Male & Waterloo & 2912 & 42 \\ \hline - 2011-01-01T02:00:00Z & Ke\$ha & Helz & Male & Calgary & 1953 & 17 \\ \hline - 2011-01-01T02:00:00Z & Ke\$ha & Xeno & Male & Taiyuan & 3194 & 170 \\ \hline - \end{tabular} - \caption{Sample Druid data for edits that have occurred on Wikipedia.} - \label{tab:sample_data} -\end{table*} - Our goal is to rapidly compute drill-downs and aggregates over this data. We want to answer questions like “How many edits were made on the page Justin Bieber from males in San Francisco?” and “What is the average number of @@ -146,7 +184,7 @@ Relational Database Management Systems (RDBMS) and NoSQL key/value stores were unable to provide a low latency data ingestion and query platform for interactive applications \cite{tschetter2011druid}. In the early days of Metamarkets, we were focused on building a hosted dashboard that would allow -users to arbitrary explore and visualize event streams. The data store +users to arbitrarily explore and visualize event streams. The data store powering the dashboard needed to return queries fast enough that the data visualizations built on top of it could provide users with an interactive experience. @@ -178,12 +216,13 @@ designed to perform a specific set of things. We believe this design separates concerns and simplifies the complexity of the system. The different node types operate fairly independent of each other and there is minimal interaction among them. Hence, intra-cluster communication failures have minimal impact -on data availability. To solve complex data analysis problems, the different -node types come together to form a fully working system. The name Druid comes -from the Druid class in many role-playing games: it is a shape-shifter, capable -of taking on many different forms to fulfill various different roles in a -group. The composition of and flow of data in a Druid cluster are shown in -Figure~\ref{fig:cluster}. +on data availability. + +To solve complex data analysis problems, the different +node types come together to form a fully working system. The composition of and +flow of data in a Druid cluster are shown in Figure~\ref{fig:cluster}. The name Druid comes from the Druid class in many role-playing games: it is a +shape-shifter, capable of taking on many different forms to fulfill various +different roles in a group. \begin{figure*} \centering @@ -192,13 +231,12 @@ Figure~\ref{fig:cluster}. \label{fig:cluster} \end{figure*} -\newpage \subsection{Real-time Nodes} \label{sec:realtime} Real-time nodes encapsulate the functionality to ingest and query event streams. Events indexed via these nodes are immediately available for querying. The nodes are only concerned with events for some small time range and -periodically hand off immutable batches of events they've collected over this +periodically hand off immutable batches of events they have collected over this small time range to other nodes in the Druid cluster that are specialized in dealing with batches of immutable events. Real-time nodes leverage Zookeeper \cite{hunt2010zookeeper} for coordination with the rest of the Druid cluster. @@ -220,10 +258,11 @@ in \cite{o1996log} and is illustrated in Figure~\ref{fig:realtime_flow}. \begin{figure} \centering \includegraphics[width = 2.6in]{realtime_flow} -\caption{Real-time nodes first buffer events in memory. On a periodic basis, -the in-memory index is persisted to disk. On another periodic basis, all -persisted indexes are merged together and handed off. Queries will hit the -in-memory index and the persisted indexes.} +\caption{Real-time nodes buffer events to an in-memory index, which is +regularly persisted to disk. On a periodic basis, persisted indexes are then merged +together before getting handed off. +Queries will hit both the in-memory and persisted indexes. +} \label{fig:realtime_flow} \end{figure} @@ -388,9 +427,7 @@ caching the results would be unreliable. \begin{figure*} \centering \includegraphics[width = 4.5in]{caching} -\caption{Broker nodes cache per segment results. Every Druid query is mapped to -a set of segments. Queries often combine cached segment results with those that -need to be computed on historical and real-time nodes.} +\caption{Results are cached per segment. Queries combine cached results with results computed on historical and real-time nodes.} \label{fig:caching} \end{figure*} @@ -762,7 +799,7 @@ involving all columns are very rare. \begin{table} \centering - \begin{tabular}{| l | l | l |} + \scriptsize\begin{tabular}{| l | l | l |} \hline \textbf{Data Source} & \textbf{Dimensions} & \textbf{Metrics} \\ \hline \texttt{a} & 25 & 21 \\ \hline @@ -774,6 +811,7 @@ involving all columns are very rare. \texttt{g} & 26 & 18 \\ \hline \texttt{h} & 78 & 14 \\ \hline \end{tabular} + \normalsize \caption{Characteristics of production data sources.} \label{tab:datasources} \end{table} @@ -789,7 +827,7 @@ approximately 10TB of segments loaded. Collectively, there are about 50 billion Druid rows in this tier. Results for every data source are not shown. -\item The hot tier uses Xeon E5-2670 processors and consists of 1302 processing +\item The hot tier uses Intel Xeon E5-2670 processors and consists of 1302 processing threads and 672 total cores (hyperthreaded). \item A memory-mapped storage engine was used (the machine was configured to @@ -828,7 +866,7 @@ comparison, we also provide the results of the same queries using MySQL using th MyISAM engine (InnoDB was slower in our experiments). We selected MySQL to benchmark -against because of its universal popularity. We choose not to select another +against because of its universal popularity. We chose not to select another open source column store because we were not confident we could correctly tune it for optimal performance. @@ -871,6 +909,7 @@ well. \begin{figure} \centering \includegraphics[width = 2.3in]{tpch_scaling} +\includegraphics[width = 2.3in]{tpch_scaling_factor} \caption{Druid scaling benchmarks -- 100GB TPC-H data.} \label{fig:tpch_scaling} \end{figure} @@ -933,9 +972,9 @@ running an Amazon \texttt{cc2.8xlarge} instance. \label{fig:ingestion_rate} \end{figure} -The latency measurements we presented are sufficient to address the our stated +The latency measurements we presented are sufficient to address the stated problems of interactivity. We would prefer the variability in the latencies to -be less. It is still very possible to possible to decrease latencies by adding +be less. It is still very possible to decrease latencies by adding additional hardware, but we have not chosen to do so because infrastructure costs are still a consideration to us. @@ -1017,7 +1056,7 @@ data centers as well. The tier configuration in Druid coordinator nodes allow for segments to be replicated across multiple tiers. Hence, segments can be exactly replicated across historical nodes in multiple data centers. Similarily, query preference can be assigned to different tiers. It is possible -to have nodes in one data center act as a primary cluster (and recieve all +to have nodes in one data center act as a primary cluster (and receive all queries) and have a redundant cluster in another data center. Such a setup may be desired if one data center is situated much closer to users. diff --git a/publications/whitepaper/figures/caching.pdf b/publications/whitepaper/figures/caching.pdf new file mode 100644 index 00000000000..e42b317148d Binary files /dev/null and b/publications/whitepaper/figures/caching.pdf differ diff --git a/publications/whitepaper/figures/caching.png b/publications/whitepaper/figures/caching.png deleted file mode 100644 index e3ee4dd94df..00000000000 Binary files a/publications/whitepaper/figures/caching.png and /dev/null differ diff --git a/publications/whitepaper/figures/cluster.pdf b/publications/whitepaper/figures/cluster.pdf new file mode 100644 index 00000000000..649be901107 Binary files /dev/null and b/publications/whitepaper/figures/cluster.pdf differ diff --git a/publications/whitepaper/figures/cluster.png b/publications/whitepaper/figures/cluster.png deleted file mode 100644 index 1a07213e2cd..00000000000 Binary files a/publications/whitepaper/figures/cluster.png and /dev/null differ diff --git a/publications/whitepaper/figures/historical_download.pdf b/publications/whitepaper/figures/historical_download.pdf new file mode 100644 index 00000000000..20c2c733831 Binary files /dev/null and b/publications/whitepaper/figures/historical_download.pdf differ diff --git a/publications/whitepaper/figures/historical_download.png b/publications/whitepaper/figures/historical_download.png deleted file mode 100644 index a9db3fd3609..00000000000 Binary files a/publications/whitepaper/figures/historical_download.png and /dev/null differ diff --git a/publications/whitepaper/figures/realtime_flow.pdf b/publications/whitepaper/figures/realtime_flow.pdf new file mode 100644 index 00000000000..182e64d2c4c Binary files /dev/null and b/publications/whitepaper/figures/realtime_flow.pdf differ diff --git a/publications/whitepaper/figures/realtime_flow.png b/publications/whitepaper/figures/realtime_flow.png deleted file mode 100644 index 4f50a5c54b5..00000000000 Binary files a/publications/whitepaper/figures/realtime_flow.png and /dev/null differ diff --git a/publications/whitepaper/figures/realtime_pipeline.pdf b/publications/whitepaper/figures/realtime_pipeline.pdf new file mode 100644 index 00000000000..581671d58e2 Binary files /dev/null and b/publications/whitepaper/figures/realtime_pipeline.pdf differ diff --git a/publications/whitepaper/figures/realtime_pipeline.png b/publications/whitepaper/figures/realtime_pipeline.png deleted file mode 100644 index f338d239117..00000000000 Binary files a/publications/whitepaper/figures/realtime_pipeline.png and /dev/null differ diff --git a/publications/whitepaper/figures/realtime_timeline.pdf b/publications/whitepaper/figures/realtime_timeline.pdf new file mode 100644 index 00000000000..339502b66cd Binary files /dev/null and b/publications/whitepaper/figures/realtime_timeline.pdf differ diff --git a/publications/whitepaper/figures/realtime_timeline.png b/publications/whitepaper/figures/realtime_timeline.png deleted file mode 100644 index 76806bc9aa2..00000000000 Binary files a/publications/whitepaper/figures/realtime_timeline.png and /dev/null differ diff --git a/publications/whitepaper/figures/tpch_scaling.pdf b/publications/whitepaper/figures/tpch_scaling.pdf new file mode 100644 index 00000000000..428e2d60c0f Binary files /dev/null and b/publications/whitepaper/figures/tpch_scaling.pdf differ diff --git a/publications/whitepaper/figures/tpch_scaling.png b/publications/whitepaper/figures/tpch_scaling.png deleted file mode 100644 index e929da0c5dd..00000000000 Binary files a/publications/whitepaper/figures/tpch_scaling.png and /dev/null differ diff --git a/publications/whitepaper/figures/tpch_scaling_factor.pdf b/publications/whitepaper/figures/tpch_scaling_factor.pdf new file mode 100644 index 00000000000..7317fb4fac7 Binary files /dev/null and b/publications/whitepaper/figures/tpch_scaling_factor.pdf differ diff --git a/publications/whitepaper/sgmd0658-yang.bib b/publications/whitepaper/sgmd0658-yang.bib new file mode 120000 index 00000000000..eea1ab1ec05 --- /dev/null +++ b/publications/whitepaper/sgmd0658-yang.bib @@ -0,0 +1 @@ +druid.bib \ No newline at end of file diff --git a/publications/whitepaper/sgmd0658-yang.pdf b/publications/whitepaper/sgmd0658-yang.pdf new file mode 100644 index 00000000000..f1790dd6421 Binary files /dev/null and b/publications/whitepaper/sgmd0658-yang.pdf differ diff --git a/publications/whitepaper/sgmd0658-yang.tex b/publications/whitepaper/sgmd0658-yang.tex new file mode 120000 index 00000000000..3f22d8b6a73 --- /dev/null +++ b/publications/whitepaper/sgmd0658-yang.tex @@ -0,0 +1 @@ +druid.tex \ No newline at end of file diff --git a/publications/whitepaper/sgmd0658-yang.zip b/publications/whitepaper/sgmd0658-yang.zip new file mode 100644 index 00000000000..7962d25ad33 Binary files /dev/null and b/publications/whitepaper/sgmd0658-yang.zip differ diff --git a/publications/whitepaper/sig-alternate-2013.cls b/publications/whitepaper/sig-alternate-2013.cls new file mode 100644 index 00000000000..92bf4b369d2 --- /dev/null +++ b/publications/whitepaper/sig-alternate-2013.cls @@ -0,0 +1,1649 @@ +% SIG-ALTERNATE.CLS - VERSION 2.5 +% "COMPATIBLE" WITH THE "ACM_PROC_ARTICLE-SP.CLS" V3.2SP +% Gerald Murray - May 23rd 2012 +% +% ---- Start of 'updates' ---- +% Changed $10 fee to $15 -- May 2012 -- Gerry +% Changed $5 fee to $10 -- April 2009 -- Gerry +% April 22nd. 2009 - Fixed 'Natbib' incompatibility problem - Gerry +% April 22nd. 2009 - Fixed 'Babel' incompatibility problem - Gerry +% April 22nd. 2009 - Inserted various bug-fixes and improvements - Gerry +% +% To produce Type 1 fonts in the document plus allow for 'normal LaTeX accenting' in the critical areas; +% title, author block, section-heads, confname, etc. etc. +% i.e. the whole purpose of this version update is to NOT resort to 'inelegant accent patches'. +% After much research, three extra .sty packages were added to the the tail (ae, aecompl, aeguill) to solve, +% in particular, the accenting problem(s). We _could_ ask authors (via instructions/sample file) to 'include' these in +% the source .tex file - in the preamble - but if everything is already provided ('behind the scenes' - embedded IN the .cls) +% then this is less work for authors and also makes everything appear 'vanilla'. +% NOTE: all 'patchwork accenting" has been commented out (here) and is no longer 'used' in the sample .tex file (either). +% Gerry June 2007 +% +% Patch for accenting in conference name/location. Gerry May 3rd. 2007 +% Rule widths changed to .5, author count (>6) fixed, roll-back for Type 3 problem. Gerry March 20th. 2007 +% Changes made to 'modernize' the fontnames but esp. for MikTeX users V2.4/2.5 - Nov. 30th. 2006 +% Updated the \email definition to allow for its use inside of 'shared affiliations' - Nov. 30th. 2006 +% Fixed the 'section number depth value' - Nov. 30th. 2006 +% +% Footnotes inside table cells using \minipage (Oct. 2002) +% Georgia fixed bug in sub-sub-section numbering in paragraphs (July 29th. 2002) +% JS/GM fix to vertical spacing before Proofs (July 30th. 2002) +% +% Made the Permission Statement / Conference Info / Copyright Info +% 'user definable' in the source .tex file OR automatic if +% not specified. +% +% Allowance made to switch default fonts between those systems using +% normal/modern font names and those using 'Type 1' or 'Truetype' fonts. +% See LINE NUMBER 255 for details. +% Also provided for enumerated/annotated Corollaries 'surrounded' by +% enumerated Theorems (line 848). +% Gerry November 11th. 1999 +% +% ---- End of 'updates' ---- +% +\def\fileversion{v2.5} % for ACM's tracking purposes +\def\filedate{May 23, 2012} % Gerry Murray's tracking data +\def\docdate {Wednesday 23rd. May 2012} % Gerry Murray (with deltas to doc} +\usepackage{epsfig} +\usepackage{amssymb} +\usepackage{amsmath} +\usepackage{amsfonts} +% Need this for accents in Arial/Helvetica +%\usepackage[T1]{fontenc} % Gerry March 12, 2007 - causes Type 3 problems (body text) +%\usepackage{textcomp} +% +% SIG-ALTERNATE DOCUMENT STYLE +% G.K.M. Tobin August-October 1999 +% adapted from ARTICLE document style by Ken Traub, Olin Shivers +% also using elements of esub2acm.cls +% HEAVILY MODIFIED, SUBSEQUENTLY, BY GERRY MURRAY 2000 +% ARTICLE DOCUMENT STYLE -- Released 16 March 1988 +% for LaTeX version 2.09 +% Copyright (C) 1988 by Leslie Lamport +% +% +%%% sig-alternate.cls is an 'ALTERNATE' document style for producing +%%% two-column camera-ready pages for ACM conferences. +%%% THIS FILE DOES NOT STRICTLY ADHERE TO THE SIGS (BOARD-ENDORSED) +%%% PROCEEDINGS STYLE. It has been designed to produce a 'tighter' +%%% paper in response to concerns over page budgets. +%%% The main features of this style are: +%%% +%%% 1) Two columns. +%%% 2) Side and top margins of 4.5pc, bottom margin of 6pc, column gutter of +%%% 2pc, hence columns are 20pc wide and 55.5pc tall. (6pc =3D 1in, approx) +%%% 3) First page has title information, and an extra 6pc of space at the +%%% bottom of the first column for the ACM copyright notice. +%%% 4) Text is 9pt on 10pt baselines; titles (except main) are 9pt bold. +%%% +%%% +%%% There are a few restrictions you must observe: +%%% +%%% 1) You cannot change the font size; ACM wants you to use 9pt. +%%% 3) You must start your paper with the \maketitle command. Prior to the +%%% \maketitle you must have \title and \author commands. If you have a +%%% \date command it will be ignored; no date appears on the paper, since +%%% the proceedings will have a date on the front cover. +%%% 4) Marginal paragraphs, tables of contents, lists of figures and tables, +%%% and page headings are all forbidden. +%%% 5) The `figure' environment will produce a figure one column wide; if you +%%% want one that is two columns wide, use `figure*'. +%%% +% +%%% Copyright Space: +%%% This style automatically reserves 1" blank space at the bottom of page 1/ +%%% column 1. This space can optionally be filled with some text using the +%%% \toappear{...} command. If used, this command must be BEFORE the \maketitle +%%% command. If this command is defined AND [preprint] is on, then the +%%% space is filled with the {...} text (at the bottom); otherwise, it is +%%% blank. If you use \toappearbox{...} instead of \toappear{...} then a +%%% box will be drawn around the text (if [preprint] is on). +%%% +%%% A typical usage looks like this: +%%% \toappear{To appear in the Ninth AES Conference on Medievil Lithuanian +%%% Embalming Technique, June 1991, Alfaretta, Georgia.} +%%% This will be included in the preprint, and left out of the conference +%%% version. +%%% +%%% WARNING: +%%% Some dvi-ps converters heuristically allow chars to drift from their +%%% true positions a few pixels. This may be noticeable with the 9pt sans-serif +%%% bold font used for section headers. +%%% You may turn this hackery off via the -e option: +%%% dvips -e 0 foo.dvi >foo.ps +%%% +\typeout{Document Class 'sig-alternate' <23rd. May '12>. Modified by G.K.M. Tobin/Gerry Murray} +\typeout{Based in part upon document Style `acmconf' <22 May 89>. Hacked 4/91 by} +\typeout{shivers@cs.cmu.edu, 4/93 by theobald@cs.mcgill.ca} +\typeout{Excerpts were taken from (Journal Style) 'esub2acm.cls'.} +\typeout{****** Bugs/comments/suggestions/technicalities to Gerry Murray -- murray@hq.acm.org ******} +\typeout{Questions on the style, SIGS policies, etc. to Adrienne Griscti griscti@acm.org} +\oddsidemargin 4.5pc +\evensidemargin 4.5pc +\advance\oddsidemargin by -1in % Correct for LaTeX gratuitousness +\advance\evensidemargin by -1in % Correct for LaTeX gratuitousness +\marginparwidth 0pt % Margin pars are not allowed. +\marginparsep 11pt % Horizontal space between outer margin and + % marginal note + + % Top of page: +\topmargin 4.5pc % Nominal distance from top of page to top of + % box containing running head. +\advance\topmargin by -1in % Correct for LaTeX gratuitousness +\headheight 0pt % Height of box containing running head. +\headsep 0pt % Space between running head and text. + % Bottom of page: +\footskip 30pt % Distance from baseline of box containing foot + % to baseline of last line of text. +\@ifundefined{footheight}{\newdimen\footheight}{}% this is for LaTeX2e +\footheight 12pt % Height of box containing running foot. + +%% Must redefine the top margin so there's room for headers and +%% page numbers if you are using the preprint option. Footers +%% are OK as is. Olin. +\advance\topmargin by -37pt % Leave 37pt above text for headers +\headheight 12pt % Height of box containing running head. +\headsep 25pt % Space between running head and text. + +\textheight 666pt % 9 1/4 column height +\textwidth 42pc % Width of text line. + % For two-column mode: +\columnsep 2pc % Space between columns +\columnseprule 0pt % Width of rule between columns. +\hfuzz 1pt % Allow some variation in column width, otherwise it's + % too hard to typeset in narrow columns. + +\footnotesep 5.6pt % Height of strut placed at the beginning of every + % footnote =3D height of normal \footnotesize strut, + % so no extra space between footnotes. + +\skip\footins 8.1pt plus 4pt minus 2pt % Space between last line of text and + % top of first footnote. +\floatsep 11pt plus 2pt minus 2pt % Space between adjacent floats moved + % to top or bottom of text page. +\textfloatsep 18pt plus 2pt minus 4pt % Space between main text and floats + % at top or bottom of page. +\intextsep 11pt plus 2pt minus 2pt % Space between in-text figures and + % text. +\@ifundefined{@maxsep}{\newdimen\@maxsep}{}% this is for LaTeX2e +\@maxsep 18pt % The maximum of \floatsep, + % \textfloatsep and \intextsep (minus + % the stretch and shrink). +\dblfloatsep 11pt plus 2pt minus 2pt % Same as \floatsep for double-column + % figures in two-column mode. +\dbltextfloatsep 18pt plus 2pt minus 4pt% \textfloatsep for double-column + % floats. +\@ifundefined{@dblmaxsep}{\newdimen\@dblmaxsep}{}% this is for LaTeX2e +\@dblmaxsep 18pt % The maximum of \dblfloatsep and + % \dbltexfloatsep. +\@fptop 0pt plus 1fil % Stretch at top of float page/column. (Must be + % 0pt plus ...) +\@fpsep 8pt plus 2fil % Space between floats on float page/column. +\@fpbot 0pt plus 1fil % Stretch at bottom of float page/column. (Must be + % 0pt plus ... ) +\@dblfptop 0pt plus 1fil % Stretch at top of float page. (Must be 0pt plus ...) +\@dblfpsep 8pt plus 2fil % Space between floats on float page. +\@dblfpbot 0pt plus 1fil % Stretch at bottom of float page. (Must be + % 0pt plus ... ) +\marginparpush 5pt % Minimum vertical separation between two marginal + % notes. + +\parskip 0pt plus 1pt % Extra vertical space between paragraphs. +\parindent 9pt % GM July 2000 / was 0pt - width of paragraph indentation. +\partopsep 2pt plus 1pt minus 1pt% Extra vertical space, in addition to + % \parskip and \topsep, added when user + % leaves blank line before environment. + +\@lowpenalty 51 % Produced by \nopagebreak[1] or \nolinebreak[1] +\@medpenalty 151 % Produced by \nopagebreak[2] or \nolinebreak[2] +\@highpenalty 301 % Produced by \nopagebreak[3] or \nolinebreak[3] + +\@beginparpenalty -\@lowpenalty % Before a list or paragraph environment. +\@endparpenalty -\@lowpenalty % After a list or paragraph environment. +\@itempenalty -\@lowpenalty % Between list items. + +%\@namedef{ds@10pt}{\@latexerr{The `10pt' option is not allowed in the `acmconf' +\@namedef{ds@10pt}{\ClassError{The `10pt' option is not allowed in the `acmconf' % January 2008 + document style.}\@eha} +%\@namedef{ds@11pt}{\@latexerr{The `11pt' option is not allowed in the `acmconf' +\@namedef{ds@11pt}{\ClassError{The `11pt' option is not allowed in the `acmconf' % January 2008 + document style.}\@eha} +%\@namedef{ds@12pt}{\@latexerr{The `12pt' option is not allowed in the `acmconf' +\@namedef{ds@12pt}{\ClassError{The `12pt' option is not allowed in the `acmconf' % January 2008 + document style.}\@eha} + +\@options + +\lineskip 2pt % \lineskip is 1pt for all font sizes. +\normallineskip 2pt +\def\baselinestretch{1} + +\abovedisplayskip 9pt plus2pt minus4.5pt% +\belowdisplayskip \abovedisplayskip +\abovedisplayshortskip \z@ plus3pt% +\belowdisplayshortskip 5.4pt plus3pt minus3pt% +\let\@listi\@listI % Setting of \@listi added 9 Jun 87 + +\def\small{\@setsize\small{9pt}\viiipt\@viiipt +\abovedisplayskip 7.6pt plus 3pt minus 4pt% +\belowdisplayskip \abovedisplayskip +\abovedisplayshortskip \z@ plus2pt% +\belowdisplayshortskip 3.6pt plus2pt minus 2pt +\def\@listi{\leftmargin\leftmargini %% Added 22 Dec 87 +\topsep 4pt plus 2pt minus 2pt\parsep 2pt plus 1pt minus 1pt +\itemsep \parsep}} + +\def\footnotesize{\@setsize\footnotesize{9pt}\ixpt\@ixpt +\abovedisplayskip 6.4pt plus 2pt minus 4pt% +\belowdisplayskip \abovedisplayskip +\abovedisplayshortskip \z@ plus 1pt% +\belowdisplayshortskip 2.7pt plus 1pt minus 2pt +\def\@listi{\leftmargin\leftmargini %% Added 22 Dec 87 +\topsep 3pt plus 1pt minus 1pt\parsep 2pt plus 1pt minus 1pt +\itemsep \parsep}} + +\newcount\aucount +\newcount\originalaucount +\newdimen\auwidth +\auwidth=\textwidth +\newdimen\auskip +\newcount\auskipcount +\newdimen\auskip +\global\auskip=1pc +\newdimen\allauboxes +\allauboxes=\auwidth +\newtoks\addauthors +\newcount\addauflag +\global\addauflag=0 %Haven't shown additional authors yet + +\newtoks\subtitletext +\gdef\subtitle#1{\subtitletext={#1}} + +\gdef\additionalauthors#1{\addauthors={#1}} + +\gdef\numberofauthors#1{\global\aucount=#1 +\ifnum\aucount>3\global\originalaucount=\aucount \global\aucount=3\fi %g} % 3 OK - Gerry March 2007 +\global\auskipcount=\aucount\global\advance\auskipcount by 1 +\global\multiply\auskipcount by 2 +\global\multiply\auskip by \auskipcount +\global\advance\auwidth by -\auskip +\global\divide\auwidth by \aucount} + +% \and was modified to count the number of authors. GKMT 12 Aug 1999 +\def\alignauthor{% % \begin{tabular} +\end{tabular}% + \begin{tabular}[t]{p{\auwidth}}\centering}% + +% *** NOTE *** NOTE *** NOTE *** NOTE *** +% If you have 'font problems' then you may need +% to change these, e.g. 'arialb' instead of "arialbd". +% Gerry Murray 11/11/1999 +% *** OR ** comment out block A and activate block B or vice versa. +% ********************************************** +% +% -- Start of block A -- (Type 1 or Truetype fonts) +%\newfont{\secfnt}{timesbd at 12pt} % was timenrb originally - now is timesbd +%\newfont{\secit}{timesbi at 12pt} %13 Jan 00 gkmt +%\newfont{\subsecfnt}{timesi at 11pt} % was timenrri originally - now is timesi +%\newfont{\subsecit}{timesbi at 11pt} % 13 Jan 00 gkmt -- was times changed to timesbi gm 2/4/2000 +% % because "normal" is italic, "italic" is Roman +%\newfont{\ttlfnt}{arialbd at 18pt} % was arialb originally - now is arialbd +%\newfont{\ttlit}{arialbi at 18pt} % 13 Jan 00 gkmt +%\newfont{\subttlfnt}{arial at 14pt} % was arialr originally - now is arial +%\newfont{\subttlit}{ariali at 14pt} % 13 Jan 00 gkmt +%\newfont{\subttlbf}{arialbd at 14pt} % 13 Jan 00 gkmt +%\newfont{\aufnt}{arial at 12pt} % was arialr originally - now is arial +%\newfont{\auit}{ariali at 12pt} % 13 Jan 00 gkmt +%\newfont{\affaddr}{arial at 10pt} % was arialr originally - now is arial +%\newfont{\affaddrit}{ariali at 10pt} %13 Jan 00 gkmt +%\newfont{\eaddfnt}{arial at 12pt} % was arialr originally - now is arial +%\newfont{\ixpt}{times at 9pt} % was timenrr originally - now is times +%\newfont{\confname}{timesi at 8pt} % was timenrri - now is timesi +%\newfont{\crnotice}{times at 8pt} % was timenrr originally - now is times +%\newfont{\ninept}{times at 9pt} % was timenrr originally - now is times + +% ********************************************* +% -- End of block A -- +% +% +% -- Start of block B -- UPDATED FONT NAMES +% ********************************************* +% Gerry Murray 11/30/2006 +% ********************************************* +\newfont{\secfnt}{ptmb8t at 12pt} +\newfont{\secit}{ptmbi8t at 12pt} %13 Jan 00 gkmt +\newfont{\subsecfnt}{ptmri8t at 11pt} +\newfont{\subsecit}{ptmbi8t at 11pt} % +\newfont{\ttlfnt}{phvb8t at 18pt} +\newfont{\ttlit}{phvbo8t at 18pt} % GM 2/4/2000 +\newfont{\subttlfnt}{phvr8t at 14pt} +\newfont{\subttlit}{phvro8t at 14pt} % GM 2/4/2000 +\newfont{\subttlbf}{phvb8t at 14pt} % 13 Jan 00 gkmt +\newfont{\aufnt}{phvr8t at 12pt} +\newfont{\auit}{phvro8t at 12pt} % GM 2/4/2000 +\newfont{\affaddr}{phvr8t at 10pt} +\newfont{\affaddrit}{phvro8t at 10pt} % GM 2/4/2000 +\newfont{\eaddfnt}{phvr8t at 12pt} +\newfont{\ixpt}{ptmr8t at 9pt} +\newfont{\confname}{ptmri8t at 8pt} +\newfont{\crnotice}{ptmr8t at 8pt} +\newfont{\ninept}{ptmr8t at 9pt} +% +++++++++++++++++++++++++++++++++++++++++++++ +% -- End of block B -- + +%\def\email#1{{{\eaddfnt{\vskip 4pt#1}}}} +% If we have an email, inside a "shared affiliation" then we need the following instead +\def\email#1{{{\eaddfnt{\par #1}}}} % revised - GM - 11/30/2006 + +\def\addauthorsection{\ifnum\originalaucount>6 % was 3 - Gerry March 2007 + \section{Additional Authors}\the\addauthors + \fi} + +\newcount\savesection +\newcount\sectioncntr +\global\sectioncntr=1 + +\setcounter{secnumdepth}{3} + +\def\appendix{\par +\section*{APPENDIX} +\setcounter{section}{0} + \setcounter{subsection}{0} + \def\thesection{\Alph{section}} } + +\leftmargini 22.5pt +\leftmarginii 19.8pt % > \labelsep + width of '(m)' +\leftmarginiii 16.8pt % > \labelsep + width of 'vii.' +\leftmarginiv 15.3pt % > \labelsep + width of 'M.' +\leftmarginv 9pt +\leftmarginvi 9pt + +\leftmargin\leftmargini +\labelsep 4.5pt +\labelwidth\leftmargini\advance\labelwidth-\labelsep + +\def\@listI{\leftmargin\leftmargini \parsep 3.6pt plus 2pt minus 1pt% +\topsep 7.2pt plus 2pt minus 4pt% +\itemsep 3.6pt plus 2pt minus 1pt} + +\let\@listi\@listI +\@listi + +\def\@listii{\leftmargin\leftmarginii + \labelwidth\leftmarginii\advance\labelwidth-\labelsep + \topsep 3.6pt plus 2pt minus 1pt + \parsep 1.8pt plus 0.9pt minus 0.9pt + \itemsep \parsep} + +\def\@listiii{\leftmargin\leftmarginiii + \labelwidth\leftmarginiii\advance\labelwidth-\labelsep + \topsep 1.8pt plus 0.9pt minus 0.9pt + \parsep \z@ \partopsep 1pt plus 0pt minus 1pt + \itemsep \topsep} + +\def\@listiv{\leftmargin\leftmarginiv + \labelwidth\leftmarginiv\advance\labelwidth-\labelsep} + +\def\@listv{\leftmargin\leftmarginv + \labelwidth\leftmarginv\advance\labelwidth-\labelsep} + +\def\@listvi{\leftmargin\leftmarginvi + \labelwidth\leftmarginvi\advance\labelwidth-\labelsep} + +\def\labelenumi{\theenumi.} +\def\theenumi{\arabic{enumi}} + +\def\labelenumii{(\theenumii)} +\def\theenumii{\alph{enumii}} +\def\p@enumii{\theenumi} + +\def\labelenumiii{\theenumiii.} +\def\theenumiii{\roman{enumiii}} +\def\p@enumiii{\theenumi(\theenumii)} + +\def\labelenumiv{\theenumiv.} +\def\theenumiv{\Alph{enumiv}} +\def\p@enumiv{\p@enumiii\theenumiii} + +\def\labelitemi{$\bullet$} +\def\labelitemii{\bf --} +\def\labelitemiii{$\ast$} +\def\labelitemiv{$\cdot$} + +\def\verse{\let\\=\@centercr + \list{}{\itemsep\z@ \itemindent -1.5em\listparindent \itemindent + \rightmargin\leftmargin\advance\leftmargin 1.5em}\item[]} +\let\endverse\endlist + +\def\quotation{\list{}{\listparindent 1.5em + \itemindent\listparindent + \rightmargin\leftmargin \parsep 0pt plus 1pt}\item[]} +\let\endquotation=\endlist + +\def\quote{\list{}{\rightmargin\leftmargin}\item[]} +\let\endquote=\endlist + +\def\descriptionlabel#1{\hspace\labelsep \bf #1} +\def\description{\list{}{\labelwidth\z@ \itemindent-\leftmargin + \let\makelabel\descriptionlabel}} + +\let\enddescription\endlist + +\def\theequation{\arabic{equation}} + +\arraycolsep 4.5pt % Half the space between columns in an array environment. +\tabcolsep 5.4pt % Half the space between columns in a tabular environment. +\arrayrulewidth .5pt % Width of rules in array and tabular environment. % (was .4) updated Gerry March 20 2007 +\doublerulesep 1.8pt % Space between adjacent rules in array or tabular env. + +\tabbingsep \labelsep % Space used by the \' command. (See LaTeX manual.) + +\skip\@mpfootins =\skip\footins + +\fboxsep =2.7pt % Space left between box and text by \fbox and \framebox. +\fboxrule =.5pt % Width of rules in box made by \fbox and \framebox. % (was .4) updated Gerry March 20 2007 + +\def\thepart{\Roman{part}} % Roman numeral part numbers. +\def\thesection {\arabic{section}} +\def\thesubsection {\thesection.\arabic{subsection}} +%\def\thesubsubsection {\thesubsection.\arabic{subsubsection}} % GM 7/30/2002 +%\def\theparagraph {\thesubsubsection.\arabic{paragraph}} % GM 7/30/2002 +\def\thesubparagraph {\theparagraph.\arabic{subparagraph}} + +\def\@pnumwidth{1.55em} +\def\@tocrmarg {2.55em} +\def\@dotsep{4.5} +\setcounter{tocdepth}{3} + +%\def\tableofcontents{\@latexerr{\tableofcontents: Tables of contents are not +% allowed in the `acmconf' document style.}\@eha} + +\def\tableofcontents{\ClassError{% + \string\tableofcontents\space is not allowed in the `acmconf' document % January 2008 + style}\@eha} + +\def\l@part#1#2{\addpenalty{\@secpenalty} + \addvspace{2.25em plus 1pt} % space above part line + \begingroup + \@tempdima 3em % width of box holding part number, used by + \parindent \z@ \rightskip \@pnumwidth %% \numberline + \parfillskip -\@pnumwidth + {\large \bf % set line in \large boldface + \leavevmode % TeX command to enter horizontal mode. + #1\hfil \hbox to\@pnumwidth{\hss #2}}\par + \nobreak % Never break after part entry + \endgroup} + +\def\l@section#1#2{\addpenalty{\@secpenalty} % good place for page break + \addvspace{1.0em plus 1pt} % space above toc entry + \@tempdima 1.5em % width of box holding section number + \begingroup + \parindent \z@ \rightskip \@pnumwidth + \parfillskip -\@pnumwidth + \bf % Boldface. + \leavevmode % TeX command to enter horizontal mode. + \advance\leftskip\@tempdima %% added 5 Feb 88 to conform to + \hskip -\leftskip %% 25 Jan 88 change to \numberline + #1\nobreak\hfil \nobreak\hbox to\@pnumwidth{\hss #2}\par + \endgroup} + + +\def\l@subsection{\@dottedtocline{2}{1.5em}{2.3em}} +\def\l@subsubsection{\@dottedtocline{3}{3.8em}{3.2em}} +\def\l@paragraph{\@dottedtocline{4}{7.0em}{4.1em}} +\def\l@subparagraph{\@dottedtocline{5}{10em}{5em}} + +%\def\listoffigures{\@latexerr{\listoffigures: Lists of figures are not +% allowed in the `acmconf' document style.}\@eha} + +\def\listoffigures{\ClassError{% + \string\listoffigures\space is not allowed in the `acmconf' document % January 2008 + style}\@eha} + +\def\l@figure{\@dottedtocline{1}{1.5em}{2.3em}} + +%\def\listoftables{\@latexerr{\listoftables: Lists of tables are not +% allowed in the `acmconf' document style.}\@eha} +%\let\l@table\l@figure + +\def\listoftables{\ClassError{% + \string\listoftables\space is not allowed in the `acmconf' document % January 2008 + style}\@eha} + \let\l@table\l@figure + +\def\footnoterule{\kern-3\p@ + \hrule width .5\columnwidth % (was .4) updated Gerry March 20 2007 + \kern 2.6\p@} % The \hrule has default height of .4pt % (was .4) updated Gerry March 20 2007 +% ------ +\long\def\@makefntext#1{\noindent +%\hbox to .5em{\hss$^{\@thefnmark}$}#1} % original +\hbox to .5em{\hss\textsuperscript{\@thefnmark}}#1} % C. Clifton / GM Oct. 2nd. 2002 +% ------- + +\long\def\@maketntext#1{\noindent +#1} + +\long\def\@maketitlenotetext#1#2{\noindent + \hbox to 1.8em{\hss$^{#1}$}#2} + +\setcounter{topnumber}{2} +\def\topfraction{.7} +\setcounter{bottomnumber}{1} +\def\bottomfraction{.3} +\setcounter{totalnumber}{3} +\def\textfraction{.2} +\def\floatpagefraction{.5} +\setcounter{dbltopnumber}{2} +\def\dbltopfraction{.7} +\def\dblfloatpagefraction{.5} + +% +\long\def\@makecaption#1#2{ + \vskip \baselineskip + \setbox\@tempboxa\hbox{\textbf{#1: #2}} + \ifdim \wd\@tempboxa >\hsize % IF longer than one line: + \textbf{#1: #2}\par % THEN set as ordinary paragraph. + \else % ELSE center. + \hbox to\hsize{\hfil\box\@tempboxa\hfil}\par + \fi} + +% + +\long\def\@makecaption#1#2{ + \vskip 10pt + \setbox\@tempboxa\hbox{\textbf{#1: #2}} + \ifdim \wd\@tempboxa >\hsize % IF longer than one line: + \textbf{#1: #2}\par % THEN set as ordinary paragraph. + \else % ELSE center. + \hbox to\hsize{\hfil\box\@tempboxa\hfil} + \fi} + +\@ifundefined{figure}{\newcounter {figure}} % this is for LaTeX2e + +\def\fps@figure{tbp} +\def\ftype@figure{1} +\def\ext@figure{lof} +\def\fnum@figure{Figure \thefigure} +\def\figure{\@float{figure}} +%\let\endfigure\end@float +\def\endfigure{\end@float} % Gerry January 2008 +\@namedef{figure*}{\@dblfloat{figure}} +\@namedef{endfigure*}{\end@dblfloat} + +\@ifundefined{table}{\newcounter {table}} % this is for LaTeX2e + +\def\fps@table{tbp} +\def\ftype@table{2} +\def\ext@table{lot} +\def\fnum@table{Table \thetable} +\def\table{\@float{table}} +%\let\endtable\end@float +\def\endtable{\end@float} % Gerry January 2008 +\@namedef{table*}{\@dblfloat{table}} +\@namedef{endtable*}{\end@dblfloat} + +\newtoks\titleboxnotes +\newcount\titleboxnoteflag + +\def\maketitle{\par + \begingroup + \def\thefootnote{\fnsymbol{footnote}} + \def\@makefnmark{\hbox + to 0pt{$^{\@thefnmark}$\hss}} + \twocolumn[\@maketitle] +\@thanks + \endgroup + \setcounter{footnote}{0} + \let\maketitle\relax + \let\@maketitle\relax + \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\gdef\@subtitle{}\let\thanks\relax + \@copyrightspace} + +%% CHANGES ON NEXT LINES +\newif\if@ll % to record which version of LaTeX is in use + +\expandafter\ifx\csname LaTeXe\endcsname\relax % LaTeX2.09 is used +\else% LaTeX2e is used, so set ll to true +\global\@lltrue +\fi + +\if@ll + \NeedsTeXFormat{LaTeX2e} + \ProvidesClass{sig-alternate} [2012/05/23 - V2.5 - based on acmproc.cls V1.3 ] + \RequirePackage{latexsym}% QUERY: are these two really needed? + \let\dooptions\ProcessOptions +\else + \let\dooptions\@options +\fi +%% END CHANGES + +\def\@height{height} +\def\@width{width} +\def\@minus{minus} +\def\@plus{plus} +\def\hb@xt@{\hbox to} +\newif\if@faircopy +\@faircopyfalse +\def\ds@faircopy{\@faircopytrue} + +\def\ds@preprint{\@faircopyfalse} + +\@twosidetrue +\@mparswitchtrue +\def\ds@draft{\overfullrule 5\p@} +%% CHANGE ON NEXT LINE +\dooptions + +\lineskip \p@ +\normallineskip \p@ +\def\baselinestretch{1} +\def\@ptsize{0} %needed for amssymbols.sty + +%% CHANGES ON NEXT LINES +\if@ll% allow use of old-style font change commands in LaTeX2e +\@maxdepth\maxdepth +% +\DeclareOldFontCommand{\rm}{\ninept\rmfamily}{\mathrm} +\DeclareOldFontCommand{\sf}{\normalfont\sffamily}{\mathsf} +\DeclareOldFontCommand{\tt}{\normalfont\ttfamily}{\mathtt} +\DeclareOldFontCommand{\bf}{\normalfont\bfseries}{\mathbf} +\DeclareOldFontCommand{\it}{\normalfont\itshape}{\mathit} +\DeclareOldFontCommand{\sl}{\normalfont\slshape}{\@nomath\sl} +\DeclareOldFontCommand{\sc}{\normalfont\scshape}{\@nomath\sc} +\DeclareRobustCommand*{\cal}{\@fontswitch{\relax}{\mathcal}} +\DeclareRobustCommand*{\mit}{\@fontswitch{\relax}{\mathnormal}} +\fi +% +\if@ll + \renewcommand{\rmdefault}{cmr} % was 'ttm' +% Note! I have also found 'mvr' to work ESPECIALLY well. +% Gerry - October 1999 +% You may need to change your LV1times.fd file so that sc is +% mapped to cmcsc - -for smallcaps -- that is if you decide +% to change {cmr} to {times} above. (Not recommended) + \renewcommand{\@ptsize}{} + \renewcommand{\normalsize}{% + \@setfontsize\normalsize\@ixpt{10.5\p@}%\ninept% + \abovedisplayskip 6\p@ \@plus2\p@ \@minus\p@ + \belowdisplayskip \abovedisplayskip + \abovedisplayshortskip 6\p@ \@minus 3\p@ + \belowdisplayshortskip 6\p@ \@minus 3\p@ + \let\@listi\@listI + } +\else + \def\@normalsize{%changed next to 9 from 10 + \@setsize\normalsize{9\p@}\ixpt\@ixpt + \abovedisplayskip 6\p@ \@plus2\p@ \@minus\p@ + \belowdisplayskip \abovedisplayskip + \abovedisplayshortskip 6\p@ \@minus 3\p@ + \belowdisplayshortskip 6\p@ \@minus 3\p@ + \let\@listi\@listI + }% +\fi +\if@ll + \newcommand\scriptsize{\@setfontsize\scriptsize\@viipt{8\p@}} + \newcommand\tiny{\@setfontsize\tiny\@vpt{6\p@}} + \newcommand\large{\@setfontsize\large\@xiipt{14\p@}} + \newcommand\Large{\@setfontsize\Large\@xivpt{18\p@}} + \newcommand\LARGE{\@setfontsize\LARGE\@xviipt{20\p@}} + \newcommand\huge{\@setfontsize\huge\@xxpt{25\p@}} + \newcommand\Huge{\@setfontsize\Huge\@xxvpt{30\p@}} +\else + \def\scriptsize{\@setsize\scriptsize{8\p@}\viipt\@viipt} + \def\tiny{\@setsize\tiny{6\p@}\vpt\@vpt} + \def\large{\@setsize\large{14\p@}\xiipt\@xiipt} + \def\Large{\@setsize\Large{18\p@}\xivpt\@xivpt} + \def\LARGE{\@setsize\LARGE{20\p@}\xviipt\@xviipt} + \def\huge{\@setsize\huge{25\p@}\xxpt\@xxpt} + \def\Huge{\@setsize\Huge{30\p@}\xxvpt\@xxvpt} +\fi +\normalsize + +% make aubox hsize/number of authors up to 3, less gutter +% then showbox gutter showbox gutter showbox -- GKMT Aug 99 +\newbox\@acmtitlebox +\def\@maketitle{\newpage + \null + \setbox\@acmtitlebox\vbox{% +\baselineskip 20pt +\vskip 2em % Vertical space above title. + \begin{center} + {\ttlfnt \@title\par} % Title set in 18pt Helvetica (Arial) bold size. + \vskip 1.5em % Vertical space after title. +%This should be the subtitle. +{\subttlfnt \the\subtitletext\par}\vskip 1.25em%\fi + {\baselineskip 16pt\aufnt % each author set in \12 pt Arial, in a + \lineskip .5em % tabular environment + \begin{tabular}[t]{c}\@author + \end{tabular}\par} + \vskip 1.5em % Vertical space after author. + \end{center}} + \dimen0=\ht\@acmtitlebox + \advance\dimen0 by -12.75pc\relax % Increased space for title box -- KBT + \unvbox\@acmtitlebox + \ifdim\dimen0<0.0pt\relax\vskip-\dimen0\fi} + + +\newcount\titlenotecount +\global\titlenotecount=0 +\newtoks\tntoks +\newtoks\tntokstwo +\newtoks\tntoksthree +\newtoks\tntoksfour +\newtoks\tntoksfive + +\def\abstract{ +\ifnum\titlenotecount>0 % was =1 + \insert\footins{% + \reset@font\footnotesize + \interlinepenalty\interfootnotelinepenalty + \splittopskip\footnotesep + \splitmaxdepth \dp\strutbox \floatingpenalty \@MM + \hsize\columnwidth \@parboxrestore + \protected@edef\@currentlabel{% + }% + \color@begingroup +\ifnum\titlenotecount=1 + \@maketntext{% + \raisebox{4pt}{$\ast$}\rule\z@\footnotesep\ignorespaces\the\tntoks\@finalstrut\strutbox}% +\fi +\ifnum\titlenotecount=2 + \@maketntext{% + \raisebox{4pt}{$\ast$}\rule\z@\footnotesep\ignorespaces\the\tntoks\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\dagger$}\rule\z@\footnotesep\ignorespaces\the\tntokstwo\@finalstrut\strutbox}% +\fi +\ifnum\titlenotecount=3 + \@maketntext{% + \raisebox{4pt}{$\ast$}\rule\z@\footnotesep\ignorespaces\the\tntoks\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\dagger$}\rule\z@\footnotesep\ignorespaces\the\tntokstwo\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\ddagger$}\rule\z@\footnotesep\ignorespaces\the\tntoksthree\@finalstrut\strutbox}% +\fi +\ifnum\titlenotecount=4 + \@maketntext{% + \raisebox{4pt}{$\ast$}\rule\z@\footnotesep\ignorespaces\the\tntoks\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\dagger$}\rule\z@\footnotesep\ignorespaces\the\tntokstwo\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\ddagger$}\rule\z@\footnotesep\ignorespaces\the\tntoksthree\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\S$}\rule\z@\footnotesep\ignorespaces\the\tntoksfour\@finalstrut\strutbox}% +\fi +\ifnum\titlenotecount=5 + \@maketntext{% + \raisebox{4pt}{$\ast$}\rule\z@\footnotesep\ignorespaces\the\tntoks\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\dagger$}\rule\z@\footnotesep\ignorespaces\the\tntokstwo\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\ddagger$}\rule\z@\footnotesep\ignorespaces\the\tntoksthree\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\S$}\rule\z@\footnotesep\ignorespaces\the\tntoksfour\par\@finalstrut\strutbox}% +\@maketntext{% + \raisebox{4pt}{$\P$}\rule\z@\footnotesep\ignorespaces\the\tntoksfive\@finalstrut\strutbox}% +\fi + \color@endgroup} %g} +\fi +\setcounter{footnote}{0} +\section*{ABSTRACT}\normalsize%\ninept +} + +\def\endabstract{\if@twocolumn\else\endquotation\fi} + +\def\keywords{\if@twocolumn +\section*{Keywords} +\else \small +\quotation +\fi} + +\def\terms{\if@twocolumn +\section*{General Terms} +\else \small +\quotation +\fi} + +% -- Classification needs to be a bit smart due to optionals - Gerry/Georgia November 2nd. 1999 +\newcount\catcount +\global\catcount=1 + +\def\category#1#2#3{% +\ifnum\catcount=1 +\section*{Categories and Subject Descriptors} +\advance\catcount by 1\else{\unskip; }\fi + \@ifnextchar [{\@category{#1}{#2}{#3}}{\@category{#1}{#2}{#3}[]}% +} + +\def\@category#1#2#3[#4]{% + \begingroup + \let\and\relax + #1 [\textbf{#2}]% + \if!#4!% + \if!#3!\else : #3\fi + \else + :\space + \if!#3!\else #3\kern\z@---\hskip\z@\fi + \textit{#4}% + \fi + \endgroup +} +% + +%%% This section (written by KBT) handles the 1" box in the lower left +%%% corner of the left column of the first page by creating a picture, +%%% and inserting the predefined string at the bottom (with a negative +%%% displacement to offset the space allocated for a non-existent +%%% caption). +%%% +\newtoks\copyrightnotice +\def\ftype@copyrightbox{8} +\def\@copyrightspace{ +\@float{copyrightbox}[b] +\begin{center} +\setlength{\unitlength}{1pc} +\begin{picture}(20,6) %Space for copyright notice +\put(0,-0.95){\crnotice{\@toappear}} +\end{picture} +\end{center} +\end@float} + +\def\@toappear{} % Default setting blank - commands below change this. +\long\def\toappear#1{\def\@toappear{\parbox[b]{20pc}{\baselineskip 9pt#1}}} +\def\toappearbox#1{\def\@toappear{\raisebox{5pt}{\framebox[20pc]{\parbox[b]{19pc}{#1}}}}} + +\newtoks\conf +\newtoks\confinfo +\def\conferenceinfo#1#2{\global\conf={#1}\global\confinfo{#2}} + + +%\def\marginpar{\@latexerr{The \marginpar command is not allowed in the +% `acmconf' document style.}\@eha} + +\def\marginpar{\ClassError{% + \string\marginpar\space is not allowed in the `acmconf' document % January 2008 + style}\@eha} + +\mark{{}{}} % Initializes TeX's marks + +\def\today{\ifcase\month\or + January\or February\or March\or April\or May\or June\or + July\or August\or September\or October\or November\or December\fi + \space\number\day, \number\year} + +\def\@begintheorem#1#2{% + \parskip 0pt % GM July 2000 (for tighter spacing) + \trivlist + \item[% + \hskip 10\p@ + \hskip \labelsep + {{\sc #1}\hskip 5\p@\relax#2.}% + ] + \it +} +\def\@opargbegintheorem#1#2#3{% + \parskip 0pt % GM July 2000 (for tighter spacing) + \trivlist + \item[% + \hskip 10\p@ + \hskip \labelsep + {\sc #1\ #2\ % This mod by Gerry to enumerate corollaries + \setbox\@tempboxa\hbox{(#3)} % and bracket the 'corollary title' + \ifdim \wd\@tempboxa>\z@ % and retain the correct numbering of e.g. theorems + \hskip 5\p@\relax % if they occur 'around' said corollaries. + \box\@tempboxa % Gerry - Nov. 1999. + \fi.}% + ] + \it +} +\newif\if@qeded +\global\@qededfalse + +% -- original +%\def\proof{% +% \vspace{-\parskip} % GM July 2000 (for tighter spacing) +% \global\@qededfalse +% \@ifnextchar[{\@xproof}{\@proof}% +%} +% -- end of original + +% (JSS) Fix for vertical spacing bug - Gerry Murray July 30th. 2002 +\def\proof{% +\vspace{-\lastskip}\vspace{-\parsep}\penalty-51% +\global\@qededfalse +\@ifnextchar[{\@xproof}{\@proof}% +} + +\def\endproof{% + \if@qeded\else\qed\fi + \endtrivlist +} +\def\@proof{% + \trivlist + \item[% + \hskip 10\p@ + \hskip \labelsep + {\sc Proof.}% + ] + \ignorespaces +} +\def\@xproof[#1]{% + \trivlist + \item[\hskip 10\p@\hskip \labelsep{\sc Proof #1.}]% + \ignorespaces +} +\def\qed{% + \unskip + \kern 10\p@ + \begingroup + \unitlength\p@ + \linethickness{.4\p@}% + \framebox(6,6){}% + \endgroup + \global\@qededtrue +} + +\def\newdef#1#2{% + \expandafter\@ifdefinable\csname #1\endcsname + {\@definecounter{#1}% + \expandafter\xdef\csname the#1\endcsname{\@thmcounter{#1}}% + \global\@namedef{#1}{\@defthm{#1}{#2}}% + \global\@namedef{end#1}{\@endtheorem}% + }% +} +\def\@defthm#1#2{% + \refstepcounter{#1}% + \@ifnextchar[{\@ydefthm{#1}{#2}}{\@xdefthm{#1}{#2}}% +} +\def\@xdefthm#1#2{% + \@begindef{#2}{\csname the#1\endcsname}% + \ignorespaces +} +\def\@ydefthm#1#2[#3]{% + \trivlist + \item[% + \hskip 10\p@ + \hskip \labelsep + {\it #2% +% \savebox\@tempboxa{#3}% + \saveb@x\@tempboxa{#3}% % January 2008 + \ifdim \wd\@tempboxa>\z@ + \ \box\@tempboxa + \fi.% + }]% + \ignorespaces +} +\def\@begindef#1#2{% + \trivlist + \item[% + \hskip 10\p@ + \hskip \labelsep + {\it #1\ \rm #2.}% + ]% +} +\def\theequation{\arabic{equation}} + +\newcounter{part} +\newcounter{section} +\newcounter{subsection}[section] +\newcounter{subsubsection}[subsection] +\newcounter{paragraph}[subsubsection] +\def\thepart{\Roman{part}} +\def\thesection{\arabic{section}} +\def\thesubsection{\thesection.\arabic{subsection}} +\def\thesubsubsection{\thesubsection.\arabic{subsubsection}} %removed \subsecfnt 29 July 2002 gkmt +\def\theparagraph{\thesubsubsection.\arabic{paragraph}} %removed \subsecfnt 29 July 2002 gkmt +\newif\if@uchead +\@ucheadfalse + +%% CHANGES: NEW NOTE +%% NOTE: OK to use old-style font commands below, since they were +%% suitably redefined for LaTeX2e +%% END CHANGES +\setcounter{secnumdepth}{3} +\def\part{% + \@startsection{part}{9}{\z@}{-10\p@ \@plus -4\p@ \@minus -2\p@} + {4\p@}{\normalsize\@ucheadtrue}% +} +\def\section{% + \@startsection{section}{1}{\z@}{-10\p@ \@plus -4\p@ \@minus -2\p@}% GM + {4\p@}{\baselineskip 14pt\secfnt\@ucheadtrue}% +} + +\def\subsection{% + \@startsection{subsection}{2}{\z@}{-8\p@ \@plus -2\p@ \@minus -\p@} + {4\p@}{\secfnt}% +} +\def\subsubsection{% + \@startsection{subsubsection}{3}{\z@}{-8\p@ \@plus -2\p@ \@minus -\p@}% + {4\p@}{\subsecfnt}% +} +%\def\paragraph{% +% \vskip 12pt\@startsection{paragraph}{3}{\z@}{6\p@ \@plus \p@}% original +% {-5\p@}{\subsecfnt}% +%} +% If one wants sections, subsections and subsubsections numbered, +% but not paragraphs, one usually sets secnumepth to 3. +% For that, the "depth" of paragraphs must be given correctly +% in the definition (``4'' instead of ``3'' as second argument +% of @startsection): +\def\paragraph{% + \vskip 12pt\@startsection{paragraph}{4}{\z@}{6\p@ \@plus \p@}% % GM and Wolfgang May - 11/30/06 + {-5\p@}{\subsecfnt}% +} +\let\@period=. +\def\@startsection#1#2#3#4#5#6{% + \if@noskipsec %gkmt, 11 aug 99 + \global\let\@period\@empty + \leavevmode + \global\let\@period.% + \fi + \par % + \@tempskipa #4\relax + \@afterindenttrue + \ifdim \@tempskipa <\z@ + \@tempskipa -\@tempskipa + \@afterindentfalse + \fi + \if@nobreak + \everypar{}% + \else + \addpenalty\@secpenalty + \addvspace\@tempskipa + \fi +\parskip=0pt % GM July 2000 (non numbered) section heads + \@ifstar + {\@ssect{#3}{#4}{#5}{#6}} + {\@dblarg{\@sect{#1}{#2}{#3}{#4}{#5}{#6}}}% +} +\def\@sect#1#2#3#4#5#6[#7]#8{% + \ifnum #2>\c@secnumdepth + \let\@svsec\@empty + \else + \refstepcounter{#1}% + \edef\@svsec{% + \begingroup + %\ifnum#2>2 \noexpand\rm \fi % changed to next 29 July 2002 gkmt + \ifnum#2>2 \noexpand#6 \fi + \csname the#1\endcsname + \endgroup + \ifnum #2=1\relax .\fi + \hskip 1em + }% + \fi + \@tempskipa #5\relax + \ifdim \@tempskipa>\z@ + \begingroup + #6\relax + \@hangfrom{\hskip #3\relax\@svsec}% + \begingroup + \interlinepenalty \@M + \if@uchead + \uppercase{#8}% + \else + #8% + \fi + \par + \endgroup + \endgroup + \csname #1mark\endcsname{#7}% + \vskip -12pt %gkmt, 11 aug 99 and GM July 2000 (was -14) - numbered section head spacing +\addcontentsline{toc}{#1}{% + \ifnum #2>\c@secnumdepth \else + \protect\numberline{\csname the#1\endcsname}% + \fi + #7% + }% + \else + \def\@svsechd{% + #6% + \hskip #3\relax + \@svsec + \if@uchead + \uppercase{#8}% + \else + #8% + \fi + \csname #1mark\endcsname{#7}% + \addcontentsline{toc}{#1}{% + \ifnum #2>\c@secnumdepth \else + \protect\numberline{\csname the#1\endcsname}% + \fi + #7% + }% + }% + \fi + \@xsect{#5}\hskip 1pt + \par +} +\def\@xsect#1{% + \@tempskipa #1\relax + \ifdim \@tempskipa>\z@ + \par + \nobreak + \vskip \@tempskipa + \@afterheading + \else + \global\@nobreakfalse + \global\@noskipsectrue + \everypar{% + \if@noskipsec + \global\@noskipsecfalse + \clubpenalty\@M + \hskip -\parindent + \begingroup + \@svsechd + \@period + \endgroup + \unskip + \@tempskipa #1\relax + \hskip -\@tempskipa + \else + \clubpenalty \@clubpenalty + \everypar{}% + \fi + }% + \fi + \ignorespaces +} +\def\@trivlist{% + \@topsepadd\topsep + \if@noskipsec + \global\let\@period\@empty + \leavevmode + \global\let\@period.% + \fi + \ifvmode + \advance\@topsepadd\partopsep + \else + \unskip + \par + \fi + \if@inlabel + \@noparitemtrue + \@noparlisttrue + \else + \@noparlistfalse + \@topsep\@topsepadd + \fi + \advance\@topsep \parskip + \leftskip\z@skip + \rightskip\@rightskip + \parfillskip\@flushglue + \@setpar{\if@newlist\else{\@@par}\fi} + \global\@newlisttrue + \@outerparskip\parskip +} + +%%% Actually, 'abbrev' works just fine as the default +%%% Bibliography style. + +\typeout{Using 'Abbrev' bibliography style} +\newcommand\bibyear[2]{% + \unskip\quad\ignorespaces#1\unskip + \if#2..\quad \else \quad#2 \fi +} +\newcommand{\bibemph}[1]{{\em#1}} +\newcommand{\bibemphic}[1]{{\em#1\/}} +\newcommand{\bibsc}[1]{{\sc#1}} +\def\@normalcite{% + \def\@cite##1##2{[##1\if@tempswa , ##2\fi]}% +} +\def\@citeNB{% + \def\@cite##1##2{##1\if@tempswa , ##2\fi}% +} +\def\@citeRB{% + \def\@cite##1##2{##1\if@tempswa , ##2\fi]}% +} +\def\start@cite#1#2{% + \edef\citeauthoryear##1##2##3{% + ###1% + \ifnum#2=\z@ \else\ ###2\fi + }% + \ifnum#1=\thr@@ + \let\@@cite\@citeyear + \else + \let\@@cite\@citenormal + \fi + \@ifstar{\@citeNB\@@cite}{\@normalcite\@@cite}% +} +%\def\cite{\start@cite23} +\DeclareRobustCommand\cite{\start@cite23} % January 2008 +\def\citeNP{\cite*} % No Parentheses e.g. 5 +%\def\citeA{\start@cite10} +\DeclareRobustCommand\citeA{\start@cite10} % January 2008 +\def\citeANP{\citeA*} +%\def\shortcite{\start@cite23} +\DeclareRobustCommand\shortcite{\start@cite23} % January 2008 +\def\shortciteNP{\shortcite*} +%\def\shortciteA{\start@cite20} +\DeclareRobustCommand\shortciteA{\start@cite20} % January 2008 +\def\shortciteANP{\shortciteA*} +%\def\citeyear{\start@cite30} +\DeclareRobustCommand\citeyear{\start@cite30} % January 2008 +\def\citeyearNP{\citeyear*} +%\def\citeN{% +\DeclareRobustCommand\citeN{% % January 2008 + \@citeRB + \def\citeauthoryear##1##2##3{##1\ [##3% + \def\reserved@a{##1}% + \def\citeauthoryear####1####2####3{% + \def\reserved@b{####1}% + \ifx\reserved@a\reserved@b + ####3% + \else + \errmessage{Package acmart Error: author mismatch + in \string\citeN^^J^^J% + See the acmart package documentation for explanation}% + \fi + }% + }% + \@ifstar\@citeyear\@citeyear +} +%\def\shortciteN{% +\DeclareRobustCommand\shortciteN{% % January 2008 + \@citeRB + \def\citeauthoryear##1##2##3{##2\ [##3% + \def\reserved@a{##2}% + \def\citeauthoryear####1####2####3{% + \def\reserved@b{####2}% + \ifx\reserved@a\reserved@b + ####3% + \else + \errmessage{Package acmart Error: author mismatch + in \string\shortciteN^^J^^J% + See the acmart package documentation for explanation}% + \fi + }% + }% + \@ifstar\@citeyear\@citeyear % GM July 2000 +} + +\def\@citenormal{% + \@ifnextchar [{\@tempswatrue\@citex;}% +% original {\@tempswafalse\@citex,[]}% was ; Gerry 2/24/00 +{\@tempswafalse\@citex[]}% % GERRY FIX FOR BABEL 3/20/2009 +} + +\def\@citeyear{% + \@ifnextchar [{\@tempswatrue\@citex,}% +% original {\@tempswafalse\@citex,[]}% +{\@tempswafalse\@citex[]}% % GERRY FIX FOR BABEL 3/20/2009 +} + +\def\@citex#1[#2]#3{% + \let\@citea\@empty + \@cite{% + \@for\@citeb:=#3\do{% + \@citea +% original \def\@citea{#1 }% + \def\@citea{#1, }% % GERRY FIX FOR BABEL 3/20/2009 -- SO THAT YOU GET [1, 2] IN THE BODY TEXT + \edef\@citeb{\expandafter\@iden\@citeb}% + \if@filesw + \immediate\write\@auxout{\string\citation{\@citeb}}% + \fi + \@ifundefined{b@\@citeb}{% + {\bf ?}% + \@warning{% + Citation `\@citeb' on page \thepage\space undefined% + }% + }% + {\csname b@\@citeb\endcsname}% + }% + }{#2}% +} +%\let\@biblabel\@gobble % Dec. 2008 - Gerry +% ---- +\def\@biblabelnum#1{[#1]} % Gerry's solution #1 - for Natbib -- April 2009 +\let\@biblabel=\@biblabelnum % Gerry's solution #1 - for Natbib -- April 2009 +\def\newblock{\relax} % Gerry Dec. 2008 +% --- +\newdimen\bibindent +\setcounter{enumi}{1} +\bibindent=0em +\def\thebibliography#1{% +\ifnum\addauflag=0\addauthorsection\global\addauflag=1\fi + \section[References]{% <=== OPTIONAL ARGUMENT ADDED HERE + {References} % was uppercased but this affects pdf bookmarks (SP/GM October 2004) + {\vskip -9pt plus 1pt} % GM Nov. 2006 / GM July 2000 (for somewhat tighter spacing) + \@mkboth{{\refname}}{{\refname}}% + }% + \list{[\arabic{enumi}]}{% + \settowidth\labelwidth{[#1]}% + \leftmargin\labelwidth + \advance\leftmargin\labelsep + \advance\leftmargin\bibindent + \parsep=0pt\itemsep=1pt % GM July 2000 + \itemindent -\bibindent + \listparindent \itemindent + \usecounter{enumi} + }% + \let\newblock\@empty + \raggedright % GM July 2000 + \sloppy + \sfcode`\.=1000\relax +} + + +\gdef\balancecolumns +{\vfill\eject +\global\@colht=\textheight +\global\ht\@cclv=\textheight +} + +\newcount\colcntr +\global\colcntr=0 +%\newbox\savebox +\newbox\saveb@x % January 2008 + +\gdef \@makecol {% +\global\advance\colcntr by 1 +\ifnum\colcntr>2 \global\colcntr=1\fi + \ifvoid\footins + \setbox\@outputbox \box\@cclv + \else + \setbox\@outputbox \vbox{% +\boxmaxdepth \@maxdepth + \@tempdima\dp\@cclv + \unvbox \@cclv + \vskip-\@tempdima + \vskip \skip\footins + \color@begingroup + \normalcolor + \footnoterule + \unvbox \footins + \color@endgroup + }% + \fi + \xdef\@freelist{\@freelist\@midlist}% + \global \let \@midlist \@empty + \@combinefloats + \ifvbox\@kludgeins + \@makespecialcolbox + \else + \setbox\@outputbox \vbox to\@colht {% +\@texttop + \dimen@ \dp\@outputbox + \unvbox \@outputbox + \vskip -\dimen@ + \@textbottom + }% + \fi + \global \maxdepth \@maxdepth +} +\def\titlenote{\@ifnextchar[\@xtitlenote{\stepcounter\@mpfn +\global\advance\titlenotecount by 1 +\ifnum\titlenotecount=1 + \raisebox{9pt}{$\ast$} +\fi +\ifnum\titlenotecount=2 + \raisebox{9pt}{$\dagger$} +\fi +\ifnum\titlenotecount=3 + \raisebox{9pt}{$\ddagger$} +\fi +\ifnum\titlenotecount=4 +\raisebox{9pt}{$\S$} +\fi +\ifnum\titlenotecount=5 +\raisebox{9pt}{$\P$} +\fi + \@titlenotetext +}} + +\long\def\@titlenotetext#1{\insert\footins{% +\ifnum\titlenotecount=1\global\tntoks={#1}\fi +\ifnum\titlenotecount=2\global\tntokstwo={#1}\fi +\ifnum\titlenotecount=3\global\tntoksthree={#1}\fi +\ifnum\titlenotecount=4\global\tntoksfour={#1}\fi +\ifnum\titlenotecount=5\global\tntoksfive={#1}\fi + \reset@font\footnotesize + \interlinepenalty\interfootnotelinepenalty + \splittopskip\footnotesep + \splitmaxdepth \dp\strutbox \floatingpenalty \@MM + \hsize\columnwidth \@parboxrestore + \protected@edef\@currentlabel{% + }% + \color@begingroup + \color@endgroup}} + +%%%%%%%%%%%%%%%%%%%%%%%%% +\ps@plain +\baselineskip=11pt +\let\thepage\relax % For NO page numbers - GM Nov. 30th. 1999 and July 2000 +\def\setpagenumber#1{\global\setcounter{page}{#1}} +%\pagenumbering{arabic} % Arabic page numbers GM July 2000 +\twocolumn % Double column. +\flushbottom % Even bottom -- alas, does not balance columns at end of document +\pagestyle{plain} + +% Need Copyright Year and Copyright Data to be user definable (in .tex file). +% Gerry Nov. 30th. 1999 +\newtoks\copyrtyr +\newtoks\acmcopyr +\newtoks\boilerplate +\global\acmcopyr={X-XXXXX-XX-X/XX/XX} % Default - 5/11/2001 *** Gerry +\global\copyrtyr={20XX} % Default - 3/3/2003 *** Gerry +\def\CopyrightYear#1{\global\copyrtyr{#1}} +\def\crdata#1{\global\acmcopyr{#1}} +\def\permission#1{\global\boilerplate{#1}} +% +\global\boilerplate={Permission to make digital or hard copies of all or part of this work for personal or classroom use is granted without fee provided that copies are not made or distributed for profit or commercial advantage and that copies bear this notice and the full citation on the first page. Copyrights for components of this work owned by others than ACM must be honored. Abstracting with credit is permitted. To copy otherwise, or republish, to post on servers or to redistribute to lists, requires prior specific permission and/or a fee. Request permissions from permissions@acm.org.} +\newtoks\copyrightetc +\global\copyrightetc{Copyright \the\copyrtyr\ ACM \the\acmcopyr\ ...\$15.00} % Gerry changed to 15 May 2012 +\toappear{\the\boilerplate\par +{\confname{\the\conf}} \the\confinfo\par \the\copyrightetc.} +%\DeclareFixedFont{\altcrnotice}{OT1}{tmr}{m}{n}{8} % << patch needed for accenting e.g. Montreal - Gerry, May 2007 +%\DeclareFixedFont{\altconfname}{OT1}{tmr}{m}{it}{8} % << patch needed for accenting in italicized confname - Gerry, May 2007 +% +%{\altconfname{{\the\conf}}} {\altcrnotice\the\confinfo\par} \the\copyrightetc.} % << Gerry, May 2007 +% +% The following section (i.e. 3 .sty inclusions) was added in May 2007 so as to fix the problems that many +% authors were having with accents. Sometimes accents would occur, but the letter-character would be of a different +% font. Conversely the letter-character font would be correct but, e.g. a 'bar' would appear superimposed on the +% character instead of, say, an unlaut/diaresis. Sometimes the letter-character would NOT appear at all. +% Using [T1]{fontenc} outright was not an option as this caused 99% of the authors to 'produce' a Type-3 (bitmapped) +% PDF file - useless for production. +% +% For proper (font) accenting we NEED these packages to be part of the .cls file i.e. 'ae', 'aecompl' and 'aeguil' +% ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +%% This is file `ae.sty' +\def\fileversion{1.3} +\def\filedate{2001/02/12} +\NeedsTeXFormat{LaTeX2e} +%\ProvidesPackage{ae}[\filedate\space\fileversion\space % GM +% Almost European Computer Modern] % GM - keeping the log file clean(er) +\newif\if@ae@slides \@ae@slidesfalse +\DeclareOption{slides}{\@ae@slidestrue} +\ProcessOptions +\fontfamily{aer} +\RequirePackage[T1]{fontenc} +\if@ae@slides + \renewcommand{\sfdefault}{laess} + \renewcommand{\rmdefault}{laess} % no roman + \renewcommand{\ttdefault}{laett} +\else + \renewcommand{\sfdefault}{aess} + \renewcommand{\rmdefault}{aer} + \renewcommand{\ttdefault}{aett} +\fi +\endinput +%% +%% End of file `ae.sty'. +% +% +\def\fileversion{0.9} +\def\filedate{1998/07/23} +\NeedsTeXFormat{LaTeX2e} +%\ProvidesPackage{aecompl}[\filedate\space\fileversion\space % GM +%T1 Complements for AE fonts (D. Roegel)] % GM -- keeping the log file clean(er) + +\def\@ae@compl#1{{\fontencoding{T1}\fontfamily{cmr}\selectfont\symbol{#1}}} +\def\guillemotleft{\@ae@compl{19}} +\def\guillemotright{\@ae@compl{20}} +\def\guilsinglleft{\@ae@compl{14}} +\def\guilsinglright{\@ae@compl{15}} +\def\TH{\@ae@compl{222}} +\def\NG{\@ae@compl{141}} +\def\ng{\@ae@compl{173}} +\def\th{\@ae@compl{254}} +\def\DJ{\@ae@compl{208}} +\def\dj{\@ae@compl{158}} +\def\DH{\@ae@compl{208}} +\def\dh{\@ae@compl{240}} +\def\@perthousandzero{\@ae@compl{24}} +\def\textperthousand{\%\@perthousandzero} +\def\textpertenthousand{\%\@perthousandzero\@perthousandzero} +\endinput +% +% +%% This is file `aeguill.sty' +% This file gives french guillemets (and not guillemots!) +% built with the Polish CMR fonts (default), WNCYR fonts, the LASY fonts +% or with the EC fonts. +% This is useful in conjunction with the ae package +% (this package loads the ae package in case it has not been loaded) +% and with or without the french(le) package. +% +% In order to get the guillemets, it is necessary to either type +% \guillemotleft and \guillemotright, or to use an 8 bit encoding +% (such as ISO-Latin1) which selects these two commands, +% or, if you use the french package (but not the frenchle package), +% to type << or >>. +% +% By default, you get the Polish CMR guillemets; if this package is loaded +% with the `cm' option, you get the LASY guillemets; with `ec,' you +% get the EC guillemets, and with `cyr,' you get the cyrillic guillemets. +% +% In verbatim mode, you always get the EC/TT guillemets. +% +% The default option is interesting in conjunction with PDF, +% because there is a Type 1 version of the Polish CMR fonts +% and these guillemets are very close in shape to the EC guillemets. +% There are no free Type 1 versions of the EC fonts. +% +% Support for Polish CMR guillemets was kindly provided by +% Rolf Niepraschk in version 0.99 (2000/05/22). +% Bernd Raichle provided extensive simplifications to the code +% for version 1.00. +% +% This package is released under the LPPL. +% +% Changes: +% Date version +% 2001/04/12 1.01 the frenchle and french package are now distinguished. +% +\def\fileversion{1.01} +\def\filedate{2001/04/12} +\NeedsTeXFormat{LaTeX2e} +%\ProvidesPackage{aeguill}[2001/04/12 1.01 % % GM +%AE fonts with french guillemets (D. Roegel)] % GM - keeping the log file clean(er) +%\RequirePackage{ae} % GM May 2007 - already embedded here + +\newcommand{\@ae@switch}[4]{#4} +\DeclareOption{ec}{\renewcommand\@ae@switch[4]{#1}} +\DeclareOption{cm}{\renewcommand\@ae@switch[4]{#2}} +\DeclareOption{cyr}{\renewcommand\@ae@switch[4]{#3}} +\DeclareOption{pl}{\renewcommand\@ae@switch[4]{#4}} +\ExecuteOptions{pl} +\ProcessOptions + +% +% Load necessary packages +% +\@ae@switch{% ec + % do nothing +}{% cm + \RequirePackage{latexsym}% GM - May 2007 - already 'mentioned as required' up above +}{% cyr + \RequirePackage[OT2,T1]{fontenc}% +}{% pl + \RequirePackage[OT4,T1]{fontenc}% +} + +% The following command will be compared to \frenchname, +% as defined in french.sty and frenchle.sty. +\def\aeguillfrenchdefault{french}% + +\let\guill@verbatim@font\verbatim@font +\def\verbatim@font{\guill@verbatim@font\ecguills{cmtt}% + \let\guillemotleft\@oguills\let\guillemotright\@fguills} + +\begingroup \catcode`\<=13 \catcode`\>=13 +\def\x{\endgroup + \def\ae@lfguill{<<}% + \def\ae@rfguill{>>}% +}\x + +\newcommand{\ecguills}[1]{% + \def\selectguillfont{\fontencoding{T1}\fontfamily{#1}\selectfont}% + \def\@oguills{{\selectguillfont\symbol{19}}}% + \def\@fguills{{\selectguillfont\symbol{20}}}% + } + +\newcommand{\aeguills}{% + \ae@guills + % We redefine \guillemotleft and \guillemotright + % in order to catch them when they are used + % with \DeclareInputText (in latin1.def for instance) + % We use \auxWARNINGi as a safe indicator that french.sty is used. + \gdef\guillemotleft{\ifx\auxWARNINGi\undefined + \@oguills % neither french.sty nor frenchle.sty + \else + \ifx\aeguillfrenchdefault\frenchname + \ae@lfguill % french.sty + \else + \@oguills % frenchle.sty + \fi + \fi}% + \gdef\guillemotright{\ifx\auxWARNINGi\undefined + \@fguills % neither french.sty nor frenchle.sty + \else + \ifx\aeguillfrenchdefault\frenchname + \ae@rfguill % french.sty + \else + \@fguills % frenchle.sty + \fi + \fi}% + } + +% +% Depending on the class option +% define the internal command \ae@guills +\@ae@switch{% ec + \newcommand{\ae@guills}{% + \ecguills{cmr}}% +}{% cm + \newcommand{\ae@guills}{% + \def\selectguillfont{\fontencoding{U}\fontfamily{lasy}% + \fontseries{m}\fontshape{n}\selectfont}% + \def\@oguills{\leavevmode\nobreak + \hbox{\selectguillfont (\kern-.20em(\kern.20em}\nobreak}% + \def\@fguills{\leavevmode\nobreak + \hbox{\selectguillfont \kern.20em)\kern-.2em)}% + \ifdim\fontdimen\@ne\font>\z@\/\fi}}% +}{% cyr + \newcommand{\ae@guills}{% + \def\selectguillfont{\fontencoding{OT2}\fontfamily{wncyr}\selectfont}% + \def\@oguills{{\selectguillfont\symbol{60}}}% + \def\@fguills{{\selectguillfont\symbol{62}}}} +}{% pl + \newcommand{\ae@guills}{% + \def\selectguillfont{\fontencoding{OT4}\fontfamily{cmr}\selectfont}% + \def\@oguills{{\selectguillfont\symbol{174}}}% + \def\@fguills{{\selectguillfont\symbol{175}}}} +} + + +\AtBeginDocument{% + \ifx\GOfrench\undefined + \aeguills + \else + \let\aeguill@GOfrench\GOfrench + \gdef\GOfrench{\aeguill@GOfrench \aeguills}% + \fi + } + +\endinput +% + diff --git a/rabbitmq/pom.xml b/rabbitmq/pom.xml index d5a6a1e6f27..3f9547fbad2 100644 --- a/rabbitmq/pom.xml +++ b/rabbitmq/pom.xml @@ -9,7 +9,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/s3-extensions/pom.xml b/s3-extensions/pom.xml index 96a37e90983..bb77b189789 100644 --- a/s3-extensions/pom.xml +++ b/s3-extensions/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/server/pom.xml b/server/pom.xml index f887711ce1a..45a7cea01e9 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -28,7 +28,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/server/src/main/java/io/druid/client/CachePopulatingQueryRunner.java b/server/src/main/java/io/druid/client/CachePopulatingQueryRunner.java new file mode 100644 index 00000000000..581ebd474fb --- /dev/null +++ b/server/src/main/java/io/druid/client/CachePopulatingQueryRunner.java @@ -0,0 +1,93 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheConfig; +import io.druid.client.cache.MapCache; +import io.druid.query.CacheStrategy; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.SegmentDescriptor; + +import java.util.ArrayList; + +public class CachePopulatingQueryRunner implements QueryRunner +{ + + private final String segmentIdentifier; + private final SegmentDescriptor segmentDescriptor; + private final QueryRunner base; + private final QueryToolChest toolChest; + private final Cache cache; + private final ObjectMapper mapper; + private final CacheConfig cacheConfig; + + public CachePopulatingQueryRunner( + String segmentIdentifier, + SegmentDescriptor segmentDescriptor, ObjectMapper mapper, + Cache cache, + QueryToolChest toolchest, + QueryRunner base, + CacheConfig cacheConfig + ) + { + this.base = base; + this.segmentIdentifier = segmentIdentifier; + this.segmentDescriptor = segmentDescriptor; + this.toolChest = toolchest; + this.cache = cache; + this.mapper = mapper; + this.cacheConfig = cacheConfig; + } + + @Override + public Sequence run(Query query) + { + + final CacheStrategy strategy = toolChest.getCacheStrategy(query); + + final boolean populateCache = Boolean.parseBoolean(query.getContextValue(CacheConfig.POPULATE_CACHE, "true")) + && strategy != null + && cacheConfig.isPopulateCache() + // historical only populates distributed cache since the cache lookups are done at broker. + && !(cache instanceof MapCache) ; + Sequence results = base.run(query); + if (populateCache) { + Cache.NamedKey key = CacheUtil.computeSegmentCacheKey( + segmentIdentifier, + segmentDescriptor, + strategy.computeCacheKey(query) + ); + CacheUtil.populate( + cache, + mapper, + key, + Sequences.toList(Sequences.map(results, strategy.prepareForCache()), new ArrayList()) + ); + } + return results; + + } +} diff --git a/server/src/main/java/io/druid/client/CacheUtil.java b/server/src/main/java/io/druid/client/CacheUtil.java new file mode 100644 index 00000000000..cbc03f64474 --- /dev/null +++ b/server/src/main/java/io/druid/client/CacheUtil.java @@ -0,0 +1,80 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2014 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import io.druid.client.cache.Cache; +import io.druid.query.SegmentDescriptor; +import org.joda.time.Interval; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class CacheUtil +{ + public static Cache.NamedKey computeSegmentCacheKey( + String segmentIdentifier, + SegmentDescriptor descriptor, + byte[] queryCacheKey + ) + { + final Interval segmentQueryInterval = descriptor.getInterval(); + final byte[] versionBytes = descriptor.getVersion().getBytes(); + + return new Cache.NamedKey( + segmentIdentifier, ByteBuffer + .allocate(16 + versionBytes.length + 4 + queryCacheKey.length) + .putLong(segmentQueryInterval.getStartMillis()) + .putLong(segmentQueryInterval.getEndMillis()) + .put(versionBytes) + .putInt(descriptor.getPartitionNumber()) + .put(queryCacheKey).array() + ); + } + + public static void populate(Cache cache, ObjectMapper mapper, Cache.NamedKey key, Iterable results) + { + try { + List bytes = Lists.newArrayList(); + int size = 0; + for (Object result : results) { + final byte[] array = mapper.writeValueAsBytes(result); + size += array.length; + bytes.add(array); + } + + byte[] valueBytes = new byte[size]; + int offset = 0; + for (byte[] array : bytes) { + System.arraycopy(array, 0, valueBytes, offset, array.length); + offset += array.length; + } + + cache.put(key, valueBytes); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } + +} diff --git a/server/src/main/java/io/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java index a9d6538b200..65ac6bea415 100644 --- a/server/src/main/java/io/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -40,6 +40,7 @@ import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.emitter.EmittingLogger; import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheConfig; import io.druid.client.selector.QueryableDruidServer; import io.druid.client.selector.ServerSelector; import io.druid.guice.annotations.Smile; @@ -61,8 +62,8 @@ import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Interval; +import javax.annotation.Nullable; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -76,24 +77,26 @@ import java.util.concurrent.Executors; public class CachingClusteredClient implements QueryRunner { private static final EmittingLogger log = new EmittingLogger(CachingClusteredClient.class); - private final QueryToolChestWarehouse warehouse; private final TimelineServerView serverView; private final Cache cache; private final ObjectMapper objectMapper; + private final CacheConfig cacheConfig; @Inject public CachingClusteredClient( QueryToolChestWarehouse warehouse, TimelineServerView serverView, Cache cache, - @Smile ObjectMapper objectMapper + @Smile ObjectMapper objectMapper, + CacheConfig cacheConfig ) { this.warehouse = warehouse; this.serverView = serverView; this.cache = cache; this.objectMapper = objectMapper; + this.cacheConfig = cacheConfig; serverView.registerSegmentCallback( Executors.newFixedThreadPool( @@ -122,18 +125,21 @@ public class CachingClusteredClient implements QueryRunner final List> cachedResults = Lists.newArrayList(); final Map cachePopulatorMap = Maps.newHashMap(); - final boolean useCache = Boolean.parseBoolean(query.getContextValue("useCache", "true")) && strategy != null; - final boolean populateCache = Boolean.parseBoolean(query.getContextValue("populateCache", "true")) - && strategy != null; + final boolean useCache = Boolean.parseBoolean(query.getContextValue(CacheConfig.USE_CACHE, "true")) + && strategy != null + && cacheConfig.isUseCache(); + final boolean populateCache = Boolean.parseBoolean(query.getContextValue(CacheConfig.POPULATE_CACHE, "true")) + && strategy != null && cacheConfig.isPopulateCache(); final boolean isBySegment = Boolean.parseBoolean(query.getContextValue("bySegment", "false")); - ImmutableMap.Builder contextBuilder = new ImmutableMap.Builder<>(); + ImmutableMap.Builder contextBuilder = new ImmutableMap.Builder<>(); final String priority = query.getContextValue("priority", "0"); contextBuilder.put("priority", priority); if (populateCache) { + contextBuilder.put(CacheConfig.POPULATE_CACHE, "false"); contextBuilder.put("bySegment", "true"); } contextBuilder.put("intermediate", "true"); @@ -180,7 +186,7 @@ public class CachingClusteredClient implements QueryRunner if (queryCacheKey != null) { Map, Cache.NamedKey> cacheKeys = Maps.newHashMap(); for (Pair segment : segments) { - final Cache.NamedKey segmentCacheKey = computeSegmentCacheKey( + final Cache.NamedKey segmentCacheKey = CacheUtil.computeSegmentCacheKey( segment.lhs.getSegment().getIdentifier(), segment.rhs, queryCacheKey @@ -286,7 +292,8 @@ public class CachingClusteredClient implements QueryRunner objectMapper.getFactory().createParser(cachedResult), cacheObjectClazz ); - } catch (IOException e) { + } + catch (IOException e) { throw Throwables.propagate(e); } } @@ -372,26 +379,6 @@ public class CachingClusteredClient implements QueryRunner ); } - private Cache.NamedKey computeSegmentCacheKey( - String segmentIdentifier, - SegmentDescriptor descriptor, - byte[] queryCacheKey - ) - { - final Interval segmentQueryInterval = descriptor.getInterval(); - final byte[] versionBytes = descriptor.getVersion().getBytes(); - - return new Cache.NamedKey( - segmentIdentifier, ByteBuffer - .allocate(16 + versionBytes.length + 4 + queryCacheKey.length) - .putLong(segmentQueryInterval.getStartMillis()) - .putLong(segmentQueryInterval.getEndMillis()) - .put(versionBytes) - .putInt(descriptor.getPartitionNumber()) - .put(queryCacheKey).array() - ); - } - private static class CachePopulator { private final Cache cache; @@ -407,26 +394,7 @@ public class CachingClusteredClient implements QueryRunner public void populate(Iterable results) { - try { - List bytes = Lists.newArrayList(); - int size = 0; - for (Object result : results) { - final byte[] array = mapper.writeValueAsBytes(result); - size += array.length; - bytes.add(array); - } - - byte[] valueBytes = new byte[size]; - int offset = 0; - for (byte[] array : bytes) { - System.arraycopy(array, 0, valueBytes, offset, array.length); - offset += array.length; - } - - cache.put(key, valueBytes); - } catch (IOException e) { - throw Throwables.propagate(e); - } + CacheUtil.populate(cache, mapper, key, results); } } } diff --git a/server/src/main/java/io/druid/client/cache/CacheConfig.java b/server/src/main/java/io/druid/client/cache/CacheConfig.java new file mode 100644 index 00000000000..817cdb4070a --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/CacheConfig.java @@ -0,0 +1,42 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class CacheConfig +{ + public static String USE_CACHE = "useCache"; + public static String POPULATE_CACHE = "populateCache"; + @JsonProperty + private boolean useCache = true; + @JsonProperty + private boolean populateCache = true; + + public boolean isPopulateCache() + { + return populateCache; + } + + public boolean isUseCache() + { + return useCache; + } +} diff --git a/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java index 4029331361a..716f5abab93 100644 --- a/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java +++ b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java @@ -39,7 +39,6 @@ public class LocalCacheProvider implements CacheProvider @Min(0) private int logEvictionCount = 0; - @Override public Cache get() { diff --git a/server/src/main/java/io/druid/client/cache/MapCache.java b/server/src/main/java/io/druid/client/cache/MapCache.java index dda2929bc89..f745fbf206f 100644 --- a/server/src/main/java/io/druid/client/cache/MapCache.java +++ b/server/src/main/java/io/druid/client/cache/MapCache.java @@ -54,7 +54,6 @@ public class MapCache implements Cache ) { this.byteCountingLRUMap = byteCountingLRUMap; - this.baseMap = Collections.synchronizedMap(byteCountingLRUMap); namespaceId = Maps.newHashMap(); diff --git a/server/src/main/java/io/druid/client/cache/MemcachedCache.java b/server/src/main/java/io/druid/client/cache/MemcachedCache.java index 8ffe995ee76..2dc3c031775 100644 --- a/server/src/main/java/io/druid/client/cache/MemcachedCache.java +++ b/server/src/main/java/io/druid/client/cache/MemcachedCache.java @@ -83,9 +83,7 @@ public class MemcachedCache implements Cache .build(), AddrUtil.getAddresses(config.getHosts()) ), - config.getMemcachedPrefix(), - config.getTimeout(), - config.getExpiration() + config ); } catch(IOException e) { throw Throwables.propagate(e); @@ -103,15 +101,16 @@ public class MemcachedCache implements Cache private final AtomicLong timeoutCount = new AtomicLong(0); private final AtomicLong errorCount = new AtomicLong(0); - MemcachedCache(MemcachedClientIF client, String memcachedPrefix, int timeout, int expiration) { - Preconditions.checkArgument(memcachedPrefix.length() <= MAX_PREFIX_LENGTH, + + MemcachedCache(MemcachedClientIF client, MemcachedCacheConfig config) { + Preconditions.checkArgument(config.getMemcachedPrefix().length() <= MAX_PREFIX_LENGTH, "memcachedPrefix length [%d] exceeds maximum length [%d]", - memcachedPrefix.length(), + config.getMemcachedPrefix().length(), MAX_PREFIX_LENGTH); - this.timeout = timeout; - this.expiration = expiration; + this.timeout = config.getTimeout(); + this.expiration = config.getExpiration(); this.client = client; - this.memcachedPrefix = memcachedPrefix; + this.memcachedPrefix = config.getMemcachedPrefix(); } @Override diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManager.java b/server/src/main/java/io/druid/db/DatabaseRuleManager.java index 9a1c014b36f..5882c964346 100644 --- a/server/src/main/java/io/druid/db/DatabaseRuleManager.java +++ b/server/src/main/java/io/druid/db/DatabaseRuleManager.java @@ -159,7 +159,7 @@ public class DatabaseRuleManager this.exec = Execs.scheduledSingleThreaded("DatabaseRuleManager-Exec--%d"); - createDefaultRule(dbi, getRulesTable(), config.get().getDefaultTier(), jsonMapper); + createDefaultRule(dbi, getRulesTable(), config.get().getDefaultRule(), jsonMapper); ScheduledExecutors.scheduleWithFixedDelay( exec, new Duration(0), @@ -274,8 +274,8 @@ public class DatabaseRuleManager if (theRules.get(dataSource) != null) { retVal.addAll(theRules.get(dataSource)); } - if (theRules.get(config.get().getDefaultTier()) != null) { - retVal.addAll(theRules.get(config.get().getDefaultTier())); + if (theRules.get(config.get().getDefaultRule()) != null) { + retVal.addAll(theRules.get(config.get().getDefaultRule())); } return retVal; } diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java b/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java index 8b770a57955..63f506649b5 100644 --- a/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java +++ b/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java @@ -27,14 +27,14 @@ import org.joda.time.Period; public class DatabaseRuleManagerConfig { @JsonProperty - private String defaultTier = "_default"; + private String defaultRule = "_default"; @JsonProperty private Period pollDuration = new Period("PT1M"); - public String getDefaultTier() + public String getDefaultRule() { - return defaultTier; + return defaultRule; } public Period getPollDuration() diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java b/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java index dfc25bea283..0296f25867e 100644 --- a/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java +++ b/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java @@ -64,7 +64,7 @@ public class DatabaseRuleManagerProvider implements Provider + * * @return Returns the set of modules loaded. */ - public static Set getLoadedModules(Class clazz) + public static Set getLoadedModules(Class clazz) { Set retVal = extensionsMap.get(clazz); if (retVal == null) { @@ -190,22 +191,29 @@ public class Initialization ) ); - final List artifacts = aether.resolveArtifacts(dependencyRequest); - List urls = Lists.newArrayListWithExpectedSize(artifacts.size()); - for (Artifact artifact : artifacts) { - if (!exclusions.contains(artifact.getGroupId())) { - urls.add(artifact.getFile().toURI().toURL()); - } else { - log.debug("Skipped Artifact[%s]", artifact); + try { + final List artifacts = aether.resolveArtifacts(dependencyRequest); + + List urls = Lists.newArrayListWithExpectedSize(artifacts.size()); + for (Artifact artifact : artifacts) { + if (!exclusions.contains(artifact.getGroupId())) { + urls.add(artifact.getFile().toURI().toURL()); + } else { + log.debug("Skipped Artifact[%s]", artifact); + } } - } - for (URL url : urls) { - log.info("Added URL[%s]", url); - } + for (URL url : urls) { + log.info("Added URL[%s]", url); + } - loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader()); - loadersMap.put(coordinate, loader); + loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader()); + loadersMap.put(coordinate, loader); + } + catch (Exception e) { + log.error(e, "Unable to resolve artifacts for [%s].", dependencyRequest); + throw Throwables.propagate(e); + } } return loader; } @@ -232,9 +240,9 @@ public class Initialization URI u = new URI(uri); Repository r = new Repository(uri); - if(u.getUserInfo() != null) { + if (u.getUserInfo() != null) { String[] auth = u.getUserInfo().split(":", 2); - if(auth.length == 2) { + if (auth.length == 2) { r.setUsername(auth[0]); r.setPassword(auth[1]); } else { @@ -247,7 +255,7 @@ public class Initialization } remoteRepositories.add(r); } - catch(URISyntaxException e) { + catch (URISyntaxException e) { throw Throwables.propagate(e); } } @@ -261,28 +269,30 @@ public class Initialization PrintStream oldOut = System.out; try { - System.setOut(new PrintStream( - new OutputStream() - { - @Override - public void write(int b) throws IOException - { + System.setOut( + new PrintStream( + new OutputStream() + { + @Override + public void write(int b) throws IOException + { - } + } - @Override - public void write(byte[] b) throws IOException - { + @Override + public void write(byte[] b) throws IOException + { - } + } - @Override - public void write(byte[] b, int off, int len) throws IOException - { + @Override + public void write(byte[] b, int off, int len) throws IOException + { - } - } - )); + } + } + ) + ); return new DefaultTeslaAether( config.getLocalRepository(), remoteRepositories.toArray(new Repository[remoteRepositories.size()]) diff --git a/server/src/main/java/io/druid/server/coordination/ServerManager.java b/server/src/main/java/io/druid/server/coordination/ServerManager.java index 3ee0c108f8a..b49fade0404 100644 --- a/server/src/main/java/io/druid/server/coordination/ServerManager.java +++ b/server/src/main/java/io/druid/server/coordination/ServerManager.java @@ -19,6 +19,7 @@ package io.druid.server.coordination; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.collect.Ordering; @@ -28,8 +29,12 @@ import com.metamx.common.guava.FunctionalIterable; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.CachePopulatingQueryRunner; +import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheConfig; import io.druid.collections.CountingMap; import io.druid.guice.annotations.Processing; +import io.druid.guice.annotations.Smile; import io.druid.query.BySegmentQueryRunner; import io.druid.query.DataSource; import io.druid.query.FinalizeResultsQueryRunner; @@ -44,7 +49,6 @@ import io.druid.query.QueryToolChest; import io.druid.query.ReferenceCountingSegmentQueryRunner; import io.druid.query.SegmentDescriptor; import io.druid.query.TableDataSource; -import io.druid.query.spec.QuerySegmentSpec; import io.druid.query.spec.SpecificSegmentQueryRunner; import io.druid.query.spec.SpecificSegmentSpec; import io.druid.segment.ReferenceCountingSegment; @@ -70,24 +74,27 @@ import java.util.concurrent.ExecutorService; public class ServerManager implements QuerySegmentWalker { private static final EmittingLogger log = new EmittingLogger(ServerManager.class); - private final Object lock = new Object(); - private final SegmentLoader segmentLoader; private final QueryRunnerFactoryConglomerate conglomerate; private final ServiceEmitter emitter; private final ExecutorService exec; - private final Map> dataSources; private final CountingMap dataSourceSizes = new CountingMap(); private final CountingMap dataSourceCounts = new CountingMap(); + private final Cache cache; + private final ObjectMapper objectMapper; + private final CacheConfig cacheConfig; @Inject public ServerManager( SegmentLoader segmentLoader, QueryRunnerFactoryConglomerate conglomerate, ServiceEmitter emitter, - @Processing ExecutorService exec + @Processing ExecutorService exec, + @Smile ObjectMapper objectMapper, + Cache cache, + CacheConfig cacheConfig ) { this.segmentLoader = segmentLoader; @@ -95,8 +102,11 @@ public class ServerManager implements QuerySegmentWalker this.emitter = emitter; this.exec = exec; + this.cache = cache; + this.objectMapper = objectMapper; this.dataSources = new HashMap<>(); + this.cacheConfig = cacheConfig; } public Map getDataSourceSizes() @@ -122,7 +132,9 @@ public class ServerManager implements QuerySegmentWalker * Load a single segment. * * @param segment segment to load + * * @return true if the segment was newly loaded, false if it was already loaded + * * @throws SegmentLoadingException if the segment cannot be loaded */ public boolean loadSegment(final DataSegment segment) throws SegmentLoadingException @@ -130,10 +142,12 @@ public class ServerManager implements QuerySegmentWalker final Segment adapter; try { adapter = segmentLoader.getSegment(segment); - } catch (SegmentLoadingException e) { + } + catch (SegmentLoadingException e) { try { segmentLoader.cleanup(segment); - } catch (SegmentLoadingException e1) { + } + catch (SegmentLoadingException e1) { // ignore } throw e; @@ -205,11 +219,12 @@ public class ServerManager implements QuerySegmentWalker try { log.info("Attempting to close segment %s", segment.getIdentifier()); oldQueryable.close(); - } catch (IOException e) { + } + catch (IOException e) { log.makeAlert(e, "Exception closing segment") - .addData("dataSource", dataSource) - .addData("segmentId", segment.getIdentifier()) - .emit(); + .addData("dataSource", dataSource) + .addData("segmentId", segment.getIdentifier()) + .emit(); } } else { log.info( @@ -241,7 +256,8 @@ public class ServerManager implements QuerySegmentWalker String dataSourceName; try { dataSourceName = ((TableDataSource) query.getDataSource()).getName(); - } catch (ClassCastException e) { + } + catch (ClassCastException e) { throw new UnsupportedOperationException("Subqueries are only supported in the broker"); } @@ -287,13 +303,12 @@ public class ServerManager implements QuerySegmentWalker factory, toolChest, input.getObject(), - new SpecificSegmentSpec( - new SegmentDescriptor( - holder.getInterval(), - holder.getVersion(), - input.getChunkNumber() - ) + new SegmentDescriptor( + holder.getInterval(), + holder.getVersion(), + input.getChunkNumber() ) + ); } } @@ -316,8 +331,8 @@ public class ServerManager implements QuerySegmentWalker final QueryRunnerFactory> factory = conglomerate.findFactory(query); if (factory == null) { log.makeAlert("Unknown query type, [%s]", query.getClass()) - .addData("dataSource", query.getDataSource()) - .emit(); + .addData("dataSource", query.getDataSource()) + .emit(); return new NoopQueryRunner(); } @@ -326,7 +341,8 @@ public class ServerManager implements QuerySegmentWalker String dataSourceName; try { dataSourceName = ((TableDataSource) query.getDataSource()).getName(); - } catch (ClassCastException e) { + } + catch (ClassCastException e) { throw new UnsupportedOperationException("Subqueries are only supported in the broker"); } @@ -360,7 +376,7 @@ public class ServerManager implements QuerySegmentWalker final ReferenceCountingSegment adapter = chunk.getObject(); return Arrays.asList( - buildAndDecorateQueryRunner(factory, toolChest, adapter, new SpecificSegmentSpec(input)) + buildAndDecorateQueryRunner(factory, toolChest, adapter, input) ); } } @@ -376,9 +392,10 @@ public class ServerManager implements QuerySegmentWalker final QueryRunnerFactory> factory, final QueryToolChest> toolChest, final ReferenceCountingSegment adapter, - final QuerySegmentSpec segmentSpec + final SegmentDescriptor segmentDescriptor ) { + SpecificSegmentSpec segmentSpec = new SpecificSegmentSpec(segmentDescriptor); return new SpecificSegmentQueryRunner( new MetricsEmittingQueryRunner( emitter, @@ -393,7 +410,15 @@ public class ServerManager implements QuerySegmentWalker new BySegmentQueryRunner( adapter.getIdentifier(), adapter.getDataInterval().getStart(), - new ReferenceCountingSegmentQueryRunner(factory, adapter) + new CachePopulatingQueryRunner( + adapter.getIdentifier(), + segmentDescriptor, + objectMapper, + cache, + toolChest, + new ReferenceCountingSegmentQueryRunner(factory, adapter), + cacheConfig + ) ) ).withWaitMeasuredFromNow(), segmentSpec diff --git a/server/src/main/java/io/druid/server/coordinator/rules/ForeverDropRule.java b/server/src/main/java/io/druid/server/coordinator/rules/ForeverDropRule.java index 510cb29c55a..a96c7f2cf80 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/ForeverDropRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/ForeverDropRule.java @@ -22,6 +22,7 @@ package io.druid.server.coordinator.rules; import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.timeline.DataSegment; import org.joda.time.DateTime; +import org.joda.time.Interval; /** */ @@ -39,4 +40,10 @@ public class ForeverDropRule extends DropRule { return true; } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) + { + return true; + } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/ForeverLoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/ForeverLoadRule.java index 2150150cb03..2909741906e 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/ForeverLoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/ForeverLoadRule.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.druid.timeline.DataSegment; import org.joda.time.DateTime; +import org.joda.time.Interval; import java.util.Map; @@ -66,4 +67,10 @@ public class ForeverLoadRule extends LoadRule { return true; } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) + { + return true; + } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/IntervalDropRule.java b/server/src/main/java/io/druid/server/coordinator/rules/IntervalDropRule.java index 8ea0bd30574..30e75b7d31c 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/IntervalDropRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/IntervalDropRule.java @@ -55,6 +55,12 @@ public class IntervalDropRule extends DropRule @Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { - return interval.contains(segment.getInterval()); + return appliesTo(segment.getInterval(), referenceTimestamp); + } + + @Override + public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) + { + return interval.contains(theInterval); } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/IntervalLoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/IntervalLoadRule.java index 9b2f67c652d..3164da690be 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/IntervalLoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/IntervalLoadRule.java @@ -85,7 +85,13 @@ public class IntervalLoadRule extends LoadRule @Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) { - return interval.contains(segment.getInterval()); + return appliesTo(segment.getInterval(), referenceTimestamp); + } + + @Override + public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) + { + return interval.contains(theInterval); } @Override diff --git a/server/src/main/java/io/druid/server/coordinator/rules/PeriodDropRule.java b/server/src/main/java/io/druid/server/coordinator/rules/PeriodDropRule.java index 1e7b1208628..c45345804d0 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/PeriodDropRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/PeriodDropRule.java @@ -55,8 +55,14 @@ public class PeriodDropRule extends DropRule @Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) + { + return appliesTo(segment.getInterval(), referenceTimestamp); + } + + @Override + public boolean appliesTo(Interval theInterval, DateTime referenceTimestamp) { final Interval currInterval = new Interval(period, referenceTimestamp); - return currInterval.contains(segment.getInterval()); + return currInterval.contains(theInterval); } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/PeriodLoadRule.java b/server/src/main/java/io/druid/server/coordinator/rules/PeriodLoadRule.java index bfad025b4a5..e996fa9e464 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/PeriodLoadRule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/PeriodLoadRule.java @@ -86,8 +86,14 @@ public class PeriodLoadRule extends LoadRule @Override public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp) + { + return appliesTo(segment.getInterval(), referenceTimestamp); + } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) { final Interval currInterval = new Interval(period, referenceTimestamp); - return currInterval.overlaps(segment.getInterval()); + return currInterval.overlaps(interval); } } diff --git a/server/src/main/java/io/druid/server/coordinator/rules/Rule.java b/server/src/main/java/io/druid/server/coordinator/rules/Rule.java index d3d809232dc..c3da9eccfde 100644 --- a/server/src/main/java/io/druid/server/coordinator/rules/Rule.java +++ b/server/src/main/java/io/druid/server/coordinator/rules/Rule.java @@ -26,6 +26,7 @@ import io.druid.server.coordinator.DruidCoordinator; import io.druid.server.coordinator.DruidCoordinatorRuntimeParams; import io.druid.timeline.DataSegment; import org.joda.time.DateTime; +import org.joda.time.Interval; /** */ @@ -37,13 +38,15 @@ import org.joda.time.DateTime; @JsonSubTypes.Type(name = "dropByPeriod", value = PeriodDropRule.class), @JsonSubTypes.Type(name = "dropByInterval", value = IntervalDropRule.class), @JsonSubTypes.Type(name = "dropForever", value = ForeverDropRule.class) - }) + public interface Rule { public String getType(); public boolean appliesTo(DataSegment segment, DateTime referenceTimestamp); + public boolean appliesTo(Interval interval, DateTime referenceTimestamp); + public CoordinatorStats run(DruidCoordinator coordinator, DruidCoordinatorRuntimeParams params, DataSegment segment); } diff --git a/server/src/main/java/io/druid/server/router/BrokerSelector.java b/server/src/main/java/io/druid/server/router/BrokerSelector.java new file mode 100644 index 00000000000..e27acf09c10 --- /dev/null +++ b/server/src/main/java/io/druid/server/router/BrokerSelector.java @@ -0,0 +1,178 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.metamx.common.Pair; +import com.metamx.common.concurrent.ScheduledExecutors; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.emitter.EmittingLogger; +import io.druid.concurrent.Execs; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.query.Query; +import io.druid.server.coordinator.rules.LoadRule; +import io.druid.server.coordinator.rules.Rule; +import org.joda.time.DateTime; +import org.joda.time.Duration; +import org.joda.time.Interval; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + */ +public class BrokerSelector +{ + private static EmittingLogger log = new EmittingLogger(BrokerSelector.class); + + private final CoordinatorRuleManager ruleManager; + private final TierConfig tierConfig; + private final ServerDiscoveryFactory serverDiscoveryFactory; + private final ConcurrentHashMap selectorMap = new ConcurrentHashMap(); + + private final Object lock = new Object(); + + private volatile boolean started = false; + + @Inject + public BrokerSelector( + CoordinatorRuleManager ruleManager, + TierConfig tierConfig, + ServerDiscoveryFactory serverDiscoveryFactory + ) + { + this.ruleManager = ruleManager; + this.tierConfig = tierConfig; + this.serverDiscoveryFactory = serverDiscoveryFactory; + } + + @LifecycleStart + public void start() + { + synchronized (lock) { + if (started) { + return; + } + + try { + for (Map.Entry entry : tierConfig.getTierToBrokerMap().entrySet()) { + ServerDiscoverySelector selector = serverDiscoveryFactory.createSelector(entry.getValue()); + selector.start(); + selectorMap.put(entry.getValue(), selector); + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + started = true; + } + } + + + @LifecycleStop + public void stop() + { + synchronized (lock) { + if (!started) { + return; + } + + try { + for (ServerDiscoverySelector selector : selectorMap.values()) { + selector.stop(); + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + started = false; + } + } + + public Pair select(final Query query) + { + synchronized (lock) { + if (!ruleManager.isStarted() || !started) { + return null; + } + } + + List rules = ruleManager.getRulesWithDefault((query.getDataSource()).getName()); + + // find the rule that can apply to the entire set of intervals + DateTime now = new DateTime(); + int lastRulePosition = -1; + LoadRule baseRule = null; + + for (Interval interval : query.getIntervals()) { + int currRulePosition = 0; + for (Rule rule : rules) { + if (rule instanceof LoadRule && currRulePosition > lastRulePosition && rule.appliesTo(interval, now)) { + lastRulePosition = currRulePosition; + baseRule = (LoadRule) rule; + break; + } + currRulePosition++; + } + } + + if (baseRule == null) { + return null; + } + + // in the baseRule, find the broker of highest priority + String brokerServiceName = null; + for (Map.Entry entry : tierConfig.getTierToBrokerMap().entrySet()) { + if (baseRule.getTieredReplicants().containsKey(entry.getKey())) { + brokerServiceName = entry.getValue(); + break; + } + } + + if (brokerServiceName == null) { + log.makeAlert( + "WTF?! No brokerServiceName found for datasource[%s], intervals[%s]. Using default[%s].", + query.getDataSource(), + query.getIntervals(), + tierConfig.getDefaultBrokerServiceName() + ).emit(); + brokerServiceName = tierConfig.getDefaultBrokerServiceName(); + } + + ServerDiscoverySelector retVal = selectorMap.get(brokerServiceName); + + if (retVal == null) { + log.makeAlert( + "WTF?! No selector found for brokerServiceName[%s]. Using default selector for[%s]", + brokerServiceName, + tierConfig.getDefaultBrokerServiceName() + ).emit(); + retVal = selectorMap.get(tierConfig.getDefaultBrokerServiceName()); + } + + return new Pair<>(brokerServiceName, retVal); + } +} diff --git a/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java b/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java new file mode 100644 index 00000000000..9acf5f94858 --- /dev/null +++ b/server/src/main/java/io/druid/server/router/CoordinatorRuleManager.java @@ -0,0 +1,193 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.util.Charsets; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; +import com.google.inject.Inject; +import com.metamx.common.concurrent.ScheduledExecutors; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.logger.Logger; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.response.StatusResponseHandler; +import com.metamx.http.client.response.StatusResponseHolder; +import io.druid.client.selector.Server; +import io.druid.concurrent.Execs; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.annotations.Global; +import io.druid.guice.annotations.Json; +import io.druid.server.coordinator.rules.Rule; +import org.joda.time.Duration; + +import java.net.URL; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicReference; + +/** + */ +@ManageLifecycle +public class CoordinatorRuleManager +{ + private static final Logger log = new Logger(CoordinatorRuleManager.class); + + private final HttpClient httpClient; + private final ObjectMapper jsonMapper; + private final Supplier config; + private final ServerDiscoverySelector selector; + + private final StatusResponseHandler responseHandler; + private final AtomicReference>> rules; + + private volatile ScheduledExecutorService exec; + + private final Object lock = new Object(); + + private volatile boolean started = false; + + @Inject + public CoordinatorRuleManager( + @Global HttpClient httpClient, + @Json ObjectMapper jsonMapper, + Supplier config, + ServerDiscoverySelector selector + ) + { + this.httpClient = httpClient; + this.jsonMapper = jsonMapper; + this.config = config; + this.selector = selector; + + this.responseHandler = new StatusResponseHandler(Charsets.UTF_8); + this.rules = new AtomicReference<>( + new ConcurrentHashMap>() + ); + } + + @LifecycleStart + public void start() + { + synchronized (lock) { + if (started) { + return; + } + + this.exec = Execs.scheduledSingleThreaded("CoordinatorRuleManager-Exec--%d"); + + ScheduledExecutors.scheduleWithFixedDelay( + exec, + new Duration(0), + config.get().getPollPeriod().toStandardDuration(), + new Runnable() + { + @Override + public void run() + { + poll(); + } + } + ); + + started = true; + } + } + + @LifecycleStop + public void stop() + { + synchronized (lock) { + if (!started) { + return; + } + + rules.set(new ConcurrentHashMap>()); + + started = false; + exec.shutdownNow(); + exec = null; + } + } + + public boolean isStarted() + { + return started; + } + + public void poll() + { + try { + String url = getRuleURL(); + if (url == null) { + return; + } + + StatusResponseHolder response = httpClient.get(new URL(url)) + .go(responseHandler) + .get(); + + ConcurrentHashMap> newRules = new ConcurrentHashMap>( + (Map>) jsonMapper.readValue( + response.getContent(), new TypeReference>>() + { + } + ) + ); + + log.info("Got [%,d] rules", newRules.keySet().size()); + + rules.set(newRules); + } + catch (Exception e) { + log.error(e, "Exception while polling for rules"); + } + } + + public List getRulesWithDefault(final String dataSource) + { + List retVal = Lists.newArrayList(); + Map> theRules = rules.get(); + if (theRules.get(dataSource) != null) { + retVal.addAll(theRules.get(dataSource)); + } + if (theRules.get(config.get().getDefaultRule()) != null) { + retVal.addAll(theRules.get(config.get().getDefaultRule())); + } + return retVal; + } + + private String getRuleURL() + { + Server server = selector.pick(); + + if (server == null) { + log.error("No instances found for [%s]!", config.get().getCoordinatorServiceName()); + return null; + } + + return String.format("http://%s%s", server.getHost(), config.get().getRulesEndpoint()); + } +} diff --git a/server/src/main/java/io/druid/server/router/RouterQuerySegmentWalker.java b/server/src/main/java/io/druid/server/router/RouterQuerySegmentWalker.java new file mode 100644 index 00000000000..92bf43135fe --- /dev/null +++ b/server/src/main/java/io/druid/server/router/RouterQuerySegmentWalker.java @@ -0,0 +1,82 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; +import com.metamx.http.client.HttpClient; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.guice.annotations.Global; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.SegmentDescriptor; +import org.joda.time.Interval; + +/** + */ +public class RouterQuerySegmentWalker implements QuerySegmentWalker +{ + private final QueryToolChestWarehouse warehouse; + private final ObjectMapper objectMapper; + private final HttpClient httpClient; + private final BrokerSelector brokerSelector; + private final TierConfig tierConfig; + + @Inject + public RouterQuerySegmentWalker( + QueryToolChestWarehouse warehouse, + ObjectMapper objectMapper, + @Global HttpClient httpClient, + BrokerSelector brokerSelector, + TierConfig tierConfig + ) + { + this.warehouse = warehouse; + this.objectMapper = objectMapper; + this.httpClient = httpClient; + this.brokerSelector = brokerSelector; + this.tierConfig = tierConfig; + } + + @Override + public QueryRunner getQueryRunnerForIntervals(Query query, Iterable intervals) + { + return makeRunner(); + } + + @Override + public QueryRunner getQueryRunnerForSegments(Query query, Iterable specs) + { + return makeRunner(); + } + + private QueryRunner makeRunner() + { + return new TierAwareQueryRunner( + warehouse, + objectMapper, + httpClient, + brokerSelector, + tierConfig + ); + } +} diff --git a/server/src/main/java/io/druid/server/router/TierAwareQueryRunner.java b/server/src/main/java/io/druid/server/router/TierAwareQueryRunner.java new file mode 100644 index 00000000000..3a098e0faed --- /dev/null +++ b/server/src/main/java/io/druid/server/router/TierAwareQueryRunner.java @@ -0,0 +1,121 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.metamx.common.Pair; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import com.metamx.emitter.EmittingLogger; +import com.metamx.http.client.HttpClient; +import io.druid.client.DirectDruidClient; +import io.druid.client.selector.Server; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChestWarehouse; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + */ +public class TierAwareQueryRunner implements QueryRunner +{ + private static EmittingLogger log = new EmittingLogger(TierAwareQueryRunner.class); + + private final QueryToolChestWarehouse warehouse; + private final ObjectMapper objectMapper; + private final HttpClient httpClient; + private final BrokerSelector brokerSelector; + private final TierConfig tierConfig; + + private final ConcurrentHashMap serverBackup = new ConcurrentHashMap(); + + public TierAwareQueryRunner( + QueryToolChestWarehouse warehouse, + ObjectMapper objectMapper, + HttpClient httpClient, + BrokerSelector brokerSelector, + TierConfig tierConfig + ) + { + this.warehouse = warehouse; + this.objectMapper = objectMapper; + this.httpClient = httpClient; + this.brokerSelector = brokerSelector; + this.tierConfig = tierConfig; + } + + public Server findServer(Query query) + { + final Pair selected = brokerSelector.select(query); + final String brokerServiceName = selected.lhs; + final ServerDiscoverySelector selector = selected.rhs; + + Server server = selector.pick(); + if (server == null) { + log.error( + "WTF?! No server found for brokerServiceName[%s]. Using backup", + brokerServiceName + ); + + server = serverBackup.get(brokerServiceName); + + if (server == null) { + log.makeAlert( + "WTF?! No backup found for brokerServiceName[%s]. Using default[%s]", + brokerServiceName, + tierConfig.getDefaultBrokerServiceName() + ).emit(); + + server = serverBackup.get(tierConfig.getDefaultBrokerServiceName()); + } + } else { + serverBackup.put(brokerServiceName, server); + } + + return server; + } + + @Override + public Sequence run(Query query) + { + Server server = findServer(query); + + if (server == null) { + log.makeAlert( + "Catastrophic failure! No servers found at all! Failing request!" + ).emit(); + return Sequences.empty(); + } + + QueryRunner client = new DirectDruidClient( + warehouse, + objectMapper, + httpClient, + server.getHost() + ); + + return client.run(query); + } +} diff --git a/server/src/main/java/io/druid/server/router/TierConfig.java b/server/src/main/java/io/druid/server/router/TierConfig.java new file mode 100644 index 00000000000..c819edfce23 --- /dev/null +++ b/server/src/main/java/io/druid/server/router/TierConfig.java @@ -0,0 +1,91 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableMap; +import io.druid.client.DruidServer; +import org.joda.time.Period; + +import javax.validation.constraints.NotNull; +import java.util.LinkedHashMap; + +/** + */ +public class TierConfig +{ + @JsonProperty + @NotNull + private String defaultBrokerServiceName = ""; + + @JsonProperty + private LinkedHashMap tierToBrokerMap; + + @JsonProperty + @NotNull + private String defaultRule = "_default"; + + @JsonProperty + @NotNull + private String rulesEndpoint = "/druid/coordinator/v1/rules"; + + @JsonProperty + @NotNull + private String coordinatorServiceName = null; + + @JsonProperty + @NotNull + private Period pollPeriod = new Period("PT1M"); + + // tier, + public LinkedHashMap getTierToBrokerMap() + { + return tierToBrokerMap == null ? new LinkedHashMap<>( + ImmutableMap.of( + DruidServer.DEFAULT_TIER, defaultBrokerServiceName + ) + ) : tierToBrokerMap; + } + + public String getDefaultBrokerServiceName() + { + return defaultBrokerServiceName; + } + + public String getDefaultRule() + { + return defaultRule; + } + + public String getRulesEndpoint() + { + return rulesEndpoint; + } + + public String getCoordinatorServiceName() + { + return coordinatorServiceName; + } + + public Period getPollPeriod() + { + return pollPeriod; + } +} diff --git a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java index 40045ec8fa7..cfea29f9a8e 100644 --- a/server/src/test/java/io/druid/client/CachingClusteredClientTest.java +++ b/server/src/test/java/io/druid/client/CachingClusteredClientTest.java @@ -36,6 +36,7 @@ import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.guava.nary.TrinaryFn; import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheConfig; import io.druid.client.cache.MapCache; import io.druid.client.selector.QueryableDruidServer; import io.druid.client.selector.RandomServerSelectorStrategy; @@ -121,7 +122,7 @@ public class CachingClusteredClientTest */ private static final int RANDOMNESS = 10; - public static final ImmutableMap CONTEXT = ImmutableMap.of(); + public static final ImmutableMap CONTEXT = ImmutableMap.of(); public static final MultipleIntervalSegmentSpec SEG_SPEC = new MultipleIntervalSegmentSpec(ImmutableList.of()); public static final String DATA_SOURCE = "test"; @@ -325,7 +326,7 @@ public class CachingClusteredClientTest testQueryCaching( 1, true, - builder.context(ImmutableMap.of("useCache", "false", + builder.context(ImmutableMap.of("useCache", "false", "populateCache", "true")).build(), new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -339,7 +340,7 @@ public class CachingClusteredClientTest testQueryCaching( 1, false, - builder.context(ImmutableMap.of("useCache", "false", + builder.context(ImmutableMap.of("useCache", "false", "populateCache", "false")).build(), new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -351,7 +352,7 @@ public class CachingClusteredClientTest testQueryCaching( 1, false, - builder.context(ImmutableMap.of("useCache", "true", + builder.context(ImmutableMap.of("useCache", "true", "populateCache", "false")).build(), new Interval("2011-01-01/2011-01-02"), makeTimeResults(new DateTime("2011-01-01"), 50, 5000) ); @@ -1193,7 +1194,8 @@ public class CachingClusteredClientTest } }, cache, - jsonMapper + jsonMapper, + new CacheConfig() ); } diff --git a/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java index 7ef6047d9ac..6867fcc07c4 100644 --- a/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java @@ -38,19 +38,23 @@ import java.util.concurrent.TimeUnit; public class MemcachedCacheBenchmark extends SimpleBenchmark { - private static final String BASE_KEY = "test_2012-11-26T00:00:00.000Z_2012-11-27T00:00:00.000Z_2012-11-27T04:11:25.979Z_"; public static final String NAMESPACE = "default"; - + private static final String BASE_KEY = "test_2012-11-26T00:00:00.000Z_2012-11-27T00:00:00.000Z_2012-11-27T04:11:25.979Z_"; + private static byte[] randBytes; + @Param({"localhost:11211"}) + String hosts; + // object size in kB + @Param({"1", "5", "10", "40"}) + int objectSize; + @Param({"100", "1000"}) + int objectCount; private MemcachedCache cache; private MemcachedClientIF client; - private static byte[] randBytes; - - @Param({"localhost:11211"}) String hosts; - - // object size in kB - @Param({"1", "5", "10", "40"}) int objectSize; - @Param({"100", "1000"}) int objectCount; + public static void main(String[] args) throws Exception + { + Runner.main(MemcachedCacheBenchmark.class, args); + } @Override protected void setUp() throws Exception @@ -73,11 +77,29 @@ public class MemcachedCacheBenchmark extends SimpleBenchmark AddrUtil.getAddresses(hosts) ); + cache = new MemcachedCache( client, - "druid-memcached-benchmark", - 30000, // 30 seconds - 3600 // 1 hour + new MemcachedCacheConfig() + { + @Override + public String getMemcachedPrefix() + { + return "druid-memcached-benchmark"; + } + + @Override + public int getTimeout() + { + return 30000; + } + + @Override + public int getExpiration() + { + return 3600; + } + } ); randBytes = new byte[objectSize * 1024]; @@ -90,9 +112,10 @@ public class MemcachedCacheBenchmark extends SimpleBenchmark client.shutdown(1, TimeUnit.MINUTES); } - public void timePutObjects(int reps) { - for(int i = 0; i < reps; ++i) { - for(int k = 0; k < objectCount; ++k) { + public void timePutObjects(int reps) + { + for (int i = 0; i < reps; ++i) { + for (int k = 0; k < objectCount; ++k) { String key = BASE_KEY + k; cache.put(new Cache.NamedKey(NAMESPACE, key.getBytes()), randBytes); } @@ -101,11 +124,12 @@ public class MemcachedCacheBenchmark extends SimpleBenchmark } } - public long timeGetObject(int reps) { + public long timeGetObject(int reps) + { byte[] bytes = null; long count = 0; for (int i = 0; i < reps; i++) { - for(int k = 0; k < objectCount; ++k) { + for (int k = 0; k < objectCount; ++k) { String key = BASE_KEY + k; bytes = cache.get(new Cache.NamedKey(NAMESPACE, key.getBytes())); count += bytes.length; @@ -114,24 +138,21 @@ public class MemcachedCacheBenchmark extends SimpleBenchmark return count; } - public long timeBulkGetObjects(int reps) { + public long timeBulkGetObjects(int reps) + { long count = 0; for (int i = 0; i < reps; i++) { List keys = Lists.newArrayList(); - for(int k = 0; k < objectCount; ++k) { + for (int k = 0; k < objectCount; ++k) { String key = BASE_KEY + k; keys.add(new Cache.NamedKey(NAMESPACE, key.getBytes())); } Map results = cache.getBulk(keys); - for(Cache.NamedKey key : keys) { + for (Cache.NamedKey key : keys) { byte[] bytes = results.get(key); count += bytes.length; } } return count; } - - public static void main(String[] args) throws Exception { - Runner.main(MemcachedCacheBenchmark.class, args); - } } diff --git a/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java index 55c85f967f0..d55127288c3 100644 --- a/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java @@ -60,7 +60,29 @@ public class MemcachedCacheTest public void setUp() throws Exception { MemcachedClientIF client = new MockMemcachedClient(); - cache = new MemcachedCache(client, "druid-memcached-test", 500, 3600); + cache = new MemcachedCache( + client, new MemcachedCacheConfig() + { + @Override + public String getMemcachedPrefix() + { + return "druid-memcached-test"; + } + + @Override + public int getTimeout() + { + return 500; + } + + @Override + public int getExpiration() + { + return 3600; + } + + } + ); } @Test @@ -396,64 +418,64 @@ class MockMemcachedClient implements MemcachedClientIF public BulkFuture> asyncGetBulk(final Iterator keys, final Transcoder tc) { return new BulkFuture>() - { - @Override - public boolean isTimeout() - { - return false; - } + { + @Override + public boolean isTimeout() + { + return false; + } - @Override - public Map getSome(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException - { - return get(); - } + @Override + public Map getSome(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException + { + return get(); + } - @Override - public OperationStatus getStatus() - { - return null; - } + @Override + public OperationStatus getStatus() + { + return null; + } - @Override - public boolean cancel(boolean b) - { - return false; - } + @Override + public boolean cancel(boolean b) + { + return false; + } - @Override - public boolean isCancelled() - { - return false; - } + @Override + public boolean isCancelled() + { + return false; + } - @Override - public boolean isDone() - { - return true; - } + @Override + public boolean isDone() + { + return true; + } - @Override - public Map get() throws InterruptedException, ExecutionException - { - Map retVal = Maps.newHashMap(); + @Override + public Map get() throws InterruptedException, ExecutionException + { + Map retVal = Maps.newHashMap(); - while(keys.hasNext()) { - String key = keys.next(); - CachedData data = theMap.get(key); - retVal.put(key, data != null ? tc.decode(data) : null); - } + while (keys.hasNext()) { + String key = keys.next(); + CachedData data = theMap.get(key); + retVal.put(key, data != null ? tc.decode(data) : null); + } - return retVal; - } + return retVal; + } - @Override - public Map get(long l, TimeUnit timeUnit) - throws InterruptedException, ExecutionException, TimeoutException - { - return get(); - } - }; + @Override + public Map get(long l, TimeUnit timeUnit) + throws InterruptedException, ExecutionException, TimeoutException + { + return get(); + } + }; } @Override diff --git a/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java index 013544dabf1..223c720c7d8 100644 --- a/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java @@ -37,7 +37,10 @@ import com.metamx.common.guava.YieldingAccumulator; import com.metamx.common.guava.YieldingSequenceBase; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.cache.CacheConfig; +import io.druid.client.cache.LocalCacheProvider; import io.druid.granularity.QueryGranularity; +import io.druid.jackson.DefaultObjectMapper; import io.druid.query.ConcatQueryRunner; import io.druid.query.Druids; import io.druid.query.NoopQueryRunner; @@ -136,7 +139,8 @@ public class ServerManagerTest } }, new NoopServiceEmitter(), - serverManagerExec + serverManagerExec, new DefaultObjectMapper(), new LocalCacheProvider().get(), + new CacheConfig() ); loadQueryable("test", "1", new Interval("P1d/2011-04-01")); @@ -592,9 +596,7 @@ public class ServerManagerTest { private final String version; private final Interval interval; - private final Object lock = new Object(); - private volatile boolean closed = false; SegmentForTesting( diff --git a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index 77b4c9f8144..d0a7b62f4be 100644 --- a/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -24,6 +24,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.logger.Logger; +import io.druid.client.cache.CacheConfig; +import io.druid.client.cache.LocalCacheProvider; import io.druid.concurrent.Execs; import io.druid.curator.CuratorTestBase; import io.druid.curator.announcement.Announcer; @@ -52,12 +54,12 @@ import java.util.List; */ public class ZkCoordinatorTest extends CuratorTestBase { + private static final Logger log = new Logger(ZkCoordinatorTest.class); + private final ObjectMapper jsonMapper = new DefaultObjectMapper(); private ZkCoordinator zkCoordinator; private ServerManager serverManager; private DataSegmentAnnouncer announcer; private File infoDir; - private final ObjectMapper jsonMapper = new DefaultObjectMapper(); - private static final Logger log = new Logger(ZkCoordinatorTest.class); @Before public void setUp() throws Exception @@ -80,7 +82,10 @@ public class ZkCoordinatorTest extends CuratorTestBase new CacheTestSegmentLoader(), new NoopQueryRunnerFactoryConglomerate(), new NoopServiceEmitter(), - MoreExecutors.sameThreadExecutor() + MoreExecutors.sameThreadExecutor(), + new DefaultObjectMapper(), + new LocalCacheProvider().get(), + new CacheConfig() ); final DruidServerMetadata me = new DruidServerMetadata("dummyServer", "dummyHost", 0, "dummyType", "normal", 0); @@ -100,7 +105,8 @@ public class ZkCoordinatorTest extends CuratorTestBase zkCoordinator = new ZkCoordinator( jsonMapper, - new SegmentLoaderConfig(){ + new SegmentLoaderConfig() + { @Override public File getInfoDir() { @@ -214,7 +220,7 @@ public class ZkCoordinatorTest extends CuratorTestBase List sortedFiles = Lists.newArrayList(files); Collections.sort(sortedFiles); - + Assert.assertEquals(segments.size(), sortedFiles.size()); for (int i = 0; i < sortedFiles.size(); i++) { DataSegment segment = jsonMapper.readValue(sortedFiles.get(i), DataSegment.class); diff --git a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java index 3e75d1b79c3..24542e70df2 100644 --- a/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java +++ b/server/src/test/java/io/druid/server/coordinator/rules/LoadRuleTest.java @@ -122,6 +122,12 @@ public class LoadRuleTest { return true; } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) + { + return true; + } }; DruidCluster druidCluster = new DruidCluster( @@ -214,6 +220,12 @@ public class LoadRuleTest { return true; } + + @Override + public boolean appliesTo(Interval interval, DateTime referenceTimestamp) + { + return true; + } }; DruidServer server1 = new DruidServer( diff --git a/server/src/test/java/io/druid/server/router/BrokerSelectorTest.java b/server/src/test/java/io/druid/server/router/BrokerSelectorTest.java new file mode 100644 index 00000000000..55e5dda44ea --- /dev/null +++ b/server/src/test/java/io/druid/server/router/BrokerSelectorTest.java @@ -0,0 +1,230 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.metamx.common.Pair; +import com.metamx.http.client.HttpClient; +import io.druid.client.DruidServer; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.guice.annotations.Global; +import io.druid.guice.annotations.Json; +import io.druid.query.Druids; +import io.druid.query.TableDataSource; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.server.coordinator.rules.IntervalLoadRule; +import io.druid.server.coordinator.rules.Rule; +import junit.framework.Assert; +import org.easymock.EasyMock; +import org.joda.time.Interval; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; + +/** + */ +public class BrokerSelectorTest +{ + private ServerDiscoveryFactory factory; + private ServerDiscoverySelector selector; + private BrokerSelector brokerSelector; + + @Before + public void setUp() throws Exception + { + factory = EasyMock.createMock(ServerDiscoveryFactory.class); + selector = EasyMock.createMock(ServerDiscoverySelector.class); + + brokerSelector = new BrokerSelector( + new TestRuleManager(null, null, null, null), + new TierConfig() + { + @Override + public LinkedHashMap getTierToBrokerMap() + { + return new LinkedHashMap( + ImmutableMap.of( + "hot", "hotBroker", + "medium", "mediumBroker", + DruidServer.DEFAULT_TIER, "coldBroker" + ) + ); + } + + @Override + public String getDefaultBrokerServiceName() + { + return "hotBroker"; + } + }, + factory + ); + EasyMock.expect(factory.createSelector(EasyMock.anyObject())).andReturn(selector).atLeastOnce(); + EasyMock.replay(factory); + + selector.start(); + EasyMock.expectLastCall().atLeastOnce(); + selector.stop(); + EasyMock.expectLastCall().atLeastOnce(); + EasyMock.replay(selector); + + brokerSelector.start(); + + } + + @After + public void tearDown() throws Exception + { + brokerSelector.stop(); + + EasyMock.verify(selector); + EasyMock.verify(factory); + } + + @Test + public void testBasicSelect() throws Exception + { + String brokerName = (String) brokerSelector.select( + new TimeBoundaryQuery( + new TableDataSource("test"), + new MultipleIntervalSegmentSpec(Arrays.asList(new Interval("2011-08-31/2011-09-01"))), + null + ) + ).lhs; + + Assert.assertEquals("coldBroker", brokerName); + } + + + @Test + public void testBasicSelect2() throws Exception + { + String brokerName = (String) brokerSelector.select( + new TimeBoundaryQuery( + new TableDataSource("test"), + new MultipleIntervalSegmentSpec(Arrays.asList(new Interval("2013-08-31/2013-09-01"))), + null + ) + ).lhs; + + Assert.assertEquals("hotBroker", brokerName); + } + + @Test + public void testSelectMatchesNothing() throws Exception + { + Pair retVal = brokerSelector.select( + new TimeBoundaryQuery( + new TableDataSource("test"), + new MultipleIntervalSegmentSpec(Arrays.asList(new Interval("2010-08-31/2010-09-01"))), + null + ) + ); + + Assert.assertEquals(null, retVal); + } + + + @Test + public void testSelectMultiInterval() throws Exception + { + String brokerName = (String) brokerSelector.select( + Druids.newTimeseriesQueryBuilder() + .dataSource("test") + .aggregators(Arrays.asList(new CountAggregatorFactory("count"))) + .intervals( + new MultipleIntervalSegmentSpec( + Arrays.asList( + new Interval("2013-08-31/2013-09-01"), + new Interval("2012-08-31/2012-09-01"), + new Interval("2011-08-31/2011-09-01") + ) + ) + ).build() + ).lhs; + + Assert.assertEquals("coldBroker", brokerName); + } + + @Test + public void testSelectMultiInterval2() throws Exception + { + String brokerName = (String) brokerSelector.select( + Druids.newTimeseriesQueryBuilder() + .dataSource("test") + .aggregators(Arrays.asList(new CountAggregatorFactory("count"))) + .intervals( + new MultipleIntervalSegmentSpec( + Arrays.asList( + new Interval("2011-08-31/2011-09-01"), + new Interval("2012-08-31/2012-09-01"), + new Interval("2013-08-31/2013-09-01") + ) + ) + ).build() + ).lhs; + + Assert.assertEquals("coldBroker", brokerName); + } + + private static class TestRuleManager extends CoordinatorRuleManager + { + public TestRuleManager( + @Global HttpClient httpClient, + @Json ObjectMapper jsonMapper, + Supplier config, + ServerDiscoverySelector selector + ) + { + super(httpClient, jsonMapper, config, selector); + } + + @Override + public boolean isStarted() + { + return true; + } + + @Override + public List getRulesWithDefault(String dataSource) + { + return Arrays.asList( + new IntervalLoadRule(new Interval("2013/2014"), ImmutableMap.of("hot", 1), null, null), + new IntervalLoadRule(new Interval("2012/2013"), ImmutableMap.of("medium", 1), null, null), + new IntervalLoadRule( + new Interval("2011/2012"), + ImmutableMap.of(DruidServer.DEFAULT_TIER, 1), + null, + null + ) + ); + } + } +} diff --git a/server/src/test/java/io/druid/server/router/TierAwareQueryRunnerTest.java b/server/src/test/java/io/druid/server/router/TierAwareQueryRunnerTest.java new file mode 100644 index 00000000000..d788daa2f2b --- /dev/null +++ b/server/src/test/java/io/druid/server/router/TierAwareQueryRunnerTest.java @@ -0,0 +1,139 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.router; + +import com.google.common.collect.ImmutableMap; +import com.metamx.common.Pair; +import io.druid.client.DruidServer; +import io.druid.client.selector.Server; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.query.Query; +import io.druid.query.TableDataSource; +import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import junit.framework.Assert; +import org.easymock.EasyMock; +import org.joda.time.Interval; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.LinkedHashMap; + +/** + */ +public class TierAwareQueryRunnerTest +{ + private ServerDiscoverySelector selector; + private BrokerSelector brokerSelector; + private TierConfig config; + private Server server; + + @Before + public void setUp() throws Exception + { + selector = EasyMock.createMock(ServerDiscoverySelector.class); + brokerSelector = EasyMock.createMock(BrokerSelector.class); + + config = new TierConfig() + { + @Override + public LinkedHashMap getTierToBrokerMap() + { + return new LinkedHashMap<>( + ImmutableMap.of( + "hot", "hotBroker", + "medium", "mediumBroker", + DruidServer.DEFAULT_TIER, "coldBroker" + ) + ); + } + + @Override + public String getDefaultBrokerServiceName() + { + return "hotBroker"; + } + }; + + server = new Server() + { + @Override + public String getScheme() + { + return null; + } + + @Override + public String getHost() + { + return "foo"; + } + + @Override + public String getAddress() + { + return null; + } + + @Override + public int getPort() + { + return 0; + } + }; + } + + @After + public void tearDown() throws Exception + { + EasyMock.verify(brokerSelector); + EasyMock.verify(selector); + } + + @Test + public void testFindServer() throws Exception + { + EasyMock.expect(brokerSelector.select(EasyMock.anyObject())).andReturn(new Pair("hotBroker", selector)); + EasyMock.replay(brokerSelector); + + EasyMock.expect(selector.pick()).andReturn(server).once(); + EasyMock.replay(selector); + + TierAwareQueryRunner queryRunner = new TierAwareQueryRunner( + null, + null, + null, + brokerSelector, + config + ); + + Server server = queryRunner.findServer( + new TimeBoundaryQuery( + new TableDataSource("test"), + new MultipleIntervalSegmentSpec(Arrays.asList(new Interval("2011-08-31/2011-09-01"))), + null + ) + ); + + Assert.assertEquals("foo", server.getHost()); + } +} diff --git a/services/pom.xml b/services/pom.xml index a07d2a0749d..fb95248e791 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -27,7 +27,7 @@ io.druid druid - 0.6.74-SNAPSHOT + 0.6.75-SNAPSHOT diff --git a/services/src/main/java/io/druid/cli/CliBroker.java b/services/src/main/java/io/druid/cli/CliBroker.java index 2e582d16afa..9690003cbba 100644 --- a/services/src/main/java/io/druid/cli/CliBroker.java +++ b/services/src/main/java/io/druid/cli/CliBroker.java @@ -28,6 +28,7 @@ import io.druid.client.BrokerServerView; import io.druid.client.CachingClusteredClient; import io.druid.client.TimelineServerView; import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheConfig; import io.druid.client.cache.CacheMonitor; import io.druid.client.cache.CacheProvider; import io.druid.client.selector.ServerSelectorStrategy; @@ -81,7 +82,7 @@ public class CliBroker extends ServerRunnable binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); - + JsonConfigProvider.bind(binder, "druid.broker.cache", CacheConfig.class); JsonConfigProvider.bind(binder, "druid.broker.balancer", ServerSelectorStrategy.class); binder.bind(QuerySegmentWalker.class).to(ClientQuerySegmentWalker.class).in(LazySingleton.class); diff --git a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java index d78a67e73a6..0d2ccff8463 100644 --- a/services/src/main/java/io/druid/cli/CliHadoopIndexer.java +++ b/services/src/main/java/io/druid/cli/CliHadoopIndexer.java @@ -54,6 +54,10 @@ public class CliHadoopIndexer implements Runnable description = "The maven coordinates to the version of hadoop to run with. Defaults to org.apache.hadoop:hadoop-core:1.0.3") private String hadoopCoordinates = "org.apache.hadoop:hadoop-core:1.0.3"; + @Option(name = "hadoopDependencies", + description = "The maven coordinates to the version of hadoop and all dependencies to run with. Defaults to using org.apache.hadoop:hadoop-core:1.0.3") + private List hadoopDependencyCoordinates = Arrays.asList("org.apache.hadoop:hadoop-core:1.0.3"); + @Inject private ExtensionsConfig extensionsConfig = null; @@ -63,9 +67,6 @@ public class CliHadoopIndexer implements Runnable { try { final DefaultTeslaAether aetherClient = Initialization.getAetherClient(extensionsConfig); - final ClassLoader hadoopLoader = Initialization.getClassLoaderForCoordinates( - aetherClient, hadoopCoordinates - ); final List extensionURLs = Lists.newArrayList(); for (String coordinate : extensionsConfig.getCoordinates()) { @@ -81,7 +82,12 @@ public class CliHadoopIndexer implements Runnable final List driverURLs = Lists.newArrayList(); driverURLs.addAll(nonHadoopURLs); // put hadoop dependencies last to avoid jets3t & apache.httpcore version conflicts - driverURLs.addAll(Arrays.asList(((URLClassLoader) hadoopLoader).getURLs())); + for (String coordinate : hadoopDependencyCoordinates) { + final ClassLoader hadoopLoader = Initialization.getClassLoaderForCoordinates( + aetherClient, coordinate + ); + driverURLs.addAll(Arrays.asList(((URLClassLoader) hadoopLoader).getURLs())); + } final URLClassLoader loader = new URLClassLoader(driverURLs.toArray(new URL[driverURLs.size()]), null); Thread.currentThread().setContextClassLoader(loader); diff --git a/services/src/main/java/io/druid/cli/CliHistorical.java b/services/src/main/java/io/druid/cli/CliHistorical.java index 0ed56e73530..0dc535aa024 100644 --- a/services/src/main/java/io/druid/cli/CliHistorical.java +++ b/services/src/main/java/io/druid/cli/CliHistorical.java @@ -24,7 +24,11 @@ import com.google.inject.Binder; import com.google.inject.Module; import com.metamx.common.logger.Logger; import io.airlift.command.Command; +import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheConfig; +import io.druid.client.cache.CacheProvider; import io.druid.guice.Jerseys; +import io.druid.guice.JsonConfigProvider; import io.druid.guice.LazySingleton; import io.druid.guice.LifecycleModule; import io.druid.guice.ManageLifecycle; @@ -73,6 +77,10 @@ public class CliHistorical extends ServerRunnable LifecycleModule.register(binder, ZkCoordinator.class); LifecycleModule.register(binder, Server.class); + binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.historical.cache", CacheProvider.class); + JsonConfigProvider.bind(binder, "druid.historical.cache", CacheConfig.class); + } } ); diff --git a/services/src/main/java/io/druid/cli/CliRouter.java b/services/src/main/java/io/druid/cli/CliRouter.java new file mode 100644 index 00000000000..740edd8b55b --- /dev/null +++ b/services/src/main/java/io/druid/cli/CliRouter.java @@ -0,0 +1,105 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.cli; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.logger.Logger; +import io.airlift.command.Command; +import io.druid.curator.discovery.DiscoveryModule; +import io.druid.curator.discovery.ServerDiscoveryFactory; +import io.druid.curator.discovery.ServerDiscoverySelector; +import io.druid.guice.Jerseys; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.LifecycleModule; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.annotations.Self; +import io.druid.query.MapQueryToolChestWarehouse; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.server.QueryResource; +import io.druid.server.initialization.JettyServerInitializer; +import io.druid.server.router.BrokerSelector; +import io.druid.server.router.CoordinatorRuleManager; +import io.druid.server.router.RouterQuerySegmentWalker; +import io.druid.server.router.TierConfig; +import org.eclipse.jetty.server.Server; + +import java.util.List; + +/** + */ +@Command( + name = "router", + description = "Experimental! Understands tiers and routes things to different brokers" +) +public class CliRouter extends ServerRunnable +{ + private static final Logger log = new Logger(CliRouter.class); + + public CliRouter() + { + super(log); + } + + @Override + protected List getModules() + { + return ImmutableList.of( + new Module() + { + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.router", TierConfig.class); + + binder.bind(CoordinatorRuleManager.class); + LifecycleModule.register(binder, CoordinatorRuleManager.class); + + binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class); + + binder.bind(BrokerSelector.class).in(ManageLifecycle.class); + binder.bind(QuerySegmentWalker.class).to(RouterQuerySegmentWalker.class).in(LazySingleton.class); + + binder.bind(JettyServerInitializer.class).to(QueryJettyServerInitializer.class).in(LazySingleton.class); + Jerseys.addResource(binder, QueryResource.class); + LifecycleModule.register(binder, QueryResource.class); + + LifecycleModule.register(binder, Server.class); + DiscoveryModule.register(binder, Self.class); + } + + @Provides + @ManageLifecycle + public ServerDiscoverySelector getCoordinatorServerDiscoverySelector( + TierConfig config, + ServerDiscoveryFactory factory + + ) + { + return factory.createSelector(config.getCoordinatorServiceName()); + } + } + ); + } +} diff --git a/services/src/main/java/io/druid/cli/Main.java b/services/src/main/java/io/druid/cli/Main.java index 7012e3a3f79..450aa36afe9 100644 --- a/services/src/main/java/io/druid/cli/Main.java +++ b/services/src/main/java/io/druid/cli/Main.java @@ -52,7 +52,7 @@ public class Main .withCommands( CliCoordinator.class, CliHistorical.class, CliBroker.class, CliRealtime.class, CliOverlord.class, CliMiddleManager.class, - CliBridge.class + CliBridge.class, CliRouter.class ); builder.withGroup("example")