diff --git a/.gitignore b/.gitignore index 845a2ddc007..c7f1290850b 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ target .idea .project .settings/ -examples/rand/RealtimeNode.out -examples/twitter/RealtimeNode.out *.log *.DS_Store +_site diff --git a/client/pom.xml b/client/pom.xml deleted file mode 100644 index d7ce942253a..00000000000 --- a/client/pom.xml +++ /dev/null @@ -1,243 +0,0 @@ - - - - - 4.0.0 - com.metamx.druid - druid-client - druid-client - druid-client - - - com.metamx - druid - 0.5.59-SNAPSHOT - - - - - com.metamx.druid - druid-common - ${project.parent.version} - - - com.metamx.druid - druid-indexing-common - ${project.parent.version} - - - - com.metamx - emitter - - - com.metamx - http-client - - - com.metamx - java-util - - - com.metamx - server-metrics - - - - com.davekoelle - alphanum - - - commons-codec - commons-codec - - - commons-httpclient - commons-httpclient - - - org.skife.config - config-magic - - - org.apache.curator - curator-client - - - org.apache.curator - curator-framework - - - org.apache.curator - curator-x-discovery - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - com.google.guava - guava - - - com.google.inject - guice - - - com.ibm.icu - icu4j - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - javax.inject - javax.inject - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey.contribs - jersey-guice - - - joda-time - joda-time - - - com.google.code.findbugs - jsr305 - - - com.google.inject.extensions - guice-servlet - - - log4j - log4j - - - org.jboss.netty - netty - 3.2.4.Final - - - javax.servlet - servlet-api - - - org.slf4j - slf4j-log4j12 - - - com.google.code.simple-spring-memcached - spymemcached - - - org.antlr - antlr4-runtime - - - commons-cli - commons-cli - - - com.metamx - bytebuffer-collections - - - net.jpountz.lz4 - lz4 - - - - - junit - junit - test - - - org.easymock - easymock - test - - - com.google.caliper - caliper - test - - - org.apache.curator - curator-test - test - - - - - - - - maven-jar-plugin - - - - test-jar - - - - - - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - - - - diff --git a/client/src/main/java/com/metamx/druid/QueryableNode.java b/client/src/main/java/com/metamx/druid/QueryableNode.java deleted file mode 100644 index ca638f44727..00000000000 --- a/client/src/main/java/com/metamx/druid/QueryableNode.java +++ /dev/null @@ -1,618 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.common.IAE; -import com.metamx.common.ISE; -import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.common.concurrent.ScheduledExecutors; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.lifecycle.LifecycleStart; -import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.BatchServerInventoryView; -import com.metamx.druid.client.DruidServerConfig; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.ServerInventoryView; -import com.metamx.druid.client.ServerInventoryViewConfig; -import com.metamx.druid.client.ServerView; -import com.metamx.druid.client.SingleServerInventoryView; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.coordination.AbstractDataSegmentAnnouncer; -import com.metamx.druid.coordination.BatchDataSegmentAnnouncer; -import com.metamx.druid.coordination.DataSegmentAnnouncer; -import com.metamx.druid.coordination.DruidServerMetadata; -import com.metamx.druid.coordination.MultipleDataSegmentAnnouncerDataSegmentAnnouncer; -import com.metamx.druid.coordination.SingleDataSegmentAnnouncer; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.http.NoopRequestLogger; -import com.metamx.druid.http.RequestLogger; -import com.metamx.druid.initialization.CuratorConfig; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServerConfig; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.utils.PropUtils; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.Emitters; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; -import com.metamx.metrics.JvmMonitor; -import com.metamx.metrics.Monitor; -import com.metamx.metrics.MonitorScheduler; -import com.metamx.metrics.MonitorSchedulerConfig; -import com.metamx.metrics.SysMonitor; -import org.apache.curator.framework.CuratorFramework; -import org.joda.time.Duration; -import org.mortbay.jetty.Server; -import org.skife.config.ConfigurationObjectFactory; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -/** - */ -public abstract class QueryableNode extends RegisteringNode -{ - private final Logger log; - - private final Lifecycle lifecycle; - private final ObjectMapper jsonMapper; - private final ObjectMapper smileMapper; - private final Properties props; - private final ConfigurationObjectFactory configFactory; - private final String nodeType; - - private DruidServerMetadata druidServerMetadata = null; - private ServiceEmitter emitter = null; - private List monitors = null; - private Server server = null; - private CuratorFramework curator = null; - private DataSegmentAnnouncer announcer = null; - private ZkPathsConfig zkPaths = null; - private ScheduledExecutorFactory scheduledExecutorFactory = null; - private RequestLogger requestLogger = null; - private ServerInventoryView serverInventoryView = null; - private ServerView serverView = null; - private InventoryView inventoryView = null; - - private boolean initialized = false; - - public QueryableNode( - String nodeType, - Logger log, - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super(Arrays.asList(jsonMapper, smileMapper)); - - this.log = log; - this.configFactory = configFactory; - this.props = props; - this.jsonMapper = jsonMapper; - this.lifecycle = lifecycle; - this.smileMapper = smileMapper; - - Preconditions.checkNotNull(props, "props"); - Preconditions.checkNotNull(lifecycle, "lifecycle"); - Preconditions.checkNotNull(jsonMapper, "jsonMapper"); - Preconditions.checkNotNull(smileMapper, "smileMapper"); - Preconditions.checkNotNull(configFactory, "configFactory"); - - Preconditions.checkState(smileMapper.getJsonFactory() instanceof SmileFactory, "smileMapper should use smile."); - this.nodeType = nodeType; - } - - public T setDruidServerMetadata(DruidServerMetadata druidServerMetadata) - { - checkFieldNotSetAndSet("druidServerMetadata", druidServerMetadata); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setCuratorFramework(CuratorFramework curator) - { - checkFieldNotSetAndSet("curator", curator); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setAnnouncer(DataSegmentAnnouncer announcer) - { - checkFieldNotSetAndSet("announcer", announcer); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setEmitter(ServiceEmitter emitter) - { - checkFieldNotSetAndSet("emitter", emitter); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setMonitors(List monitors) - { - checkFieldNotSetAndSet("monitors", monitors); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setServer(Server server) - { - checkFieldNotSetAndSet("server", server); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setZkPaths(ZkPathsConfig zkPaths) - { - checkFieldNotSetAndSet("zkPaths", zkPaths); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setScheduledExecutorFactory(ScheduledExecutorFactory factory) - { - checkFieldNotSetAndSet("scheduledExecutorFactory", factory); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setRequestLogger(RequestLogger requestLogger) - { - checkFieldNotSetAndSet("requestLogger", requestLogger); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setInventoryView(InventoryView inventoryView) - { - checkFieldNotSetAndSet("inventoryView", inventoryView); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setServerView(ServerView serverView) - { - checkFieldNotSetAndSet("serverView", serverView); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T registerJacksonSubtype(Class... clazzes) - { - jsonMapper.registerSubtypes(clazzes); - smileMapper.registerSubtypes(clazzes); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T registerJacksonSubtype(NamedType... namedTypes) - { - jsonMapper.registerSubtypes(namedTypes); - smileMapper.registerSubtypes(namedTypes); - return (T) this; - } - - public Lifecycle getLifecycle() - { - return lifecycle; - } - - public ObjectMapper getJsonMapper() - { - return jsonMapper; - } - - public ObjectMapper getSmileMapper() - { - return smileMapper; - } - - public Properties getProps() - { - return props; - } - - public ConfigurationObjectFactory getConfigFactory() - { - return configFactory; - } - - public DruidServerMetadata getDruidServerMetadata() - { - initializeDruidServerMetadata(); - return druidServerMetadata; - } - - public CuratorFramework getCuratorFramework() - { - initializeCuratorFramework(); - return curator; - } - - public DataSegmentAnnouncer getAnnouncer() - { - initializeAnnouncer(); - return announcer; - } - - public ServiceEmitter getEmitter() - { - initializeEmitter(); - return emitter; - } - - public List getMonitors() - { - initializeMonitors(); - return monitors; - } - - public Server getServer() - { - initializeServer(); - return server; - } - - public ZkPathsConfig getZkPaths() - { - initializeZkPaths(); - return zkPaths; - } - - public ScheduledExecutorFactory getScheduledExecutorFactory() - { - initializeScheduledExecutorFactory(); - return scheduledExecutorFactory; - } - - public RequestLogger getRequestLogger() - { - initializeRequestLogger(); - return requestLogger; - } - - public ServerView getServerView() - { - initializeServerView(); - return serverView; - } - - public InventoryView getInventoryView() - { - initializeInventoryView(); - return inventoryView; - } - - private void initializeDruidServerMetadata() - { - if (druidServerMetadata == null) { - final DruidServerConfig serverConfig = getConfigFactory().build(DruidServerConfig.class); - setDruidServerMetadata( - new DruidServerMetadata( - serverConfig.getServerName(), - serverConfig.getHost(), - serverConfig.getMaxSize(), - nodeType, - serverConfig.getTier() - ) - ); - } - } - - private void initializeServerView() - { - if (serverView == null) { - initializeServerInventoryView(); - serverView = serverInventoryView; - } - } - - private void initializeInventoryView() - { - if (inventoryView == null) { - initializeServerInventoryView(); - inventoryView = serverInventoryView; - } - } - - private void initializeServerInventoryView() - { - if (serverInventoryView == null) { - final ExecutorService exec = Executors.newFixedThreadPool( - 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ServerInventoryView-%s").build() - ); - - final ServerInventoryViewConfig serverInventoryViewConfig = getConfigFactory().build(ServerInventoryViewConfig.class); - final String announcerType = serverInventoryViewConfig.getAnnouncerType(); - - if ("legacy".equalsIgnoreCase(announcerType)) { - serverInventoryView = new SingleServerInventoryView( - serverInventoryViewConfig, - getZkPaths(), - getCuratorFramework(), - exec, - getJsonMapper() - ); - } else if ("batch".equalsIgnoreCase(announcerType)) { - serverInventoryView = new BatchServerInventoryView( - serverInventoryViewConfig, - getZkPaths(), - getCuratorFramework(), - exec, - getJsonMapper() - ); - } else { - throw new IAE("Unknown type %s", announcerType); - } - lifecycle.addManagedInstance(serverInventoryView); - } - } - - private void initializeRequestLogger() - { - if (requestLogger == null) { - try { - final String loggingType = props.getProperty("druid.request.logging.type"); - if ("emitter".equals(loggingType)) { - setRequestLogger( - Initialization.makeEmittingRequestLogger( - getProps(), - getEmitter() - ) - ); - } else if ("file".equalsIgnoreCase(loggingType)) { - setRequestLogger( - Initialization.makeFileRequestLogger( - getJsonMapper(), - getScheduledExecutorFactory(), - getProps() - ) - ); - } else { - setRequestLogger(new NoopRequestLogger()); - } - } - catch (IOException e) { - throw Throwables.propagate(e); - } - lifecycle.addManagedInstance(requestLogger); - } - } - - private void initializeZkPaths() - { - if (zkPaths == null) { - setZkPaths(getConfigFactory().build(ZkPathsConfig.class)); - } - } - - private void initializeScheduledExecutorFactory() - { - if (scheduledExecutorFactory == null) { - setScheduledExecutorFactory(ScheduledExecutors.createFactory(getLifecycle())); - } - } - - private void initializeCuratorFramework() - { - if (curator == null) { - try { - setCuratorFramework(Initialization.makeCuratorFramework(configFactory.build(CuratorConfig.class), lifecycle)); - } - catch (IOException e) { - throw Throwables.propagate(e); - } - } - } - - private void initializeAnnouncer() - { - if (announcer == null) { - final Announcer announcer = new Announcer(getCuratorFramework(), Execs.singleThreaded("Announcer-%s")); - lifecycle.addManagedInstance(announcer); - - final ZkDataSegmentAnnouncerConfig config = getConfigFactory().build(ZkDataSegmentAnnouncerConfig.class); - final String announcerType = config.getAnnouncerType(); - - final DataSegmentAnnouncer dataSegmentAnnouncer; - if ("batch".equalsIgnoreCase(announcerType)) { - dataSegmentAnnouncer = new BatchDataSegmentAnnouncer( - getDruidServerMetadata(), - config, - announcer, - getJsonMapper() - ); - } else if ("legacy".equalsIgnoreCase(announcerType)) { - dataSegmentAnnouncer = new MultipleDataSegmentAnnouncerDataSegmentAnnouncer( - Arrays.asList( - new BatchDataSegmentAnnouncer( - getDruidServerMetadata(), - config, - announcer, - getJsonMapper() - ), - new SingleDataSegmentAnnouncer( - getDruidServerMetadata(), - getZkPaths(), - announcer, - getJsonMapper() - ) - ) - ); - } else { - throw new ISE("Unknown announcer type [%s]", announcerType); - } - - setAnnouncer(dataSegmentAnnouncer); - - lifecycle.addManagedInstance(getAnnouncer(), Lifecycle.Stage.LAST); - } - } - - private void initializeServer() - { - if (server == null) { - setServer(Initialization.makeJettyServer(configFactory.build(ServerConfig.class))); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - log.info("Starting Jetty"); - server.start(); - } - - @Override - public void stop() - { - log.info("Stopping Jetty"); - try { - server.stop(); - } - catch (Exception e) { - log.error(e, "Exception thrown while stopping Jetty"); - } - } - } - ); - } - } - - private void initializeMonitors() - { - if (monitors == null) { - List theMonitors = Lists.newArrayList(); - theMonitors.add(new JvmMonitor()); - if (Boolean.parseBoolean(props.getProperty("druid.monitoring.monitorSystem", "false"))) { - theMonitors.add(new SysMonitor()); - } - - setMonitors(theMonitors); - } - } - - private void initializeEmitter() - { - if (emitter == null) { - final HttpClientConfig.Builder configBuilder = HttpClientConfig.builder().withNumConnections(1); - - final String emitterTimeoutDuration = props.getProperty("druid.emitter.timeOut"); - if (emitterTimeoutDuration != null) { - configBuilder.withReadTimeout(new Duration(emitterTimeoutDuration)); - } - - final HttpClient httpClient = HttpClientInit.createClient(configBuilder.build(), lifecycle); - - setEmitter( - new ServiceEmitter( - PropUtils.getProperty(props, "druid.service"), - PropUtils.getProperty(props, "druid.host"), - Emitters.create(props, httpClient, jsonMapper, lifecycle) - ) - ); - } - EmittingLogger.registerEmitter(emitter); - } - - protected void init() throws Exception - { - doInit(); - initialized = true; - } - - protected abstract void doInit() throws Exception; - - @LifecycleStart - public synchronized void start() throws Exception - { - if (!initialized) { - init(); - } - - lifecycle.start(); - } - - @LifecycleStop - public synchronized void stop() - { - lifecycle.stop(); - } - - protected ScheduledExecutorService startMonitoring(List monitors) - { - final ScheduledExecutorService globalScheduledExec = getScheduledExecutorFactory().create(1, "Global--%d"); - final MonitorScheduler monitorScheduler = new MonitorScheduler( - getConfigFactory().build(MonitorSchedulerConfig.class), - globalScheduledExec, - getEmitter(), - monitors - ); - getLifecycle().addManagedInstance(monitorScheduler); - return globalScheduledExec; - } - - protected void checkFieldNotSetAndSet(String fieldName, Object value) - { - Class theClazz = this.getClass(); - while (theClazz != null && theClazz != Object.class) { - try { - final Field field = theClazz.getDeclaredField(fieldName); - field.setAccessible(true); - Preconditions.checkState(field.get(this) == null, "Cannot set %s once it has already been set.", fieldName); - - field.set(this, value); - return; - } - catch (NoSuchFieldException e) { - // Perhaps it is inherited? - theClazz = theClazz.getSuperclass(); - } - catch (IllegalAccessException e) { - throw Throwables.propagate(e); - } - } - - throw new ISE("Unknown field[%s] on class[%s]", fieldName, this.getClass()); - } -} diff --git a/client/src/main/java/com/metamx/druid/client/DataSegment.java b/client/src/main/java/com/metamx/druid/client/DataSegment.java deleted file mode 100644 index fec47128cf7..00000000000 --- a/client/src/main/java/com/metamx/druid/client/DataSegment.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.client; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.metamx.common.Granularity; -import com.metamx.druid.jackson.CommaListJoinDeserializer; -import com.metamx.druid.jackson.CommaListJoinSerializer; -import com.metamx.druid.query.segment.SegmentDescriptor; -import com.metamx.druid.shard.NoneShardSpec; -import com.metamx.druid.shard.ShardSpec; - -import org.joda.time.DateTime; -import org.joda.time.Interval; - -import javax.annotation.Nullable; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -/** - */ -public class DataSegment implements Comparable -{ - public static String delimiter = "_"; - private final Integer binaryVersion; - - public static String makeDataSegmentIdentifier( - String dataSource, - DateTime start, - DateTime end, - String version, - ShardSpec shardSpec - ) - { - StringBuilder sb = new StringBuilder(); - - sb.append(dataSource).append(delimiter) - .append(start).append(delimiter) - .append(end).append(delimiter) - .append(version); - - if (shardSpec.getPartitionNum() != 0) { - sb.append(delimiter).append(shardSpec.getPartitionNum()); - } - - return sb.toString(); - } - - private final String dataSource; - private final Interval interval; - private final String version; - private final Map loadSpec; - private final List dimensions; - private final List metrics; - private final ShardSpec shardSpec; - private final long size; - private final String identifier; - - @JsonCreator - public DataSegment( - @JsonProperty("dataSource") String dataSource, - @JsonProperty("interval") Interval interval, - @JsonProperty("version") String version, - @JsonProperty("loadSpec") Map loadSpec, - @JsonProperty("dimensions") @JsonDeserialize(using = CommaListJoinDeserializer.class) List dimensions, - @JsonProperty("metrics") @JsonDeserialize(using = CommaListJoinDeserializer.class) List metrics, - @JsonProperty("shardSpec") ShardSpec shardSpec, - @JsonProperty("binaryVersion") Integer binaryVersion, - @JsonProperty("size") long size - ) - { - final Predicate nonEmpty = new Predicate() - { - @Override - public boolean apply(@Nullable String input) - { - return input != null && !input.isEmpty(); - } - }; - - this.dataSource = dataSource.toLowerCase(); - this.interval = interval; - this.loadSpec = loadSpec; - this.version = version; - this.dimensions = dimensions == null - ? ImmutableList.of() - : ImmutableList.copyOf(Iterables.filter(dimensions, nonEmpty)); - this.metrics = metrics == null - ? ImmutableList.of() - : ImmutableList.copyOf(Iterables.filter(metrics, nonEmpty)); - this.shardSpec = (shardSpec == null) ? new NoneShardSpec() : shardSpec; - this.binaryVersion = binaryVersion; - this.size = size; - - this.identifier = makeDataSegmentIdentifier( - this.dataSource, - this.interval.getStart(), - this.interval.getEnd(), - this.version, - this.shardSpec - ); - } - - /** - * Get dataSource - * - * @return the dataSource - */ - @JsonProperty - public String getDataSource() - { - return dataSource; - } - - @JsonProperty - public Interval getInterval() - { - return interval; - } - - @JsonProperty - public Map getLoadSpec() - { - return loadSpec; - } - - @JsonProperty - public String getVersion() - { - return version; - } - - @JsonProperty - @JsonSerialize(using = CommaListJoinSerializer.class) - public List getDimensions() - { - return dimensions; - } - - @JsonProperty - @JsonSerialize(using = CommaListJoinSerializer.class) - public List getMetrics() - { - return metrics; - } - - @JsonProperty - public ShardSpec getShardSpec() - { - return shardSpec; - } - - @JsonProperty - public Integer getBinaryVersion() - { - return binaryVersion; - } - - @JsonProperty - public long getSize() - { - return size; - } - - @JsonProperty - public String getIdentifier() - { - return identifier; - } - - public SegmentDescriptor toDescriptor() - { - return new SegmentDescriptor(interval, version, shardSpec.getPartitionNum()); - } - - public DataSegment withLoadSpec(Map loadSpec) - { - return builder(this).loadSpec(loadSpec).build(); - } - - public DataSegment withDimensions(List dimensions) - { - return builder(this).dimensions(dimensions).build(); - } - - public DataSegment withSize(long size) - { - return builder(this).size(size).build(); - } - - public DataSegment withVersion(String version) - { - return builder(this).version(version).build(); - } - - public DataSegment withBinaryVersion(int binaryVersion) - { - return builder(this).binaryVersion(binaryVersion).build(); - } - - @Override - public int compareTo(DataSegment dataSegment) - { - return getIdentifier().compareTo(dataSegment.getIdentifier()); - } - - @Override - public boolean equals(Object o) - { - if (o instanceof DataSegment) { - return getIdentifier().equals(((DataSegment) o).getIdentifier()); - } - return false; - } - - @Override - public int hashCode() - { - return getIdentifier().hashCode(); - } - - @Override - public String toString() - { - return "DataSegment{" + - "size=" + size + - ", shardSpec=" + shardSpec + - ", metrics=" + metrics + - ", dimensions=" + dimensions + - ", version='" + version + '\'' + - ", loadSpec=" + loadSpec + - ", interval=" + interval + - ", dataSource='" + dataSource + '\'' + - ", binaryVersion='" + binaryVersion + '\'' + - '}'; - } - - public static Comparator bucketMonthComparator() - { - return new Comparator() - { - @Override - public int compare(DataSegment lhs, DataSegment rhs) - { - int retVal; - - DateTime lhsMonth = Granularity.MONTH.truncate(lhs.getInterval().getStart()); - DateTime rhsMonth = Granularity.MONTH.truncate(rhs.getInterval().getStart()); - - retVal = lhsMonth.compareTo(rhsMonth); - - if (retVal != 0) { - return retVal; - } - - return lhs.compareTo(rhs); - } - }; - } - - public static Builder builder() - { - return new Builder(); - } - - public static Builder builder(DataSegment segment) - { - return new Builder(segment); - } - - public static class Builder - { - private String dataSource; - private Interval interval; - private String version; - private Map loadSpec; - private List dimensions; - private List metrics; - private ShardSpec shardSpec; - private Integer binaryVersion; - private long size; - - public Builder() - { - this.loadSpec = ImmutableMap.of(); - this.dimensions = ImmutableList.of(); - this.metrics = ImmutableList.of(); - this.shardSpec = new NoneShardSpec(); - this.size = -1; - } - - public Builder(DataSegment segment) - { - this.dataSource = segment.getDataSource(); - this.interval = segment.getInterval(); - this.version = segment.getVersion(); - this.loadSpec = segment.getLoadSpec(); - this.dimensions = segment.getDimensions(); - this.metrics = segment.getMetrics(); - this.shardSpec = segment.getShardSpec(); - this.binaryVersion = segment.getBinaryVersion(); - this.size = segment.getSize(); - } - - public Builder dataSource(String dataSource) - { - this.dataSource = dataSource; - return this; - } - - public Builder interval(Interval interval) - { - this.interval = interval; - return this; - } - - public Builder version(String version) - { - this.version = version; - return this; - } - - public Builder loadSpec(Map loadSpec) - { - this.loadSpec = loadSpec; - return this; - } - - public Builder dimensions(List dimensions) - { - this.dimensions = dimensions; - return this; - } - - public Builder metrics(List metrics) - { - this.metrics = metrics; - return this; - } - - public Builder shardSpec(ShardSpec shardSpec) - { - this.shardSpec = shardSpec; - return this; - } - - public Builder binaryVersion(Integer binaryVersion) - { - this.binaryVersion = binaryVersion; - return this; - } - - public Builder size(long size) - { - this.size = size; - return this; - } - - public DataSegment build() - { - // Check stuff that goes into the identifier, at least. - Preconditions.checkNotNull(dataSource, "dataSource"); - Preconditions.checkNotNull(interval, "interval"); - Preconditions.checkNotNull(version, "version"); - Preconditions.checkNotNull(shardSpec, "shardSpec"); - - return new DataSegment( - dataSource, - interval, - version, - loadSpec, - dimensions, - metrics, - shardSpec, - binaryVersion, - size - ); - } - } -} diff --git a/client/src/main/java/com/metamx/druid/client/InventoryView.java b/client/src/main/java/com/metamx/druid/client/InventoryView.java deleted file mode 100644 index bbf7f6e46cc..00000000000 --- a/client/src/main/java/com/metamx/druid/client/InventoryView.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.metamx.druid.client; - -/** - */ -public interface InventoryView -{ - public DruidServer getInventoryValue(String string); - public Iterable getInventory(); -} diff --git a/client/src/main/java/com/metamx/druid/client/ServerInventoryViewConfig.java b/client/src/main/java/com/metamx/druid/client/ServerInventoryViewConfig.java deleted file mode 100644 index 6130a96a66c..00000000000 --- a/client/src/main/java/com/metamx/druid/client/ServerInventoryViewConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.client; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class ServerInventoryViewConfig -{ - @Config("druid.master.removedSegmentLifetime") - @Default("1") - public abstract int getRemovedSegmentLifetime(); - - @Config("druid.announcer.type") - @Default("legacy") - public abstract String getAnnouncerType(); -} \ No newline at end of file diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheConfig.java b/client/src/main/java/com/metamx/druid/client/cache/CacheConfig.java deleted file mode 100644 index e970363e0e5..00000000000 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheConfig.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.metamx.druid.client.cache; - -import org.skife.config.Config; -import org.skife.config.Default; - -public abstract class CacheConfig -{ - @Config("druid.bard.cache.type") - @Default("local") - public abstract String getType(); -} diff --git a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java b/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java deleted file mode 100644 index 2bd7b84e4cc..00000000000 --- a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.metamx.druid.client.cache; - -import org.skife.config.Config; -import org.skife.config.Default; - -public abstract class MemcachedCacheConfig -{ - @Config("${prefix}.expiration") - @Default("2592000") - public abstract int getExpiration(); - - @Config("${prefix}.timeout") - @Default("500") - public abstract int getTimeout(); - - @Config("${prefix}.hosts") - public abstract String getHosts(); - - @Config("${prefix}.maxObjectSize") - @Default("52428800") - public abstract int getMaxObjectSize(); - - @Config("${prefix}.memcachedPrefix") - @Default("druid") - public abstract String getMemcachedPrefix(); -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/AddressPortServiceInstanceFactory.java b/client/src/main/java/com/metamx/druid/curator/discovery/AddressPortServiceInstanceFactory.java deleted file mode 100644 index 3f4fa988655..00000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/AddressPortServiceInstanceFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.metamx.druid.curator.discovery; - -import com.google.common.base.Throwables; -import org.apache.curator.x.discovery.ServiceInstance; - -public class AddressPortServiceInstanceFactory implements ServiceInstanceFactory -{ - private final String address; - private final int port; - - public AddressPortServiceInstanceFactory(String address, int port) - { - this.address = address; - this.port = port; - } - - @Override - public ServiceInstance create(String service) - { - try { - return ServiceInstance.builder().name(service).address(address).port(port).build(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/CuratorServiceAnnouncer.java b/client/src/main/java/com/metamx/druid/curator/discovery/CuratorServiceAnnouncer.java deleted file mode 100644 index a3156b3205d..00000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/CuratorServiceAnnouncer.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.metamx.druid.curator.discovery; - -import com.google.common.collect.Maps; -import com.metamx.common.logger.Logger; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.apache.curator.x.discovery.ServiceInstance; - -import java.util.Map; - -/** - * Uses the Curator Service Discovery recipe to announce services. - */ -public class CuratorServiceAnnouncer implements ServiceAnnouncer -{ - private static final Logger log = new Logger(CuratorServiceAnnouncer.class); - - private final ServiceDiscovery discovery; - private final ServiceInstanceFactory instanceFactory; - private final Map> instanceMap = Maps.newHashMap(); - private final Object monitor = new Object(); - - public CuratorServiceAnnouncer( - ServiceDiscovery discovery, - ServiceInstanceFactory instanceFactory - ) - { - this.discovery = discovery; - this.instanceFactory = instanceFactory; - } - - @Override - public void announce(String service) throws Exception - { - final ServiceInstance instance; - - synchronized (monitor) { - if (instanceMap.containsKey(service)) { - log.warn("Ignoring request to announce service[%s]", service); - return; - } else { - instance = instanceFactory.create(service); - instanceMap.put(service, instance); - } - } - - try { - log.info("Announcing service[%s]", service); - discovery.registerService(instance); - } catch (Exception e) { - log.warn("Failed to announce service[%s]", service); - synchronized (monitor) { - instanceMap.remove(service); - } - } - } - - @Override - public void unannounce(String service) throws Exception - { - final ServiceInstance instance; - - synchronized (monitor) { - instance = instanceMap.get(service); - if (instance == null) { - log.warn("Ignoring request to unannounce service[%s]", service); - return; - } - } - - log.info("Unannouncing service[%s]", service); - try { - discovery.unregisterService(instance); - } catch (Exception e) { - log.warn(e, "Failed to unannounce service[%s]", service); - } finally { - synchronized (monitor) { - instanceMap.remove(service); - } - } - } -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/NoopServiceAnnouncer.java b/client/src/main/java/com/metamx/druid/curator/discovery/NoopServiceAnnouncer.java deleted file mode 100644 index 782739f7a24..00000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/NoopServiceAnnouncer.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.metamx.druid.curator.discovery; - -/** - * Does nothing. - */ -public class NoopServiceAnnouncer implements ServiceAnnouncer -{ - @Override - public void unannounce(String service) - { - - } - - @Override - public void announce(String service) - { - - } -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceAnnouncer.java b/client/src/main/java/com/metamx/druid/curator/discovery/ServiceAnnouncer.java deleted file mode 100644 index 4e91122423a..00000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceAnnouncer.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.metamx.druid.curator.discovery; - -/** - * Announces our ability to serve a particular function. Multiple users may announce the same service, in which - * case they are treated as interchangeable instances of that service. - */ -public interface ServiceAnnouncer -{ - public void announce(String service) throws Exception; - public void unannounce(String service) throws Exception; -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceInstanceFactory.java b/client/src/main/java/com/metamx/druid/curator/discovery/ServiceInstanceFactory.java deleted file mode 100644 index 8f8cc19c0bd..00000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceInstanceFactory.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.metamx.druid.curator.discovery; - -import org.apache.curator.x.discovery.ServiceInstance; - -public interface ServiceInstanceFactory -{ - public ServiceInstance create(String service); -} diff --git a/client/src/main/java/com/metamx/druid/http/BrokerNode.java b/client/src/main/java/com/metamx/druid/http/BrokerNode.java deleted file mode 100644 index 75b69fe5d94..00000000000 --- a/client/src/main/java/com/metamx/druid/http/BrokerNode.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.servlet.GuiceFilter; -import com.metamx.common.ISE; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.QueryableNode; -import com.metamx.druid.client.BrokerServerView; -import com.metamx.druid.client.CachingClusteredClient; -import com.metamx.druid.client.cache.Cache; -import com.metamx.druid.client.cache.CacheConfig; -import com.metamx.druid.client.cache.CacheMonitor; -import com.metamx.druid.client.cache.MapCache; -import com.metamx.druid.client.cache.MapCacheConfig; -import com.metamx.druid.client.cache.MemcachedCache; -import com.metamx.druid.client.cache.MemcachedCacheConfig; -import com.metamx.druid.curator.discovery.ServiceAnnouncer; -import com.metamx.druid.curator.discovery.ServiceInstanceFactory; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServiceDiscoveryConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.ReflectionQueryToolChestWarehouse; -import com.metamx.druid.utils.PropUtils; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; -import com.metamx.metrics.Monitor; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.servlet.GzipFilter; -import org.skife.config.ConfigurationObjectFactory; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - */ - -public class BrokerNode extends QueryableNode -{ - private static final Logger log = new Logger(BrokerNode.class); - - public static final String CACHE_TYPE_LOCAL = "local"; - public static final String CACHE_TYPE_MEMCACHED = "memcached"; - public static final String CACHE_PROPERTY_PREFIX = "druid.bard.cache"; - - private final List extraModules = Lists.newArrayList(); - private final List pathsForGuiceFilter = Lists.newArrayList(); - - private QueryToolChestWarehouse warehouse = null; - private HttpClient brokerHttpClient = null; - private Cache cache = null; - - private boolean useDiscovery = true; - - public static Builder builder() - { - return new Builder(); - } - - public BrokerNode( - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super("broker", log, props, lifecycle, jsonMapper, smileMapper, configFactory); - } - - public QueryToolChestWarehouse getWarehouse() - { - initializeWarehouse(); - return warehouse; - } - - public BrokerNode setWarehouse(QueryToolChestWarehouse warehouse) - { - checkFieldNotSetAndSet("warehouse", warehouse); - return this; - } - - public HttpClient getBrokerHttpClient() - { - initializeBrokerHttpClient(); - return brokerHttpClient; - } - - public BrokerNode setBrokerHttpClient(HttpClient brokerHttpClient) - { - checkFieldNotSetAndSet("brokerHttpClient", brokerHttpClient); - return this; - } - - public Cache getCache() - { - initializeCacheBroker(); - return cache; - } - - public BrokerNode setCache(Cache cache) - { - checkFieldNotSetAndSet("cache", cache); - return this; - } - - public BrokerNode useDiscovery(boolean useDiscovery) - { - this.useDiscovery = useDiscovery; - return this; - } - - /** - * This method allows you to specify more Guice modules to use primarily for injected extra Jersey resources. - * I'd like to remove the Guice dependency for this, but I don't know how to set up Jersey without Guice... - * - * This is deprecated because at some point in the future, we will eliminate the Guice dependency and anything - * that uses this will break. Use at your own risk. - * - * @param module the module to register with Guice - * - * @return this - */ - @Deprecated - public BrokerNode addModule(Module module) - { - extraModules.add(module); - return this; - } - - /** - * This method is used to specify extra paths that the GuiceFilter should pay attention to. - * - * This is deprecated for the same reason that addModule is deprecated. - * - * @param path the path that the GuiceFilter should pay attention to. - * - * @return this - */ - @Deprecated - public BrokerNode addPathForGuiceFilter(String path) - { - pathsForGuiceFilter.add(path); - return this; - } - - @Override - protected void doInit() throws Exception - { - initializeWarehouse(); - initializeBrokerHttpClient(); - initializeCacheBroker(); - initializeDiscovery(); - - final Lifecycle lifecycle = getLifecycle(); - - final List monitors = getMonitors(); - monitors.add(new CacheMonitor(cache)); - startMonitoring(monitors); - - final ExecutorService viewExec = Executors.newFixedThreadPool( - 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("BrokerServerView-%s").build() - ); - final BrokerServerView view = new BrokerServerView( - warehouse, getSmileMapper(), brokerHttpClient, getServerView(), viewExec - ); - - final CachingClusteredClient baseClient = new CachingClusteredClient(warehouse, view, cache, getSmileMapper()); - lifecycle.addManagedInstance(baseClient); - - final ClientQuerySegmentWalker texasRanger = new ClientQuerySegmentWalker(warehouse, getEmitter(), baseClient); - - List theModules = Lists.newArrayList(); - theModules.add(new ClientServletModule(texasRanger, getInventoryView(), getJsonMapper())); - theModules.addAll(extraModules); - - final Injector injector = Guice.createInjector(theModules); - final Context root = new Context(getServer(), "/", Context.SESSIONS); - root.addServlet(new ServletHolder(new StatusServlet()), "/status"); - root.addServlet( - new ServletHolder(new QueryServlet(getJsonMapper(), getSmileMapper(), texasRanger, getEmitter(), getRequestLogger())), - "/druid/v2/*" - ); - root.addFilter(GzipFilter.class, "/*", 0); - - root.addEventListener(new GuiceServletConfig(injector)); - root.addFilter(GuiceFilter.class, "/druid/v2/datasources/*", 0); - - for (String path : pathsForGuiceFilter) { - root.addFilter(GuiceFilter.class, path, 0); - } - } - - private void initializeDiscovery() throws Exception - { - if (useDiscovery) { - final Lifecycle lifecycle = getLifecycle(); - final ServiceDiscoveryConfig serviceDiscoveryConfig = getConfigFactory().build(ServiceDiscoveryConfig.class); - final CuratorFramework curatorFramework = Initialization.makeCuratorFramework( - serviceDiscoveryConfig, lifecycle - ); - final ServiceDiscovery serviceDiscovery = Initialization.makeServiceDiscoveryClient( - curatorFramework, serviceDiscoveryConfig, lifecycle - ); - final ServiceAnnouncer serviceAnnouncer = Initialization.makeServiceAnnouncer( - serviceDiscoveryConfig, serviceDiscovery - ); - Initialization.announceDefaultService(serviceDiscoveryConfig, serviceAnnouncer, lifecycle); - } - } - - private void initializeCacheBroker() - { - if (cache == null) { - String cacheType = getConfigFactory() - .build(CacheConfig.class) - .getType(); - - if (cacheType.equals(CACHE_TYPE_LOCAL)) { - setCache( - MapCache.create( - getConfigFactory().buildWithReplacements( - MapCacheConfig.class, - ImmutableMap.of("prefix", CACHE_PROPERTY_PREFIX) - ) - ) - ); - } else if (cacheType.equals(CACHE_TYPE_MEMCACHED)) { - setCache( - MemcachedCache.create( - getConfigFactory().buildWithReplacements( - MemcachedCacheConfig.class, - ImmutableMap.of("prefix", CACHE_PROPERTY_PREFIX) - ) - ) - ); - } else { - throw new ISE("Unknown cache type [%s]", cacheType); - } - } - } - - private void initializeBrokerHttpClient() - { - if (brokerHttpClient == null) { - setBrokerHttpClient( - HttpClientInit.createClient( - HttpClientConfig - .builder() - .withNumConnections(PropUtils.getPropertyAsInt(getProps(), "druid.client.http.connections")) - .build(), - getLifecycle() - ) - ); - } - } - - private void initializeWarehouse() - { - if (warehouse == null) { - setWarehouse(new ReflectionQueryToolChestWarehouse()); - } - } - - public static class Builder - { - private ObjectMapper jsonMapper = null; - private ObjectMapper smileMapper = null; - private Lifecycle lifecycle = null; - private Properties props = null; - private ConfigurationObjectFactory configFactory = null; - - public Builder withMappers(ObjectMapper jsonMapper, ObjectMapper smileMapper) - { - this.jsonMapper = jsonMapper; - this.smileMapper = smileMapper; - return this; - } - - public Builder withProps(Properties props) - { - this.props = props; - return this; - } - - public Builder withConfigFactory(ConfigurationObjectFactory configFactory) - { - this.configFactory = configFactory; - return this; - } - - public BrokerNode build() - { - if (jsonMapper == null && smileMapper == null) { - jsonMapper = new DefaultObjectMapper(); - smileMapper = new DefaultObjectMapper(new SmileFactory()); - smileMapper.getJsonFactory().setCodec(smileMapper); - } - else if (jsonMapper == null || smileMapper == null) { - throw new ISE("Only jsonMapper[%s] or smileMapper[%s] was set, must set neither or both.", jsonMapper, smileMapper); - } - - if (lifecycle == null) { - lifecycle = new Lifecycle(); - } - - if (props == null) { - props = Initialization.loadProperties(); - } - - if (configFactory == null) { - configFactory = Config.createFactory(props); - } - - return new BrokerNode(props, lifecycle, jsonMapper, smileMapper, configFactory); - } - } -} diff --git a/client/src/main/java/com/metamx/druid/http/ClientMain.java b/client/src/main/java/com/metamx/druid/http/ClientMain.java deleted file mode 100644 index 1beda5c405d..00000000000 --- a/client/src/main/java/com/metamx/druid/http/ClientMain.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -/** - */ -@Deprecated -public class ClientMain -{ - public static void main(String[] args) throws Exception - { - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("K thx bye."); - BrokerMain.main(args); - } -} diff --git a/client/src/main/java/com/metamx/druid/http/ClientServletModule.java b/client/src/main/java/com/metamx/druid/http/ClientServletModule.java deleted file mode 100644 index 8de99f59496..00000000000 --- a/client/src/main/java/com/metamx/druid/http/ClientServletModule.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; -import com.google.inject.Provides; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.sun.jersey.guice.JerseyServletModule; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; - -import javax.inject.Singleton; - -/** - */ -public class ClientServletModule extends JerseyServletModule -{ - private final QuerySegmentWalker texasRanger; - private final InventoryView serverInventoryView; - private final ObjectMapper jsonMapper; - - public ClientServletModule( - QuerySegmentWalker texasRanger, - InventoryView serverInventoryView, - ObjectMapper jsonMapper - ) - { - this.texasRanger = texasRanger; - this.serverInventoryView = serverInventoryView; - this.jsonMapper = jsonMapper; - } - - @Override - protected void configureServlets() - { - bind(ClientInfoResource.class); - bind(QuerySegmentWalker.class).toInstance(texasRanger); - bind(InventoryView.class).toInstance(serverInventoryView); - - serve("/*").with(GuiceContainer.class); - } - - @Provides - @Singleton - public JacksonJsonProvider getJacksonJsonProvider() - { - final JacksonJsonProvider provider = new JacksonJsonProvider(); - provider.setMapper(jsonMapper); - return provider; - } -} - diff --git a/client/src/main/java/com/metamx/druid/http/StatusServlet.java b/client/src/main/java/com/metamx/druid/http/StatusServlet.java deleted file mode 100644 index 95a40c543b8..00000000000 --- a/client/src/main/java/com/metamx/druid/http/StatusServlet.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; - -/** - */ -public class StatusServlet extends HttpServlet -{ - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException - { - ByteArrayOutputStream retVal = new ByteArrayOutputStream(); - PrintWriter out = new PrintWriter(new OutputStreamWriter(retVal)); - - Runtime runtime = Runtime.getRuntime(); - long maxMemory = runtime.maxMemory(); - long totalMemory = runtime.totalMemory(); - long freeMemory = runtime.freeMemory(); - - out.printf("Max Memory:\t%,18d\t%1$d%n", maxMemory); - out.printf("Total Memory:\t%,18d\t%1$d%n", totalMemory); - out.printf("Free Memory:\t%,18d\t%1$d%n", freeMemory); - out.printf("Used Memory:\t%,18d\t%1$d%n", totalMemory - freeMemory); - - out.flush(); - - resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentType("text/plain"); - resp.getOutputStream().write(retVal.toByteArray()); - } -} diff --git a/client/src/main/java/com/metamx/druid/initialization/Initialization.java b/client/src/main/java/com/metamx/druid/initialization/Initialization.java deleted file mode 100644 index de29eabd550..00000000000 --- a/client/src/main/java/com/metamx/druid/initialization/Initialization.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.initialization; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.base.Throwables; -import com.google.common.io.Closeables; -import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.PotentiallyGzippedCompressionProvider; -import com.metamx.druid.curator.discovery.AddressPortServiceInstanceFactory; -import com.metamx.druid.curator.discovery.CuratorServiceAnnouncer; -import com.metamx.druid.curator.discovery.ServiceAnnouncer; -import com.metamx.druid.curator.discovery.ServiceInstanceFactory; -import com.metamx.druid.http.EmittingRequestLogger; -import com.metamx.druid.http.FileRequestLogger; -import com.metamx.druid.http.RequestLogger; -import com.metamx.druid.utils.PropUtils; -import com.metamx.emitter.core.Emitter; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.BoundedExponentialBackoffRetry; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.apache.curator.x.discovery.ServiceDiscoveryBuilder; -import org.apache.curator.x.discovery.ServiceInstance; -import org.apache.curator.x.discovery.ServiceProvider; -import org.apache.zookeeper.data.Stat; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.nio.SelectChannelConnector; -import org.mortbay.thread.QueuedThreadPool; -import org.skife.config.ConfigurationObjectFactory; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Properties; - -/** - */ -public class Initialization -{ - private static final Logger log = new Logger(Initialization.class); - - private static final String PROPERTIES_FILE = "runtime.properties"; - private static final Properties zkProps = new Properties(); - private static final Properties fileProps = new Properties(zkProps); - private static Properties props = null; - - /** - * Load properties. - * Properties are layered: - *

- * # stored in zookeeper - * # runtime.properties file, - * # cmdLine -D - *

- * command line overrides runtime.properties which overrides zookeeper - *

- * Idempotent. Thread-safe. Properties are only loaded once. - * If property druid.zk.service.host is not set then do not load properties from zookeeper. - * - * @return Properties ready to use. - */ - public synchronized static Properties loadProperties() - { - if (props != null) { - return props; - } - - // Note that zookeeper coordinates must be either in cmdLine or in runtime.properties - Properties sp = System.getProperties(); - - Properties tmp_props = new Properties(fileProps); // the head of the 3 level Properties chain - tmp_props.putAll(sp); - - final InputStream stream = ClassLoader.getSystemResourceAsStream(PROPERTIES_FILE); - if (stream == null) { - log.info("%s not found on classpath, relying only on system properties and zookeeper.", PROPERTIES_FILE); - } else { - log.info("Loading properties from %s", PROPERTIES_FILE); - try { - try { - fileProps.load(stream); - } - catch (IOException e) { - throw Throwables.propagate(e); - } - } - finally { - Closeables.closeQuietly(stream); - } - } - - // log properties from file; stringPropertyNames() would normally cascade down into the sub Properties objects, but - // zkProps (the parent level) is empty at this point so it will only log properties from runtime.properties - for (String prop : fileProps.stringPropertyNames()) { - log.info("Loaded(runtime.properties) Property[%s] as [%s]", prop, fileProps.getProperty(prop)); - } - - final String zkHostsProperty = "druid.zk.service.host"; - - if (tmp_props.getProperty(zkHostsProperty) != null) { - final ConfigurationObjectFactory factory = Config.createFactory(tmp_props); - - ZkPathsConfig config; - try { - config = factory.build(ZkPathsConfig.class); - } - catch (IllegalArgumentException e) { - log.warn(e, "Unable to build ZkPathsConfig. Cannot load properties from ZK."); - config = null; - } - - if (config != null) { - Lifecycle lifecycle = new Lifecycle(); - try { - CuratorFramework curator = makeCuratorFramework(factory.build(CuratorConfig.class), lifecycle); - - lifecycle.start(); - - final Stat stat = curator.checkExists().forPath(config.getPropertiesPath()); - if (stat != null) { - final byte[] data = curator.getData().forPath(config.getPropertiesPath()); - zkProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - } - - // log properties from zk - for (String prop : zkProps.stringPropertyNames()) { - log.info("Loaded(zk) Property[%s] as [%s]", prop, zkProps.getProperty(prop)); - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - finally { - lifecycle.stop(); - } - } - } else { - log.warn("property[%s] not set, skipping ZK-specified properties.", zkHostsProperty); - } - - props = tmp_props; - - return props; - } - - public static Server makeJettyServer(ServerConfig config) - { - final QueuedThreadPool threadPool = new QueuedThreadPool(); - threadPool.setMinThreads(config.getNumThreads()); - threadPool.setMaxThreads(config.getNumThreads()); - - final Server server = new Server(); - server.setThreadPool(threadPool); - - SelectChannelConnector connector = new SelectChannelConnector(); - connector.setPort(config.getPort()); - connector.setMaxIdleTime(config.getMaxIdleTimeMillis()); - connector.setStatsOn(true); - - server.setConnectors(new Connector[]{connector}); - - return server; - } - - public static CuratorFramework makeCuratorFramework( - CuratorConfig curatorConfig, - Lifecycle lifecycle - ) throws IOException - { - final CuratorFramework framework = - CuratorFrameworkFactory.builder() - .connectString(curatorConfig.getZkHosts()) - .sessionTimeoutMs(curatorConfig.getZkSessionTimeoutMs()) - .retryPolicy(new BoundedExponentialBackoffRetry(1000, 45000, 30)) - .compressionProvider(new PotentiallyGzippedCompressionProvider(curatorConfig.enableCompression())) - .build(); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - framework.start(); - } - - @Override - public void stop() - { - framework.close(); - } - } - ); - - return framework; - } - - public static ServiceDiscovery makeServiceDiscoveryClient( - CuratorFramework discoveryClient, - ServiceDiscoveryConfig config, - Lifecycle lifecycle - ) - throws Exception - { - final ServiceDiscovery serviceDiscovery = - ServiceDiscoveryBuilder.builder(Void.class) - .basePath(config.getDiscoveryPath()) - .client(discoveryClient) - .build(); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - serviceDiscovery.start(); - } - - @Override - public void stop() - { - try { - serviceDiscovery.close(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - } - ); - - return serviceDiscovery; - } - - public static ServiceAnnouncer makeServiceAnnouncer( - ServiceDiscoveryConfig config, - ServiceDiscovery serviceDiscovery - ) - { - final ServiceInstanceFactory serviceInstanceFactory = makeServiceInstanceFactory(config); - return new CuratorServiceAnnouncer(serviceDiscovery, serviceInstanceFactory); - } - - public static void announceDefaultService( - final ServiceDiscoveryConfig config, - final ServiceAnnouncer serviceAnnouncer, - final Lifecycle lifecycle - ) throws Exception - { - final String service = config.getServiceName().replace('/', ':'); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - serviceAnnouncer.announce(service); - } - - @Override - public void stop() - { - try { - serviceAnnouncer.unannounce(service); - } - catch (Exception e) { - log.warn(e, "Failed to unannouce default service[%s]", service); - } - } - } - ); - } - - public static ServiceProvider makeServiceProvider( - String serviceName, - ServiceDiscovery serviceDiscovery, - Lifecycle lifecycle - ) - { - final ServiceProvider serviceProvider = serviceDiscovery.serviceProviderBuilder() - .serviceName(serviceName) - .build(); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - serviceProvider.start(); - } - - @Override - public void stop() - { - try { - serviceProvider.close(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - } - ); - - return serviceProvider; - } - - public static RequestLogger makeFileRequestLogger( - ObjectMapper objectMapper, - ScheduledExecutorFactory factory, - Properties props - ) throws IOException - { - return new FileRequestLogger( - objectMapper, - factory.create(1, "RequestLogger-%s"), - new File(PropUtils.getProperty(props, "druid.request.logging.dir")) - ); - } - - public static RequestLogger makeEmittingRequestLogger(Properties props, Emitter emitter) - { - return new EmittingRequestLogger( - PropUtils.getProperty(props, "druid.service"), - PropUtils.getProperty(props, "druid.host"), - emitter, - PropUtils.getProperty(props, "druid.request.logging.feed") - ); - } - - public static ServiceInstanceFactory makeServiceInstanceFactory(ServiceDiscoveryConfig config) - { - final String host = config.getHost(); - final String address; - final int colon = host.indexOf(':'); - if (colon < 0) { - address = host; - } else { - address = host.substring(0, colon); - } - - return new AddressPortServiceInstanceFactory(address, config.getPort()); - } -} diff --git a/client/src/main/java/com/metamx/druid/initialization/ZkDataSegmentAnnouncerConfig.java b/client/src/main/java/com/metamx/druid/initialization/ZkDataSegmentAnnouncerConfig.java deleted file mode 100644 index e14c65027eb..00000000000 --- a/client/src/main/java/com/metamx/druid/initialization/ZkDataSegmentAnnouncerConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.metamx.druid.initialization; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class ZkDataSegmentAnnouncerConfig extends ZkPathsConfig -{ - @Config("druid.zk.segmentsPerNode") - @Default("50") - public abstract int getSegmentsPerNode(); - - @Config("druid.zk.maxNumBytesPerNode") - @Default("512000") - public abstract long getMaxNumBytes(); - - @Config("druid.announcer.type") - @Default("legacy") - public abstract String getAnnouncerType(); -} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java deleted file mode 100644 index 3fa27b99ace..00000000000 --- a/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.metamx.druid.query.group.having; - -import com.metamx.druid.input.Row; - -/** - */ -public class AlwaysHavingSpec implements HavingSpec -{ - @Override - public boolean eval(Row row) - { - return true; - } -} diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java deleted file mode 100644 index 3f383430b12..00000000000 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.metamx.druid.query.group.orderby; - -import com.google.common.base.Function; -import com.google.common.base.Functions; -import com.metamx.common.guava.Sequence; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.dimension.DimensionSpec; - -import java.util.List; - -/** - */ -public class NoopLimitSpec implements LimitSpec -{ - @Override - public Function, Sequence> build( - List dimensions, List aggs, List postAggs - ) - { - return Functions.identity(); - } - - @Override - public String toString() - { - return "NoopLimitSpec"; - } -} diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java b/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java deleted file mode 100644 index 5b5a3f48cb6..00000000000 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.metamx.druid.query.group.orderby; - -import com.google.common.collect.MinMaxPriorityQueue; - -import java.util.Iterator; - -/** - * Utility class that supports iterating a priority queue in sorted order. - */ -class OrderedPriorityQueueItems implements Iterable -{ - private MinMaxPriorityQueue rows; - - public OrderedPriorityQueueItems(MinMaxPriorityQueue rows) - { - this.rows = rows; - } - - @Override - public Iterator iterator() - { - return new Iterator() { - - @Override - public boolean hasNext() - { - return !rows.isEmpty(); - } - - @Override - public T next() - { - return rows.poll(); - } - - @Override - public void remove() - { - throw new UnsupportedOperationException("Can't remove any item from an intermediary heap for orderBy/limit"); - } - }; - } -} diff --git a/client/src/main/java/com/metamx/druid/query/segment/SegmentDescriptor.java b/client/src/main/java/com/metamx/druid/query/segment/SegmentDescriptor.java deleted file mode 100644 index 1e2ac83484a..00000000000 --- a/client/src/main/java/com/metamx/druid/query/segment/SegmentDescriptor.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.query.segment; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.joda.time.Interval; - -/** -*/ -public class SegmentDescriptor -{ - private final Interval interval; - private final String version; - private final int partitionNumber; - - @JsonCreator - public SegmentDescriptor( - @JsonProperty("itvl") Interval interval, - @JsonProperty("ver") String version, - @JsonProperty("part") int partitionNumber) - { - this.interval = interval; - this.version = version; - this.partitionNumber = partitionNumber; - } - - @JsonProperty("itvl") - public Interval getInterval() - { - return interval; - } - - @JsonProperty("ver") - public String getVersion() - { - return version; - } - - @JsonProperty("part") - public int getPartitionNumber() - { - return partitionNumber; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SegmentDescriptor that = (SegmentDescriptor) o; - - if (partitionNumber != that.partitionNumber) { - return false; - } - if (interval != null ? !interval.equals(that.interval) : that.interval != null) { - return false; - } - if (version != null ? !version.equals(that.version) : that.version != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() - { - int result = interval != null ? interval.hashCode() : 0; - result = 31 * result + (version != null ? version.hashCode() : 0); - result = 31 * result + partitionNumber; - return result; - } - - @Override - public String toString() - { - return "SegmentDescriptor{" + - "interval=" + interval + - ", version='" + version + '\'' + - ", partitionNumber=" + partitionNumber + - '}'; - } -} diff --git a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java b/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java deleted file mode 100644 index 4d77a19086d..00000000000 --- a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.metamx.druid.shard; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.LinearPartitionChunk; -import com.metamx.druid.partition.PartitionChunk; - -import java.util.Map; - -public class LinearShardSpec implements ShardSpec { - private int partitionNum; - - public LinearShardSpec() { - this(-1); - } - - public LinearShardSpec(int partitionNum) { - this.partitionNum = partitionNum; - } - - @JsonProperty("partitionNum") - @Override - public int getPartitionNum() { - return partitionNum; - } - - @Override - public PartitionChunk createChunk(T obj) { - return new LinearPartitionChunk(partitionNum, obj); - } - - @Override - public boolean isInChunk(Map dimensions) { - return true; - } - - @Override - public boolean isInChunk(InputRow inputRow) { - return true; - } -} diff --git a/client/src/main/java/com/metamx/druid/shard/ShardSpec.java b/client/src/main/java/com/metamx/druid/shard/ShardSpec.java deleted file mode 100644 index b5b778283b5..00000000000 --- a/client/src/main/java/com/metamx/druid/shard/ShardSpec.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.shard; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.PartitionChunk; - -import java.util.Map; - -/** - * A Marker interface that exists to combine ShardSpec objects together for Jackson - */ -@JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property="type", include=JsonTypeInfo.As.PROPERTY) -@JsonSubTypes({ - @JsonSubTypes.Type(name="single", value=SingleDimensionShardSpec.class), - @JsonSubTypes.Type(name="none", value=NoneShardSpec.class), - @JsonSubTypes.Type(name="linear", value=LinearShardSpec.class), - @JsonSubTypes.Type(name="numbered", value=NumberedShardSpec.class) -}) -public interface ShardSpec -{ - public PartitionChunk createChunk(T obj); - public boolean isInChunk(Map dimensions); - public boolean isInChunk(InputRow inputRow); - public int getPartitionNum(); -} diff --git a/common/pom.xml b/common/pom.xml index 8d264bfb57e..af29a79f989 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -1,7 +1,7 @@ - - -DruidDataFlow - -1 - -REST query - -5 - -Realtime.working_set - -1->5 - - - query - -2 - -realtime_data_src - -2->5 - - -Realtime.Firehose - -4 - -indexed segments -blob_store (S3, HDFS) - -5->1 - - - results - -6 - -Realtime.spill_disk - -5->6 - - -Realtime.Firehose - -9 - -Realtime.local_disk - -5->9 - - -Realtime.Indexer - -10 - -metadata_store (mysql) - -5->10 - - -Realtime.MetadataUpdater - -6->9 - - -Realtime.Indexer - -7 - -HDFS_data_src - -11 - -HadoopDruidIndexer.working_set - -7->11 - - -HadoopDruidIndexer - -8 - -external_data_src - -12 - -IndexerService.working_set - -8->12 - - -IndexerService - -9->4 - - -Realtime.segmentPusher - -11->4 - - -HadoopDruidIndexer - -11->10 - - -HadoopDruidIndexer - -12->4 - - -IndexerService - -12->10 - - -IndexerService - - - diff --git a/doc/data_flow_simple.dot b/doc/data_flow_simple.dot deleted file mode 100644 index a30f8ad73de..00000000000 --- a/doc/data_flow_simple.dot +++ /dev/null @@ -1,27 +0,0 @@ -digraph DruidDataFlow { - graph [bgcolor="#fffef5", clusterrank=global, rankdir=TB]; - node [color="#0a0701", fillcolor="#fdf4c6", fontname=Helvetica, shape=box, style=filled, label="\N"]; - edge [color="#377d18"]; - - 1 [label = "REST query"]; - 10 [label = "Broker.query_cache"]; - 2 [label = "realtime_data_src"]; - 3 [label = "Compute.disk_cache"]; - 4 [label = "indexed segments\nblob_store (S3, HDFS)"]; - 5 [label = "Realtime.working_set"]; - 6 [label = "Compute.working_set"]; - -2 -> 5 [label = "Realtime.Firehose"]; -5 -> 4 [label = "Realtime.segmentPusher"]; - -4 -> 3 [label = "Compute.load"]; -3 -> 6 [label = "Compute.map"]; - -5 -> 10 [label = " results "]; -6 -> 10 [label = " results "]; -10 -> 1 [label = " results "]; - -10 -> 5 [label = " query "]; -10 -> 6 [label = " query "]; -1 -> 10 [label = " query "]; -} diff --git a/doc/data_flow_simple.png b/doc/data_flow_simple.png deleted file mode 100644 index f397953b651..00000000000 Binary files a/doc/data_flow_simple.png and /dev/null differ diff --git a/doc/data_flow_simple.svg b/doc/data_flow_simple.svg deleted file mode 100644 index 012a06c2977..00000000000 --- a/doc/data_flow_simple.svg +++ /dev/null @@ -1,92 +0,0 @@ - - - - - -DruidDataFlow - -1 - -REST query - -10 - -Broker.query_cache - -1->10 - - - query - -10->1 - - - results - -5 - -Realtime.working_set - -10->5 - - - query - -6 - -Compute.working_set - -10->6 - - - query - -2 - -realtime_data_src - -2->5 - - -Realtime.Firehose - -3 - -Compute.disk_cache - -3->6 - - -Compute.map - -4 - -indexed segments -blob_store (S3, HDFS) - -4->3 - - -Compute.load - -5->10 - - - results - -5->4 - - -Realtime.segmentPusher - -6->10 - - - results - - - diff --git a/doc/publications/vldb/README.md b/doc/publications/vldb/README.md deleted file mode 100644 index 44025ab1dbf..00000000000 --- a/doc/publications/vldb/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Download [MacTeX](http://tug.org/mactex/) -```bash -make -``` diff --git a/doc/publications/vldb/druid.pdf b/doc/publications/vldb/druid.pdf deleted file mode 100644 index 3553c0c4b5b..00000000000 Binary files a/doc/publications/vldb/druid.pdf and /dev/null differ diff --git a/doc/segment_propagation.png b/doc/segment_propagation.png deleted file mode 100644 index e9d3e1a17a6..00000000000 Binary files a/doc/segment_propagation.png and /dev/null differ diff --git a/doc/segmentation_propagation.txt b/doc/segmentation_propagation.txt deleted file mode 100644 index e70f4d37aa7..00000000000 --- a/doc/segmentation_propagation.txt +++ /dev/null @@ -1,13 +0,0 @@ -# http://www.websequencediagrams.com/ -title Segment Propagation - -Realtime->Deep Storage: push(segment) -Realtime->MySQL: write(metadata) -Master->+MySQL: segments? -MySQL-->-Master: metadata -note over Master: rules select segment owner -Master->Zookeeper: writes(ephemeral node) -Zookeeper->Compute: reads(ephemeral node) -Compute->+Deep Storage: pull(segment) -Deep Storage-->-Compute: transfer(segment) -Compute->Zookeeper: delete(ephemeral node) diff --git a/docs/Aggregations.md b/docs/Aggregations.md new file mode 100644 index 00000000000..37b99aeffc2 --- /dev/null +++ b/docs/Aggregations.md @@ -0,0 +1,90 @@ +--- +layout: default +--- +Aggregations are specifications of processing over metrics available in Druid. +Available aggregations are: + +### Sum aggregators + +#### `longSum` aggregator + +computes the sum of values as a 64-bit, signed integer + + { + "type" : "longSum", + "name" : , + "fieldName" : + } + +`name` – output name for the summed value +`fieldName` – name of the metric column to sum over + +#### `doubleSum` aggregator + +Computes the sum of values as 64-bit floating point value. Similar to `longSum` + + { + "type" : "doubleSum", + "name" : , + "fieldName" : + } + +### Count aggregator + +`count` computes the row count that match the filters + + { + "type" : "count", + "name" : , + } + +### Min / Max aggregators + +#### `min` aggregator + +`min` computes the minimum metric value + + { + "type" : "min", + "name" : , + "fieldName" : + } + +#### `max` aggregator + +`max` computes the maximum metric value + + { + "type" : "max", + "name" : , + "fieldName" : + } + +### JavaScript aggregator + +Computes an arbitrary JavaScript function over a set of columns (both metrics and dimensions). + +All JavaScript functions must return numerical values. + + { + "type": "javascript", + "name": "", + "fieldNames" : [ , , ... ], + "fnAggregate" : "function(current, column1, column2, ...) { + + return + }" + "fnCombine" : "function(partialA, partialB) { return ; }" + "fnReset" : "function() { return ; }" + } + +**Example** + + { + "type": "javascript", + "name": "sum(log(x)/y) + 10", + "fieldNames": ["x", "y"], + "fnAggregate" : "function(current, a, b) { return current + (Math.log(a) * b); }" + "fnCombine" : "function(partialA, partialB) { return partialA + partialB; }" + "fnReset" : "function() { return 10; }" + } \ No newline at end of file diff --git a/docs/Batch-ingestion.md b/docs/Batch-ingestion.md new file mode 100644 index 00000000000..42a42ac7b29 --- /dev/null +++ b/docs/Batch-ingestion.md @@ -0,0 +1,141 @@ +--- +layout: default +--- +Batch Data Ingestion +==================== + +There are two choices for batch data ingestion to your Druid cluster, you can use the [Indexing service](Indexing-service.html) or you can use the `HadoopDruidIndexerMain`. This page describes how to use the `HadoopDruidIndexerMain`. + +Which should I use? +------------------- + +The [Indexing service](Indexing-service.html) is a node that can run as part of your Druid cluster and can accomplish a number of different types of indexing tasks. Even if all you care about is batch indexing, it provides for the encapsulation of things like the Database that is used for segment metadata and other things, so that your indexing tasks do not need to include such information. Long-term, the indexing service is going to be the preferred method of ingesting data. + +The `HadoopDruidIndexerMain` runs hadoop jobs in order to separate and index data segments. It takes advantage of Hadoop as a job scheduling and distributed job execution platform. It is a simple method if you already have Hadoop running and don’t want to spend the time configuring and deploying the [Indexing service](Indexing service.html) just yet. + +HadoopDruidIndexer +------------------ + +Located at `com.metamx.druid.indexer.HadoopDruidIndexerMain` can be run like + + + java -cp hadoop_config_path:druid_indexer_selfcontained_jar_path com.metamx.druid.indexer.HadoopDruidIndexerMain + + +The interval is the [ISO8601 interval](http://en.wikipedia.org/wiki/ISO_8601#Time_intervals) of the data you are processing. The config\_file is a path to a file (the “specFile”) that contains JSON and an example looks like: + + + { + "dataSource": "the_data_source", + "timestampColumn": "ts", + "timestampFormat": "", + "dataSpec": { + "format": "", + "columns": ["ts", "column_1", "column_2", "column_3", "column_4", "column_5"], + "dimensions": ["column_1", "column_2", "column_3"] + }, + "granularitySpec": { + "type":"uniform", + "intervals":[""], + "gran":"day" + }, + "pathSpec": { "type": "granularity", + "dataGranularity": "hour", + "inputPath": "s3n://billy-bucket/the/data/is/here", + "filePattern": ".*" }, + "rollupSpec": { "aggs": [ + { "type": "count", "name":"event_count" }, + { "type": "doubleSum", "fieldName": "column_4", "name": "revenue" }, + { "type": "longSum", "fieldName" : "column_5", "name": "clicks" } + ], + "rollupGranularity": "minute"}, + "workingPath": "/tmp/path/on/hdfs", + "segmentOutputPath": "s3n://billy-bucket/the/segments/go/here", + "leaveIntermediate": "false", + "partitionsSpec": { + "targetPartitionSize": 5000000 + }, + "updaterJobSpec": { + "type":"db", + "connectURI":"jdbc:mysql://localhost:7980/test_db", + "user":"username", + "password":"passmeup", + "segmentTable":"segments" + } + } + + +### Hadoop indexer config + +|property|description|required?| +|--------|-----------|---------| +|dataSource|name of the dataSource the data will belong to|yes| +|timestampColumn|the column that is to be used as the timestamp column|yes| +|timestampFormat|the format of timestamps; auto = either iso or millis, Joda time formats:http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html|yes| +|dataSpec|a specification of the data format and an array that names all of the columns in the input data|yes| +|dimensions|the columns that are to be used as dimensions|yes| +|granularitySpec|the time granularity and interval to chunk segments up into|yes| +|pathSpec|a specification of where to pull the data in from|yes| +|rollupSpec|a specification of the rollup to perform while processing the data|yes| +|workingPath|the working path to use for intermediate results (results between Hadoop jobs)|yes| +|segmentOutputPath|the path to dump segments into|yes| +|leaveIntermediate|leave behind files in the workingPath when job completes or fails (debugging tool)|no| +|partitionsSpec|a specification of how to partition each time bucket into segments, absence of this property means no partitioning will occur|no| +|updaterJobSpec|a specification of how to update the metadata for the druid cluster these segments belong to|yes| +|registererers|a list of serde handler classnames|no| + +### Path specification + +There are multiple types of path specification: + +##### `granularity` + +Is a type of data loader that expects data to be laid out in a specific path format. Specifically, it expects it to be segregated by day in this directory format `y=XXXX/m=XX/d=XX/H=XX/M=XX/S=XX` (dates are represented by lowercase, time is represented by uppercase). + +|property|description|required?| +|--------|-----------|---------| +|dataGranularity|specifies the granularity to expect the data at, e.g. hour means to expect directories `y=XXXX/m=XX/d=XX/H=XX`|yes| +|inputPath|Base path to append the expected time path to|yes| +|filePattern|Pattern that files should match to be included|yes| + +For example, if the sample config were run with the interval 2012-06-01/2012-06-02, it would expect data at the paths + + s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=00 + s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=01 + ... + s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=23 + +### Rollup specification + +The indexing process has the ability to roll data up as it processes the incoming data. If data has already been summarized, summarizing it again will produce the same results so either way is not a problem. This specifies how that rollup should take place. + +|property|description|required?| +|--------|-----------|---------| +|aggs|specifies a list of aggregators to aggregate for each bucket (a bucket is defined by the tuple of the truncated timestamp and the dimensions). Aggregators available here are the same as available when querying.|yes| +|rollupGranularity|The granularity to use when truncating incoming timestamps for bucketization|yes| + +### Partitioning specification + +Segments are always partitioned based on timestamp (according to the granularitySpec) and may be further partitioned in some other way. For example, data for a day may be split by the dimension “last\_name” into two segments: one with all values from A-M and one with all values from N-Z. + +To use this option, the indexer must be given a target partition size. It can then find a good set of partition ranges on its own. + +|property|description|required?| +|--------|-----------|---------| +|targetPartitionSize|target number of rows to include in a partition, should be a number that targets segments of 700MB\~1GB.|yes| +|partitionDimension|the dimension to partition on. Leave blank to select a dimension automatically.|no| +|assumeGrouped|assume input data has already been grouped on time and dimensions. This is faster, but can choose suboptimal partitions if the assumption is violated.|no| + +### Updater job spec + +This is a specification of the properties that tell the job how to update metadata such that the Druid cluster will see the output segments and load them. + +|property|description|required?| +|--------|-----------|---------| +|type|“db” is the only value available|yes| +|connectURI|a valid JDBC url to MySQL|yes| +|user|username for db|yes| +|password|password for db|yes| +|segmentTable|table to use in DB|yes| + +These properties should parrot what you have configured for your [Master](Master.html). diff --git a/docs/Booting-a-production-cluster.md b/docs/Booting-a-production-cluster.md new file mode 100644 index 00000000000..f7e5444ab8e --- /dev/null +++ b/docs/Booting-a-production-cluster.md @@ -0,0 +1,30 @@ +--- +layout: default +--- +# Booting a Single Node Cluster # + +[Loading Your Data](Loading-Your-Data.html) and [Querying Your Data](Querying-Your-Data.html) contain recipes to boot a small druid cluster on localhost. Here we will boot a small cluster on EC2. You can checkout the code, or download a tarball from [here](http://static.druid.io/artifacts/druid-services-0.5.51-SNAPSHOT-bin.tar.gz). + +The [ec2 run script](https://github.com/metamx/druid/blob/master/examples/bin/run_ec2.sh), run_ec2.sh, is located at 'examples/bin' if you have checked out the code, or at the root of the project if you've downloaded a tarball. The scripts rely on the [Amazon EC2 API Tools](http://aws.amazon.com/developertools/351), and you will need to set three environment variables: + +```bash +# Setup environment for ec2-api-tools +export EC2_HOME=/path/to/ec2-api-tools-1.6.7.4/ +export PATH=$PATH:$EC2_HOME/bin +export AWS_ACCESS_KEY= +export AWS_SECRET_KEY= +``` + +Then, booting an ec2 instance running one node of each type is as simple as running the script, run_ec2.sh :) + +# Apache Whirr # + +Apache Whirr is a set of libraries for launching cloud services. You can clone a version of Whirr that includes Druid as a service from git@github.com:rjurney/whirr.git: + +```bash +git clone git@github.com:rjurney/whirr.git +cd whirr +git checkout trunk +mvn clean install -Dmaven.test.failure.ignore=true -Dcheckstyle.skip +sp;bin/whirr launch-cluster --config recipes/druid.properties +``` \ No newline at end of file diff --git a/docs/Broker.md b/docs/Broker.md new file mode 100644 index 00000000000..fee33bedc83 --- /dev/null +++ b/docs/Broker.md @@ -0,0 +1,30 @@ +--- +layout: default +--- +Broker +====== + +The Broker is the node to route queries to if you want to run a distributed cluster. It understands the metadata published to ZooKeeper about what segments exist on what nodes and routes queries such that they hit the right nodes. This node also merges the result sets from all of the individual nodes together. + +Forwarding Queries +------------------ + +Most druid queries contain an interval object that indicates a span of time for which data is requested. Likewise, Druid [Segments](Segments.html) are partitioned to contain data for some interval of time and segments are distributed across a cluster. Consider a simple datasource with 7 segments where each segment contains data for a given day of the week. Any query issued to the datasource for more than one day of data will hit more than one segment. These segments will likely be distributed across multiple nodes, and hence, the query will likely hit multiple nodes. + +To determine which nodes to forward queries to, the Broker node first builds a view of the world from information in Zookeeper. Zookeeper maintains information about [Compute](Compute.html) and [Realtime](Realtime.html) nodes and the segments they are serving. For every datasource in Zookeeper, the Broker node builds a timeline of segments and the nodes that serve them. When queries are received for a specific datasource and interval, the Broker node performs a lookup into the timeline associated with the query datasource for the query interval and retrieves the nodes that contain data for the query. The Broker node then forwards down the query to the selected nodes. + +Caching +------- + +Broker nodes employ a distributed cache with a LRU cache invalidation strategy. The broker cache stores per segment results. The cache can be local to each broker node or shared across multiple nodes using an external distributed cache such as [memcached](http://memcached.org/). Each time a broker node receives a query, it first maps the query to a set of segments. A subset of these segment results may already exist in the cache and the results can be directly pulled from the cache. For any segment results that do not exist in the cache, the broker node will forward the query to the +compute nodes. Once the compute nodes return their results, the broker will store those results in the cache. Real-time segments are never cached and hence requests for real-time data will always be forwarded to real-time nodes. Real-time data is perpetually changing and caching the results would be unreliable. + +Running +------- + +Broker nodes can be run using the `com.metamx.druid.http.BrokerMain` class. + +Configuration +------------- + +See [Configuration](Configuration.html). diff --git a/docs/Build-from-source.md b/docs/Build-from-source.md new file mode 100644 index 00000000000..3f323259b80 --- /dev/null +++ b/docs/Build-from-source.md @@ -0,0 +1,24 @@ +--- +layout: default +--- +### Clone and Build from Source + +The other way to setup Druid is from source via git. To do so, run these commands: + +``` +git clone git@github.com:metamx/druid.git +cd druid +./build.sh +``` + +You should see a bunch of files: + +``` +DruidCorporateCLA.pdf README common examples indexer pom.xml server +DruidIndividualCLA.pdf build.sh doc group_by.body install publications services +LICENSE client eclipse_formatting.xml index-common merger realtime +``` + +You can find the example executables in the examples/bin directory: +* run_example_server.sh +* run_example_client.sh diff --git a/docs/Cluster-setup.md b/docs/Cluster-setup.md new file mode 100644 index 00000000000..b8281e99468 --- /dev/null +++ b/docs/Cluster-setup.md @@ -0,0 +1,114 @@ +--- +layout: default +--- +A Druid cluster consists of various node types that need to be set up depending on your use case. See our [Design](Design.html) docs for a description of the different node types. + +Setup Scripts +------------- + +One of our community members, [housejester](https://github.com/housejester/), contributed some scripts to help with setting up a cluster. Checkout the [github](https://github.com/housejester/druid-test-harness) and [wiki](https://github.com/housejester/druid-test-harness/wiki/Druid-Test-Harness). + +Minimum Physical Layout: Absolute Minimum +----------------------------------------- + +As a special case, the absolute minimum setup is one of the standalone examples for realtime ingestion and querying; see [Examples](Examples.html) that can easily run on one machine with one core and 1GB RAM. This layout can be set up to try some basic queries with Druid. + +Minimum Physical Layout: Experimental Testing with 4GB of RAM +------------------------------------------------------------- + +This layout can be used to load some data from deep storage onto a Druid compute node for the first time. A minimal physical layout for a 1 or 2 core machine with 4GB of RAM is: + +1. node1: [Master](Master.html) + metadata service + zookeeper + [Compute](Compute.html) +2. transient nodes: indexer + +This setup is only reasonable to prove that a configuration works. It would not be worthwhile to use this layout for performance measurement. + +Comfortable Physical Layout: Pilot Project with Multiple Machines +----------------------------------------------------------------- + +*The machine size “flavors” are using AWS/EC2 terminology for descriptive purposes only and is not meant to imply that AWS/EC2 is required or recommended. Another cloud provider or your own hardware can also work.* + +A minimal physical layout not constrained by cores that demonstrates parallel querying and realtime, using AWS-EC2 “small”/m1.small (one core, with 1.7GB of RAM) or larger, no realtime, is: + +1. node1: [Master](Master.html) (m1.small) +2. node2: metadata service (m1.small) +3. node3: zookeeper (m1.small) +4. node4: [Broker](Broker.html) (m1.small or m1.medium or m1.large) +5. node5: [Compute](Compute.html) (m1.small or m1.medium or m1.large) +6. node6: [Compute](Compute.html) (m1.small or m1.medium or m1.large) +7. node7: [Realtime](Realtime.html) (m1.small or m1.medium or m1.large) +8. transient nodes: indexer + +This layout naturally lends itself to adding more RAM and core to Compute nodes, and to adding many more Compute nodes. Depending on the actual load, the Master, metadata server, and Zookeeper might need to use larger machines. + +High Availability Physical Layout +--------------------------------- + +*The machine size “flavors” are using AWS/EC2 terminology for descriptive purposes only and is not meant to imply that AWS/EC2 is required or recommended. Another cloud provider or your own hardware can also work.* + +An HA layout allows full rolling restarts and heavy volume: + +1. node1: [Master](Master.html) (m1.small or m1.medium or m1.large) +2. node2: [Master](Master.html) (m1.small or m1.medium or m1.large) (backup) +3. node3: metadata service (c1.medium or m1.large) +4. node4: metadata service (c1.medium or m1.large) (backup) +5. node5: zookeeper (c1.medium) +6. node6: zookeeper (c1.medium) +7. node7: zookeeper (c1.medium) +8. node8: [Broker](Broker.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +9. node9: [Broker](Broker.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) (backup) +10. node10: [Compute](Compute.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +11. node11: [Compute](Compute.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +12. node12: [Realtime](Realtime.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +13. transient nodes: indexer + +Sizing for Cores and RAM +------------------------ + +The Compute and Broker nodes will use as many cores as are available, depending on usage, so it is best to keep these on dedicated machines. The upper limit of effectively utilized cores is not well characterized yet and would depend on types of queries, query load, and the schema. Compute daemons should have a heap a size of at least 1GB per core for normal usage, but could be squeezed into a smaller heap for testing. Since in-memory caching is essential for good performance, even more RAM is better. Broker nodes will use RAM for caching, so they do more than just route queries. + +The effective utilization of cores by Zookeeper, MySQL, and Master nodes is likely to be between 1 and 2 for each process/daemon, so these could potentially share a machine with lots of cores. These daemons work with heap a size between 500MB and 1GB. + +Storage +------- + +Indexed segments should be kept in a permanent store accessible by all nodes like AWS S3 or HDFS or equivalent. Currently Druid supports S3, but this will be extended soon. + +Local disk (“ephemeral” on AWS EC2) for caching is recommended over network mounted storage (example of mounted: AWS EBS, Elastic Block Store) in order to avoid network delays during times of heavy usage. If your data center is suitably provisioned for networked storage, perhaps with separate LAN/NICs just for storage, then mounted might work fine. + +Setup +----- + +Setting up a cluster is essentially just firing up all of the nodes you want with the proper [configuration](configuration.html). One thing to be aware of is that there are a few properties in the configuration that potentially need to be set individually for each process: + + + druid.server.type=historical|realtime + druid.host=someHostOrIPaddrWithPort + druid.port=8080 + + +`druid.server.type` should be set to “historical” for your compute nodes and realtime for the realtime nodes. The master will only assign segments to a “historical” node and the broker has some intelligence around its ability to cache results when talking to a realtime node. This does not need to be set for the master or the broker. + +`druid.host` should be set to the hostname and port that can be used to talk to the given server process. Basically, someone should be able to send a request to http://\${druid.host}/ and actually talk to the process. + +`druid.port` should be set to the port that the server should listen on. In the vast majority of cases, this port should be the same as what is on `druid.host`. + +Build/Run +--------- + +The simplest way to build and run from the repository is to run `mvn package` from the base directory and then take `druid-services/target/druid-services-*-selfcontained.jar` and push that around to your machines; the jar does not need to be expanded, and since it contains the main() methods for each kind of service, it is **not** invoked with java ~~jar. It can be run from a normal java command-line by just including it on the classpath and then giving it the main class that you want to run. For example one instance of the Compute node/service can be started like this: +\ + +java~~Duser.timezone=UTC ~~Dfile.encoding=UTF-8~~cp compute/:druid-services/target/druid-services~~\*~~selfcontained.jar com.metamx.druid.http.ComputeMain + + + +The following table shows the possible services and fully qualified class for main(). + +|service|main class| +|-------|----------| +|[ Realtime ]( Realtime .html)|com.metamx.druid.realtime.RealtimeMain| +|[ Master ]( Master .html)|com.metamx.druid.http.MasterMain| +|[ Broker ]( Broker .html)|com.metamx.druid.http.BrokerMain| +|[ Compute ]( Compute .html)|com.metamx.druid.http.ComputeMain| + diff --git a/docs/Compute.md b/docs/Compute.md new file mode 100644 index 00000000000..e7df17ebbd5 --- /dev/null +++ b/docs/Compute.md @@ -0,0 +1,40 @@ +--- +layout: default +--- +Compute +======= + +Compute nodes are the work horses of a cluster. They load up historical segments and expose them for querying. + +Loading and Serving Segments +---------------------------- + +Each compute node maintains a constant connection to Zookeeper and watches a configurable set of Zookeeper paths for new segment information. Compute nodes do not communicate directly with each other or with the master nodes but instead rely on Zookeeper for coordination. + +The [Master](Master.html) node is responsible for assigning new segments to compute nodes. Assignment is done by creating an ephemeral Zookeeper entry under a load queue path associated with a compute node. For more information on how the master assigns segments to compute nodes, please see [Master](Master.html). + +When a compute node notices a new load queue entry in its load queue path, it will first check a local disk directory (cache) for the information about segment. If no information about the segment exists in the cache, the compute node will download metadata about the new segment to serve from Zookeeper. This metadata includes specifications about where the segment is located in deep storage and about how to decompress and process the segment. For more information about segment metadata and Druid segments in general, please see [Segments](Segments.html). Once a compute node completes processing a segment, the segment is announced in Zookeeper under a served segments path associated with the node. At this point, the segment is available for querying. + +Loading and Serving Segments From Cache +--------------------------------------- + +Recall that when a compute node notices a new segment entry in its load queue path, the compute node first checks a configurable cache directory on its local disk to see if the segment had been previously downloaded. If a local cache entry already exists, the compute node will directly read the segment binary files from disk and load the segment. + +The segment cache is also leveraged when a compute node is first started. On startup, a compute node will search through its cache directory and immediately load and serve all segments that are found. This feature allows compute nodes to be queried as soon they come online. + +Querying Segments +----------------- + +Please see [Querying](Querying.html) for more information on querying compute nodes. + +For every query that a compute node services, it will log the query and report metrics on the time taken to run the query. + +Running +------- + +Compute nodes can be run using the `com.metamx.druid.http.ComputeMain` class. + +Configuration +------------- + +See [Configuration](Configuration.html). diff --git a/docs/Concepts-and-Terminology.md b/docs/Concepts-and-Terminology.md new file mode 100644 index 00000000000..925941dd8c5 --- /dev/null +++ b/docs/Concepts-and-Terminology.md @@ -0,0 +1,15 @@ +--- +layout: default +--- +Concepts and Terminology +======================== + +- **Aggregators:** A mechanism for combining records during realtime incremental indexing, Hadoop batch indexing, and in queries. +- **DataSource:** A table-like view of data; specified in a “specFile” and in a query. +- **Granularity:** The time interval corresponding to aggregation by time. + - The *indexGranularity* setting in a schema is used to aggregate input (ingest) records within an interval into a single output (internal) record. + - The *segmentGranularity* is the interval specifying how internal records are stored together in a single file. + +- **Segment:** A collection of (internal) records that are stored and processed together. +- **Shard:** A unit of partitioning data across machine. TODO: clarify; by time or other dimensions? +- **specFile** is specification for services in JSON format; see [Realtime](Realtime.html) and [Batch-ingestion](Batch-ingestion.html) diff --git a/docs/Configuration.md b/docs/Configuration.md new file mode 100644 index 00000000000..4042d02d825 --- /dev/null +++ b/docs/Configuration.md @@ -0,0 +1,162 @@ +--- +layout: default +--- +This describes the basic server configuration that is loaded by all the server processes; the same file is loaded by all. See also the json “specFile” descriptions in [Realtime](Realtime.html) and [Batch-ingestion](Batch-ingestion.html). + +JVM Configuration Best Practices +================================ + +There are three JVM parameters that we set on all of our processes: + +1. `-Duser.timezone=UTC` This sets the default timezone of the JVM to UTC. We always set this and do not test with other default timezones, so local timezones might work, but they also might uncover weird and interesting bugs +2. `-Dfile.encoding=UTF-8` This is similar to timezone, we test assuming UTF-8. Local encodings might work, but they also might result in weird and interesting bugs +3. `-Djava.io.tmpdir=` Various parts of the system that interact with the file system do it via temporary files, these files can get somewhat large. Many production systems are setup to have small (but fast) `/tmp` directories, these can be problematic with Druid so we recommend pointing the JVM’s tmp directory to something with a little more meat. + +Basic Service Configuration +=========================== + +Configuration of the various nodes is done via Java properties. These can either be provided as `-D` system properties on the java command line or they can be passed in via a file called `runtime.properties` that exists on the classpath. Note: as a future item, I’d like to consolidate all of the various configuration into a yaml/JSON based configuration files. + +The periodic time intervals (like “PT1M”) are [ISO8601 intervals](http://en.wikipedia.org/wiki/ISO_8601#Time_intervals) + +An example runtime.properties is as follows: + + + # S3 access + com.metamx.aws.accessKey= + com.metamx.aws.secretKey= + + # thread pool size for servicing queries + druid.client.http.connections=30 + + # JDBC connection string for metadata database + druid.database.connectURI= + druid.database.user=user + druid.database.password=password + # time between polling for metadata database + druid.database.poll.duration=PT1M + druid.database.segmentTable=prod_segments + + # Path on local FS for storage of segments; dir will be created if needed + druid.paths.indexCache=/tmp/druid/indexCache + # Path on local FS for storage of segment metadata; dir will be created if needed + druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache + + druid.request.logging.dir=/tmp/druid/log + + druid.server.maxSize=300000000000 + + # ZK quorum IPs + druid.zk.service.host= + # ZK path prefix for Druid-usage of zookeeper, Druid will create multiple paths underneath this znode + druid.zk.paths.base=/druid + # ZK path for discovery, the only path not to default to anything + druid.zk.paths.discoveryPath=/druid/discoveryPath + + # the host:port as advertised to clients + druid.host=someHostOrIPaddrWithPort + # the port on which to listen, this port should line up with the druid.host value + druid.port=8080 + + com.metamx.emitter.logging=true + com.metamx.emitter.logging.level=debug + + druid.processing.formatString=processing_%s + druid.processing.numThreads=3 + + + druid.computation.buffer.size=100000000 + + # S3 dest for realtime indexer + druid.pusher.s3.bucket= + druid.pusher.s3.baseKey= + + druid.bard.cache.sizeInBytes=40000000 + druid.master.merger.service=blah_blah + + +Configuration groupings +----------------------- + +### S3 Access + +These properties are for connecting with S3 and using it to pull down segments. In the future, we plan on being able to use other deep storage file systems as well, like HDFS. The file system is actually only accessed by the [Compute](Compute.html), [Realtime](Realtime.html) and [Indexing service](Indexing service.html) nodes. + +|Property|Description|Default| +|--------|-----------|-------| +|`com.metamx.aws.accessKey`|The access key to use to access S3.|none| +|`com.metamx.aws.secretKey`|The secret key to use to access S3.|none| +|`druid.pusher.s3.bucket`|The bucket to store segments, this is used by Realtime and the Indexing service.|none| +|`druid.pusher.s3.baseKey`|The base key to use when storing segments, this is used by Realtime and the Indexing service|none| + +### JDBC connection + +These properties specify the jdbc connection and other configuration around the “segments table” database. The only processes that connect to the DB with these properties are the [Master](Master.html) and [Indexing service](Indexing-service.html). This is tested on MySQL. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.database.connectURI`|The jdbc connection uri|none| +|`druid.database.user`|The username to connect with|none| +|`druid.database.password`|The password to connect with|none| +|`druid.database.poll.duration`|The duration between polls the Master does for updates to the set of active segments. Generally defines the amount of lag time it can take for the master to notice new segments|PT1M| +|`druid.database.segmentTable`|The table to use to look for segments.|none| +|`druid.database.ruleTable`|The table to use to look for segment load/drop rules.|none| +|`druid.database.configTable`|The table to use to look for configs.|none| + +### Master properties + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.master.period`|The run period for the master. The master’s operates by maintaining the current state of the world in memory and periodically looking at the set of segments available and segments being served to make decisions about whether any changes need to be made to the data topology. This property sets the delay between each of these runs|PT60S| +|`druid.master.removedSegmentLifetime`|When a node disappears, the master can provide a grace period for how long it waits before deciding that the node really isn’t going to come back and it really should declare that all segments from that node are no longer available. This sets that grace period in number of runs of the master.|1| +|`druid.master.startDelay`|The operation of the Master works on the assumption that it has an up-to-date view of the state of the world when it runs, the current ZK interaction code, however, is written in a way that doesn’t allow the Master to know for a fact that it’s done loading the current state of the world. This delay is a hack to give it enough time to believe that it has all the data|PT600S| + +### Zk properties + +See [ZooKeeper](ZooKeeper.html) for a description of these properties. + +### Service properties + +These are properties that define various service/HTTP server aspects + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.client.http.connections`|Size of connection pool for the Broker to connect to compute nodes. If there are more queries than this number that all need to speak to the same node, then they will queue up.|none| +|`druid.paths.indexCache`|Segments assigned to a compute node are first stored on the local file system and then served by the compute node. This path defines where that local cache resides. Directory will be created if needed|none| +|`druid.paths.segmentInfoCache`|Compute nodes keep track of the segments they are serving so that when the process is restarted they can reload the same segments without waiting for the master to reassign. This path defines where this metadata is kept. Directory will be created if needed|none| +|`druid.http.numThreads`|The number of HTTP worker threads.|10| +|`druid.http.maxIdleTimeMillis`|The amount of time a connection can remain idle before it is terminated|300000 (5 min)| +|`druid.request.logging.dir`|Compute, Realtime and Broker nodes maintain request logs of all of the requests they get (interacton is via POST, so normal request logs don’t generally capture information about the actual query), this specifies the directory to store the request logs in|none| +|`druid.host`|The host for the current node. This is used to advertise the current processes location as reachable from another node and should generally be specified such that `http://${druid.host}/` could actually talk to this process|none| +|`druid.port`|This is the port to actually listen on; unless port mapping is used, this will be the same port as is on `druid.host`|none| +|`druid.processing.formatString`|Realtime and Compute nodes use this format string to name their processing threads.|none| +|`druid.processing.numThreads`|The number of processing threads to have available for parallel processing of segments. Our rule of thumb is `num_cores - 1`, this means that even under heavy load there will still be one core available to do background tasks like talking with ZK and pulling down segments.|none| +|`druid.computation.buffer.size`|This specifies a buffer size for the storage of intermediate results. The computation engine in both the Compute and Realtime nodes will use a scratch buffer of this size to do all of their intermediate computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can require more passes depending on the query that is being executed.|1073741824 (1GB)| +|`druid.service`|The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services|none| +|`druid.bard.cache.sizeInBytes`|The Broker (called Bard internally) instance has the ability to store results of queries in an in-memory cache. This specifies the number of bytes to use for that cache|none| + +### Compute Properties + +These are properties that the compute nodes use + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.server.maxSize`|The maximum number of bytes worth of segment that the node wants assigned to it. This is not a limit that the compute nodes actually enforce, they just publish it to the master and trust the master to do the right thing|none| +|`druid.server.type`|Specifies the type of the node. This is published via ZK and depending on the value the node will be treated specially by the Master/Broker. Allowed values are “realtime” or “historical”. This is a configuration parameter because the plan is to allow for a more configurable cluster composition. At the current time, all realtime nodes should just be “realtime” and all compute nodes should just be “compute”|none| + +### Emitter Properties + +The Druid servers emit various metrics and alerts via something we call an [Emitter](Emitter.html). There are two emitter implementations included with the code, one that just logs to log4j and one that does POSTs of JSON events to a server. More information can be found on the [Emitter](Emitter.html) page. The properties for using the logging emitter are described below. + +|Property|Description|Default| +|--------|-----------|-------| +|`com.metamx.emitter.logging`|Set to “true” to use the logging emitter|none| +|`com.metamx.emitter.logging.level`|Sets the level to log at|debug| +|`com.metamx.emitter.logging.class`|Sets the class to log at|com.metamx.emiter.core.LoggingEmitter| + +### Realtime Properties + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.realtime.specFile`|The file with realtime specifications in it. See [Realtime](Realtime.html).|none| + diff --git a/docs/Contribute.md b/docs/Contribute.md new file mode 100644 index 00000000000..a853eb430d2 --- /dev/null +++ b/docs/Contribute.md @@ -0,0 +1,8 @@ +--- +layout: default +--- +If you are interested in contributing to the code, we accept [pull requests](https://help.github.com/articles/using-pull-requests). Note: we have only just completed decoupling our Metamarkets-specific code from the code base and we took some short-cuts in interface design to make it happen. So, there are a number of interfaces that exist right now which are likely to be in flux. If you are embedding Druid in your system, it will be safest for the time being to only extend/implement interfaces that this wiki describes, as those are intended as stable (unless otherwise mentioned). + +For issue tracking, we are using the github issue tracker. Please fill out an issue from the Issues tab on the github screen. + +We also have a [Libraries](Libraries.html) page that lists external libraries that people have created for working with Druid. diff --git a/docs/Deep-Storage.md b/docs/Deep-Storage.md new file mode 100644 index 00000000000..bd9a0ec8a66 --- /dev/null +++ b/docs/Deep-Storage.md @@ -0,0 +1,42 @@ +--- +layout: default +--- +Deep storage is where segments are stored. It is a storage mechanism that Druid does not provide. This deep storage infrastructure defines the level of durability of your data, as long as Druid nodes can see this storage infrastructure and get at the segments stored on it, you will not lose data no matter how many Druid nodes you lose. If segments disappear from this storage layer, then you will lose whatever data those segments represented. + +The currently supported types of deep storage follow. + +## S3-compatible + +S3-compatible deep storage is basically either S3 or something like riak-cs which exposes the same API as S3. This is the default deep storage implementation. + +S3 configuration parameters are + + com.metamx.aws.accessKey= + com.metamx.aws.secretKey= + druid.pusher.s3.bucket= + druid.pusher.s3.baseKey= + +## HDFS + +As of 0.4.0, HDFS can be used for storage of segments as well. + +In order to use hdfs for deep storage, you need to set the following configuration on your realtime nodes. + + druid.pusher.hdfs=true + druid.pusher.hdfs.storageDirectory= + +If you are using the Hadoop indexer, set your output directory to be a location on Hadoop and it will work + + +## Local Mount + +A local mount can be used for storage of segments as well. This allows you to use just your local file system or anything else that can be mount locally like NFS, Ceph, etc. + +In order to use a local mount for deep storage, you need to set the following configuration on your realtime nodes. + + druid.pusher.local=true + druid.pusher.local.storageDirectory= + +Note that you should generally set `druid.pusher.local.storageDirectory` to something different from `druid.paths.indexCache`. + +If you are using the Hadoop indexer in local mode, then just give it a local file as your output directory and it will work. \ No newline at end of file diff --git a/docs/Design.md b/docs/Design.md new file mode 100644 index 00000000000..25d69d95fe8 --- /dev/null +++ b/docs/Design.md @@ -0,0 +1,88 @@ +--- +layout: default +--- +For a comprehensive look at the architecture of Druid, read the [White Paper](http://static.druid.io/docs/druid.pdf). + +What is Druid? +============== + +Druid is a system built to allow fast (“real-time”) access to large sets of seldom-changing data. It was designed with the intent of being a service and maintaining 100% uptime in the face of code deployments, machine failures and other eventualities of a production system. It can be useful for back-office use cases as well, but design decisions were made explicitly targetting an always-up service. + +Druid currently allows for single-table queries in a similar manner to [Dremel](http://research.google.com/pubs/pub36632.html) and [PowerDrill](http://www.vldb.org/pvldb/vol5/p1436_alexanderhall_vldb2012.pdf). It adds to the mix + +1. columnar storage format for partially nested data structures +2. hierarchical query distribution with intermediate pruning +3. indexing for quick filtering +4. realtime ingestion (ingested data is immediately available for querying) +5. fault-tolerant distributed architecture that doesn’t lose data + +As far as a comparison of systems is concerned, Druid sits in between PowerDrill and Dremel on the spectrum of functionality. It implements almost everything Dremel offers (Dremel handles arbitrary nested data structures while Druid only allows for a single level of array-based nesting) and gets into some of the interesting data layout and compression methods from PowerDrill. + +Druid is a good fit for products that require real-time data ingestion of a single, large data stream. Especially if you are targetting no-downtime operation and are building your product on top of a time-oriented summarization of the incoming data stream. Druid is probably not the right solution if you care more about query flexibility and raw data access than query speed and no-downtime operation. When talking about query speed it is important to clarify what “fast” means, with Druid it is entirely within the realm of possibility (we have done it) to achieve queries that run in single-digit seconds across a 6TB data set. + +### Architecture + +Druid is architected as a grouping of systems each with a distinct role and together they form a working system. The name comes from the Druid class in many role-playing games: it is a shape-shifter, capable of taking many different forms to fulfill various different roles in a group. + +The node types that currently exist are: +\* **Compute** nodes are the workhorses that handle storage and querying on “historical” data (non-realtime) +\* **Realtime** nodes ingest data in real-time, they are in charge of listening to a stream of incoming data and making it available immediately inside the Druid system. As data they have ingested ages, they hand it off to the compute nodes. +\* **Master** nodes act as coordinators. They look over the grouping of computes and make sure that data is available, replicated and in a generally “optimal” configuration. +\* **Broker** nodes understand the topology of data across all of the other nodes in the cluster and re-write and route queries accordingly +\* **Indexer** nodes form a cluster of workers to load batch and real-time data into the system as well as allow for alterations to the data stored in the system (also known as the Indexing Service) + +This separation allows each node to only care about what it is best at. By separating Compute and Realtime, we separate the memory concerns of listening on a real-time stream of data and processing it for entry into the system. By separating the Master and Broker, we separate the needs for querying from the needs for maintaining “good” data distribution across the cluster. + +All nodes can be run in some highly available fashion. Either as symmetric peers in a share-nothing cluster or as hot-swap failover nodes. + +Aside from these nodes, there are 3 external dependencies to the system: + +1. A running [ZooKeeper](http://zookeeper.apache.org/) cluster for cluster service discovery and maintenance of current data topology +2. A MySQL instance for maintenance of metadata about the data segments that should be served by the system +3. A “deep storage” LOB store/file system to hold the stored segments + +### Data Storage + +Getting data into the Druid system requires an indexing process. This gives the system a chance to analyze the data, add indexing structures, compress and adjust the layout in an attempt to optimize query speed. A quick list of what happens to the data follows. + +- Converted to columnar format +- Indexed with bitmap indexes +- Compressed using various algorithms + - LZF (switching to Snappy is on the roadmap, not yet implemented) + - Dictionary encoding w/ id storage minimization + - Bitmap compression + - RLE (on the roadmap, but not yet implemented) + +The output of the indexing process is stored in a “deep storage” LOB store/file system ([Deep Storage](Deep Storage.html) for information about potential options). Data is then loaded by compute nodes by first downloading the data to their local disk and then memory mapping it before serving queries. + +If a compute node dies, it will no longer serve its segments, but given that the segments are still available on the “deep storage” any other node can simply download the segment and start serving it. This means that it is possible to actually remove all compute nodes from the cluster and then re-provision them without any data loss. It also means that if the “deep storage” is not available, the nodes can continue to serve the segments they have already pulled down (i.e. the cluster goes stale, not down). + +In order for a segment to exist inside of the cluster, an entry has to be added to a table in a MySQL instance. This entry is a self-describing bit of metadata about the segment, it includes things like the schema of the segment, the size, and the location on deep storage. These entries are what the Master uses to know what data **should** be available on the cluster. + +### Fault Tolerance + +- **Compute** As discussed above, if a compute node dies, another compute node can take its place and there is no fear of data loss +- **Master** Can be run in a hot fail-over configuration. If no masters are running, then changes to the data topology will stop happening (no new data and no data balancing decisions), but the system will continue to run. +- **Broker** Can be run in parallel or in hot fail-over. +- **Realtime** Depending on the semantics of the delivery stream, multiple of these can be run in parallel processing the exact same stream. They periodically checkpoint to disk and eventually push out to the Computes. Steps are taken to be able to recover from process death, but loss of access to the local disk can result in data loss if this is the only method of adding data to the system. +- **“deep storage” file system** If this is not available, new data will not be able to enter the cluster, but the cluster will continue operating as is. +- **MySQL** If this is not available, the master will be unable to find out about new segments in the system, but it will continue with its current view of the segments that should exist in the cluster. +- **ZooKeeper** If this is not available, data topology changes will not be able to be made, but the Brokers will maintain their most recent view of the data topology and continue serving requests accordingly. + +### Query processing + +A query first enters the Broker, where the broker will match the query with the data segments that are known to exist. It will then pick a set of machines that are serving those segments and rewrite the query for each server to specify the segment(s) targetted. The Compute/Realtime nodes will take in the query, process them and return results. The Broker then takes the results and merges them together to get the final answer, which it returns. In this way, the broker can prune all of the data that doesn’t match a query before ever even looking at a single row of data. + +For filters at a more granular level than what the Broker can prune based on, the indexing structures inside each segment allows the compute nodes to figure out which (if any) rows match the filter set before looking at any row of data. It can do all of the boolean algebra of the filter on the bitmap indices and never actually look directly at a row of data. + +Once it knows the rows that match the current query, it can access the columns it cares about for those rows directly without having to load data that it is just going to throw away. + +The following diagram shows the data flow for queries without showing batch indexing: + +![Simple Data Flow](https://raw.github.com/metamx/druid/master/doc/data_flow_simple.png "Simple Data Flow") + +### In-memory? + +Druid is not always and only in-memory. When we first built it, it is true that it was all in-memory all the time, but as time went on the price-performance tradeoff ended up swinging towards keeping all of our customers data in memory all the time a non-starter. We then added the ability to memory map data and allow the OS to handle paging data in and out of memory on demand. Our production cluster is primarily configured to operate with this memory mapping behavior and we are definitely over-subscribed in terms of memory available vs. data a node is serving. + +As you read some of the old blog posts or other literature about the project, you will see “in-memory” often touted as that is the history of where Druid came from, but the technical reality is that there is a spectrum of price vs. performance and being able to slide along it from all in-memory (high cost, great performance) to mostly on disk (low cost, low performance) is the important knob to be able to adjust. diff --git a/docs/Download.md b/docs/Download.md new file mode 100644 index 00000000000..1bf1352de58 --- /dev/null +++ b/docs/Download.md @@ -0,0 +1,14 @@ +--- +layout: default +--- +A version may be declared as a release candidate if it has been deployed to a sizable production cluster. Release candidates are declared as stable after we feel fairly confident there are no major bugs in the version. Check out the [Versioning](Versioning.html) section for how we describe software versions. + +Release Candidate +----------------- + +There is no release candidate at this time. + +Stable Release +-------------- + +The current stable is tagged at version [0.5.49](https://github.com/metamx/druid/tree/druid-0.5.49). diff --git a/docs/Druid-Personal-Demo-Cluster.md b/docs/Druid-Personal-Demo-Cluster.md new file mode 100644 index 00000000000..0ef9834f198 --- /dev/null +++ b/docs/Druid-Personal-Demo-Cluster.md @@ -0,0 +1,80 @@ +--- +layout: default +--- +# Druid Personal Demo Cluster (DPDC) + +Note, there are currently some issues with the CloudFormation. We are working through them and will update the documentation here when things work properly. In the meantime, the simplest way to get your feet wet with a cluster setup is to run through the instructions at [housejester/druid-test-harness](https://github.com/housejester/druid-test-harness), though it is based on an older version. If you just want to get a feel for the types of data and queries that you can issue, check out [Realtime Examples](Realtime-Examples.html) + +## Introduction +To make it easy for you to get started with Druid, we created an AWS (Amazon Web Services) [CloudFormation](http://aws.amazon.com/cloudformation/) Template that allows you to create a small pre-configured Druid cluster using your own AWS account. The cluster contains a pre-loaded sample workload, the Wikipedia edit stream, and a basic query interface that gets you familiar with Druid capabilities like drill-downs and filters. + + +This guide walks you through the steps to create the cluster and then how to create basic queries. (The cluster setup should take you about 15-20 minutes depending on AWS response times). + + +## What’s in this Druid Demo Cluster? + +1. A single "Master" node. This node co-locates the [Master](Master.html) process, the [Broker](Broker.html) process, Zookeeper, and the MySQL instance. You can read more about Druid architecture [Design](Design.html). + +1. Three compute nodes; these compute nodes, have been pre-configured to work with the Master node and should automatically load up the Wikipedia edit stream data (no specific setup is required). + +## Setup Instructions +1. Log in to your AWS account: Start by logging into the [Console page](https://console.aws.amazon.com) of your AWS account; if you don’t have one, follow this link to sign up for one [http://aws.amazon.com/](http://aws.amazon.com/). + +![AWS Console Page](images/demo/setup-01-console.png) + +1. If you have a [Key Pair](http://docs.aws.amazon.com/gettingstarted/latest/wah/getting-started-create-key-pair.html) already created you may skip this step. Note: this is required to create the demo cluster and is generally not used unless instances need to be accessed directly (e.g. via SSH). + + 1. Click **EC2** to go to the EC2 Dashboard. From there, click **Key Pairs** under Network & Security. + ![EC2 Dashboard](images/demo/setup-02a-keypair.png) + + 1. Click on the button **Create Key Pair**. A dialog box will appear prompting you to enter a Key Pair name (as long as you remember it, the name is arbitrary, for this example we entered `Druid`). Click **Create**. You will be prompted to download a .pam; store this file in a safe place. + ![Create Key Pair](images/demo/setup-02b-keypair.png) + +1. Unless you’re there already, go back to the Console page, or follow this link: https://console.aws.amazon.com. Click **CloudFormation** under Deployment & Management. +![CloudFormation](images/demo/setup-03-ec2.png) + +1. Click **Create New Stack**, which will bring up the **Create Stack** dialog. +![Create New Stack](images/demo/setup-04-newstack.png) + +1. Enter a **Stack Name** (it’s arbitrary, we chose, `DruidStack`). Click **Provide a Template URL** type in the following template URL: _**https://s3.amazonaws.com/cf-templates-jm2ikmzj3y6x-us-east-1/2013081cA9-Druid04012013.template**_. Press **Continue**, this will take you to the Create Stack dialog. +![Stack Name & URL](images/demo/setup-05-createstack.png) + +1. Enter `Druid` (or the Key Pair name you created in Step 2) in the **KeyPairName** field; click **Continue**. This should bring up another dialog prompting you to enter a **Key** and **Value**. +![Stack Parameters](images/demo/setup-06-parameters.png) + +1. While the inputs are arbitrary, it’s important to remember this information; we chose to enter `version` for **Key** and `1` for **Value**. Press **Continue** to bring up a confirmation dialog. +![Add Tags](images/demo/setup-07a-tags.png) + +1. Click **Continue** to start creating your Druid Demo environment (this will bring up another dialog box indicating your environment is being created; click **Close** to take you to a more detailed view of the Stack creation process). Note: depending on AWS, this step could take over 15 minutes – initialization continues even after the instances are created. (So yes, now would be a good time to grab that cup of coffee). +![Review](images/demo/setup-07b-review.png) +![Create Stack Complete](images/demo/setup-07c-complete.png) + +1. Click and expand the **Events** tab in the CloudFormation Stacks window to get a more detailed view of the Druid Demo Cluster setup. + +![CloudFormations](images/demo/setup-09-events.png) + +1. Get the IP address of your Druid Master Node: + 1. Go to the following URL: [https://console.aws.amazon.com/ec2](https://console.aws.amazon.com/ec2) + 1. Click **Instances** in the left pane – you should see something similar to the following figure. + 1. Select the **DruidMaster** instance + 1. Your IP address is right under the heading: **EC2 Instance: DruidMaster**. Select and copy that entire line, which ends with `amazonaws.com`. + +![EC2 Instances](images/demo/setup-10-ip.png) + +## Querying Data + +1. Use the following URL to bring up the Druid Demo Cluster query interface (replace **IPAddressDruidMaster** with the actual druid master IP Address): + +**`http://IPAddressDruidMaster:8082/druid/v3/demoServlet`** + +As you can see from the image below, there are default values in the Dimensions and Granularity fields. Clicking **Execute** will produce a basic query result. +![Demo Query Interface](images/demo/query-1.png) + +1. Note: when the Query is in running the **Execute** button will be disabled and read: **Fetching…** +![Demo Query](images/demo/query-2.png) + +1. You can add multiple Aggregation values, adjust Granularity, and Dimensions; query results will appear at the bottom of the window. + + +Enjoy! And for sure, please send along your comments and feedback or, aspirations on expanding and developing this demo. https://groups.google.com/d/forum/druid-development. Attention R users: we just open-sourced our R Druid connector: https://github.com/metamx/RDruid. diff --git a/docs/Druid-vs-Cassandra.md b/docs/Druid-vs-Cassandra.md new file mode 100644 index 00000000000..e191dde2af7 --- /dev/null +++ b/docs/Druid-vs-Cassandra.md @@ -0,0 +1,8 @@ +--- +layout: default +--- +We are not experts on Cassandra, if anything is incorrect about our portrayal, please let us know on the mailing list or via some other means. We will fix this page. + +Druid is highly optimized for scans and aggregations, it supports arbitrarily deep drill downs into data sets without the need to pre-compute, and it can ingest event streams in real-time and allow users to query events as they come in. Cassandra is a great key-value store and it has some features that allow you to use it to do more interesting things than what you can do with a pure key-value store. But, it is not built for the same use cases that Druid handles, namely regularly scanning over billions of entries per query. + +Furthermore, Druid is fully read-consistent. Druid breaks down a data set into immutable chunks known as segments. All replicants always present the exact same view for the piece of data they are holding and we don’t have to worry about data synchronization. The tradeoff is that Druid has limited semantics for write and update operations. Cassandra, similar to Amazon’s Dynamo, has an eventually consistent data model. Writes are always supported but updates to data may take some time before all replicas sync up (data reconciliation is done at read time). This model favors availability and scalability over consistency. \ No newline at end of file diff --git a/docs/Druid-vs-Hadoop.md b/docs/Druid-vs-Hadoop.md new file mode 100644 index 00000000000..37559b1da8f --- /dev/null +++ b/docs/Druid-vs-Hadoop.md @@ -0,0 +1,6 @@ +--- +layout: default +--- +Druid is a complementary addition to Hadoop. Hadoop is great at storing and making accessible large amounts of individually low-value data. Unfortunately, Hadoop is not great at providing query speed guarantees on top of that data, nor does it have very good operational characteristics for a customer-facing production system. Druid, on the other hand, excels at taking high-value summaries of the low-value data on Hadoop, making it available in a fast and always-on fashion, such that it could be exposed directly to a customer. + +Druid also requires some infrastructure to exist for “deep storage”. HDFS is one of the implemented options for this “deep storage”. diff --git a/docs/Druid-vs-Impala-or-Shark.md b/docs/Druid-vs-Impala-or-Shark.md new file mode 100644 index 00000000000..ee59b3def0c --- /dev/null +++ b/docs/Druid-vs-Impala-or-Shark.md @@ -0,0 +1,45 @@ +--- +layout: default +--- +The question of Druid versus Impala or Shark basically comes down to your product requirements and what the systems were designed to do. + +Druid was designed to + +1. be an always on service +1. ingest data in real-time +1. handle slice-n-dice style ad-hoc queries + +Impala and Shark's primary design concerns (as far as I am aware) were to replace Hadoop MapReduce with another, faster, query layer that is completely generic and plays well with the other ecosystem of Hadoop technologies. I will caveat this discussion with the statement that I am not an expert on Impala or Shark, nor am I intimately familiar with their roadmaps. If anything is incorrect on this page, I'd be happy to change it, please send a note to the mailing list. + +What does this mean? We can talk about it in terms of four general areas + +1. Fault Tolerance +1. Query Speed +1. Data Ingestion +1. Query Flexibility + +## Fault Tolerance + +Druid pulls segments down from [Deep Storage](Deep Storage.html) before serving queries on top of it. This means that for the data to exist in the Druid cluster, it must exist as a local copy on a historical node. If deep storage becomes unavailable for any reason, new segments will not be loaded into the system, but the cluster will continue to operate exactly as it was when the backing store disappeared. + +Impala and Shark, on the other hand, pull their data in from HDFS (or some other Hadoop FileSystem) in response to a query. This has implications for the operation of queries if you need to take HDFS down for a bit (say a software upgrade). It's possible that data that has been cached in the nodes is still available when the backing file system goes down, but I'm not sure. + +This is just one example, but Druid was built to continue operating in the face of failures of any one of its various pieces. The [Design](Design.html) describes these design decisions from the Druid side in more detail. + +## Query Speed + +Druid takes control of the data given to it, storing it in a column-oriented fashion, compressing it and adding indexing structures. All of which add to the speed at which queries can be processed. The column orientation means that we only look at the data that a query asks for in order to compute the answer. Compression increases the data storage capacity of RAM and allows us to fit more data into quickly accessible RAM. Indexing structures mean that as you add boolean filters to your queries, we do less processing and you get your result faster, whereas a lot of processing engines do *more* processing when filters are added. + +Impala/Shark can basically be thought of as daemon caching layers on top of HDFS. They are processes that stay on even if there is no query running (eliminating the JVM startup costs from Hadoop MapReduce) and they have facilities to cache data locally so that it can be accessed and updated quicker. But, I do not believe they go beyond caching capabilities to actually speed up queries. So, at the end of the day, they don't change the paradigm from a brute-force, scan-everything query processing paradigm. + +## Data Ingestion + +Druid is built to allow for real-time ingestion of data. You can ingest data and query it immediately upon ingestion, the latency between how quickly the event is reflected in the data is dominated by how long it takes to deliver the event to Druid. + +Impala/Shark, being based on data in HDFS or some other backing store, are limited in their data ingestion rates by the rate at which that backing store can make data available. Generally, the backing store is the biggest bottleneck for how quickly data can become available. + +## Query Flexibility + +Druid supports timeseries and groupBy style queries. It doesn't have support for joins, which makes it a lot less flexible for generic processing. + +Impala/Shark support SQL style queries with full joins. \ No newline at end of file diff --git a/docs/Druid-vs-redshift.md b/docs/Druid-vs-redshift.md new file mode 100644 index 00000000000..8469209b10b --- /dev/null +++ b/docs/Druid-vs-redshift.md @@ -0,0 +1,40 @@ +--- +layout: default +--- +###How does Druid compare to Redshift? + +In terms of drawing a differentiation, Redshift is essentially ParAccel (Actian) which Amazon is licensing. + +Aside from potential performance differences, there are some functional differences: + +###Real-time data ingestion + +Because Druid is optimized to provide insight against massive quantities of streaming data; it is able to load and aggregate data in real-time. + +Generally traditional data warehouses including column stores work only with batch ingestion and are not optimal for streaming data in regularly. + +###Druid is a read oriented analytical data store + +It’s write semantics aren’t as fluid and does not support joins. ParAccel is a full database with SQL support including joins and insert/update statements. + +###Data distribution model + +Druid’s data distribution, is segment based which exists on highly available “deep” storage, like S3 or HDFS. Scaling up (or down) does not require massive copy actions or downtime; in fact, losing any number of compute nodes does not result in data loss because new compute nodes can always be brought up by reading data from “deep” storage. + +To contrast, ParAccel’s data distribution model is hash-based. Expanding the cluster requires re-hashing the data across the nodes, making it difficult to perform without taking downtime. Amazon’s Redshift works around this issue with a multi-step process: + +* set cluster into read-only mode +* copy data from cluster to new cluster that exists in parallel +* redirect traffic to new cluster + +###Replication strategy + +Druid employs segment-level data distribution meaning that more nodes can be added and rebalanced without having to perform a staged swap. The replication strategy also makes all replicas available for querying. + +ParAccel’s hash-based distribution generally means that replication is conducted via hot spares. This puts a numerical limit on the number of nodes you can lose without losing data, and this replication strategy often does not allow the hot spare to help share query load. + +###Indexing strategy + +Along with column oriented structures, Druid uses indexing structures to speed up query execution when a filter is provided. Indexing structures do increase storage overhead (and make it more difficult to allow for mutation), but they can also significantly speed up queries. + +ParAccel does not appear to employ indexing strategies. \ No newline at end of file diff --git a/docs/Druid-vs-vertica.md b/docs/Druid-vs-vertica.md new file mode 100644 index 00000000000..535e5e06300 --- /dev/null +++ b/docs/Druid-vs-vertica.md @@ -0,0 +1,10 @@ +--- +layout: default +--- +How does Druid compare to Vertica? + +Vertica is similar to ParAccel/Redshift ([Druid-vs-Redshift](Druid-vs-Redshift.html)) described above in that it wasn’t built for real-time streaming data ingestion and it supports full SQL. + +The other big difference is that instead of employing indexing, Vertica tries to optimize processing by leveraging run-length encoding (RLE) and other compression techniques along with a “projection” system that creates materialized copies of the data in a different sort order (to maximize the effectiveness of RLE). + +We are unclear about how Vertica handles data distribution and replication, so we cannot speak to if/how Druid is different. diff --git a/docs/Examples.md b/docs/Examples.md new file mode 100644 index 00000000000..4207911464b --- /dev/null +++ b/docs/Examples.md @@ -0,0 +1,68 @@ +--- +layout: default +--- +Examples +======== + +The examples on this page are setup in order to give you a feel for what Druid does in practice. They are quick demos of Druid based on [RealtimeStandaloneMain](https://github.com/metamx/druid/blob/master/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java). While you wouldn’t run it this way in production you should be able to see how ingestion works and the kind of exploratory queries that are possible. Everything that can be done on your box here can be scaled out to 10’s of billions of events and terabytes of data per day in a production cluster while still giving the snappy responsive exploratory queries. + +Installing Standalone Druid +--------------------------- + +There are two options for installing standalone Druid. Building from source, and downloading the Druid Standalone Kit (DSK). + +### Building from source + +Clone Druid and build it: + + git clone https://github.com/metamx/druid.git druid + cd druid + git fetch --tags + git checkout druid-0.4.30 + ./build.sh + + +### Downloading the DSK (Druid Standalone Kit) + +[Download](http://static.druid.io/data/examples/druid-services-0.4.6.tar.gz) a stand-alone tarball and run it: + + + tar -xzf druid-services-0.X.X-SNAPSHOT-bin.tar.gz + cd druid-services-0.X.X-SNAPSHOT + + +Twitter Example +--------------- + +For a full tutorial based on the twitter example, check out this [Twitter Tutorial](Twitter-Tutorial.html). + +This Example uses a feature of Twitter that allows for sampling of it’s stream. We sample the Twitter stream via our [TwitterSpritzerFirehoseFactory](https://github.com/metamx/druid/blob/master/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java) class and use it to simulate the kinds of data you might ingest into Druid. Then, with the client part, the sample shows what kinds of analytics explorations you can do during and after the data is loaded. + +### What you’ll learn +\* See how large amounts of data gets ingested into Druid in real-time +\* Learn how to do fast, interactive, analytics queries on that real-time data + +### What you need +\* A build of standalone Druid with the Twitter example (see above) +\* A Twitter username and password. + +### What you’ll do + +See [Tutorial](Tutorial.html) + +Rand Example +------------ + +This uses `RandomFirehoseFactory` which emits a stream of random numbers (outColumn, a positive double) with timestamps along with an associated token (target). This provides a timeseries that requires no network access for demonstration, characterization, and testing. The generated tuples can be thought of as asynchronously produced triples (timestamp, outColumn, target) where the timestamp varies depending on speed of processing. + +In a terminal window, (NOTE: If you are using the cloned Github repository these scripts are in ./examples/bin) start the server with: + +`./run_example_server.sh` +`# type rand when prompted` + +In another terminal window: + +`./run_example_client.sh` +`# type rand when prompted` + +The result of the client query is in JSON format. The client makes a REST request using the program `curl` which is usually installed on Linux, Unix, and OSX by default. diff --git a/docs/Filters.md b/docs/Filters.md new file mode 100644 index 00000000000..41ae91f93e2 --- /dev/null +++ b/docs/Filters.md @@ -0,0 +1,91 @@ +--- +layout: default +--- +A filter is a JSON object indicating which rows of data should be included in the computation for a query. It’s essentially the equivalent of the WHERE clause in SQL. Druid supports the following types of filters. + +### Selector filter + +The simplest filter is a selector filter. The selector filter will match a specific dimension with a specific value. Selector filters can be used as the base filters for more complex Boolean expressions of filters. + +The grammar for a SELECTOR filter is as follows: + + "filter": { + "type": "selector", + "dimension": , + "value": + } + + +This is the equivalent of `WHERE = ''`. + +### Regular expression filter + +The regular expression filter is similar to the selector filter, but using regular expressions. It matches the specified dimension with the given pattern. The pattern can be any standard [Java regular expression](http://docs.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html). + + "filter": { + "type": "regex", + "dimension": , + "pattern": + } + + +### Logical expression filters + +#### AND + +The grammar for an AND filter is as follows: + + "filter": { + "type": "and", + "fields": [, , ...] + } + + +The filters in fields can be any other filter defined on this page. + +#### OR + +The grammar for an OR filter is as follows: + + "filter": { + "type": "or", + "fields": [, , ...] + } + + +The filters in fields can be any other filter defined on this page. + +#### NOT + +The grammar for a NOT filter is as follows: + + "filter": { + "type": "not", + "field": + } + + +The filter specified at field can be any other filter defined on this page. + +### JavaScript filter + +The JavaScript filter matches a dimension against the specified JavaScript function predicate. The filter matches values for which the function returns true. + +The function takes a single argument, the dimension value, and returns either true or false. + + { + "type" : "javascript", + "dimension" : , + "function" : "function(value) { <...> }" + } + + +**Example** +The following matches any dimension values for the dimension `name` between `'bar'` and `'foo'` + + { + "type" : "javascript", + "dimension" : "name", + "function" : "function(x) { return(x >= 'bar' && x <= 'foo') }" + } + diff --git a/docs/Firehose.md b/docs/Firehose.md new file mode 100644 index 00000000000..92c5caa2386 --- /dev/null +++ b/docs/Firehose.md @@ -0,0 +1,52 @@ +--- +layout: default +--- +Firehoses describe the data stream source. They are pluggable and thus the configuration schema can and will vary based on the `type` of the firehose. + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|type|String|Specifies the type of firehose. Each value will have its own configuration schema, firehoses packaged with Druid are described [here](https://github.com/metamx/druid/wiki/Firehose#available-firehoses)|yes| + +We describe the configuration of the Kafka firehose from the example below, but check [here](https://github.com/metamx/druid/wiki/Firehose#available-firehoses) for more information about the various firehoses that are available in Druid. + +- `consumerProps` is a map of properties for the Kafka consumer. The JSON object is converted into a Properties object and passed along to the Kafka consumer. +- `feed` is the feed that the Kafka consumer should read from. +- `parser` represents a parser that knows how to convert from String representations into the required `InputRow` representation that Druid uses. This is a potentially reusable piece that can be found in many of the firehoses that are based on text streams. The spec in the example describes a JSON feed (new-line delimited objects), with a timestamp column called “timestamp” in ISO8601 format and that it should not include the dimension “value” when processing. More information about the options available for the parser are available [here](https://github.com/metamx/druid/wiki/Firehose#parsing-data). + +Available Firehoses +------------------- + +There are several firehoses readily available in Druid, some are meant for examples, others can be used directly in a production environment. + +#### KafkaFirehose + +This firehose acts as a Kafka consumer and ingests data from Kafka. + +#### StaticS3Firehose + +This firehose ingests events from a predefined list of S3 objects. + +#### TwitterSpritzerFirehose + +See [Examples](Examples.html). This firehose connects directly to the twitter spritzer data stream. + +#### RandomFirehose + +See [Examples](Examples.html). This firehose creates a stream of random numbers. + +#### RabbitMqFirehouse + +This firehose ingests events from a define rabbit-mq queue. + +Parsing Data +------------ + +There are several ways to parse data. + +#### StringInputRowParser + +This parser converts Strings. + +#### MapInputRowParser + +This parser converts flat, key/value pair maps. diff --git a/docs/Granularities.md b/docs/Granularities.md new file mode 100644 index 00000000000..0cb25a7a5df --- /dev/null +++ b/docs/Granularities.md @@ -0,0 +1,49 @@ +--- +layout: default +--- +The granularity field determines how data gets bucketed across the time dimension, i.e how it gets aggregated by hour, day, minute, etc. + +It can be specified either as a string for simple granularities or as an object for arbitrary granularities. + +### Simple Granularities + +Simple granularities are specified as a string and bucket timestamps by their UTC time (i.e. days start at 00:00 UTC). + +Supported granularity strings are: `all`, `none`, `minute`, `fifteen_minute`, `thirty_minute`, `hour` and `day` +\* **`all`** buckets everything into a single bucket +\* **`none`** does not bucket data (it actually uses the granularity of the index - minimum here is `none` which means millisecond granularity). Using `none` in a [timeseries query|TimeSeriesQuery](timeseries query|TimeSeriesQuery.html) is currently not recommended (the system will try to generate 0 values for all milliseconds that didn’t exist, which is often a lot). + +### Duration Granularities + +Duration granularities are specified as an exact duration in milliseconds and timestamps are returned as UTC. + +They also support specifying an optional origin, which defines where to start counting time buckets from (defaults to 1970-01-01T00:00:00Z). + + {"type": "duration", "duration": "7200000"} + +This chunks up every 2 hours. + + {"type": "duration", "duration": "3600000", "origin": "2012-01-01T00:30:00Z"} + +This chunks up every hour on the half-hour. + +### Period Granularities + +Period granularities are specified as arbitrary period combinations of years, months, weeks, hours, minutes and seconds (e.g. P2W, P3M, PT1H30M, PT0.750S) in ISO8601 format. + +They support specifying a time zone which determines where period boundaries start and also determines the timezone of the returned timestamps. + +By default years start on the first of January, months start on the first of the month and weeks start on Mondays unless an origin is specified. + +Time zone is optional (defaults to UTC) +Origin is optional (defaults to 1970-01-01T00:00:00 in the given time zone) + + {"type": "period", "period": "P2D", "timeZone": "America/Los_Angeles"} + +This will bucket by two day chunks in the Pacific timezone. + + {"type": "period", "period": "P3M", "timeZone": "America/Los_Angeles", "origin": "2012-02-01T00:00:00-08:00"} + +This will bucket by 3 month chunks in the Pacific timezone where the three-month quarters are defined as starting from February. + +Supported time zones: timezone support is provided by the [Joda Time library](http://www.joda.org), which uses the standard IANA time zones. [Joda Time supported timezones](http://joda-time.sourceforge.net/timezones.html) diff --git a/docs/GroupByQuery.md b/docs/GroupByQuery.md new file mode 100644 index 00000000000..01edc6bdc7e --- /dev/null +++ b/docs/GroupByQuery.md @@ -0,0 +1,134 @@ +--- +layout: default +--- +These types of queries take a groupBy query object and return an array of JSON objects where each object represents a grouping asked for by the query. + +An example groupBy query object is shown below: + +

+
+{
+ [queryType]() “groupBy”,
+ [dataSource]() “sample\_datasource”,
+ [granularity]() “day”,
+ [dimensions]() [“dim1”, “dim2”],
+ [limitSpec]() {
+ [type]() “default”,
+ [limit]() 5000,
+ [columns]() [“dim1”, “metric1”]
+ },
+ [filter]() {
+ [type]() “and”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension1”,
+ [value]() “sample\_value1”
+ },
+ {
+ [type]() “or”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension2”,
+ [value]() “sample\_value2”
+ },
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension3”,
+ [value]() “sample\_value3”
+ }
+ ]
+ }
+ ]
+ },
+ [aggregations]() [
+ {
+ [type]() “longSum”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “doubleSum”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ],
+ [postAggregations]() [
+ {
+ [type]() “arithmetic”,
+ [name]() “sample\_divide”,
+ [fn]() “/”,
+ [fields]() [
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ]
+ }
+ ],
+ [intervals]() [
+ “2012-01-01T00:00:00.000/2012-01-03T00:00:00.000”
+ ],
+ [having]() {
+ [type]() “greaterThan”,
+ [aggregation]() “sample\_name1”,
+ [value]() 0
+ }
+}
+
+
+ + +There are 9 main parts to a groupBy query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “groupBy”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|dimensions|A JSON list of dimensions to do the groupBy over|yes| +|orderBy|See [OrderBy](OrderBy.html).|no| +|having|See [Having](Having.html).|no| +|granularity|Defines the granularity of the query. See [Granularities](Granularities.html)|yes| +|filter|See [Filters](Filters.html)|no| +|aggregations|See [Aggregations](Aggregations.html)|yes| +|postAggregations|See [Post Aggregations](Post-Aggregations.html)|no| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +To pull it all together, the above query would return *n\*m* data points, up to a maximum of 5000 points, where n is the cardinality of the “dim1” dimension, m is the cardinality of the “dim2” dimension, each day between 2012-01-01 and 2012-01-03, from the “sample\_datasource” table. Each data point contains the (long) sum of sample\_fieldName1 if the value of the data point is greater than 0, the (double) sum of sample\_fieldName2 and the (double) the result of sample\_fieldName1 divided by sample\_fieldName2 for the filter set for a particular grouping of “dim1” and “dim2”. The output looks like this: + +
+
+[ {
+ “version” : “v1”,
+ “timestamp” : “2012-01-01T00:00:00.000Z”,
+ “event” : {
+ “dim1” : ,
+ “dim2” : ,
+ “sample\_name1” : ,
+ “sample\_name2” :,
+ “sample\_divide” : 
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-01-01T00:00:00.000Z”,
+ “event” : {
+ “dim1” : ,
+ “dim2” : ,
+ “sample\_name1” : ,
+ “sample\_name2” :,
+ “sample\_divide” : 
+ }
+},
+…
+]
+
+
+ diff --git a/docs/Having.md b/docs/Having.md new file mode 100644 index 00000000000..62ab4644451 --- /dev/null +++ b/docs/Having.md @@ -0,0 +1,93 @@ +--- +layout: default +--- +A having clause is a JSON object identifying which rows from a groupBy query should be returned, by specifying conditions on aggregated values. + +It is essentially the equivalent of the HAVING clause in SQL. + +Druid supports the following types of having clauses. + +### Numeric filters + +The simplest having clause is a numeric filter. +Numeric filters can be used as the base filters for more complex boolean expressions of filters. + +#### Equal To + +The equalTo filter will match rows with a specific aggregate value. +The grammar for an `equalTo` filter is as follows: + + "having": { + "type": "equalTo", + "aggregation": , + "value": + } + + +This is the equivalent of `HAVING = `. + +#### Greater Than + +The greaterThan filter will match rows with aggregate values greater than the given value. +The grammar for a `greaterThan` filter is as follows: + + "having": { + "type": "greaterThan", + "aggregation": , + "value": + } + + +This is the equivalent of `HAVING > `. + +#### Less Than + +The lessThan filter will match rows with aggregate values less than the specified value. +The grammar for a `greaterThan` filter is as follows: + + "having": { + "type": "lessThan", + "aggregation": , + "value": + } + + +This is the equivalent of `HAVING < `. + +### Logical expression filters + +#### AND + +The grammar for an AND filter is as follows: + + "having": { + "type": "and", + "havingSpecs": [, , ...] + } + + +The having clauses in `havingSpecs` can be any other having clause defined on this page. + +#### OR + +The grammar for an OR filter is as follows: + + "having": { + "type": "or", + "havingSpecs": [, , ...] + } + + +The having clauses in `havingSpecs` can be any other having clause defined on this page. + +#### NOT + +The grammar for a NOT filter is as follows: + + "having": { + "type": "not", + "havingSpec": + } + + +The having clause specified at `havingSpec` can be any other having clause defined on this page. diff --git a/docs/Home.md b/docs/Home.md new file mode 100644 index 00000000000..8587aae4749 --- /dev/null +++ b/docs/Home.md @@ -0,0 +1,53 @@ +--- +layout: default +--- +Druid is an open-source analytics datastore designed for realtime, exploratory, queries on large-scale data sets (100’s of Billions entries, 100’s TB data). Druid provides for cost effective, always-on, realtime data ingestion and arbitrary data exploration. + +- Check out some [Examples](Examples.html) +- Try out Druid with our Getting Started [Tutorial](https://github.com/metamx/druid/wiki/Tutorial%3A-A-First-Look-at-Druid) +- Learn more by reading the [White Paper](http://static.druid.io/docs/druid.pdf) + +Why Druid? +---------- + +Druid was originally created to resolve query latency issues seen with trying to use Hadoop to power an interactive service. Hadoop has shown the world that it’s possible to house your data warehouse on commodity hardware for a fraction of the price of typical solutions. As people adopt Hadoop for their data warehousing needs, they find two things. + +1. They can now query all of their data in a fairly flexible manner and answer any question they have +2. The queries take a long time + +The first one is the joy that everyone feels the first time they get Hadoop running. The latter is what they realize after they have used Hadoop interactively for a while because Hadoop is optimized for throughput, not latency. Druid is a system that you can set up in your organization next to Hadoop. It provides the ability to access your data in an interactive slice-and-dice fashion. It trades off some query flexibility and takes over the storage format in order to provide the speed. + +Druid is especially useful if you are summarizing your data sets and then querying the summarizations. If you put your summarizations into Druid, you will get quick queryability out of a system that you can be confident will scale up as your data volumes increase. Deployments have scaled up to 2TB of data per hour at peak ingested and aggregated in real-time. + +We have more details about the general design of the system and why you might want to use it in our [White Paper](http://static.druid.io/docs/druid.pdf) or in our [Design](Design.html) doc. + +The data store world is vast, confusing and constantly in flux. This page is meant to help potential evaluators decide whether Druid is a good fit for the problem one needs to solve. If anything about it is incorrect please provide that feedback on the mailing list or via some other means, we will fix this page. + +#### When Druid? +\* You need to do interactive, fast, exploration of large amounts of data +\* You need analytics (not key value store) +\* You have a lot of data (10s of Billions of events added per day, 10s of TB of data added per day) +\* You want to do your analysis on data as it’s happening (realtime) +\* Your store needs to be always-on, 24x7x365 and years into the future. + +#### Not Druid? +\* The amount of data you have can easily be handled by MySql +\* Your querying for individual entries or doing lookups (Not Analytics) +\* Batch is good enough +\* Canned queries is good enough +\* Downtime is no big deal + +#### Druid vs… +\* [Druid-vs-Impala-or-Shark](Druid-vs-Impala-or-Shark.html) +\* [Druid-vs-Redshift](Druid-vs-Redshift.html) +\* [Druid-vs-Vertica](Druid-vs-Vertica.html) +\* [Druid-vs-Cassandra](Druid-vs-Cassandra.html) +\* [Druid-vs-Hadoop](Druid-vs-Hadoop.html) + +Key Features +------------ + +- **Designed for Analytics** - Druid is built for exploratory analytics for OLAP workflows (streamalytics). It supports a variety of filters, aggregators and query types and provides a framework for plugging in new functionality. Users have leveraged Druid’s infrastructure to develop features such as top K queries and histograms. +- **Interactive Queries** - Druid’s low latency data ingestion architecture allows events to be queried milliseconds after they are created. Druid’s query latency is optimized by only reading and scanning exactly what is needed. Aggregate and filter on data without sitting around waiting for results. +- **Highly Available** - Druid is used to back SaaS implementations that need to be up all the time. Your data is still available and queryable during system updates. Scale up or down without data loss. +- **Scalable** - Existing Druid deployments handle billions of events and terabytes of data per day. Druid is designed to be petabyte scale. diff --git a/docs/Indexing-Service.md b/docs/Indexing-Service.md new file mode 100644 index 00000000000..d878e1b4176 --- /dev/null +++ b/docs/Indexing-Service.md @@ -0,0 +1,193 @@ +--- +layout: default +--- +Disclaimer: We are still in the process of finalizing the indexing service and these configs are prone to change at any time. We will announce when we feel the indexing service and the configurations described are stable. + +The indexing service is a distributed task/job queue. It accepts requests in the form of [Tasks](Tasks.html) and executes those tasks across a set of worker nodes. Worker capacity can be automatically adjusted based on the number of tasks pending in the system. The indexing service is highly available, has built in retry logic, and can backup per task logs in deep storage. + +The indexing service is composed of two main components, a coordinator node that manages task distribution and worker capacity, and worker nodes that execute tasks in separate JVMs. + +Preamble +-------- + +The truth is, the indexing service is an experience that is difficult to characterize with words. When they asked me to write this preamble, I was taken aback. I wasn’t quite sure what exactly to write or how to describe this… entity. I accepted the job, as much for the challenge and inner growth as the money, and took to the mountains for reflection. Six months later, I knew I had it, I was done and had achieved the next euphoric victory in the continuous struggle that plagues my life. But, enough about me. This is about the indexing service. + +The indexing service is philosophical transcendence, an infallible truth that will shape your soul, mold your character, and define your reality. The indexing service is creating world peace, playing with puppies, unwrapping presents on Christmas morning, cradling a loved one, and beating Goro in Mortal Kombat for the first time. The indexing service is sustainable economic growth, global propensity, and a world of transparent financial transactions. The indexing service is a true belieber. The indexing service is panicking because you forgot you signed up for a course and the big exam is in a few minutes, only to wake up and realize it was all a dream. What is the indexing service? More like what isn’t the indexing service. The indexing service is here and it is ready, but are you? + +Indexer Coordinator Node +------------------------ + +The indexer coordinator node exposes HTTP endpoints where tasks can be submitted by posting a JSON blob to specific endpoints. It can be started by launching IndexerCoordinatorMain.java. The indexer coordinator node can operate in local mode or remote mode. In local mode, the coordinator and worker run on the same host and port. In remote mode, worker processes run on separate hosts and ports. + +Tasks can be submitted via POST requests to: + + http://:/druid/indexer/v1/task + +Tasks can cancelled via POST requests to: + + http://:/druid/indexer/v1/task/{taskId}/shutdown + +Issuing the cancel request once sends a graceful shutdown request. Graceful shutdowns may not stop a task right away, but instead issue a safe stop command at a point deemed least impactful to the system. Issuing the cancel request twice in succession will kill –9 the task. + +Task statuses can be retrieved via GET requests to: + + http://:/druid/indexer/v1/task/{taskId}/status + +Task segments can be retrieved via GET requests to: + + http://:/druid/indexer/v1/task/{taskId}/segments + +When a task is submitted, the coordinator creates a lock over the data source and interval of the task. The coordinator also stores the task in a MySQL database table. The database table is read at startup time to bootstrap any tasks that may have been submitted to the coordinator but may not yet have been executed. + +The coordinator also exposes a simple UI to show what tasks are currently running on what nodes at + + http://:/static/console.html + +#### Task Execution + +The coordinator retrieves worker setup metadata from the Druid [MySQL](MySQL.html) config table. This metadata contains information about the version of workers to create, the maximum and minimum number of workers in the cluster at one time, and additional information required to automatically create workers. + +Tasks are assigned to workers by creating entries under specific /tasks paths associated with a worker, similar to how the Druid master node assigns segments to compute nodes. See [Worker Configuration](Indexing-Service#configuration-1). Once a worker picks up a task, it deletes the task entry and announces a task status under a /status path associated with the worker. Tasks are submitted to a worker until the worker hits capacity. If all workers in a cluster are at capacity, the indexer coordinator node automatically creates new worker resources. + +#### Autoscaling + +The Autoscaling mechanisms currently in place are tightly coupled with our deployment infrastructure but the framework should be in place for other implementations. We are highly open to new implementations or extensions of the existing mechanisms. In our own deployments, worker nodes are Amazon AWS EC2 nodes and they are provisioned to register themselves in a [galaxy](https://github.com/ning/galaxy) environment. + +The Coordinator node controls the number of workers in the cluster according to a worker setup spec that is submitted via a POST request to the indexer at: + + http://:/druid/indexer/v1/worker/setup + +A sample worker setup spec is shown below: + + { + "minVersion":"some_version", + "minNumWorkers":"0", + "maxNumWorkers":"10", + "nodeData": { + "type":"ec2", + "amiId":"ami-someId", + "instanceType":"m1.xlarge", + "minInstances":"1", + "maxInstances":"1", + "securityGroupIds":["securityGroupIds"], + "keyName":"keyName" + }, + "userData":{ + "classType":"galaxy", + "env":"druid", + "version":"druid_version", + "type":"sample_cluster/worker" + } + } + + +Issuing a GET request at the same URL will return the current worker setup spec that is currently in place. The worker setup spec list above is just a sample and it is possible to write worker setup specs for other deployment environments. A description of the worker setup spec is shown below. + +|Property|Description|Default| +|--------|-----------|-------| +|`minVersion`|The coordinator only assigns tasks to workers with a version greater than the minVersion. If this is not specified, the minVersion will be the same as the coordinator version.|none| +|`minNumWorkers`|The minimum number of workers that can be in the cluster at any given time.|0| +|`maxNumWorkers`|The maximum number of workers that can be in the cluster at any given time.|0| +|`nodeData`|A JSON object that contains metadata about new nodes to create.|none| +|`userData`|A JSON object that contains metadata about how the node should register itself on startup. This data is sent with node creation requests.|none| + +For more information about configuring Auto-scaling, see [Auto-Scaling Configuration](https://github.com/metamx/druid/wiki/Indexing-Service#auto-scaling-configuration). + +#### Running + +Indexer Coordinator nodes can be run using the `com.metamx.druid.indexing.coordinator.http.IndexerCoordinatorMain` class. + +#### Configuration + +Indexer Coordinator nodes require [basic service configuration](https://github.com/metamx/druid/wiki/Configuration#basic-service-configuration). In addition, there are several extra configurations that are required. + + -Ddruid.zk.paths.indexer.announcementsPath=/druid/indexer/announcements + -Ddruid.zk.paths.indexer.leaderLatchPath=/druid/indexer/leaderLatchPath + -Ddruid.zk.paths.indexer.statusPath=/druid/indexer/status + -Ddruid.zk.paths.indexer.tasksPath=/druid/demo/indexer/tasks + + -Ddruid.indexer.runner=remote + -Ddruid.indexer.taskDir=/mnt/persistent/task/ + -Ddruid.indexer.configTable=sample_config + -Ddruid.indexer.workerSetupConfigName=worker_setup + -Ddruid.indexer.strategy=ec2 + -Ddruid.indexer.hadoopWorkingPath=/tmp/druid-indexing + -Ddruid.indexer.logs.s3bucket=some_bucket + -Ddruid.indexer.logs.s3prefix=some_prefix + +The indexing service requires some additional Zookeeper configs. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.zk.paths.indexer.announcementsPath`|The base path where workers announce themselves.|none| +|`druid.zk.paths.indexer.leaderLatchPath`|The base that coordinator nodes use to determine a leader.|none| +|`druid.zk.paths.indexer.statusPath`|The base path where workers announce task statuses.|none| +|`druid.zk.paths.indexer.tasksPath`|The base path where the coordinator assigns new tasks.|none| + +There’s several additional configs that are required to run tasks. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.indexer.runner`|Indicates whether tasks should be run locally or in a distributed environment. “local” or “remote”.|local| +|`druid.indexer.taskDir`|Intermediate temporary directory that tasks may use.|none| +|`druid.indexer.configTable`|The MySQL config table where misc configs live.|none| +|`druid.indexer.strategy`|The autoscaling strategy to use.|noop| +|`druid.indexer.hadoopWorkingPath`|Intermediate temporary hadoop working directory that certain index tasks may use.|none| +|`druid.indexer.logs.s3bucket`|S3 bucket to store logs.|none| +|`druid.indexer.logs.s3prefix`|S3 key prefix to store logs.|none| + +#### Console + +The indexer console can be used to view pending tasks, running tasks, available workers, and recent worker creation and termination. The console can be accessed at: + + http://:8080/static/console.html + +Worker Node +----------- + +The worker node executes submitted tasks. Workers run tasks in separate JVMs. + +#### Running + +Worker nodes can be run using the `com.metamx.druid.indexing.worker.http.WorkerMain` class. Worker nodes can automatically be created by the Indexer Coordinator as part of autoscaling. + +#### Configuration + +Worker nodes require [basic service configuration](https://github.com/metamx/druid/wiki/Configuration#basic-service-configuration). In addition, there are several extra configurations that are required. + + -Ddruid.worker.version=0 + -Ddruid.worker.capacity=3 + + -Ddruid.indexer.threads=3 + -Ddruid.indexer.taskDir=/mnt/persistent/task/ + -Ddruid.indexer.hadoopWorkingPath=/tmp/druid-indexing + + -Ddruid.worker.masterService=druid:sample_cluster:indexer + + -Ddruid.indexer.fork.hostpattern=:%d + -Ddruid.indexer.fork.startport=8080 + -Ddruid.indexer.fork.main=com.metamx.druid.indexing.worker.executor.ExecutorMain + -Ddruid.indexer.fork.opts="-server -Xmx1g -Xms1g -XX:NewSize=256m -XX:MaxNewSize=256m" + -Ddruid.indexer.fork.property.druid.service=druid/sample_cluster/executor + + # These configs are the same configs you would set for basic service configuration, just with a different prefix + -Ddruid.indexer.fork.property.druid.monitoring.monitorSystem=false + -Ddruid.indexer.fork.property.druid.computation.buffer.size=268435456 + -Ddruid.indexer.fork.property.druid.indexer.taskDir=/mnt/persistent/task/ + -Ddruid.indexer.fork.property.druid.processing.formatString=processing-%s + -Ddruid.indexer.fork.property.druid.processing.numThreads=1 + -Ddruid.indexer.fork.property.druid.server.maxSize=0 + -Ddruid.indexer.fork.property.druid.request.logging.dir=request_logs/ + +Many of the configurations for workers are similar to those for basic service configuration":https://github.com/metamx/druid/wiki/Configuration\#basic-service-configuration, but with a different config prefix. Below we describe the unique worker configs. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.worker.version`|Version identifier for the worker.|0| +|`druid.worker.capacity`|Maximum number of tasks the worker can accept.|1| +|`druid.indexer.threads`|Number of processing threads per worker.|1| +|`druid.worker.masterService`|Name of the indexer coordinator used for service discovery.|none| +|`druid.indexer.fork.hostpattern`|The format of the host name.|none| +|`druid.indexer.fork.startport`|Port in which child JVM starts from.|none| +|`druid.indexer.fork.opts`|JVM options for child JVMs.|none| + diff --git a/docs/Libraries.md b/docs/Libraries.md new file mode 100644 index 00000000000..0c57ffab3e8 --- /dev/null +++ b/docs/Libraries.md @@ -0,0 +1,23 @@ +--- +layout: default +--- +### R + +- [RDruid](https://github.com/metamx/RDruid) - Druid connector for R + +Community Libraries +------------------- + +Some great folks have written their own libraries to interact with Druid + +#### Ruby +\* [madvertise/ruby-druid](https://github.com/madvertise/ruby-druid) - A ruby client for Druid + +#### Python +\* [metamx/pydruid](https://github.com/metamx/pydruid) - A python client for Druid + +#### Helper Libraries + +- [madvertise/druid-dumbo](https://github.com/madvertise/druid-dumbo) - Scripts to help generate batch configs for the ingestion of data into Druid + +- [housejester/druid-test-harness](https://github.com/housejester/druid-test-harness) - A set of scripts to simplify standing up some servers and seeing how things work diff --git a/docs/Loading-Your-Data.md b/docs/Loading-Your-Data.md new file mode 100644 index 00000000000..2e27fad8303 --- /dev/null +++ b/docs/Loading-Your-Data.md @@ -0,0 +1,370 @@ +--- +layout: default +--- +Once you have a realtime node working, it is time to load your own data to see how Druid performs. + +Druid can ingest data in three ways: via Kafka and a realtime node, via the indexing service, and via the Hadoop batch loader. Data is ingested in realtime using a [Firehose](Firehose.html). + +## Create Config Directories ## +Each type of node needs its own config file and directory, so create them as subdirectories under the druid directory. +```bash +mkdir config +mkdir config/realtime +mkdir config/master +mkdir config/compute +mkdir config/broker +``` + +## Loading Data with Kafka ## + +[KafkaFirehoseFactory](https://github.com/metamx/druid/blob/master/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java) is how druid communicates with Kafka. Using this [Firehose](Firehose.html) with the right configuration, we can import data into Druid in realtime without writing any code. To load data to a realtime node via Kafka, we'll first need to initialize Zookeeper and Kafka, and then configure and initialize a [Realtime](Realtime.html) node. + +### Booting Kafka ### + +Instructions for booting a Zookeeper and then Kafka cluster are available [here](http://kafka.apache.org/07/quickstart.html). + +1. Download Apache Kafka 0.7.2 from [http://kafka.apache.org/downloads.html](http://kafka.apache.org/downloads.html) +```bash +wget http://apache.spinellicreations.com/incubator/kafka/kafka-0.7.2-incubating/kafka-0.7.2-incubating-src.tgz +tar -xvzf kafka-0.7.2-incubating-src.tgz +cd kafka-0.7.2-incubating-src +``` +2. Build Kafka +```bash +./sbt update +./sbt package +``` +3. Boot Kafka +```bash +cat config/zookeeper.properties +bin/zookeeper-server-start.sh config/zookeeper.properties +# in a new console +bin/kafka-server-start.sh config/server.properties +``` +4. Launch the console producer (so you can type in JSON kafka messages in a bit) +```bash +bin/kafka-console-producer.sh --zookeeper localhost:2181 --topic druidtest +``` +### Launching a Realtime Node + +1. Create a valid configuration file similar to this called config/realtime/runtime.properties: +``` +druid.host=0.0.0.0:8080 +druid.port=8080 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=user +druid.database.password=diurd +druid.database.connectURI= +druid.host=127.0.0.1:8080 + +``` +2. Create a valid realtime configuration file similar to this called realtime.spec: +```json +[{ + "schema" : { "dataSource":"druidtest", + "aggregators":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"}], + "indexGranularity":"minute", + "shardSpec" : { "type": "none" } }, + "config" : { "maxRowsInMemory" : 500000, + "intermediatePersistPeriod" : "PT10m" }, + "firehose" : { "type" : "kafka-0.7.2", + "consumerProps" : { "zk.connect" : "localhost:2181", + "zk.connectiontimeout.ms" : "15000", + "zk.sessiontimeout.ms" : "15000", + "zk.synctime.ms" : "5000", + "groupid" : "topic-pixel-local", + "fetch.size" : "1048586", + "autooffset.reset" : "largest", + "autocommit.enable" : "false" }, + "feed" : "druidtest", + "parser" : { "timestampSpec" : { "column" : "utcdt", "format" : "iso" }, + "data" : { "format" : "json" }, + "dimensionExclusions" : ["wp"] } }, + "plumber" : { "type" : "realtime", + "windowPeriod" : "PT10m", + "segmentGranularity":"hour", + "basePersistDirectory" : "/tmp/realtime/basePersist", + "rejectionPolicy": {"type": "messageTime"} } + +}] +``` +3. Launch the realtime node +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-Ddruid.realtime.specFile=config/realtime/realtime.spec \ +-classpath lib/*:config/realtime com.metamx.druid.realtime.RealtimeMain +``` +4. Paste data into the Kafka console producer +```json +{"utcdt": "2010-01-01T01:01:01", "wp": 1000, "gender": "male", "age": 100} +{"utcdt": "2010-01-01T01:01:02", "wp": 2000, "gender": "female", "age": 50} +{"utcdt": "2010-01-01T01:01:03", "wp": 3000, "gender": "male", "age": 20} +{"utcdt": "2010-01-01T01:01:04", "wp": 4000, "gender": "female", "age": 30} +{"utcdt": "2010-01-01T01:01:05", "wp": 5000, "gender": "male", "age": 40} +``` +5. Watch the events as they are ingested by Druid's realtime node +```bash +... +2013-06-17 21:41:55,569 INFO [Global--0] com.metamx.emitter.core.LoggingEmitter - Event [{"feed":"metrics","timestamp":"2013-06-17T21:41:55.569Z","service":"example","host":"127.0.0.1","metric":"events/processed","value":5,"user2":"druidtest"}] +... +``` +6. In a new console, edit a file called query.body: +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "dimensions": [], + "aggregations": [ + { "type": "count", "name": "rows" }, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` +7. Submit the query via curl +```bash +curl -X POST "http://localhost:8080/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +8. View Result! +```json +[ { + "timestamp" : "2010-01-01T01:01:00.000Z", + "result" : { + "imps" : 20, + "wp" : 60000.0, + "rows" : 5 + } +} ] +``` +Now you're ready for [Querying Your Data](Querying-Your-Data.html)! + +## Loading Data with the HadoopDruidIndexer ## + +Historical data can be loaded via a Hadoop job. + +The setup for a single node, 'standalone' Hadoop cluster is available at [http://hadoop.apache.org/docs/stable/single_node_setup.html](http://hadoop.apache.org/docs/stable/single_node_setup.html). + +### Setup MySQL ### +1. If you don't already have it, download MySQL Community Server here: [http://dev.mysql.com/downloads/mysql/](http://dev.mysql.com/downloads/mysql/) +2. Install MySQL +3. Create a druid user and database +```bash +mysql -u root +``` +```sql +GRANT ALL ON druid.* TO 'druid'@'localhost' IDENTIFIED BY 'diurd'; +CREATE database druid; +``` +The [Master](Master.html) node will create the tables it needs based on its configuration. + +### Make sure you have ZooKeeper Running ### + +Make sure that you have a zookeeper instance running. If you followed the instructions for Kafka, it is probably running. If you are unsure if you have zookeeper running, try running + +```bash +ps auxww | grep zoo | grep -v grep +``` + +If you get any result back, then zookeeper is most likely running. If you haven't setup Kafka or do not have zookeeper running, then you can download it and start it up with + +```bash +curl http://www.motorlogy.com/apache/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz -o zookeeper-3.4.5.tar.gz +tar xzf zookeeper-3.4.5.tar.gz +cd zookeeper-3.4.5 +cp conf/zoo_sample.cfg conf/zoo.cfg +./bin/zkServer.sh start +cd .. +``` + +### Launch a Master Node ### +If you've already setup a realtime node, be aware that although you can run multiple node types on one physical computer, you must assign them unique ports. Having used 8080 for the [Realtime](Realtime.html) node, we use 8081 for the [Master](Master.html). + +1. Setup a configuration file called config/master/runtime.properties similar to: +```bash +druid.host=0.0.0.0:8081 +druid.port=8081 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.master.startDelay=PT60s +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=druid +druid.database.password=diurd +druid.database.connectURI=jdbc:mysql://localhost:3306/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath +druid.database.ruleTable=rules +druid.database.configTable=config + +# Path on local FS for storage of segments; dir will be created if needed +druid.paths.indexCache=/tmp/druid/indexCache +# Path on local FS for storage of segment metadata; dir will be created if needed +druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache +``` +2. Launch the [Master](Master.html) node +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath lib/*:config/master \ +com.metamx.druid.http.MasterMain +``` + +### Launch a Compute/Historical Node ### +1. Create a configuration file in config/compute/runtime.properties similar to: +```bash +druid.host=0.0.0.0:8082 +druid.port=8082 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=druid +druid.database.password=diurd +druid.database.connectURI=jdbc:mysql://localhost:3306/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath +druid.database.ruleTable=rules +druid.database.configTable=config + +# Path on local FS for storage of segments; dir will be created if needed +druid.paths.indexCache=/tmp/druid/indexCache +# Path on local FS for storage of segment metadata; dir will be created if needed +druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache +# Setup local storage mode +druid.pusher.local.storageDirectory=/tmp/druid/localStorage +druid.pusher.local=true +``` +2. Launch the compute node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath lib/*:config/compute \ +com.metamx.druid.http.ComputeMain +``` + +### Create a File of Records ### + +We can use the same records we have been, in a file called records.json: +```json +{"utcdt": "2010-01-01T01:01:01", "wp": 1000, "gender": "male", "age": 100} +{"utcdt": "2010-01-01T01:01:02", "wp": 2000, "gender": "female", "age": 50} +{"utcdt": "2010-01-01T01:01:03", "wp": 3000, "gender": "male", "age": 20} +{"utcdt": "2010-01-01T01:01:04", "wp": 4000, "gender": "female", "age": 30} +{"utcdt": "2010-01-01T01:01:05", "wp": 5000, "gender": "male", "age": 40} +``` + +### Run the Hadoop Job ### + +Now its time to run the Hadoop [Batch-ingestion](Batch-ingestion.html) job, HadoopDruidIndexer, which will fill a historical [Compute](Compute.html) node with data. First we'll need to configure the job. + +1. Create a config called batchConfig.json similar to: +```json +{ + "dataSource": "druidtest", + "timestampColumn": "utcdt", + "timestampFormat": "iso", + "dataSpec": { + "format": "json", + "dimensions": ["gender", "age"] + }, + "granularitySpec": { + "type":"uniform", + "intervals":["2010-01-01T01/PT1H"], + "gran":"hour" + }, + "pathSpec": { "type": "static", + "paths": "/Users/rjurney/Software/druid/records.json" }, + "rollupSpec": { "aggs":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"} + ], + "rollupGranularity": "minute"}, + "workingPath": "/tmp/working_path", + "segmentOutputPath": "/tmp/segments", + "leaveIntermediate": "false", + "partitionsSpec": { + "targetPartitionSize": 5000000 + }, + "updaterJobSpec": { + "type":"db", + "connectURI":"jdbc:mysql://localhost:3306/druid", + "user":"druid", + "password":"diurd", + "segmentTable":"prod_segments" + } +} +``` +2. Now run the job, with the config pointing at batchConfig.json: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=realtime.spec -classpath lib/* com.metamx.druid.indexer.HadoopDruidIndexerMain batchConfig.json +``` + +You can now move on to [Querying Your Data](Querying-Your-Data.html)! \ No newline at end of file diff --git a/docs/Master.md b/docs/Master.md new file mode 100644 index 00000000000..eb86a3e81fd --- /dev/null +++ b/docs/Master.md @@ -0,0 +1,106 @@ +--- +layout: default +--- +Master +====== + +The Druid master node is primarily responsible for segment management and distribution. More specifically, the Druid master node communicates to compute nodes to load or drop segments based on configurations. The Druid master is responsible for loading new segments, dropping outdated segments, managing segment replication, and balancing segment load. + +The Druid master runs periodically and the time between each run is a configurable parameter. Each time the Druid master runs, it assesses the current state of the cluster before deciding on the appropriate actions to take. Similar to the broker and compute nodes, the Druid master maintains a connection to a Zookeeper cluster for current cluster information. The master also maintains a connection to a database containing information about available segments and rules. Available segments are stored in a segment table and list all segments that should be loaded in the cluster. Rules are stored in a rule table and indicate how segments should be handled. + +Before any unassigned segments are serviced by compute nodes, the available compute nodes for each tier are first sorted in terms of capacity, with least capacity servers having the highest priority. Unassigned segments are always assigned to the nodes with least capacity to maintain a level of balance between nodes. The master does not directly communicate with a compute node when assigning it a new segment; instead the master creates some temporary information about the new segment under load queue path of the compute node. Once this request is seen, the compute node will load the segment and begin servicing it. + +Rules +----- + +Segments are loaded and dropped from the cluster based on a set of rules. Rules indicate how segments should be assigned to different compute node tiers and how many replicants of a segment should exist in each tier. Rules may also indicate when segments should be dropped entirely from the cluster. The master loads a set of rules from the database. Rules may be specific to a certain datasource and/or a default set of rules can be configured. Rules are read in order and hence the ordering of rules is important. The master will cycle through all available segments and match each segment with the first rule that applies. Each segment may only match a single rule + +For more information on rules, see [Rule Configuration](Rule-Configuration.html). + +Cleaning Up Segments +-------------------- + +Each run, the Druid master compares the list of available database segments in the database with the current segments in the cluster. Segments that are not in the database but are still being served in the cluster are flagged and appended to a removal list. Segments that are overshadowed (their versions are too old and their data has been replaced by newer segments) are also dropped. + +Segment Availability +-------------------- + +If a compute node restarts or becomes unavailable for any reason, the Druid master will notice a node has gone missing and treat all segments served by that node as being dropped. Given a sufficient period of time, the segments may be reassigned to other compute nodes in the cluster. However, each segment that is dropped is not immediately forgotten. Instead, there is a transitional data structure that stores all dropped segments with an associated lifetime. The lifetime represents a period of time in which the master will not reassign a dropped segment. Hence, if a compute node becomes unavailable and available again within a short period of time, the compute node will start up and serve segments from its cache without any those segments being reassigned across the cluster. + +Balancing Segment Load +---------------------- + +To ensure an even distribution of segments across compute nodes in the cluster, the master node will find the total size of all segments being served by every compute node each time the master runs. For every compute node tier in the cluster, the master node will determine the compute node with the highest utilization and the compute node with the lowest utilization. The percent difference in utilization between the two nodes is computed, and if the result exceeds a certain threshold, a number of segments will be moved from the highest utilized node to the lowest utilized node. There is a configurable limit on the number of segments that can be moved from one node to another each time the master runs. Segments to be moved are selected at random and only moved if the resulting utilization calculation indicates the percentage difference between the highest and lowest servers has decreased. + +HTTP Endpoints +-------------- + +The master node exposes several HTTP endpoints for interactions. + +### GET + +/info/master - returns the current true master of the cluster as a JSON object. E.g. A GET request to :8080/info/master will yield JSON of the form {[host]("IP"}) + +/info/cluster - returns JSON data about every node and segment in the cluster. E.g. A GET request to :8080/info/cluster will yield JSON data organized by nodes. Information about each node and each segment on each node will be returned. + +/info/servers (optional param ?full) - returns all segments in the cluster if the full flag is not set, otherwise returns full metadata about all servers in the cluster + +/info/servers/{serverName} - returns full metadata about a specific server + +/info/servers/{serverName}/segments (optional param ?full) - returns a list of all segments for a server if the full flag is not set, otherwise returns all segment metadata + +/info/servers/{serverName}/segments/{segmentId} - returns full metadata for a specific segment + +/info/segments (optional param ?full)- returns all segments in the cluster as a list if the full flag is not set, otherwise returns all metadata about segments in the cluster + +/info/segments/{segmentId} - returns full metadata for a specific segment + +/info/datasources (optional param ?full) - returns a list of datasources in the cluster if the full flag is not set, otherwise returns all the metadata for every datasource in the cluster + +/info/datasources/{dataSourceName} - returns full metadata for a datasource + +/info/datasources/{dataSourceName}/segments (optional param ?full) - returns a list of all segments for a datasource if the full flag is not set, otherwise returns full segment metadata for a datasource + +/info/datasources/{dataSourceName}/segments/{segmentId} - returns full segment metadata for a specific segment + +/info/rules - returns all rules for all data sources in the cluster including the default datasource. + +/info/rules/{dataSourceName} - returns all rules for a specified datasource + +### POST + +/info/rules/{dataSourceName} - POST with a list of rules in JSON form to update rules. + +The Master Console +------------------ + +The Druid master exposes a web GUI for displaying cluster information and rule configuration. After the master starts, the console can be accessed at http://HOST:PORT/static/. There exists a full cluster view, as well as views for individual compute nodes, datasources and segments themselves. Segment information can be displayed in raw JSON form or as part of a sortable and filterable table. + +The master console also exposes an interface to creating and editing rules. All valid datasources configured in the segment database, along with a default datasource, are available for configuration. Rules of different types can be added, deleted or edited. + +FAQ +--- + +1. **Do clients ever contact the master node?** + +The master is not involved in the lifecycle of a query. + +Compute nodes never directly contact the master node. The Druid master tells the compute nodes to load/drop data via Zookeeper, but the compute nodes are completely unaware of the master. + +Brokers also never contact the master. Brokers base their understanding of the data topology on metadata exposed by the compute nodes via ZK and are completely unaware of the master. + +2. **Does it matter if the master node starts up before or after other processes?** + +No. If the Druid master is not started up, no new segments will be loaded in the cluster and outdated segments will not be dropped. However, the master node can be started up at any time, and after a configurable delay, will start running master tasks. + +This also means that if you have a working cluster and all of your masters die, the cluster will continue to function, it just won’t experience any changes to its data topology. + +Running +------- + +Master nodes can be run using the `com.metamx.druid.http.MasterMain` class. + +Configuration +------------- + +See [Configuration](Configuration.html). diff --git a/docs/MySQL.md b/docs/MySQL.md new file mode 100644 index 00000000000..713ad0ab18d --- /dev/null +++ b/docs/MySQL.md @@ -0,0 +1,47 @@ +--- +layout: default +--- +MySQL is an external dependency of Druid. We use it to store various metadata about the system, but not to store the actual data. There are a number of tables used for various purposes described below. + +Segments Table +-------------- + +This is dictated by the `druid.database.segmentTable` property (Note that these properties are going to change in the next stable version after 0.4.12). + +This table stores metadata about the segments that are available in the system. The table is polled by the [Master](Master.html) to determine the set of segments that should be available for querying in the system. The table has two main functional columns, the other columns are for indexing purposes. + +The `used` column is a boolean “tombstone”. A 1 means that the segment should be “used” by the cluster (i.e. it should be loaded and available for requests). A 0 means that the segment should not be actively loaded into the cluster. We do this as a means of removing segments from the cluster without actually removing their metadata (which allows for simpler rolling back if that is ever an issue). + +The `payload` column stores a JSON blob that has all of the metadata for the segment (some of the data stored in this payload is redundant with some of the columns in the table, that is intentional). This looks something like + + { + "dataSource":"wikipedia", + "interval":"2012-05-23T00:00:00.000Z/2012-05-24T00:00:00.000Z", + "version":"2012-05-24T00:10:00.046Z", + "loadSpec":{"type":"s3_zip", + "bucket":"bucket_for_segment", + "key":"path/to/segment/on/s3"}, + "dimensions":"comma-delimited-list-of-dimension-names", + "metrics":"comma-delimited-list-of-metric-names", + "shardSpec":{"type":"none"}, + "binaryVersion":9, + "size":size_of_segment, + "identifier":"wikipedia_2012-05-23T00:00:00.000Z_2012-05-24T00:00:00.000Z_2012-05-23T00:10:00.046Z" + } + +Note that the format of this blob can and will change from time-to-time. + +Rule Table +---------- + +The rule table is used to store the various rules about where segments should land. These rules are used by the [Master](Master.html) when making segment (re-)allocation decisions about the cluster. + +Config Table +------------ + +The config table is used to store runtime configuration objects. We do not have many of these yet and we are not sure if we will keep this mechanism going forward, but it is the beginnings of a method of changing some configuration parameters across the cluster at runtime. + +Task-related Tables +------------------- + +There are also a number of tables created and used by the [Indexing Service](Indexing-Service.html) in the course of its work. diff --git a/docs/OrderBy.md b/docs/OrderBy.md new file mode 100644 index 00000000000..9dcffff7886 --- /dev/null +++ b/docs/OrderBy.md @@ -0,0 +1,27 @@ +--- +layout: default +--- +The orderBy field provides the functionality to sort and limit the set of results from a groupBy query. Available options are: + +### DefaultLimitSpec + +The default limit spec takes a limit and the list of columns to do an orderBy operation over. The grammar is: + + + { + "type" : "default", + "limit" : , + "columns" : [list of OrderByColumnSpec], + } + + +#### OrderByColumnSpec + +OrderByColumnSpecs indicate how to do order by operations. Each order by condition can be a String or a map of the following form: + + + { + "dimension" : "", + "direction" : "ASCENDING OR DESCENDING" + } + diff --git a/docs/Plumber.md b/docs/Plumber.md new file mode 100644 index 00000000000..b2123e94393 --- /dev/null +++ b/docs/Plumber.md @@ -0,0 +1,24 @@ +--- +layout: default +--- +The Plumber is the thing that handles generated segments both while they are being generated and when they are “done”. This is also technically a pluggable interface and there are multiple implementations, but there are a lot of details handled by the plumber such that it is expected that there will only be a few implementations and only more advanced third-parties will implement their own. See [here](https://github.com/metamx/druid/wiki/Plumber#available-plumbers) for a description of the plumbers included with Druid. + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|type|String|Specifies the type of plumber. Each value will have its own configuration schema, plumbers packaged with Druid are described [here](https://github.com/metamx/druid/wiki/Plumber#available-plumbers)|yes| + +We provide a brief description of the example to exemplify the types of things that are configured on the plumber. + +- `windowPeriod` is the amount of lag time to allow events. This is configured with a 10 minute window, meaning that any event more than 10 minutes ago will be thrown away and not included in the segment generated by the realtime server. +- `basePersistDirectory` is the directory to put things that need persistence. The plumber is responsible for the actual intermediate persists and this tells it where to store those persists. + +Available Plumbers +------------------ + +#### YeOldePlumber + +This plumber creates single historical segments. + +#### RealtimePlumber + +This plumber creates real-time/mutable segments. diff --git a/docs/Post-aggregations.md b/docs/Post-aggregations.md new file mode 100644 index 00000000000..2e11f98d0e0 --- /dev/null +++ b/docs/Post-aggregations.md @@ -0,0 +1,95 @@ +--- +layout: default +--- +Post-aggregations are specifications of processing that should happen on aggregated values as they come out of Druid. If you include a post aggregation as part of a query, make sure to include all aggregators the post-aggregator requires. + +There are several post-aggregators available. + +### Arithmetic post-aggregator + +The arithmetic post-aggregator applies the provided function to the given fields from left to right. The fields can be aggregators or other post aggregators. + +Supported functions are `+`, `-`, `*`, and `/` + +The grammar for an arithmetic post aggregation is: + + postAggregation : { + "type" : "arithmetic", + "name" : , + "fn" : , + "fields": [, , ...] + } + +### Field accessor post-aggregator + +This returns the value produced by the specified [aggregator|Aggregations](aggregator|Aggregations.html). + +`fieldName` refers to the output name of the aggregator given in the [aggregations|Aggregations](aggregations|Aggregations.html) portion of the query. + + field_accessor : { + "type" : "fieldAccess", + "fieldName" : + } + +### Constant post-aggregator + +The constant post-aggregator always returns the specified value. + + constant : { + "type" : "constant", + "name" : , + "value" : , + } + +### Example Usage + +In this example, let’s calculate a simple percentage using post aggregators. Let’s imagine our data set has a metric called “total”. + +The format of the query JSON is as follows: + + + { + ... + "aggregations" : [ + { + "type" : "count", + "name" : "rows" + }, + { + "type" : "doubleSum", + "name" : "tot", + "fieldName" : "total" + } + ], + "postAggregations" : { + "type" : "arithmetic", + "name" : "average", + "fn" : "*", + "fields" : [ + { + "type" : "arithmetic", + "name" : "div", + "fn" : "/", + "fields" : [ + { + "type" : "fieldAccess", + "name" : "tot", + "fieldName" : "tot" + }, + { + "type" : "fieldAccess", + "name" : "rows", + "fieldName" : "rows" + } + ] + }, + { + "type" : "constant", + "name": "const", + "value" : 100 + } + ] + } + ... + } + diff --git a/docs/Querying-your-data.md b/docs/Querying-your-data.md new file mode 100644 index 00000000000..dc3e04d645c --- /dev/null +++ b/docs/Querying-your-data.md @@ -0,0 +1,366 @@ +--- +layout: default +--- +# Setup # + +Before we start querying druid, we're going to finish setting up a complete cluster on localhost. In [Loading Your Data](Loading-Your-Data.html) we setup a [Realtime](Realtime.html), [Compute](Compute.html) and [Master](Master.html) node. If you've already completed that tutorial, you need only follow the directions for 'Booting a Broker Node'. + +## Booting a Broker Node ## + +1. Setup a config file at config/broker/runtime.properties that looks like this: +``` +druid.host=0.0.0.0:8083 +druid.port=8083 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=druid +druid.database.password=diurd +druid.database.connectURI=jdbc:mysql://localhost:3306/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath +druid.database.ruleTable=rules +druid.database.configTable=config + +# Path on local FS for storage of segments; dir will be created if needed +druid.paths.indexCache=/tmp/druid/indexCache +# Path on local FS for storage of segment metadata; dir will be created if needed +druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache +druid.pusher.local.storageDirectory=/tmp/druid/localStorage +druid.pusher.local=true + +# thread pool size for servicing queries +druid.client.http.connections=30 +``` + +2. Run the broker node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-Ddruid.realtime.specFile=realtime.spec \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/broker \ +com.metamx.druid.http.BrokerMain +``` + +## Booting a Master Node ## + +1. Setup a config file at config/master/runtime.properties that looks like this: [https://gist.github.com/rjurney/5818870](https://gist.github.com/rjurney/5818870) +2. Run the master node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/master \ +com.metamx.druid.http.MasterMain +``` + +## Booting a Realtime Node ## + +1. Setup a config file at config/realtime/runtime.properties that looks like this: [https://gist.github.com/rjurney/5818774](https://gist.github.com/rjurney/5818774) + +2. Setup a realtime.spec file like this: [https://gist.github.com/rjurney/5818779](https://gist.github.com/rjurney/5818779) +3. Run the realtime node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-Ddruid.realtime.specFile=realtime.spec \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/realtime \ +com.metamx.druid.realtime.RealtimeMain +``` + +## Booting a Compute Node ## + +1. Setup a config file at config/compute/runtime.properties that looks like this: [https://gist.github.com/rjurney/5818885](https://gist.github.com/rjurney/5818885) +2. Run the compute node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/compute \ +com.metamx.druid.http.ComputeMain +``` + +# Querying Your Data # + +Now that we have a complete cluster setup on localhost, we need to load data. To do so, refer to [Loading Your Data](Loading-Your-Data.html). Having done that, its time to query our data! For a complete specification of queries, see [Querying](Querying.html). + +## Querying Different Nodes ## + +As a shared-nothing system, there are three ways to query druid, against the [Realtime](Realtime.html), [Compute](Compute.html) or [Broker](Broker.html) node. Querying a Realtime node returns only realtime data, querying a compute node returns only historical segments. Querying the broker will query both realtime and compute segments and compose an overall result for the query. This is the normal mode of operation for queries in druid. + +### Construct a Query ### + +For constructing this query, see: Querying against the realtime.spec +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "dimensions": [], + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` + +### Querying the Realtime Node ### + +Run our query against port 8080: +```bash +curl -X POST "http://localhost:8080/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +See our result: +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 5, + "wp" : 15000.0, + "rows" : 5 + } +} ] +``` + +### Querying the Compute Node ### +Run the query against port 8082: +```bash +curl -X POST "http://localhost:8082/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +And get (similar to): +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 27, + "wp" : 77000.0, + "rows" : 9 + } +} ] +``` +### Querying both Nodes via the Broker ### +Run the query against port 8083: +```bash +curl -X POST "http://localhost:8083/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +And get: +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 5, + "wp" : 15000.0, + "rows" : 5 + } +} ] +``` + +Now that we know what nodes can be queried (although you should usually use the broker node), lets learn how to know what queries are available. + +## Querying Against the realtime.spec ## + +How are we to know what queries we can run? Although [Querying](Querying.html) is a helpful index, to get a handle on querying our data we need to look at our [Realtime](Realtime.html) node's realtime.spec file: + +```json +[{ + "schema" : { "dataSource":"druidtest", + "aggregators":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"}], + "indexGranularity":"minute", + "shardSpec" : { "type": "none" } }, + "config" : { "maxRowsInMemory" : 500000, + "intermediatePersistPeriod" : "PT10m" }, + "firehose" : { "type" : "kafka-0.7.2", + "consumerProps" : { "zk.connect" : "localhost:2181", + "zk.connectiontimeout.ms" : "15000", + "zk.sessiontimeout.ms" : "15000", + "zk.synctime.ms" : "5000", + "groupid" : "topic-pixel-local", + "fetch.size" : "1048586", + "autooffset.reset" : "largest", + "autocommit.enable" : "false" }, + "feed" : "druidtest", + "parser" : { "timestampSpec" : { "column" : "utcdt", "format" : "iso" }, + "data" : { "format" : "json" }, + "dimensionExclusions" : ["wp"] } }, + "plumber" : { "type" : "realtime", + "windowPeriod" : "PT10m", + "segmentGranularity":"hour", + "basePersistDirectory" : "/tmp/realtime/basePersist", + "rejectionPolicy": {"type": "messageTime"} } + +}] +``` + +### dataSource ### + +```json +"dataSource":"druidtest" +``` +Our dataSource tells us the name of the relation/table, or 'source of data', to query in both our realtime.spec and query.body! + +### aggregations ### + +Note the [Aggregations](Aggregations.html) in our query: + +```json + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], +``` + +this matches up to the aggregators in the schema of our realtime.spec! + +```json +"aggregators":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"}], +``` + +### dimensions ### + +Lets look back at our actual records (from [Loading Your Data](Loading Your Data.html)): + +```json +{"utcdt": "2010-01-01T01:01:01", "wp": 1000, "gender": "male", "age": 100} +{"utcdt": "2010-01-01T01:01:02", "wp": 2000, "gender": "female", "age": 50} +{"utcdt": "2010-01-01T01:01:03", "wp": 3000, "gender": "male", "age": 20} +{"utcdt": "2010-01-01T01:01:04", "wp": 4000, "gender": "female", "age": 30} +{"utcdt": "2010-01-01T01:01:05", "wp": 5000, "gender": "male", "age": 40} +``` + +Note that we have two dimensions to our data, other than our primary metric, wp. They are 'gender' and 'age'. We can specify these in our query! Note that we have added a dimension: age, below. + +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "dimensions": ["age"], + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` + +Which gets us grouped data in return! + +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "100", + "wp" : 1000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "20", + "wp" : 3000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "30", + "wp" : 4000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "40", + "wp" : 5000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "50", + "wp" : 2000.0, + "rows" : 1 + } +} ] +``` + +### filtering ### + +Now that we've observed our dimensions, we can also filter: + +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "filter": { + "type": "selector", + "dimension": "gender", + "value": "male" + }, + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` + +Which gets us just people aged 40: + +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 3, + "wp" : 9000.0, + "rows" : 3 + } +} ] +``` + +Check out [Filters](Filters.html) for more. + +## Learn More ## + +You can learn more about querying at [Querying](Querying.html)! Now check out [Booting a production cluster](Booting-a-production-cluster.html)! \ No newline at end of file diff --git a/docs/Querying.md b/docs/Querying.md new file mode 100644 index 00000000000..7e613a074ec --- /dev/null +++ b/docs/Querying.md @@ -0,0 +1,114 @@ +--- +layout: default +--- +Querying +======== + +Queries are made using an HTTP REST style request to a [Broker](Broker.html), [Compute](Compute.html), or [Realtime](Realtime.html) node. The query is expressed in JSON and each of these node types expose the same REST query interface. + +We start by describing an example query with additional comments that mention possible variations. Query operators are also summarized in a table below. + +Example Query “rand” +-------------------- + +Here is the query in the examples/rand subproject (file is query.body), followed by a commented version of the same. + +\`\`\`javascript +{ + [queryType]() “groupBy”, + [dataSource]() “randSeq”, + [granularity]() “all”, + [dimensions]() [], + [aggregations]() [ + { [type]() “count”, [name]() “rows” }, + { [type]() “doubleSum”, [fieldName]() “events”, [name]() “e” }, + { [type]() “doubleSum”, [fieldName]() “outColumn”, [name]() “randomNumberSum” } + ], + [postAggregations]() [{ + [type]() “arithmetic”, + [name]() “avg\_random”, + [fn]() “/”, + [fields]() [ + { [type]() “fieldAccess”, [fieldName]() “randomNumberSum” }, + { [type]() “fieldAccess”, [fieldName]() “rows” } + ] + }], + [intervals]() [“2012-10-01T00:00/2020-01-01T00”] +} +\`\`\` + +This query could be submitted via curl like so (assuming the query object is in a file “query.json”). + + curl -X POST "http://host:port/druid/v2/?pretty" -H 'content-type: application/json' -d @query.json + +The “pretty” query parameter gets the results formatted a bit nicer. + +Details of Example Query “rand” +------------------------------- + +The queryType JSON field identifies which kind of query operator is to be used, in this case it is groupBy, the most frequently used kind (which corresponds to an internal implementation class GroupByQuery registered as “groupBy”), and it has a set of required fields that are also part of this query. The queryType can also be “search” or “timeBoundary” which have similar or different required fields summarized below: +\`\`\`javascript +{ + [queryType]() “groupBy”, +\`\`\` +The dataSource JSON field shown next identifies where to apply the query. In this case, randSeq corresponds to the examples/rand/rand\_realtime.spec file schema: +\`\`\`javascript + [dataSource]() “randSeq”, +\`\`\` +The granularity JSON field specifies the bucket size for values. It could be a built-in time interval like “second”, “minute”, “fifteen\_minute”, “thirty\_minute”, “hour” or “day”. It can also be an expression like `{"type": "period", "period":"PT6m"}` meaning “6 minute buckets”. See [Granularities](Granularities.html) for more information on the different options for this field. In this example, it is set to the special value “all” which means [bucket all data points together into the same time bucket]() +\`\`\`javascript + [granularity]() “all”, +\`\`\` +The dimensions JSON field value is an array of zero or more fields as defined in the dataSource spec file or defined in the input records and carried forward. These are used to constrain the grouping. If empty, then one value per time granularity bucket is requested in the groupBy: +\`\`\`javascript + [dimensions]() [], +\`\`\` +A groupBy also requires the JSON field “aggregations” (See [Aggregations](Aggregations.html)), which are applied to the column specified by fieldName and the output of the aggregation will be named according to the value in the “name” field: +\`\`\`javascript + [aggregations]() [ + { [type]() “count”, [name]() “rows” }, + { [type]() “doubleSum”, [fieldName]() “events”, [name]() “e” }, + { [type]() “doubleSum”, [fieldName]() “outColumn”, [name]() “randomNumberSum” } + ], +\`\`\` +You can also specify postAggregations, which are applied after data has been aggregated for the current granularity and dimensions bucket. See [Post Aggregations](Post Aggregations.html) for a detailed description. In the rand example, an arithmetic type operation (division, as specified by “fn”) is performed with the result “name” of “avg\_random”. The “fields” field specifies the inputs from the aggregation stage to this expression. Note that identifiers corresponding to “name” JSON field inside the type “fieldAccess” are required but not used outside this expression, so they are prefixed with “dummy” for clarity: +\`\`\`javascript + [postAggregations]() [{ + [type]() “arithmetic”, + [name]() “avg\_random”, + [fn]() “/”, + [fields]() [ + { [type]() “fieldAccess”, [fieldName]() “randomNumberSum” }, + { [type]() “fieldAccess”, [fieldName]() “rows” } + ] + }], +\`\`\` +The time range(s) of the query; data outside the specified intervals will not be used; this example specifies from October 1, 2012 until January 1, 2020: +\`\`\`javascript + [intervals]() [“2012-10-01T00:00/2020-01-01T00”] +} +\`\`\` + +Query Operators +--------------- + +The following table summarizes query properties. + +|query types|property|description|required?| +|-----------|--------|-----------|---------| +|timeseries, groupBy, search, timeBoundary|dataSource|query is applied to this data source|yes| +|timeseries, groupBy, search|intervals|range of time series to include in query|yes| +|timeseries, groupBy, search, timeBoundary|context|This is a key-value map that can allow the query to alter some of the behavior of a query. It is primarily used for debugging, for example if you include `"bySegment":true` in the map, you will get results associated with the data segment they came from.|no| +|timeseries, groupBy, search|filter|Specifies the filter (the “WHERE” clause in SQL) for the query. See [Filters](Filters.html)|no| +|timeseries, groupBy, search|granularity|the timestamp granularity to bucket results into (i.e. “hour”). See [Granularities](Granularities.html) for more information.|no| +|groupBy|dimensions|constrains the groupings; if empty, then one value per time granularity bucket|yes| +|timeseries, groupBy|aggregations|aggregations that combine values in a bucket. See [Aggregations](Aggregations.html).|yes| +|timeseries, groupBy|postAggregations|aggregations of aggregations. See [Post Aggregations](Post Aggregations.html).|yes| +|search|limit|maximum number of results (default is 1000), a system-level maximum can also be set via `com.metamx.query.search.maxSearchLimit`|no| +|search|searchDimensions|Dimensions to apply the search query to. If not specified, it will search through all dimensions.|no| +|search|query|The query portion of the search query. This is essentially a predicate that specifies if something matches.|yes| + +Additional Information about Query Types +---------------------------------------- + +[TimeseriesQuery](TimeseriesQuery.html) diff --git a/docs/Realtime.md b/docs/Realtime.md new file mode 100644 index 00000000000..855607d7eb5 --- /dev/null +++ b/docs/Realtime.md @@ -0,0 +1,157 @@ +--- +layout: default +--- +Realtime +======== + +Realtime nodes provide a realtime index. Data indexed via these nodes is immediately available for querying. Realtime nodes will periodically build segments representing the data they’ve collected over some span of time and hand these segments off to [Compute](Compute.html) nodes. + +Running +------- + +Realtime nodes can be run using the `com.metamx.druid.realtime.RealtimeMain` class. + +Segment Propagation +------------------- + +The segment propagation diagram for real-time data ingestion can be seen below: + +![Segment Propagation](https://raw.github.com/metamx/druid/druid-0.5.4/doc/segment_propagation.png "Segment Propagation") + +Configuration +------------- + +Realtime nodes take a mix of base server configuration and spec files that describe how to connect, process and expose the realtime feed. See [Configuration](Configuration.html) for information about general server configuration. + +### Realtime “specFile” + +The property `druid.realtime.specFile` has the path of a file (absolute or relative path and file name) with realtime specifications in it. This “specFile” should be a JSON Array of JSON objects like the following: + + + [{ + "schema" : { "dataSource":"dataSourceName", + "aggregators":[ {"type":"count", "name":"events"}, + {"type":"doubleSum","name":"outColumn","fieldName":"inColumn"} ], + "indexGranularity":"minute", + "shardSpec" : { "type": "none" } }, + "config" : { "maxRowsInMemory" : 500000, + "intermediatePersistPeriod" : "PT10m" }, + "firehose" : { "type" : "kafka-0.7.2", + "consumerProps" : { "zk.connect" : "zk_connect_string", + "zk.connectiontimeout.ms" : "15000", + "zk.sessiontimeout.ms" : "15000", + "zk.synctime.ms" : "5000", + "groupid" : "consumer-group", + "fetch.size" : "1048586", + "autooffset.reset" : "largest", + "autocommit.enable" : "false" }, + "feed" : "your_kafka_topic", + "parser" : { "timestampSpec" : { "column" : "timestamp", "format" : "iso" }, + "data" : { "format" : "json" }, + "dimensionExclusions" : ["value"] } }, + "plumber" : { "type" : "realtime", + "windowPeriod" : "PT10m", + "segmentGranularity":"hour", + "basePersistDirectory" : "/tmp/realtime/basePersist" } + }] + + +This is a JSON Array so you can give more than one realtime stream to a given node. The number you can put in the same process depends on the exact configuration. In general, it is best to think of each realtime stream handler as requiring 2-threads: 1 thread for data consumption and aggregation, 1 thread for incremental persists and other background tasks. + +There are four parts to a realtime stream specification, `schema`, `config`, `firehose` and `plumber` which we will go into here. + +#### Schema + +This describes the data schema for the output Druid segment. More information about concepts in Druid and querying can be found at [Concepts-and-Terminology](Concepts-and-Terminology.html) and [Querying](Querying.html). + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|aggregators|Array of Objects|The list of aggregators to use to aggregate colliding rows together.|yes| +|dataSource|String|The name of the dataSource that the segment belongs to.|yes| +|indexGranularity|String|The granularity of the data inside the segment. E.g. a value of “minute” will mean that data is aggregated at minutely granularity. That is, if there are collisions in the tuple (minute(timestamp), dimensions), then it will aggregate values together using the aggregators instead of storing individual rows.|yes| +|segmentGranularity|String|The granularity of the segment as a whole. This is generally larger than the index granularity and describes the rate at which the realtime server will push segments out for historical servers to take over.|yes| +|shardSpec|Object|This describes the shard that is represented by this server. This must be specified properly in order to have multiple realtime nodes indexing the same data stream in a sharded fashion.|no| + +### Config + +This provides configuration for the data processing portion of the realtime stream processor. + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|intermediatePersistPeriod|ISO8601 Period String|The period that determines the rate at which intermediate persists occur. These persists determine how often commits happen against the incoming realtime stream. If the realtime data loading process is interrupted at time T, it should be restarted to re-read data that arrived at T minus this period.|yes| +|maxRowsInMemory|Number|The number of rows to aggregate before persisting. This number is the post-aggregation rows, so it is not equivalent to the number of input events, but the number of aggregated rows that those events result in. This is used to manage the required JVM heap size.|yes| + +### Firehose + +See [Firehose](Firehose.html). + +### Plumber + +See [Plumber](Plumber.html) + +Constraints +----------- + +The following tables summarizes constraints between settings in the spec file for the Realtime subsystem. + +|*. Name |*. Effect |*. Minimum |*. Recommended | +| windowPeriod| when reading an InputRow, events with timestamp older than now minus this window are discarded | time jitter tolerance | use this to reject outliers | +| segmentGranularity| time granularity (minute, hour, day, week, month) for loading data at query time | equal to indexGranularity| more than indexGranularity| +| indexGranularity| time granularity (minute, hour, day, week, month) of indexes | less than segmentGranularity| minute, hour, day, week, month | +| intermediatePersistPeriod| the max real time (ISO8601 Period) between flushes of InputRows from memory to disk | avoid excessive flushing | number of un-persisted rows in memory also constrained by maxRowsInMemory | +| maxRowsInMemory| the max number of InputRows to hold in memory before a flush to disk | number of un-persisted post-aggregation rows in memory is also constrained by intermediatePersistPeriod | use this to avoid running out of heap if too many rows in an intermediatePersistPeriod | + +The normal, expected use cases have the following overall constraints: `indexGranularity < intermediatePersistPeriod =< windowPeriod < segmentGranularity` + +If the RealtimeNode process runs out of heap, try adjusting druid.computation.buffer.size property which specifies a size in bytes that must fit into the heap. + +Requirements +------------ + +Realtime nodes currently require a Kafka cluster to sit in front of them and collect results. There’s more configuration required for these as well. + +Extending the code +------------------ + +Realtime integration is intended to be extended in two ways: + +1. Connect to data streams from varied systems ([Firehose](https://github.com/metamx/druid/blob/master/realtime/src/main/java/com/metamx/druid/realtime/FirehoseFactory.java)) +2. Adjust the publishing strategy to match your needs ([Plumber](https://github.com/metamx/druid/blob/master/realtime/src/main/java/com/metamx/druid/realtime/PlumberSchool.java)) + +The expectations are that the former will be very common and something that users of Druid will do on a fairly regular basis. Most users will probably never have to deal with the latter form of customization. Indeed, we hope that all potential use cases can be packaged up as part of Druid proper without requiring proprietary customization. + +Given those expectations, adding a firehose is straightforward and completely encapsulated inside of the interface. Adding a plumber is more involved and requires understanding of how the system works to get right, it’s not impossible, but it’s not intended that individuals new to Druid will be able to do it immediately. + +We will do our best to accept contributions from the community of new Firehoses and Plumbers, but we also understand the requirement for being able to plug in your own proprietary implementations. The model for doing this is by embedding the druid code in another project and writing your own `main()` method that initializes a RealtimeNode object and registers your proprietary objects with it. + + + public class MyRealtimeMain + { + private static final Logger log = new Logger(MyRealtimeMain.class); + + public static void main(String[] args) throws Exception + { + LogLevelAdjuster.register(); + + Lifecycle lifecycle = new Lifecycle(); + + lifecycle.addManagedInstance( + RealtimeNode.builder() + .build() + .registerJacksonSubtype(foo.bar.MyFirehose.class) + ); + + try { + lifecycle.start(); + } + catch (Throwable t) { + log.info(t, "Throwable caught at startup, committing seppuku"); + System.exit(2); + } + + lifecycle.join(); + } + } + + +Pluggable pieces of the system are either handled by a setter on the RealtimeNode object, or they are configuration driven and need to be setup to allow for [Jackson polymorphic deserialization](http://wiki.fasterxml.com/JacksonPolymorphicDeserialization) and registered via the relevant methods on the RealtimeNode object. diff --git a/docs/Rule-Configuration.md b/docs/Rule-Configuration.md new file mode 100644 index 00000000000..2695da646ab --- /dev/null +++ b/docs/Rule-Configuration.md @@ -0,0 +1,80 @@ +--- +layout: default +--- +Note: It is recommended that the master console is used to configure rules. However, the master node does have HTTP endpoints to programmatically configure rules. + +Load Rules +---------- + +Load rules indicate how many replicants of a segment should exist in a server tier. + +### Interval Load Rule + +Interval load rules are of the form: + + + { + "type" : "loadByInterval", + "interval" : "2012-01-01/2013-01-01", + "tier" : "hot" + } + + +type - this should always be “loadByInterval” +interval - A JSON Object representing ISO-8601 Intervals +tier - the configured compute node tier + +### Period Load Rule + +Period load rules are of the form: + + + { + "type" : "loadByInterval", + "period" : "P1M", + "tier" : "hot" + } + + +type - this should always be “loadByPeriod” +period - A JSON Object representing ISO-8601 Periods +tier - the configured compute node tier + +The interval of a segment will be compared against the specified period. The rule matches if the period overlaps the interval. + +Drop Rules +---------- + +Drop rules indicate when segments should be dropped from the cluster. + +### Interval Drop Rule + +Interval drop rules are of the form: + + + { + "type" : "dropByInterval", + "interval" : "2012-01-01/2013-01-01" + } + + +type - this should always be “dropByInterval” +interval - A JSON Object representing ISO-8601 Periods + +A segment is dropped if the interval contains the interval of the segment. + +### Period Drop Rule + +Period drop rules are of the form: + + + { + "type" : "dropByPeriod", + "period" : "P1M" + } + + +type - this should always be “dropByPeriod” +period - A JSON Object representing ISO-8601 Periods + +The interval of a segment will be compared against the specified period. The period is from some time in the past to the current time. The rule matches if the period contains the interval. diff --git a/docs/SearchQuery.md b/docs/SearchQuery.md new file mode 100644 index 00000000000..b206f652c6e --- /dev/null +++ b/docs/SearchQuery.md @@ -0,0 +1,71 @@ +--- +layout: default +--- +A search query returns dimension values that match the search specification. + + { + "queryType": "search", + "dataSource": "sample_datasource", + "granularity": "day", + "searchDimensions": [ + "dim1", + "dim2" + ], + "query": { + "type": "insensitive_contains", + "value": "Ke" + }, + "sort" : { + "type": "lexicographic" + }, + "intervals": [ + "2013-01-01T00:00:00.000/2013-01-03T00:00:00.000" + ] + } + + +There are several main parts to a search query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “search”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|granularity|Defines the granularity of the query. See [Granularities](Granularities.html)|yes| +|filter|See [Filters](Filters.html)|no| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|searchDimensions|The dimensions to run the search over. Excluding this means the search is run over all dimensions.|no| +|query|See [SearchQuerySpec](SearchQuerySpec.html).|yes| +|sort|How the results of the search should sorted. Two possible types here are “lexicographic” and “strlen”.|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +The format of the result is: + + [ + { + "timestamp": "2012-01-01T00:00:00.000Z", + "result": [ + { + "dimension": "dim1", + "value": "Ke$ha" + }, + { + "dimension": "dim2", + "value": "Ke$haForPresident" + } + ] + }, + { + "timestamp": "2012-01-02T00:00:00.000Z", + "result": [ + { + "dimension": "dim1", + "value": "SomethingThatContainsKe" + }, + { + "dimension": "dim2", + "value": "SomethingElseThatContainsKe" + } + ] + } + ] + diff --git a/docs/SearchQuerySpec.md b/docs/SearchQuerySpec.md new file mode 100644 index 00000000000..9b9db04b8e6 --- /dev/null +++ b/docs/SearchQuerySpec.md @@ -0,0 +1,26 @@ +--- +layout: default +--- +Search query specs define how a “match” is defined between a search value and a dimension value. The available search query specs are: + +InsensitiveContainsSearchQuerySpec +---------------------------------- + +If any part of a dimension value contains the value specified in this search query spec, regardless of case, a “match” occurs. The grammar is: + + { + "type" : "insensitive_contains", + "value" : "some_value" + } + + +FragmentSearchQuerySpec +----------------------- + +If any part of a dimension value contains any of the values specified in this search query spec, regardless of case, a “match” occurs. The grammar is: + + { + "type" : "fragment", + "values" : ["fragment1", "fragment2"] + } + diff --git a/docs/SegmentMetadataQuery.md b/docs/SegmentMetadataQuery.md new file mode 100644 index 00000000000..0e6eefb78e1 --- /dev/null +++ b/docs/SegmentMetadataQuery.md @@ -0,0 +1,58 @@ +--- +layout: default +--- +Segment metadata queries return per segment information about: +\* Cardinality of all columns in the segment +\* Estimated byte size for the segment columns in TSV format +\* Interval the segment covers +\* Column type of all the columns in the segment +\* Estimated total segment byte size in TSV format +\* Segment id + + { + "queryType":"segmentMetadata", + "dataSource":"sample_datasource", + "intervals":["2013-01-01/2014-01-01"], + } + + +There are several main parts to a segment metadata query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “segmentMetadata”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|merge|Merge all individual segment metadata results into a single result|no| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +The format of the result is: + + [ { + "id" : "some_id", + "intervals" : [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ], + "columns" : { + "__time" : { + "type" : "LONG", + "size" : 407240380, + "cardinality" : null + }, + "dim1" : { + "type" : "STRING", + "size" : 100000, + "cardinality" : 1944 + }, + "dim2" : { + "type" : "STRING", + "size" : 100000, + "cardinality" : 1504 + }, + "metric1" : { + "type" : "FLOAT", + "size" : 100000, + "cardinality" : null + } + }, + "size" : 300000 + } ] + diff --git a/docs/Segments.md b/docs/Segments.md new file mode 100644 index 00000000000..5af50cd8e48 --- /dev/null +++ b/docs/Segments.md @@ -0,0 +1,63 @@ +--- +layout: default +--- +Segments +======== + +Segments are the fundamental structure to store data in Druid. [Compute](Compute.html) and [Realtime](Realtime.html) nodes load and serve segments for querying. To construct segments, Druid will always shard data by a time partition. Data may be further sharded based on dimension cardinality and row count. + +The latest Druid segment version is `v9`. + +Naming Convention +----------------- + +Identifiers for segments are typically constructed using the segment datasource, interval start time (in ISO 8601 format), interval end time (in ISO 8601 format), and a version. If data is additionally sharded beyond a time range, the segment identifier will also contain a partition number. + +An example segment identifier may be: +datasource\_intervalStart\_intervalEnd\_version\_partitionNum + +Segment Components +------------------ + +A segment is compromised of several files, listed below. + +### `version.bin` + +4 bytes representing the current segment version as an integer. E.g., for v9 segments, the version is 0x0, 0x0, 0x0, 0x9 + +### `meta.smoosh` + +A file with metadata (filenames and offsets) about the contents of the other `smoosh` files + +### `XXXXX.smoosh` + +There are some number of these files, which are concatenated binary data + +The `smoosh` files represent multiple files “smooshed” together in order to minimize the number of file descriptors that must be open to house the data. They are files of up to 2GB in size (to match the limit of a memory mapped ByteBuffer in Java). The `smoosh` files house individual files for each of the columns in the data as well as an `index.drd` file with extra metadata about the segment. + +There is also a special column called `__time` that refers to the time column of the segment. This will hopefully become less and less special as the code evolves, but for now it’s as special as my Mommy always told me I am. + +### `index.drd` + +The `index.drd` file houses 3 pieces of data in order + +1. The names of all of the columns of the data +2. The names of the “dimensions” of the data (these are the dictionary-encoded, string columns. This is here to support some legacy APIs and will be superfluous in the future) +3. The data interval represented by this segment stored as the start and end timestamps as longs + +Format of a column +------------------ + +Each column is stored as two parts: + +1. A Jackson-serialized ColumnDescriptor +2. The rest of the binary for the column + +A ColumnDescriptor is essentially an object that allows us to use jackson’s polymorphic deserialization to add new and interesting methods of serialization with minimal impact to the code. It consists of some metadata about the column (what type is it, is it multi-valued, etc.) and then a list of serde logic that can deserialize the rest of the binary. + +Sharding Data to Create Segments +-------------------------------- + +### Sharding Data by Dimension + +If the cumulative total number of rows for the different values of a given column exceed some configurable threshold, multiple segments representing the same time interval for the same datasource may be created. These segments will contain some partition number as part of their identifier. Sharding by dimension reduces some of the the costs associated with operations over high cardinality dimensions. diff --git a/docs/Spatial-Filters.md b/docs/Spatial-Filters.md new file mode 100644 index 00000000000..2ca83b9a3f9 --- /dev/null +++ b/docs/Spatial-Filters.md @@ -0,0 +1,35 @@ +--- +layout: default +--- +Note: This feature is highly experimental and only works with spatially indexed dimensions. + +The grammar for a spatial filter is as follows: + + + { + "dimension": "spatialDim", + "bound": { + "type": "rectangular", + "minCoords": [10.0, 20.0], + "maxCoords": [30.0, 40.0] + } + } + + +Bounds +------ + +### Rectangular + +|property|description|required?| +|--------|-----------|---------| +|minCoords|List of minimum dimension coordinates for coordinates [x, y, z, …]|yes| +|maxCoords|List of maximum dimension coordinates for coordinates [x, y, z, …]|yes| + +### Radius + +|property|description|required?| +|--------|-----------|---------| +|coords|Origin coordinates in the form [x, y, z, …]|yes| +|radius|The float radius value|yes| + diff --git a/docs/Spatial-Indexing.md b/docs/Spatial-Indexing.md new file mode 100644 index 00000000000..1df36593433 --- /dev/null +++ b/docs/Spatial-Indexing.md @@ -0,0 +1,26 @@ +--- +layout: default +--- +Note: This feature is highly experimental. + +In any of the data specs, there is now the option of providing spatial dimensions. For example, for a JSON data spec, spatial dimensions can be specified as follows: + + + { + "type": "JSON", + "dimensions": , + "spatialDimensions": [ + { + "dimName": "coordinates", + "dims": ["lat", "long"] + }, + ... + ] + } + + +|property|description|required?| +|--------|-----------|---------| +|dimName|The name of the spatial dimension. A spatial dimension may be constructed from multiple other dimensions or it may already exist as part of an event. If a spatial dimension already exists, it must be an array of dimension values.|yes| +|dims|A list of dimension names that comprise a spatial dimension.|no| + diff --git a/docs/Stand-Alone-With-Riak-CS.md b/docs/Stand-Alone-With-Riak-CS.md new file mode 100644 index 00000000000..d1dc8f780d8 --- /dev/null +++ b/docs/Stand-Alone-With-Riak-CS.md @@ -0,0 +1,226 @@ +--- +layout: default +--- +This page describes how to use Riak-CS for deep storage instead of S3. We are still setting up some of the peripheral stuff (file downloads, etc.). + +This guide provided by Pablo Nebrera, thanks! + +## The VMWare instance + +A VMWare [image](http://static.druid.io/artifacts/vmware/druid_riak.tgz) based on Druid 0.3.27.2 and built according to the instructions below has also been provided by Pablo Nebrera. + +The provided vmware machine has access with the following credentials: + + username: root + password: riakdruid + + +## The Setup + +We started with a minimal CentOS installation but you can use any other compatible installation. At the end of this setup you will one node that is running: + +1. A Kafka Broker +1. A single-node Zookeeper ensemble +1. A single-node Riak-CS cluster +1. A Druid [Master](Master.html) +1. A Druid [Broker](Broker.html) +1. A Druid [Compute](Compute.html) +1. A Druid [Realtime](Realtime.html) + +This just walks through getting the relevant software installed and running. You will then need to configure the [Realtime](Realtime.html) node to take in your data. + +### Configure System + +1. Install `CentOS-6.4-x86_64-minimal.iso` ("RedHat v6.4" is the name of the AWS AMI) or your favorite Linux OS (if you use a different OS, some of the installation instructions for peripheral services might differ, please adjust them according to the system you are using). The rest of these instructions assume that you have a running instance and are running as the root user. + +1. Configure the network. We used dhcp executing: + + dhclient eth0 + +1. Disable firewall for now + + service iptables stop + chkconfig iptables off + +1. Change the limits on the number of open files a process can have: + + cat >> /etc/security/limits.conf <<- _RBEOF_ + # ulimit settings for Riak CS + root soft nofile 65536 + root hard nofile 65536 + riak soft nofile 65536 + riak hard nofile 65536 + _RBEOF_ + + ulimit -n 65536 + +### Install base software packages + +1. Install necessary software with yum + + yum install -y java-1.7.0-openjdk-devel git wget mysql-server + +1. Install maven + + wget http://apache.rediris.es/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz + tar xzf apache-maven-3.0.5-bin.tar.gz -C /usr/local + pushd /usr/local + sudo ln -s apache-maven-3.0.5 maven + popd + echo 'export M2_HOME=/usr/local/maven' >> /etc/profile.d/maven.sh + echo 'export PATH=${M2_HOME}/bin:${PATH}' >> /etc/profile.d/maven.sh + source /etc/profile.d/maven.sh + +1. Install erlang + + wget http://binaries.erlang-solutions.com/rpm/centos/6/x86_64/esl-erlang-R15B01-1.x86_64.rpm + yum localinstall -y esl-erlang-R15B01-1.x86_64.rpm + +### Install Kafka And Zookeeper + +1. Install kafka and zookeeper: + + wget http://apache.rediris.es/incubator/kafka/kafka-0.7.2-incubating/kafka-0.7.2-incubating-src.tgz + tar zxvf kafka-0.7.2-incubating-src.tgz + pushd kafka-0.7.2-incubating-src/ + ./sbt update + ./sbt package + mkdir -p /var/lib/kafka + rsync -a * /var/lib/kafka/ + popd + +### Install Riak-CS + +1. Install s3cmd to manage riak s3 + + wget http://downloads.sourceforge.net/project/s3tools/s3cmd/1.5.0-alpha3/s3cmd-1.5.0-alpha3.tar.gz + tar xzvf s3cmd-1.5.0-alpha3.tar.gz + cd s3cmd-1.5.0-alpha3 + cp -r s3cmd S3 /usr/local/bin/ + +1. Install riak, riak-cs and stanchion. Note: riak-cs-control is optional + + wget http://s3.amazonaws.com/downloads.basho.com/riak/1.3/1.3.1/rhel/6/riak-1.3.1-1.el6.x86_64.rpm + wget http://s3.amazonaws.com/downloads.basho.com/riak-cs/1.3/1.3.1/rhel/6/riak-cs-1.3.1-1.el6.x86_64.rpm + wget http://s3.amazonaws.com/downloads.basho.com/stanchion/1.3/1.3.1/rhel/6/stanchion-1.3.1-1.el6.x86_64.rpm + wget http://s3.amazonaws.com/downloads.basho.com/riak-cs-control/1.0/1.0.0/rhel/6/riak-cs-control-1.0.0-1.el6.x86_64.rpm + yum localinstall -y riak-*.rpm stanchion-*.rpm + +### Install Druid + +1. Clone the git repository for druid, checkout a "stable" tag and build + + git clone https://github.com/metamx/druid.git druid + pushd druid + git checkout druid-0.4.12 + export LANGUAGE=C + export LC_MESSAGE=C + export LC_ALL=C + export LANG=en_US + ./build.sh + mkdir -p /var/lib/druid/app + cp ./services/target/druid-services-*-selfcontained.jar /var/lib/druid/app + ln -s /var/lib/druid/app/druid-services-*-selfcontained.jar /var/lib/druid/app/druid-services.jar + popd + + +### Configure stuff + +1. Add this line to /etc/hosts + + echo "127.0.0.1 s3.amazonaws.com bucket.s3.amazonaws.com `hostname`" >> /etc/hosts + + NOTE: the bucket name in this case is "bucket", but you might need to update it to your bucket name if you want to use a different bucket name. + +1. Download and extract run scripts and configuration files: + + wget http://static.druid.io/artifacts/scripts/druid_scripts_nebrera.tar / + pushd / + tar xvf ~/druid_scripts_nebrera.tar + popd + + +1. Start Riak in order to create a user: + + /etc/init.d/riak start + /etc/init.d/riak-cs start + /etc/init.d/stanchion start + + You can check riak status using: + + riak-admin member-status + + You should expect results like + + Attempting to restart script through sudo -H -u riak + ================================= Membership ================================== + Status Ring Pending Node + ------------------------------------------------------------------------------- + valid 100.0% -- 'riak@127.0.0.1' + ------------------------------------------------------------------------------- + Valid:1 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + + +1. Create riak-cs user and yoink out credentials. + + curl -H 'Content-Type: application/json' -X POST http://127.0.0.1:8088/riak-cs/user --data '{"email":"example@domain.com", "name":"admin"}' >> /tmp/riak_user.json + export RIAK_KEY_ID=`sed 's/^.*"key_id":"//' /tmp/riak_user.json | cut -d '"' -f 1` + export RIAK_KEY_SECRET=`sed 's/^.*"key_secret":"//' /tmp/riak_user.json | cut -d '"' -f 1` + sed -i "s/<%=[ ]*@key_id[ ]*%>/${RIAK_KEY_ID}/" /etc/riak-cs/app.config /etc/riak-cs-control/app.config /etc/stanchion/app.config /etc/druid/config.sh /etc/druid/base.properties /root/.s3cfg + sed -i "s/<%=[ ]*@key_secret[ ]*%>/${RIAK_KEY_SECRET}/" /etc/riak-cs/app.config /etc/riak-cs-control/app.config /etc/stanchion/app.config /etc/druid/config.sh /etc/druid/base.properties /root/.s3cfg + + This will store the result of creating the user into `/tmp/riak_user.json`. You can look at it if you are interested. It will look something like this + + {"email":"example@domain.com", + "display_name":"example", + "name":"admin", + "key_id":"DOXKZYR_QM2S-7HSKAEU", + "key_secret":"GtvVJow068RM-_viHIYR9DWMAXsFcL1SmjuNfA==", + "id":"4c5b5468c180f3efafd531b6cd8e2bb24371d99640aad5ced5fbbc0604fc473d", + "status":"enabled"} + +1. Stop riak-cs: + + /etc/init.d/riak-cs stop + /etc/init.d/stanchion stop + /etc/init.d/riak stop + +1. Disable anonymous user creation + + sed 's/{[ ]*anonymous_user_creation[ ]*,[ ]*true[ ]*}/{anonymous_user_creation, false}/' /etc/riak-cs/app.config |grep anonymous_user_creation + +1. Restart riak-cs services: + + /etc/init.d/riak start + /etc/init.d/riak-cs start + /etc/init.d/stanchion start + + +1. Create your bucket. The example name and in config files is "bucket" + + s3cmd mb s3://bucket + + You can verify that the bucket is created with: + + s3cmd ls + +1. Start MySQL server + + service mysqld start + chkconfig mysqld on + /usr/bin/mysqladmin -u root password 'riakdruid' + + NOTE: If you don't like "riakdruid" as your password, feel free to change it around. + NOTE: If you have used root user to connect to database. It should be changed by other user but I have used this one to simplify it + +1. Start zookeeper and kafka + + /etc/init.d/zookeeper start + /etc/init.d/kafka start + +1. Start druid + + /etc/init.d/druid_master start + /etc/init.d/druid_realtime start + /etc/init.d/druid_broker start + /etc/init.d/druid_compute start \ No newline at end of file diff --git a/docs/Support.md b/docs/Support.md new file mode 100644 index 00000000000..3dd512e050f --- /dev/null +++ b/docs/Support.md @@ -0,0 +1,16 @@ +--- +layout: default +--- +Numerous backend engineers at [Metamarkets](http://www.metamarkets.com) work on Druid full-time. If you any questions about usage or code, feel free to contact any of us. + +Google Groups Mailing List +-------------------------- + +The best place for questions is through our mailing list: +[druid-development@googlegroups.com](mailto:druid-development@googlegroups.com) +[https://groups.google.com/d/forum/druid-development](https://groups.google.com/d/forum/druid-development) + +IRC +--- + +Several of us also hang out in the channel \#druid-dev on irc.freenode.net. diff --git a/docs/Tasks.md b/docs/Tasks.md new file mode 100644 index 00000000000..95341f581ec --- /dev/null +++ b/docs/Tasks.md @@ -0,0 +1,71 @@ +--- +layout: default +--- +Tasks are run on workers and always operate on a single datasource. Once an indexer coordinator node accepts a task, a lock is created for the datasource and interval specified in the task. Tasks do not need to explicitly release locks, they are released upon task completion. Tasks may potentially release locks early if they desire. Tasks ids are unique by naming them using UUIDs or the timestamp in which the task was created. Tasks are also part of a “task group”, which is a set of tasks that can share interval locks. + +There are several different types of tasks. + +Append Task +----------- + +Append tasks append a list of segments together into a single segment (one after the other). The grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Merge Task +---------- + +Merge tasks merge a list of segments together. Any common timestamps are merged. The grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Delete Task +----------- + +Delete tasks create empty segments with no data. The grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Kill Task +--------- + +Kill tasks delete all information about a segment and removes it from deep storage. Killable segments must be disabled (used==0) in the Druid segment table. The available grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Index Task +---------- + +Index Partitions Task +--------------------- + +Index Generator Task +-------------------- + +Index Hadoop Task +----------------- + +Index Realtime Task +------------------- + +Version Converter Task +---------------------- + +Version Converter SubTask +------------------------- diff --git a/docs/Thanks.md b/docs/Thanks.md new file mode 100644 index 00000000000..cb1c873cca0 --- /dev/null +++ b/docs/Thanks.md @@ -0,0 +1,11 @@ +--- +layout: default +--- +YourKit supports the Druid open source projects with its +full-featured Java Profiler. +YourKit, LLC is the creator of innovative and intelligent tools for profiling +Java and .NET applications. Take a look at YourKit's software products: +YourKit Java +Profiler and +YourKit .NET +Profiler. \ No newline at end of file diff --git a/docs/TimeBoundaryQuery.md b/docs/TimeBoundaryQuery.md new file mode 100644 index 00000000000..bde4ca1c812 --- /dev/null +++ b/docs/TimeBoundaryQuery.md @@ -0,0 +1,29 @@ +--- +layout: default +--- +Time boundary queries return the earliest and latest data points of a data set. The grammar is: + + { + "queryType" : "timeBoundary", + "dataSource": "sample_datasource" + } + + +There are 3 main parts to a time boundary query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “timeBoundary”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +The format of the result is: + + [ { + "timestamp" : "2013-05-09T18:24:00.000Z", + "result" : { + "minTime" : "2013-05-09T18:24:00.000Z", + "maxTime" : "2013-05-09T18:37:00.000Z" + } + } ] + diff --git a/docs/TimeseriesQuery.md b/docs/TimeseriesQuery.md new file mode 100644 index 00000000000..62ebcee59f1 --- /dev/null +++ b/docs/TimeseriesQuery.md @@ -0,0 +1,118 @@ +--- +layout: default +--- +Timeseries queries +================== + +These types of queries take a timeseries query object and return an array of JSON objects where each object represents a value asked for by the timeseries query. + +An example timeseries query object is shown below: + +
+
+{
+ [queryType]() “timeseries”,
+ [dataSource]() “sample\_datasource”,
+ [granularity]() “day”,
+ [filter]() {
+ [type]() “and”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension1”,
+ [value]() “sample\_value1”
+ },
+ {
+ [type]() “or”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension2”,
+ [value]() “sample\_value2”
+ },
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension3”,
+ [value]() “sample\_value3”
+ }
+ ]
+ }
+ ]
+ },
+ [aggregations]() [
+ {
+ [type]() “longSum”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “doubleSum”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ],
+ [postAggregations]() [
+ {
+ [type]() “arithmetic”,
+ [name]() “sample\_divide”,
+ [fn]() “/”,
+ [fields]() [
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ]
+ }
+ ],
+ [intervals]() [
+ “2012-01-01T00:00:00.000/2012-01-03T00:00:00.000”
+ ]
+}
+
+
+ + +There are 7 main parts to a timeseries query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “timeseries”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|granularity|Defines the granularity of the query. See [Granularities](Granularities.html)|yes| +|filter|See [Filters](Filters.html)|no| +|aggregations|See [Aggregations](Aggregations.html)|yes| +|postAggregations|See [Post Aggregations](Post-Aggregations.html)|no| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +To pull it all together, the above query would return 2 data points, one for each day between 2012-01-01 and 2012-01-03, from the “sample\_datasource” table. Each data point would be the (long) sum of sample\_fieldName1, the (double) sum of sample\_fieldName2 and the (double) the result of sample\_fieldName1 divided by sample\_fieldName2 for the filter set. The output looks like this: + +
+
+[
+ {
+ [timestamp]() “2012-01-01T00:00:00.000Z”,
+ [result]() {
+ [sample\_name1]() ,
+ [sample\_name2]() ,
+ [sample\_divide]() 
+ }
+ },
+ {
+ [timestamp]() “2012-01-02T00:00:00.000Z”,
+ [result]() {
+ [sample\_name1]() ,
+ [sample\_name2]() ,
+ [sample\_divide]() 
+ }
+ }
+]
+
+
+ diff --git a/docs/Tutorial:-A-First-Look-at-Druid.md b/docs/Tutorial:-A-First-Look-at-Druid.md new file mode 100644 index 00000000000..987cf89fa28 --- /dev/null +++ b/docs/Tutorial:-A-First-Look-at-Druid.md @@ -0,0 +1,369 @@ +--- +layout: default +--- +Greetings! This tutorial will help clarify some core Druid concepts. We will use a realtime dataset and issue some basic Druid queries. If you are ready to explore Druid, and learn a thing or two, read on! + +About the data +-------------- + +The data source we’ll be working with is Wikipedia edits. Each time an edit is made in Wikipedia, an event gets pushed to an IRC channel associated with the language of the Wikipedia page. We scrape IRC channels for several different languages and load this data into Druid. + +Each event has a timestamp indicating the time of the edit (in UTC time), a list of dimensions indicating various metadata about the event (such as information about the user editing the page and where the user resides), and a list of metrics associated with the event (such as the number of characters added and deleted). + +Specifically. the data schema looks like so: + +Dimensions (things to filter on): +\`\`\`json +“page” +“language” +“user” +“unpatrolled” +“newPage” +“robot” +“anonymous” +“namespace” +“continent” +“country” +“region” +“city” +\`\`\` + +Metrics (things to aggregate over): +\`\`\`json +“count” +“added” +“delta” +“deleted” +\`\`\` + +These metrics track the number of characters added, deleted, and changed. + +Setting Up +---------- + +There are two ways to setup Druid: download a tarball, or [Build From Source](Build From Source.html). You only need to do one of these. + +### Download a Tarball + +We’ve built a tarball that contains everything you’ll need. You’ll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.5.54-bin.tar.gz) +Download this file to a directory of your choosing. + +You can extract the awesomeness within by issuing: + + tar -zxvf druid-services-*-bin.tar.gz + +Not too lost so far right? That’s great! If you cd into the directory: + + cd druid-services-0.5.54 + +You should see a bunch of files: +\* run\_example\_server.sh +\* run\_example\_client.sh +\* LICENSE, config, examples, lib directories + +Running Example Scripts +----------------------- + +Let’s start doing stuff. You can start a Druid [Realtime](Realtime.html) node by issuing: + + ./run_example_server.sh + +Select “wikipedia”. + +Once the node starts up you will see a bunch of logs about setting up properties and connecting to the data source. If everything was successful, you should see messages of the form shown below. + + + 2013-07-19 21:54:05,154 INFO [main] com.metamx.druid.realtime.RealtimeNode - Starting Jetty + 2013-07-19 21:54:05,154 INFO [main] org.mortbay.log - jetty-6.1.x + 2013-07-19 21:54:05,171 INFO [chief-wikipedia] com.metamx.druid.realtime.plumber.RealtimePlumberSchool - Expect to run at [2013-07-19T22:03:00.000Z] + 2013-07-19 21:54:05,246 INFO [main] org.mortbay.log - Started SelectChannelConnector@0.0.0.0:8083 + + +The Druid real time-node ingests events in an in-memory buffer. Periodically, these events will be persisted to disk. If you are interested in the details of our real-time architecture and why we persist indexes to disk, I suggest you read our [White Paper](http://static.druid.io/docs/druid.pdf). + +Okay, things are about to get real(~~time). To query the real-time node you’ve spun up, you can issue: +\./run\_example\_client.sh\ +Select “wikipedia” once again. This script issues ]s to the data we’ve been ingesting. The query looks like this: +\`\`\`json +{ + [queryType]("groupBy"), + [dataSource]("wikipedia"), + [granularity]("minute"), + [dimensions]([) + “page” + ], + [aggregations]([) + { + [type]("count"), + [name]("rows") + }, + { + [type]("longSum"), + [fieldName]("edit_count"), + [name]("count") + } + ], + [filter]({) + [type]("selector"), + [dimension]("namespace"), + [value]("article") + }, + [intervals]([) + “2013-06-01T00:00/2020-01-01T00” + ] +} +\`\`\` +This is a **groupBy** query, which you may be familiar with from SQL. We are grouping, or aggregating, via the **dimensions** field: . We are **filtering** via the **“namespace”** dimension, to only look at edits on **“articles”**. Our **aggregations** are what we are calculating: a count of the number of data rows, and a count of the number of edits that have occurred. +The result looks something like this: +\`\`\`json +[ + { + [version]() “v1”, + [timestamp]() “2013-09-04T21:44:00.000Z”, + [event]() { + [count]() 0, + [page]() “2013\\u201314\_Brentford\_F.C.*season", + [rows]() 1 + } + }, + { + [version]() "v1", + [timestamp]() "2013-09-04T21:44:00.000Z", + [event]() { + [count]() 0, + [page]() "8e*00e9tape\_du\_Tour\_de\_France\_2013”, + [rows]() 1 + } + }, + { + [version]() “v1”, + [timestamp]() “2013-09-04T21:44:00.000Z”, + [event]() { + [count]() 0, + [page]() “Agenda\_of\_the\_Tea\_Party\_movement”, + [rows]() 1 + } + }, +… +\`\`\` +This groupBy query is a bit complicated and we’ll return to it later. For the time being, just make sure you are getting some blocks of data back. If you are having problems, make sure you have [curl](http://curl.haxx.se/) installed. Control+C to break out of the client script. +h2. Querying Druid +In your favorite editor, create the file: +\time\_boundary\_query.body\ +Druid queries are JSON blobs which are relatively painless to create programmatically, but an absolute pain to write by hand. So anyway, we are going to create a Druid query by hand. Add the following to the file you just created: +\ +{ + [queryType]() “timeBoundary”, + [dataSource]() “wikipedia” +} +\ +The ] is one of the simplest Druid queries. To run the query, you can issue: +\ curl~~X POST ‘http://localhost:8083/druid/v2/?pretty’ ~~H ‘content-type: application/json’~~d ```` time_boundary_query.body + +We get something like this JSON back: + +```json +[ { + "timestamp" : "2013-09-04T21:44:00.000Z", + "result" : { + "minTime" : "2013-09-04T21:44:00.000Z", + "maxTime" : "2013-09-04T21:47:00.000Z" + } +} ] +``` +As you can probably tell, the result is indicating the maximum and minimum timestamps we've seen thus far (summarized to a minutely granularity). Let's explore a bit further. + +Return to your favorite editor and create the file: +
timeseries_query.body
+ +We are going to make a slightly more complicated query, the [TimeseriesQuery](TimeseriesQuery.html). Copy and paste the following into the file: +

+{
+    "queryType": "timeseries", 
+    "dataSource": "wikipedia", 
+    "intervals": [
+        "2010-01-01/2020-01-01"
+    ], 
+    "granularity": "all", 
+    "aggregations": [
+        {
+            "type": "longSum", 
+            "fieldName": "count",
+            "name": "edit_count"
+        }, 
+        {
+            "type": "doubleSum", 
+            "fieldName": "added", 
+            "name": "chars_added"
+        }
+    ]
+}
+
+ +You are probably wondering, what are these [Granularities](Granularities.html) and [Aggregations](Aggregations.html) things? What the query is doing is aggregating some metrics over some span of time. +To issue the query and get some results, run the following in your command line: +
curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d  ````timeseries\_query.body
+
+
+Once again, you should get a JSON blob of text back with your results, that looks something like this: + +\`\`\`json +[ { + “timestamp” : “2013-09-04T21:44:00.000Z”, + “result” : { + “chars\_added” : 312670.0, + “edit\_count” : 733 + } +} ] +\`\`\` + +If you issue the query again, you should notice your results updating. + +Right now all the results you are getting back are being aggregated into a single timestamp bucket. What if we wanted to see our aggregations on a per minute basis? What field can we change in the query to accomplish this? + +If you loudly exclaimed “we can change granularity to minute”, you are absolutely correct! We can specify different granularities to bucket our results, like so: + + + { + "queryType": "timeseries", + "dataSource": "wikipedia", + "intervals": [ + "2010-01-01/2020-01-01" + ], + "granularity": "minute", + "aggregations": [ + { + "type": "longSum", + "fieldName": "count", + "name": "edit_count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "chars_added" + } + ] + } + + +This gives us something like the following: + +\`\`\`json +[ + { + “timestamp” : “2013-09-04T21:44:00.000Z”, + “result” : { + “chars\_added” : 30665.0, + “edit\_count” : 128 + } + }, { + “timestamp” : “2013-09-04T21:45:00.000Z”, + “result” : { + “chars\_added” : 122637.0, + “edit\_count” : 167 + } + }, { + “timestamp” : “2013-09-04T21:46:00.000Z”, + “result” : { + “chars\_added” : 78938.0, + “edit\_count” : 159 + } + }, +… +\`\`\` + +Solving a Problem +----------------- + +One of Druid’s main powers is to provide answers to problems, so let’s pose a problem. What if we wanted to know what the top pages in the US are, ordered by the number of edits over the last few minutes you’ve been going through this tutorial? To solve this problem, we have to return to the query we introduced at the very beginning of this tutorial, the [GroupByQuery](GroupByQuery.html). It would be nice if we could group by results by dimension value and somehow sort those results… and it turns out we can! + +Let’s create the file: + + group_by_query.body + and put the following in there: + 

+    {
+        "queryType": "groupBy", 
+        "dataSource": "wikipedia", 
+        "granularity": "all", 
+        "dimensions": [
+            "page"
+        ], 
+        "orderBy": {
+            "type": "default", 
+            "columns": [
+                {
+                    "dimension": "edit_count", 
+                    "direction": "DESCENDING"
+                }
+            ], 
+            "limit": 10
+        }, 
+        "aggregations": [
+            {
+                "type": "longSum", 
+                "fieldName": "count", 
+                "name": "edit_count"
+            }
+        ], 
+        "filter": {
+            "type": "selector", 
+            "dimension": "country", 
+            "value": "United States"
+        }, 
+        "intervals": [
+            "2012-10-01T00:00/2020-01-01T00"
+        ]
+    }
+    
+
+Woah! Our query just got a way more complicated. Now we have these [Filters](Filters.html) things and this [OrderBy](OrderBy.html) thing. Fear not, it turns out the new objects we’ve introduced to our query can help define the format of our results and provide an answer to our question.
+
+If you issue the query:
+
+    curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d @group_by_query.body
+
+You should see an answer to our question. As an example, some results are shown below:
+
+\`\`\`json
+[
+ {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “page” : “RTC\_Transit”,
+ “edit\_count” : 6
+ }
+ }, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “page” : “List\_of\_Deadly\_Women\_episodes”,
+ “edit\_count” : 4
+ }
+ }, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “page” : “User\_talk:David\_Biddulph”,
+ “edit\_count” : 4
+ }
+ },
+…
+\`\`\`
+
+Feel free to tweak other query parameters to answer other questions you may have about the data.
+
+Next Steps
+----------
+
+What to know even more information about the Druid Cluster? Check out [Tutorial: The Druid Cluster](Tutorial:-The-Druid-Cluster.html)
+
+Druid is even more fun if you load your own data into it! To learn how to load your data, see [Loading Your Data](Loading-Your-Data.html).
+
+Additional Information
+----------------------
+
+This tutorial is merely showcasing a small fraction of what Druid can do. If you are interested in more information about Druid, including setting up a more sophisticated Druid cluster, please read the other links in our wiki.
+
+And thus concludes our journey! Hopefully you learned a thing or two about Druid real-time ingestion, querying Druid, and how Druid can be used to solve problems. If you have additional questions, feel free to post in our [google groups page](http://www.groups.google.com/forum/#!forum/druid-development).
diff --git a/docs/Tutorial:-The-Druid-Cluster.md b/docs/Tutorial:-The-Druid-Cluster.md
new file mode 100644
index 00000000000..282ec9fa7f8
--- /dev/null
+++ b/docs/Tutorial:-The-Druid-Cluster.md
@@ -0,0 +1,302 @@
+---
+layout: default
+---
+Welcome back! In our first [tutorial](https://github.com/metamx/druid/wiki/Tutorial%3A-A-First-Look-at-Druid), we introduced you to the most basic Druid setup: a single realtime node. We streamed in some data and queried it. Realtime nodes collect very recent data and periodically hand that data off to the rest of the Druid cluster. Some questions about the architecture must naturally come to mind. What does the rest of Druid cluster look like? How does Druid load available static data?
+
+This tutorial will hopefully answer these questions!
+
+In this tutorial, we will set up other types of Druid nodes as well as and external dependencies for a fully functional Druid cluster. The architecture of Druid is very much like the [Megazord](http://www.youtube.com/watch?v=7mQuHh1X4H4) from the popular 90s show Mighty Morphin' Power Rangers. Each Druid node has a specific purpose and the nodes come together to form a fully functional system.
+
+## Downloading Druid ##
+
+If you followed the first tutorial, you should already have Druid downloaded. If not, let's go back and do that first.
+
+You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.5.54-bin.tar.gz)
+
+and untar the contents within by issuing:
+```bash
+tar -zxvf druid-services-*-bin.tar.gz
+cd druid-services-*
+```
+
+You can also [Build From Source](Build-From-Source.html).
+
+## External Dependencies ##
+
+Druid requires 3 external dependencies. A "deep" storage that acts as a backup data repository, a relational database such as MySQL to hold configuration and metadata information, and [Apache Zookeeper](http://zookeeper.apache.org/) for coordination among different pieces of the cluster.
+
+For deep storage, we have made a public S3 bucket (static.druid.io) available where data for this particular tutorial can be downloaded. More on the data [later](https://github.com/metamx/druid/wiki/Tutorial-Part-2#the-data).
+
+### Setting up MySQL ###
+
+1. If you don't already have it, download MySQL Community Server here: [http://dev.mysql.com/downloads/mysql/](http://dev.mysql.com/downloads/mysql/)
+2. Install MySQL
+3. Create a druid user and database
+```bash
+mysql -u root
+```
+```sql
+GRANT ALL ON druid.* TO 'druid'@'localhost' IDENTIFIED BY 'diurd';
+CREATE database druid;
+```
+
+### Setting up Zookeeper ###
+```bash
+curl http://www.motorlogy.com/apache/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz -o zookeeper-3.4.5.tar.gz
+tar xzf zookeeper-3.4.5.tar.gz
+cd zookeeper-3.4.5
+cp conf/zoo_sample.cfg conf/zoo.cfg
+./bin/zkServer.sh start
+cd ..
+```
+
+## The Data ##
+
+Similar to the first tutorial, the data we will be loading is based on edits that have occurred on Wikipedia. Every time someone edits a page in Wikipedia, metadata is generated about the editor and edited page. Druid collects each individual event and packages them together in a container known as a [segment](https://github.com/metamx/druid/wiki/Segments). Segments contain data over some span of time. We've prebuilt a segment for this tutorial and will cover making your own segments in other [pages](https://github.com/metamx/druid/wiki/Loading-Your-Data).The segment we are going to work with has the following format:
+
+Dimensions (things to filter on):
+```json
+"page"
+"language"
+"user"
+"unpatrolled"
+"newPage"
+"robot"
+"anonymous"
+"namespace"
+"continent"
+"country"
+"region"
+"city"
+```
+
+Metrics (things to aggregate over):
+```json
+"count"
+"added"
+"delta"
+"deleted"
+```
+
+## The Cluster ##
+
+Let's start up a few nodes and download our data. First things though, let's create a config directory where we will store configs for our various nodes:
+
+```
+mkdir config
+```
+
+If you are interested in learning more about Druid configuration files, check out this [link](https://github.com/metamx/druid/wiki/Configuration). Many aspects of Druid are customizable. For the purposes of this tutorial, we are going to use default values for most things.
+
+### Start a Master Node ###
+
+Master nodes are in charge of load assignment and distribution. Master nodes monitor the status of the cluster and command compute nodes to assign and drop segments.
+
+To create the master config file:
+
+```
+mkdir config/master
+```
+
+Under the directory we just created, create the file ```runtime.properties``` with the following contents:
+
+```
+druid.host=127.0.0.1:8082
+druid.port=8082
+druid.service=master
+
+# logging
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
+# aws (demo user)
+com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ
+com.metamx.aws.secretKey=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
+
+# db
+druid.database.segmentTable=segments
+druid.database.user=druid
+druid.database.password=diurd
+druid.database.connectURI=jdbc:mysql://localhost:3306/druid
+druid.database.ruleTable=rules
+druid.database.configTable=config
+
+# master runtime configs
+druid.master.startDelay=PT60S
+```
+
+To start the master node:
+
+```bash
+java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:config/master com.metamx.druid.http.MasterMain
+```
+
+### Start a Compute Node ###
+
+Compute nodes are the workhorses of a cluster and are in charge of loading historical segments and making them available for queries. Our Wikipedia segment will be downloaded by a compute node.
+
+To create the compute config file:
+
+```
+mkdir config/compute
+```
+
+Under the directory we just created, create the file ```runtime.properties``` with the following contents:
+```
+druid.host=127.0.0.1:8081
+druid.port=8081
+druid.service=compute
+
+# logging
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
+# processing
+druid.processing.buffer.sizeBytes=10000000
+
+# aws (demo user)
+com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ
+com.metamx.aws.secretKey=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
+
+# Path on local FS for storage of segments; dir will be created if needed
+druid.paths.indexCache=/tmp/druid/indexCache
+
+# Path on local FS for storage of segment metadata; dir will be created if needed
+druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache
+
+# server
+druid.server.maxSize=100000000
+```
+
+To start the compute node:
+
+```bash
+java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:config/compute com.metamx.druid.http.ComputeMain
+```
+
+### Start a Broker Node ###
+
+Broker nodes are responsible for figuring out which compute and/or realtime nodes correspond to which queries. They also merge partial results from these nodes in a scatter/gather fashion.
+
+To create the broker config file:
+
+```
+mkdir config/broker
+```
+
+Under the directory we just created, create the file ```runtime.properties``` with the following contents:
+
+```
+druid.host=127.0.0.1:8080
+druid.port=8080
+druid.service=broker
+
+# logging
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
+# thread pool size for servicing queries
+druid.client.http.connections=10
+```
+
+To start the broker node:
+
+```bash
+java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:config/broker com.metamx.druid.http.BrokerMain
+```
+
+## Loading the Data ##
+
+The MySQL dependency we introduced earlier on contains a 'segments' table that contains entries for segments that should be loaded into our cluster. The Druid master compares this table with segments that already exist in the cluster to determine what should be loaded and dropped. To load our wikipedia segment, we need to create an entry in our MySQL segment table.
+
+Usually, when new segments are created, these MySQL entries are created directly so you never have to do this by hand. For this tutorial, we can do this manually by going back into MySQL and issuing:
+
+```
+use druid;
+```
+
+``
+INSERT INTO segments (id, dataSource, created_date, start, end, partitioned, version, used, payload) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}');
+``
+
+If you look in your master node logs, you should, after a maximum of a minute or so, see logs of the following form:
+
+```
+2013-08-08 22:48:41,967 INFO [main-EventThread] com.metamx.druid.master.LoadQueuePeon - Server[/druid/loadQueue/127.0.0.1:8081] done processing [/druid/loadQueue/127.0.0.1:8081/wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z]
+2013-08-08 22:48:41,969 INFO [ServerInventoryView-0] com.metamx.druid.client.SingleServerInventoryView - Server[127.0.0.1:8081] added segment[wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z]
+```
+
+When the segment completes downloading and ready for queries, you should see the following message on your compute node logs:
+
+```
+2013-08-08 22:48:41,959 INFO [ZkCoordinator-0] com.metamx.druid.coordination.BatchDataSegmentAnnouncer - Announcing segment[wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z] at path[/druid/segments/127.0.0.1:8081/2013-08-08T22:48:41.959Z]
+```
+
+At this point, we can query the segment. For more information on querying, see this[link](https://github.com/metamx/druid/wiki/Querying).
+
+## Next Steps ##
+
+Now that you have an understanding of what the Druid clsuter looks like, why not load some of your own data?
+Check out the [Loading Your Own Data](https://github.com/metamx/druid/wiki/Loading-Your-Data) section for more info!
\ No newline at end of file
diff --git a/docs/Tutorial:-Webstream.md b/docs/Tutorial:-Webstream.md
new file mode 100644
index 00000000000..bfb7ed73bed
--- /dev/null
+++ b/docs/Tutorial:-Webstream.md
@@ -0,0 +1,357 @@
+---
+layout: default
+---
+Greetings! This tutorial will help clarify some core Druid concepts. We will use a realtime dataset and issue some basic Druid queries. If you are ready to explore Druid, and learn a thing or two, read on!
+
+About the data
+--------------
+
+The data source we’ll be working with is the Bit.ly USA Government website statistics stream. You can see the stream [here](http://developer.usa.gov/1usagov), and read about the stream [here](http://www.usa.gov/About/developer-resources/1usagov.shtml) . This is a feed of json data that gets updated whenever anyone clicks a bit.ly shortened USA.gov website. A typical event might look something like this:
+\`\`\`json
+{
+ [user\_agent]() “Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)”,
+ [country]() “US”,
+ [known\_user]() 1,
+ [timezone]() “America/New\_York”,
+ [geo\_region]() “DC”,
+ [global\_bitly\_hash]() “17ctAFs”,
+ [encoding\_user\_bitly\_hash]() “17ctAFr”,
+ [encoding\_user\_login]() “senrubiopress”,
+ [aaccept\_language]() “en-US”,
+ [short\_url\_cname]() “1.usa.gov”,
+ [referring\_url]() “http://t.co/4Av4NUFAYq”,
+ [long\_url]() “http://www.rubio.senate.gov/public/index.cfm/fighting-for-florida?ID=c8357d12-9da8-4e9d-b00d-7168e1bf3599”,
+ [timestamp]() 1372190407,
+ [timestamp of time hash was created]() 1372190097,
+ [city]() “Washington”,
+ [latitude\_longitude]() [
+ 38.893299,
+ ~~77.014603
+ ]
+}
+\`\`\`
+The “known\_user” field is always 1 or 0. It is 1 if the user is known to the server, and 0 otherwise. We will use this field extensively in this demo.
+h2. Setting Up
+There are two ways to setup Druid: download a tarball, or ]. You only need to do one of these.
+h3. Download a Tarball
+We’ve built a tarball that contains everything you’ll need. You’ll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.5.50-bin.tar.gz)
+Download this file to a directory of your choosing.
+You can extract the awesomeness within by issuing:
+\tar~~zxvf druid-services~~**~~bin.tar.gz\
+Not too lost so far right? That’s great! If you cd into the directory:
+\cd druid-services-0.5.50\
+You should see a bunch of files:
+\* run\_example\_server.sh
+\* run\_example\_client.sh
+\* LICENSE, config, examples, lib directories
+
+h2. Running Example Scripts
+Let’s start doing stuff. You can start a Druid ] node by issuing:
+\./run\_example\_server.sh\
+Select “webstream”.
+Once the node starts up you will see a bunch of logs about setting up properties and connecting to the data source. If everything was successful, you should see messages of the form shown below.
+\
+2013-07-19 21:54:05,154 INFO com.metamx.druid.realtime.RealtimeNode~~ Starting Jetty
+2013-07-19 21:54:05,154 INFO org.mortbay.log - jetty-6.1.x
+2013-07-19 21:54:05,171 INFO com.metamx.druid.realtime.plumber.RealtimePlumberSchool - Expect to run at
+2013-07-19 21:54:05,246 INFO org.mortbay.log - Started SelectChannelConnector@0.0.0.0:8083
+\
+The Druid real time-node ingests events in an in-memory buffer. Periodically, these events will be persisted to disk. If you are interested in the details of our real-time architecture and why we persist indexes to disk, I suggest you read our [White Paper](http://static.druid.io/docs/druid.pdf).
+Okay, things are about to get real. To query the real-time node you’ve spun up, you can issue:
+\./run\_example\_client.sh\
+Select “webstream” once again. This script issues ]s to the data we’ve been ingesting. The query looks like this:
+\`\`\`json
+{
+ [queryType]() “groupBy”,
+ [dataSource]() “webstream”,
+ [granularity]() “minute”,
+ [dimensions]() [
+ “timezone”
+ ],
+ [aggregations]() [
+ {
+ [type]() “count”,
+ [name]() “rows”
+ },
+ {
+ [type]() “doubleSum”,
+ [fieldName]() “known\_users”,
+ [name]() “known\_users”
+ }
+ ],
+ [filter]() {
+ [type]() “selector”,
+ [dimension]() “country”,
+ [value]() “US”
+ },
+ [intervals]() [
+ “2013-06-01T00:00/2020-01-01T00”
+ ]
+}
+\`\`\`
+This is a****groupBy**\* query, which you may be familiar with from SQL. We are grouping, or aggregating, via the **dimensions** field: . We are **filtering** via the **“country”** dimension, to only look at website hits in the US. Our **aggregations** are what we are calculating: a row count, and the sum of the number of known users in our data.
+The result looks something like this:
+\`\`\`json
+[
+ {
+ [version]() “v1”,
+ [timestamp]() “2013-07-18T19:39:00.000Z”,
+ [event]() {
+ [timezone]() “America/Chicago”,
+ [known\_users]() 10,
+ [rows]() 15
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2013-07-18T19:39:00.000Z”,
+ [event]() {
+ [timezone]() “America/Los\_Angeles”,
+ [known\_users]() 0,
+ [rows]() 3
+ }
+ },
+…
+\`\`\`
+This groupBy query is a bit complicated and we’ll return to it later. For the time being, just make sure you are getting some blocks of data back. If you are having problems, make sure you have [curl](http://curl.haxx.se/) installed. Control+C to break out of the client script.
+h2. Querying Druid
+In your favorite editor, create the file:
+\time\_boundary\_query.body\
+Druid queries are JSON blobs which are relatively painless to create programmatically, but an absolute pain to write by hand. So anyway, we are going to create a Druid query by hand. Add the following to the file you just created:
+\
+{
+ [queryType]() “timeBoundary”,
+ [dataSource]() “webstream”
+}
+\
+The ] is one of the simplest Druid queries. To run the query, you can issue:
+\ curl~~X POST ‘http://localhost:8083/druid/v2/?pretty’ ~~H ‘content-type: application/json’~~d ```` time_boundary_query.body
+ +We get something like this JSON back: + +```json +[ + { + "timestamp": "2013-07-18T19:39:00.000Z", + "result": { + "minTime": "2013-07-18T19:39:00.000Z", + "maxTime": "2013-07-18T19:46:00.000Z" + } + } +] +``` +As you can probably tell, the result is indicating the maximum and minimum timestamps we've seen thus far (summarized to a minutely granularity). Let's explore a bit further. + +Return to your favorite editor and create the file: +
timeseries_query.body
+ +We are going to make a slightly more complicated query, the [TimeseriesQuery](TimeseriesQuery.html). Copy and paste the following into the file: +

+{
+    "queryType": "timeseries", 
+    "dataSource": "webstream", 
+    "intervals": [
+        "2010-01-01/2020-01-01"
+    ], 
+    "granularity": "all", 
+    "aggregations": [
+        {
+            "type": "count", 
+            "name": "rows"
+        }, 
+        {
+            "type": "doubleSum", 
+            "fieldName": "known_users", 
+            "name": "known_users"
+        }
+    ]
+}
+
+ +You are probably wondering, what are these [Granularities](Granularities.html) and [Aggregations](Aggregations.html) things? What the query is doing is aggregating some metrics over some span of time. +To issue the query and get some results, run the following in your command line: +
curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d  ````timeseries\_query.body
+
+
+Once again, you should get a JSON blob of text back with your results, that looks something like this: + +\`\`\`json +[ +{ + “timestamp” : “2013-07-18T19:39:00.000Z”, + “result” : { + “known\_users” : 787.0, + “rows” : 2004 + } +} +] +\`\`\` + +If you issue the query again, you should notice your results updating. + +Right now all the results you are getting back are being aggregated into a single timestamp bucket. What if we wanted to see our aggregations on a per minute basis? What field can we change in the query to accomplish this? + +If you loudly exclaimed “we can change granularity to minute”, you are absolutely correct! We can specify different granularities to bucket our results, like so: + + + { + "queryType": "timeseries", + "dataSource": "webstream", + "intervals": [ + "2010-01-01/2020-01-01" + ], + "granularity": "minute", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "doubleSum", + "fieldName": "known_users", + "name": "known_users" + } + ] + } + + +This gives us something like the following: + +\`\`\`json +[ + { + [timestamp]() “2013-07-18T19:39:00.000Z”, + [result]() { + [known\_users]() 33, + [rows]() 76 + } + }, + { + [timestamp]() “2013-07-18T19:40:00.000Z”, + [result]() { + [known\_users]() 105, + [rows]() 221 + } + }, + { + [timestamp]() “2013-07-18T19:41:00.000Z”, + [result]() { + [known\_users]() 53, + [rows]() 167 + } + }, +… +\`\`\` + +Solving a Problem +----------------- + +One of Druid’s main powers is to provide answers to problems, so let’s pose a problem. What if we wanted to know what the top states in the US are, ordered by the number of visits by known users over the last few minutes? To solve this problem, we have to return to the query we introduced at the very beginning of this tutorial, the [GroupByQuery](GroupByQuery.html). It would be nice if we could group by results by dimension value and somehow sort those results… and it turns out we can! + +Let’s create the file: + + group_by_query.body + and put the following in there: + 

+    {
+        "queryType": "groupBy", 
+        "dataSource": "webstream", 
+        "granularity": "all", 
+        "dimensions": [
+            "geo_region"
+        ], 
+        "orderBy": {
+            "type": "default", 
+            "columns": [
+                {
+                    "dimension": "known_users", 
+                    "direction": "DESCENDING"
+                }
+            ], 
+            "limit": 10
+        }, 
+        "aggregations": [
+            {
+                "type": "count", 
+                "name": "rows"
+            }, 
+            {
+                "type": "doubleSum", 
+                "fieldName": "known_users", 
+                "name": "known_users"
+            }
+        ], 
+        "filter": {
+            "type": "selector", 
+            "dimension": "country", 
+            "value": "US"
+        }, 
+        "intervals": [
+            "2012-10-01T00:00/2020-01-01T00"
+        ]
+    }
+    
+
+Woah! Our query just got a way more complicated. Now we have these [Filters](Filters.html) things and this [OrderBy](OrderBy.html) thing. Fear not, it turns out the new objects we’ve introduced to our query can help define the format of our results and provide an answer to our question.
+
+If you issue the query:
+
+    curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d @group_by_query.body
+
+You should see an answer to our question. For my stream, it looks like this:
+
+\`\`\`json
+[
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “RI”,
+ [known\_users]() 359,
+ [rows]() 143
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “NY”,
+ [known\_users]() 187,
+ [rows]() 322
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “CA”,
+ [known\_users]() 145,
+ [rows]() 466
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “IL”,
+ [known\_users]() 121,
+ [rows]() 185
+ }
+ },
+…
+\`\`\`
+
+Feel free to tweak other query parameters to answer other questions you may have about the data.
+
+Next Steps
+----------
+
+What to know even more information about the Druid Cluster? Check out [Tutorial: The Druid Cluster](Tutorial:-The-Druid-Cluster.html)
+Druid is even more fun if you load your own data into it! To learn how to load your data, see [Loading Your Data](Loading-Your-Data.html).
+
+Additional Information
+----------------------
+
+This tutorial is merely showcasing a small fraction of what Druid can do. If you are interested in more information about Druid, including setting up a more sophisticated Druid cluster, please read the other links in our wiki.
+
+And thus concludes our journey! Hopefully you learned a thing or two about Druid real-time ingestion, querying Druid, and how Druid can be used to solve problems. If you have additional questions, feel free to post in our [google groups page](http://www.groups.google.com/forum/#!forum/druid-development).
diff --git a/docs/Twitter-Tutorial.md b/docs/Twitter-Tutorial.md
new file mode 100644
index 00000000000..dc2151ec3e4
--- /dev/null
+++ b/docs/Twitter-Tutorial.md
@@ -0,0 +1,329 @@
+---
+layout: default
+---
+Greetings! We see you’ve taken an interest in Druid. That’s awesome! Hopefully this tutorial will help clarify some core Druid concepts. We will go through one of the Real-time [Examples](Examples.html), and issue some basic Druid queries. The data source we’ll be working with is the [Twitter spritzer stream](https://dev.twitter.com/docs/streaming-apis/streams/public). If you are ready to explore Druid, brave its challenges, and maybe learn a thing or two, read on!
+
+Setting Up
+----------
+
+There are two ways to setup Druid: download a tarball, or build it from source.
+
+### Download a Tarball
+
+We’ve built a tarball that contains everything you’ll need. You’ll find it [here](http://static.druid.io/data/examples/druid-services-0.4.6.tar.gz).
+Download this bad boy to a directory of your choosing.
+
+You can extract the awesomeness within by issuing:
+
+    tar -zxvf druid-services-0.4.6.tar.gz
+
+Not too lost so far right? That’s great! If you cd into the directory:
+
+    cd druid-services-0.4.6-SNAPSHOT
+
+You should see a bunch of files:
+\* run\_example\_server.sh
+\* run\_example\_client.sh
+\* LICENSE, config, examples, lib directories
+
+### Clone and Build from Source
+
+The other way to setup Druid is from source via git. To do so, run these commands:
+
+\`\`\`
+git clone git@github.com:metamx/druid.git
+cd druid
+git checkout druid-0.4.32-branch
+./build.sh
+\`\`\`
+
+You should see a bunch of files:
+
+\`\`\`
+DruidCorporateCLA.pdf README common examples indexer pom.xml server
+DruidIndividualCLA.pdf build.sh doc group\_by.body install publications services
+LICENSE client eclipse\_formatting.xml index-common merger realtime
+\`\`\`
+
+You can find the example executables in the examples/bin directory:
+\* run\_example\_server.sh
+\* run\_example\_client.sh
+
+Running Example Scripts
+-----------------------
+
+Let’s start doing stuff. You can start a Druid [Realtime](Realtime.html) node by issuing:
+
+    ./run_example_server.sh
+
+Select “twitter”.
+
+You’ll need to register a new application with the twitter API, which only takes a minute. Go to [https://twitter.com/oauth\_clients/new](https://twitter.com/oauth_clients/new) and fill out the form and submit. Don’t worry, the home page and callback url can be anything. This will generate keys for the Twitter example application. Take note of the values for consumer key/secret and access token/secret.
+
+Enter your credentials when prompted.
+
+Once the node starts up you will see a bunch of logs about setting up properties and connecting to the data source. If everything was successful, you should see messages of the form shown below. If you see crazy exceptions, you probably typed in your login information incorrectly.
+
+    
+    2013-05-17 23:04:40,934 INFO [main] org.mortbay.log - Started SelectChannelConnector@0.0.0.0:8080
+    2013-05-17 23:04:40,935 INFO [main] com.metamx.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking start method[public void com.metamx.druid.http.FileRequestLogger.start()] on object[com.metamx.druid.http.FileRequestLogger@42bb0406].
+    2013-05-17 23:04:41,578 INFO [Twitter Stream consumer-1[Establishing connection]] twitter4j.TwitterStreamImpl - Connection established.
+    2013-05-17 23:04:41,578 INFO [Twitter Stream consumer-1[Establishing connection]] druid.examples.twitter.TwitterSpritzerFirehoseFactory - Connected_to_Twitter
+    2013-05-17 23:04:41,578 INFO [Twitter Stream consumer-1[Establishing connection]] twitter4j.TwitterStreamImpl - Receiving status stream.
+    
+
+Periodically, you’ll also see messages of the form:
+
+    
+    2013-05-17 23:04:59,793 INFO [chief-twitterstream] druid.examples.twitter.TwitterSpritzerFirehoseFactory - nextRow() has returned 1,000 InputRows
+    
+
+These messages indicate you are ingesting events. The Druid real time-node ingests events in an in-memory buffer. Periodically, these events will be persisted to disk. Persisting to disk generates a whole bunch of logs:
+
+    
+    2013-05-17 23:06:40,918 INFO [chief-twitterstream] com.metamx.druid.realtime.plumber.RealtimePlumberSchool - Submitting persist runnable for dataSource[twitterstream]
+    2013-05-17 23:06:40,920 INFO [twitterstream-incremental-persist] com.metamx.druid.realtime.plumber.RealtimePlumberSchool - DataSource[twitterstream], Interval[2013-05-17T23:00:00.000Z/2013-05-18T00:00:00.000Z], persisting Hydrant[FireHydrant{index=com.metamx.druid.index.v1.IncrementalIndex@126212dd, queryable=com.metamx.druid.index.IncrementalIndexSegment@64c47498, count=0}]
+    2013-05-17 23:06:40,937 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting persist for interval[2013-05-17T23:00:00.000Z/2013-05-17T23:07:00.000Z], rows[4,666]
+    2013-05-17 23:06:41,039 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed index.drd in 11 millis.
+    2013-05-17 23:06:41,070 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed dim conversions in 31 millis.
+    2013-05-17 23:06:41,275 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.CompressedPools - Allocating new chunkEncoder[1]
+    2013-05-17 23:06:41,332 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed walk through of 4,666 rows in 262 millis.
+    2013-05-17 23:06:41,334 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting dimension[htags] with cardinality[634]
+    2013-05-17 23:06:41,381 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Completed dimension[htags] in 49 millis.
+    2013-05-17 23:06:41,382 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting dimension[lang] with cardinality[19]
+    2013-05-17 23:06:41,398 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Completed dimension[lang] in 17 millis.
+    2013-05-17 23:06:41,398 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting dimension[utc_offset] with cardinality[32]
+    2013-05-17 23:06:41,413 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Completed dimension[utc_offset] in 15 millis.
+    2013-05-17 23:06:41,413 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed inverted.drd in 81 millis.
+    2013-05-17 23:06:41,425 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexIO$DefaultIndexIOHandler - Converting v8[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] to v9[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0]
+    2013-05-17 23:06:41,426 INFO [twitterstream-incremental-persist] 
+    ... ETC
+    
+
+The logs are about building different columns, probably not the most exciting stuff (they might as well be in Vulcan) if are you learning about Druid for the first time. Nevertheless, if you are interested in the details of our real-time architecture and why we persist indexes to disk, I suggest you read our [White Paper](http://static.druid.io/docs/druid.pdf).
+
+Okay, things are about to get real (~~time). To query the real-time node you’ve spun up, you can issue:
+\./run\_example\_client.sh\
+Select “twitter” once again. This script issues ]s to the twitter data we’ve been ingesting. The query looks like this:
+\`\`\`json
+{
+ [queryType]() “groupBy”,
+ [dataSource]() “twitterstream”,
+ [granularity]() “all”,
+ [dimensions]() ,
+ [aggregations]([)
+ { [type]() “count”, [name]() “rows”},
+ { [type]() “doubleSum”, [fieldName]() “tweets”, [name]() “tweets”}
+ ],
+ [filter]() { [type]() “selector”, [dimension]() “lang”, [value]() “en” },
+ [intervals](["2012-10-01T00:00/2020-01-01T00"])
+}
+\`\`\`
+This is a **groupBy** query, which you may be familiar with from SQL. We are grouping, or aggregating, via the **dimensions** field: . We are **filtering** via the **“lang”** dimension, to only look at english tweets. Our **aggregations** are what we are calculating: a row count, and the sum of the tweets in our data.
+The result looks something like this:
+\`\`\`json
+[
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [utc\_offset]() “~~10800",
+ [tweets]() 90,
+ [lang]() "en",
+ [rows]() 81
+ }
+ },
+ {
+ [version]() "v1",
+ [timestamp]() "2012-10-01T00:00:00.000Z",
+ [event]() {
+ [utc\_offset]() "~~14400”,
+ [tweets]() 177,
+ [lang]() “en”,
+ [rows]() 154
+ }
+ },
+…
+\`\`\`
+This data, plotted in a time series/distribution, looks something like this:
+![Timezone / Tweets Scatter Plot](http://metamarkets.com/wp-content/uploads/2013/06/tweets_timezone_offset.png "Timezone / Tweets Scatter Plot")
+This groupBy query is a bit complicated and we’ll return to it later. For the time being, just make sure you are getting some blocks of data back. If you are having problems, make sure you have [curl](http://curl.haxx.se/) installed. Control+C to break out of the client script.
+h2. Querying Druid
+In your favorite editor, create the file:
+\time\_boundary\_query.body\
+Druid queries are JSON blobs which are relatively painless to create programmatically, but an absolute pain to write by hand. So anyway, we are going to create a Druid query by hand. Add the following to the file you just created:
+\
+\
+The ] is one of the simplest Druid queries. To run the query, you can issue:
+\ curl~~X POST ‘http://localhost:8080/druid/v2/?pretty’ ~~H ‘content-type: application/json’~~d ```` time_boundary_query.body
+ +We get something like this JSON back: + +```json +[ { + "timestamp" : "2013-06-10T19:09:00.000Z", + "result" : { + "minTime" : "2013-06-10T19:09:00.000Z", + "maxTime" : "2013-06-10T20:50:00.000Z" + } +} ] +``` +That's the result. What information do you think the result is conveying? +... +If you said the result is indicating the maximum and minimum timestamps we've seen thus far (summarized to a minutely granularity), you are absolutely correct. I can see you are a person legitimately interested in learning about Druid. Let's explore a bit further. + +Return to your favorite editor and create the file: +
timeseries_query.body
+ +We are going to make a slightly more complicated query, the [TimeseriesQuery](TimeseriesQuery.html). Copy and paste the following into the file: +
{
+  "queryType":"timeseries",
+  "dataSource":"twitterstream",
+  "intervals":["2010-01-01/2020-01-01"],
+  "granularity":"all",
+  "aggregations":[
+      { "type": "count", "name": "rows"},
+      { "type": "doubleSum", "fieldName": "tweets", "name": "tweets"}
+  ]
+}
+
+ +You are probably wondering, what are these [Granularities](Granularities.html) and [Aggregations](Aggregations.html) things? What the query is doing is aggregating some metrics over some span of time. +To issue the query and get some results, run the following in your command line: +
curl -X POST 'http://localhost:8080/druid/v2/?pretty' -H 'content-type: application/json'  -d  ````timeseries\_query.body
+
+
+Once again, you should get a JSON blob of text back with your results, that looks something like this: + +\`\`\`json +[ { + “timestamp” : “2013-06-10T19:09:00.000Z”, + “result” : { + “tweets” : 358562.0, + “rows” : 272271 + } +} ] +\`\`\` + +If you issue the query again, you should notice your results updating. + +Right now all the results you are getting back are being aggregated into a single timestamp bucket. What if we wanted to see our aggregations on a per minute basis? What field can we change in the query to accomplish this? + +If you loudly exclaimed “we can change granularity to minute”, you are absolutely correct again! We can specify different granularities to bucket our results, like so: + +\`\`\`json +{ + [queryType]("timeseries"), + [dataSource]("twitterstream"), + [intervals](["2010-01-01/2020-01-01"]), + [granularity]("minute"), + [aggregations]([) + { [type]() “count”, [name]() “rows”}, + { [type]() “doubleSum”, [fieldName]() “tweets”, [name]() “tweets”} + ] +} +\`\`\` + +This gives us something like the following: + +\`\`\`json +[ { + “timestamp” : “2013-06-10T19:09:00.000Z”, + “result” : { + “tweets” : 2650.0, + “rows” : 2120 + } +}, { + “timestamp” : “2013-06-10T19:10:00.000Z”, + “result” : { + “tweets” : 3401.0, + “rows” : 2609 + } +}, { + “timestamp” : “2013-06-10T19:11:00.000Z”, + “result” : { + “tweets” : 3472.0, + “rows” : 2610 + } +}, +… +\`\`\` + +Solving a Problem +----------------- + +One of Druid’s main powers (see what we did there?) is to provide answers to problems, so let’s pose a problem. What if we wanted to know what the top hash tags are, ordered by the number tweets, where the language is english, over the last few minutes you’ve been reading this tutorial? To solve this problem, we have to return to the query we introduced at the very beginning of this tutorial, the [GroupByQuery](GroupByQuery.html). It would be nice if we could group by results by dimension value and somehow sort those results… and it turns out we can! + +Let’s create the file: + + group_by_query.body + and put the following in there: + 
{
+        "queryType": "groupBy",
+        "dataSource": "twitterstream",
+        "granularity": "all",
+        "dimensions": ["htags"],
+        "orderBy": {"type":"default", "columns":[{"dimension": "tweets", "direction":"DESCENDING"}], "limit":5},
+        "aggregations":[
+          { "type": "longSum", "fieldName": "tweets", "name": "tweets"}
+        ],
+        "filter": {"type": "selector", "dimension": "lang", "value": "en" },
+        "intervals":["2012-10-01T00:00/2020-01-01T00"]
+    }
+    
+
+Woah! Our query just got a way more complicated. Now we have these [Filters](Filters.html) things and this [OrderBy](OrderBy.html) thing. Fear not, it turns out the new objects we’ve introduced to our query can help define the format of our results and provide an answer to our question.
+
+If you issue the query:
+
+    curl -X POST 'http://localhost:8080/druid/v2/?pretty' -H 'content-type: application/json'  -d @group_by_query.body
+
+You should hopefully see an answer to our question. For my twitter stream, it looks like this:
+
+\`\`\`json
+[ {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 2660,
+ “htags” : “android”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1944,
+ “htags” : “E3”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1927,
+ “htags” : “15SueñosPendientes”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1717,
+ “htags” : “ipad”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1515,
+ “htags” : “IDidntTextYouBackBecause”
+ }
+} ]
+\`\`\`
+
+Feel free to tweak other query parameters to answer other questions you may have about the data.
+
+Additional Information
+----------------------
+
+This tutorial is merely showcasing a small fraction of what Druid can do. Next, continue on to [Loading Your Data](Loading Your Data.html).
+
+And thus concludes our journey! Hopefully you learned a thing or two about Druid real-time ingestion, querying Druid, and how Druid can be used to solve problems. If you have additional questions, feel free to post in our [google groups page](http://www.groups.google.com/forum/#!forum/druid-development).
diff --git a/docs/Versioning.md b/docs/Versioning.md
new file mode 100644
index 00000000000..6b9e79fe9d3
--- /dev/null
+++ b/docs/Versioning.md
@@ -0,0 +1,24 @@
+---
+layout: default
+---
+This page discusses how we do versioning and provides information on our stable releases.
+
+Versioning Strategy
+-------------------
+
+We generally follow [semantic versioning](http://semver.org/). The general idea is
+
+-   “Major” version (leftmost): backwards incompatible, no guarantees exist about APIs between the versions
+-   “Minor” version (middle number): you can move forward from a smaller number to a larger number, but moving backwards *might* be incompatible.
+-   “bug-fix” version (“patch” or the rightmost): Interchangeable. The higher the number, the more things are fixed (hopefully), but the programming interfaces are completely compatible and you should be able to just drop in a new jar and have it work.
+
+Note that this is defined in terms of programming API, **not** in terms of functionality. It is possible that a brand new awesome way of doing something is introduced in a “bug-fix” release version if it doesn’t add to the public API or change it.
+
+One exception for right now, while we are still in major version 0, we are considering the APIs to be in beta and are conflating “major” and “minor” so a minor version increase could be backwards incompatible for as long as we are at major version 0. These will be communicated via email on the group.
+
+For external deployments, we recommend running the stable release tag. Releases are considered stable after we have deployed them into our production environment and they have operated bug-free for some time.
+
+Tagging strategy
+----------------
+
+Tags of the codebase are equivalent to release candidates. We tag the code every time we want to take it through our release process, which includes some QA cycles and deployments. So, it is not safe to assume that a tag is a stable release, it is a solidification of the code as it goes through our production QA cycle and deployment. Tags will never change, but we often go through a number of iterations of tags before actually getting a stable release onto production. So, it is recommended that if you are not aware of what is on a tag, to stick to the stable releases listed on the [Download](Download.html) page.
diff --git a/docs/ZooKeeper.md b/docs/ZooKeeper.md
new file mode 100644
index 00000000000..d3e24e29ceb
--- /dev/null
+++ b/docs/ZooKeeper.md
@@ -0,0 +1,66 @@
+---
+layout: default
+---
+Druid uses ZooKeeper (ZK) for management of current cluster state. The operations that happen over ZK are
+
+1.  [Master](Master.html) leader election
+2.  Segment “publishing” protocol from [Compute](Compute.html) and [Realtime](Realtime.html)
+3.  Segment load/drop protocol between [Master](Master.html) and [Compute](Compute.html)
+
+### Property Configuration
+
+ZooKeeper paths are set via the `runtime.properties` configuration file. Druid will automatically create paths that do not exist, so typos in config files is a very easy way to become split-brained.
+
+There is a prefix path that is required and can be used as the only (well, kinda, see the note below) path-related zookeeper configuration parameter (everything else will be a default based on the prefix):
+
+    druid.zk.paths.base
+
+You can also override each individual path (defaults are shown below):
+
+    druid.zk.paths.propertiesPath=${druid.zk.paths.base}/properties
+    druid.zk.paths.announcementsPath=${druid.zk.paths.base}/announcements
+    druid.zk.paths.servedSegmentsPath=${druid.zk.paths.base}/servedSegments
+    druid.zk.paths.loadQueuePath=${druid.zk.paths.base}/loadQueue
+    druid.zk.paths.masterPath=${druid.zk.paths.base}/master
+    druid.zk.paths.indexer.announcementsPath=${druid.zk.paths.base}/indexer/announcements
+    druid.zk.paths.indexer.tasksPath=${druid.zk.paths.base}/indexer/tasks
+    druid.zk.paths.indexer.statusPath=${druid.zk.paths.base}/indexer/status
+    druid.zk.paths.indexer.leaderLatchPath=${druid.zk.paths.base}/indexer/leaderLatchPath
+
+NOTE: We also use Curator’s service discovery module to expose some services via zookeeper. This also uses a zookeeper path, but this path is **not** affected by `druid.zk.paths.base` and **must** be specified separately. This property is
+
+    druid.zk.paths.discoveryPath
+
+### Master Leader Election
+
+We use the Curator LeadershipLatch recipe to do leader election at path
+
+    ${druid.zk.paths.masterPath}/_MASTER
+
+### Segment “publishing” protocol from Compute and Realtime
+
+The `announcementsPath` and `servedSegmentsPath` are used for this.
+
+All [Compute](Compute.html) and [Realtime](Realtime.html) nodes publish themselves on the `announcementsPath`, specifically, they will create an ephemeral znode at
+
+    ${druid.zk.paths.announcementsPath}/${druid.host}
+
+Which signifies that they exist. They will also subsequently create a permanent znode at
+
+    ${druid.zk.paths.servedSegmentsPath}/${druid.host}
+
+And as they load up segments, they will attach ephemeral znodes that look like
+
+    ${druid.zk.paths.servedSegmentsPath}/${druid.host}/_segment_identifier_
+
+Nodes like the [Master](Master.html) and [Broker](Broker.html) can then watch these paths to see which nodes are currently serving which segments.
+
+### Segment load/drop protocol between Master and Compute
+
+The `loadQueuePath` is used for this.
+
+When the [Master](Master.html) decides that a [Compute](Compute.html) node should load or drop a segment, it writes an ephemeral znode to
+
+    ${druid.zk.paths.loadQueuePath}/_host_of_compute_node/_segment_identifier
+
+This node will contain a payload that indicates to the Compute node what it should do with the given segment. When the Compute node is done with the work, it will delete the znode in order to signify to the Master that it is complete.
diff --git a/docs/_config.yml b/docs/_config.yml
new file mode 100644
index 00000000000..6609d8b3dea
--- /dev/null
+++ b/docs/_config.yml
@@ -0,0 +1,5 @@
+name: Your New Jekyll Site
+pygments: true
+markdown: redcarpet
+redcarpet:
+  extensions: ["no_intra_emphasis", "fenced_code_blocks", "autolink", "tables", "with_toc_data"]
diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html
new file mode 100644
index 00000000000..12106274e0f
--- /dev/null
+++ b/docs/_layouts/default.html
@@ -0,0 +1,147 @@
+
+
+  	
+    	
+    Druid | {{page.title}}	
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  
+  	
+    

+      
+      

+        

+          

+            

+              

+                
+                Home
+                

+                  
+                

+              

+            

+          

+        

+        {% if page.id == 'home' %}
+        
Druid is open-source infrastructure for real²time exploratory analytics on large datasets.

+        
+        {% endif %}
+      

+      
+      

+
+        {{ content }}
+
+      

+    

+    
+    
+    
+  
+
+
diff --git a/docs/_layouts/docs.html b/docs/_layouts/docs.html
new file mode 100644
index 00000000000..cb38f8c336e
--- /dev/null
+++ b/docs/_layouts/docs.html
@@ -0,0 +1,8 @@
+---
+layout: default
+---
+  

+
+    {{ content }}
+      
+  

diff --git a/docs/_layouts/page.html b/docs/_layouts/page.html
new file mode 100644
index 00000000000..b5ebfb17ace
--- /dev/null
+++ b/docs/_layouts/page.html
@@ -0,0 +1,11 @@
+---
+layout: default
+---
+
+

+  

+
+    {{ content }}
+    
+  

+

diff --git a/docs/_layouts/post.html b/docs/_layouts/post.html
new file mode 100644
index 00000000000..affb233fd23
--- /dev/null
+++ b/docs/_layouts/post.html
@@ -0,0 +1,44 @@
+---
+layout: default
+sectionid: blog
+---
+
+

+  

+    
Recent posts

+    
+  

+
+  

+    

+      

+        {{ page.title }}
+        {{ page.date | date: "%B %e, %Y" }} · {{ page.author | upcase }}
+      

+
+      {% if page.image %}{{ page.title }}{% endif %}
+
+      {{ content }}
+
+    

+    
+    
+    comments powered by Disqus
+    
+    

+  

+

diff --git a/docs/_posts/2013-09-16-welcome-to-jekyll.markdown b/docs/_posts/2013-09-16-welcome-to-jekyll.markdown
new file mode 100644
index 00000000000..c50a740d847
--- /dev/null
+++ b/docs/_posts/2013-09-16-welcome-to-jekyll.markdown
@@ -0,0 +1,24 @@
+---
+layout: post
+title:  "Welcome to Jekyll!"
+date:   2013-09-16 13:06:49
+categories: jekyll update
+---
+
+You'll find this post in your `_posts` directory - edit this post and re-build (or run with the `-w` switch) to see your changes!
+To add new posts, simply add a file in the `_posts` directory that follows the convention: YYYY-MM-DD-name-of-post.ext.
+
+Jekyll also offers powerful support for code snippets:
+
+{% highlight ruby %}
+def print_hi(name)
+  puts "Hi, #{name}"
+end
+print_hi('Tom')
+#=> prints 'Hi, Tom' to STDOUT.
+{% endhighlight %}
+
+Check out the [Jekyll docs][jekyll] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll's GitHub repo][jekyll-gh].
+
+[jekyll-gh]: https://github.com/mojombo/jekyll
+[jekyll]:    http://jekyllrb.com
diff --git a/docs/contents.md b/docs/contents.md
new file mode 100644
index 00000000000..963f88926e1
--- /dev/null
+++ b/docs/contents.md
@@ -0,0 +1,71 @@
+---
+layout: default
+---
+Contents
+\* [Introduction|Home](Introduction|Home.html)
+\* [Download](Download.html)
+\* [Support](Support.html)
+\* [Contribute](Contribute.html)
+========================
+
+Getting Started
+\* [Tutorial: A First Look at Druid](Tutorial:-A-First-Look-at-Druid.html)
+\* [Tutorial: The Druid Cluster](Tutorial:-The-Druid-Cluster.html)
+\* [Loading Your Data](Loading-Your-Data.html)
+\* [Querying Your Data](Querying-Your-Data.html)
+\* [Booting a Production Cluster](Booting-a-Production-Cluster.html)
+\* [Examples](Examples.html)
+\* [Cluster Setup](Cluster-Setup.html)
+\* [Configuration](Configuration.html)
+--------------------------------------
+
+Data Ingestion
+\* [Realtime](Realtime.html)
+\* [Batch|Batch Ingestion](Batch|Batch-Ingestion.html)
+\* [Indexing Service](Indexing-Service.html)
+----------------------------
+
+Querying
+\* [Querying](Querying.html)
+**\* ]
+**\* [Aggregations](Aggregations.html)
+**\* ]
+**\* [Granularities](Granularities.html)
+\* Query Types
+**\* ]
+****\* ]
+****\* ]
+**\* [SearchQuery](SearchQuery.html)
+**\* ]
+** [SegmentMetadataQuery](SegmentMetadataQuery.html)
+**\* ]
+**\* [TimeseriesQuery](TimeseriesQuery.html)
+---------------------------
+
+Architecture
+\* [Design](Design.html)
+\* [Segments](Segments.html)
+\* Node Types
+**\* ]
+**\* [Broker](Broker.html)
+**\* ]
+****\* ]
+**\* [Realtime](Realtime.html)
+**\* ]
+**\* [Plumber](Plumber.html)
+\* External Dependencies
+**\* ]
+**\* [MySQL](MySQL.html)
+**\* ]
+** [Concepts and Terminology](Concepts-and-Terminology.html)
+-------------------------------
+
+Development
+\* [Versioning](Versioning.html)
+\* [Build From Source](Build-From-Source.html)
+\* [Libraries](Libraries.html)
+------------------------
+
+Misc
+\* [Thanks](Thanks.html)
+-------------
diff --git a/docs/css/bootstrap-responsive.css b/docs/css/bootstrap-responsive.css
new file mode 100644
index 00000000000..9570d1c2e40
--- /dev/null
+++ b/docs/css/bootstrap-responsive.css
@@ -0,0 +1,1058 @@
+/*!
+ * Bootstrap Responsive v2.1.1
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */
+
+.clearfix {
+  *zoom: 1;
+}
+
+.clearfix:before,
+.clearfix:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.clearfix:after {
+  clear: both;
+}
+
+.hide-text {
+  font: 0/0 a;
+  color: transparent;
+  text-shadow: none;
+  background-color: transparent;
+  border: 0;
+}
+
+.input-block-level {
+  display: block;
+  width: 100%;
+  min-height: 30px;
+  -webkit-box-sizing: border-box;
+     -moz-box-sizing: border-box;
+          box-sizing: border-box;
+}
+
+.hidden {
+  display: none;
+  visibility: hidden;
+}
+
+.visible-phone {
+  display: none !important;
+}
+
+.visible-tablet {
+  display: none !important;
+}
+
+.hidden-desktop {
+  display: none !important;
+}
+
+.visible-desktop {
+  display: inherit !important;
+}
+
+@media (min-width: 768px) and (max-width: 979px) {
+  .hidden-desktop {
+    display: inherit !important;
+  }
+  .visible-desktop {
+    display: none !important ;
+  }
+  .visible-tablet {
+    display: inherit !important;
+  }
+  .hidden-tablet {
+    display: none !important;
+  }
+}
+
+@media (max-width: 767px) {
+  .hidden-desktop {
+    display: inherit !important;
+  }
+  .visible-desktop {
+    display: none !important;
+  }
+  .visible-phone {
+    display: inherit !important;
+  }
+  .hidden-phone {
+    display: none !important;
+  }
+}
+
+@media (min-width: 1200px) {
+  .row {
+    margin-left: -30px;
+    *zoom: 1;
+  }
+  .row:before,
+  .row:after {
+    display: table;
+    line-height: 0;
+    content: "";
+  }
+  .row:after {
+    clear: both;
+  }
+  [class*="span"] {
+    float: left;
+    min-height: 1px;
+    margin-left: 30px;
+  }
+  .container,
+  .navbar-static-top .container,
+  .navbar-fixed-top .container,
+  .navbar-fixed-bottom .container {
+    width: 1170px;
+  }
+  .span12 {
+    width: 1170px;
+  }
+  .span11 {
+    width: 1070px;
+  }
+  .span10 {
+    width: 970px;
+  }
+  .span9 {
+    width: 870px;
+  }
+  .span8 {
+    width: 770px;
+  }
+  .span7 {
+    width: 670px;
+  }
+  .span6 {
+    width: 570px;
+  }
+  .span5 {
+    width: 470px;
+  }
+  .span4 {
+    width: 370px;
+  }
+  .span3 {
+    width: 270px;
+  }
+  .span2 {
+    width: 170px;
+  }
+  .span1 {
+    width: 70px;
+  }
+  .offset12 {
+    margin-left: 1230px;
+  }
+  .offset11 {
+    margin-left: 1130px;
+  }
+  .offset10 {
+    margin-left: 1030px;
+  }
+  .offset9 {
+    margin-left: 930px;
+  }
+  .offset8 {
+    margin-left: 830px;
+  }
+  .offset7 {
+    margin-left: 730px;
+  }
+  .offset6 {
+    margin-left: 630px;
+  }
+  .offset5 {
+    margin-left: 530px;
+  }
+  .offset4 {
+    margin-left: 430px;
+  }
+  .offset3 {
+    margin-left: 330px;
+  }
+  .offset2 {
+    margin-left: 230px;
+  }
+  .offset1 {
+    margin-left: 130px;
+  }
+  .row-fluid {
+    width: 100%;
+    *zoom: 1;
+  }
+  .row-fluid:before,
+  .row-fluid:after {
+    display: table;
+    line-height: 0;
+    content: "";
+  }
+  .row-fluid:after {
+    clear: both;
+  }
+  .row-fluid [class*="span"] {
+    display: block;
+    float: left;
+    width: 100%;
+    min-height: 30px;
+    margin-left: 2.564102564102564%;
+    *margin-left: 2.5109110747408616%;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+            box-sizing: border-box;
+  }
+  .row-fluid [class*="span"]:first-child {
+    margin-left: 0;
+  }
+  .row-fluid .span12 {
+    width: 100%;
+    *width: 99.94680851063829%;
+  }
+  .row-fluid .span11 {
+    width: 91.45299145299145%;
+    *width: 91.39979996362975%;
+  }
+  .row-fluid .span10 {
+    width: 82.90598290598291%;
+    *width: 82.8527914166212%;
+  }
+  .row-fluid .span9 {
+    width: 74.35897435897436%;
+    *width: 74.30578286961266%;
+  }
+  .row-fluid .span8 {
+    width: 65.81196581196582%;
+    *width: 65.75877432260411%;
+  }
+  .row-fluid .span7 {
+    width: 57.26495726495726%;
+    *width: 57.21176577559556%;
+  }
+  .row-fluid .span6 {
+    width: 48.717948717948715%;
+    *width: 48.664757228587014%;
+  }
+  .row-fluid .span5 {
+    width: 40.17094017094017%;
+    *width: 40.11774868157847%;
+  }
+  .row-fluid .span4 {
+    width: 31.623931623931625%;
+    *width: 31.570740134569924%;
+  }
+  .row-fluid .span3 {
+    width: 23.076923076923077%;
+    *width: 23.023731587561375%;
+  }
+  .row-fluid .span2 {
+    width: 14.52991452991453%;
+    *width: 14.476723040552828%;
+  }
+  .row-fluid .span1 {
+    width: 5.982905982905983%;
+    *width: 5.929714493544281%;
+  }
+  .row-fluid .offset12 {
+    margin-left: 105.12820512820512%;
+    *margin-left: 105.02182214948171%;
+  }
+  .row-fluid .offset12:first-child {
+    margin-left: 102.56410256410257%;
+    *margin-left: 102.45771958537915%;
+  }
+  .row-fluid .offset11 {
+    margin-left: 96.58119658119658%;
+    *margin-left: 96.47481360247316%;
+  }
+  .row-fluid .offset11:first-child {
+    margin-left: 94.01709401709402%;
+    *margin-left: 93.91071103837061%;
+  }
+  .row-fluid .offset10 {
+    margin-left: 88.03418803418803%;
+    *margin-left: 87.92780505546462%;
+  }
+  .row-fluid .offset10:first-child {
+    margin-left: 85.47008547008548%;
+    *margin-left: 85.36370249136206%;
+  }
+  .row-fluid .offset9 {
+    margin-left: 79.48717948717949%;
+    *margin-left: 79.38079650845607%;
+  }
+  .row-fluid .offset9:first-child {
+    margin-left: 76.92307692307693%;
+    *margin-left: 76.81669394435352%;
+  }
+  .row-fluid .offset8 {
+    margin-left: 70.94017094017094%;
+    *margin-left: 70.83378796144753%;
+  }
+  .row-fluid .offset8:first-child {
+    margin-left: 68.37606837606839%;
+    *margin-left: 68.26968539734497%;
+  }
+  .row-fluid .offset7 {
+    margin-left: 62.393162393162385%;
+    *margin-left: 62.28677941443899%;
+  }
+  .row-fluid .offset7:first-child {
+    margin-left: 59.82905982905982%;
+    *margin-left: 59.72267685033642%;
+  }
+  .row-fluid .offset6 {
+    margin-left: 53.84615384615384%;
+    *margin-left: 53.739770867430444%;
+  }
+  .row-fluid .offset6:first-child {
+    margin-left: 51.28205128205128%;
+    *margin-left: 51.175668303327875%;
+  }
+  .row-fluid .offset5 {
+    margin-left: 45.299145299145295%;
+    *margin-left: 45.1927623204219%;
+  }
+  .row-fluid .offset5:first-child {
+    margin-left: 42.73504273504273%;
+    *margin-left: 42.62865975631933%;
+  }
+  .row-fluid .offset4 {
+    margin-left: 36.75213675213675%;
+    *margin-left: 36.645753773413354%;
+  }
+  .row-fluid .offset4:first-child {
+    margin-left: 34.18803418803419%;
+    *margin-left: 34.081651209310785%;
+  }
+  .row-fluid .offset3 {
+    margin-left: 28.205128205128204%;
+    *margin-left: 28.0987452264048%;
+  }
+  .row-fluid .offset3:first-child {
+    margin-left: 25.641025641025642%;
+    *margin-left: 25.53464266230224%;
+  }
+  .row-fluid .offset2 {
+    margin-left: 19.65811965811966%;
+    *margin-left: 19.551736679396257%;
+  }
+  .row-fluid .offset2:first-child {
+    margin-left: 17.094017094017094%;
+    *margin-left: 16.98763411529369%;
+  }
+  .row-fluid .offset1 {
+    margin-left: 11.11111111111111%;
+    *margin-left: 11.004728132387708%;
+  }
+  .row-fluid .offset1:first-child {
+    margin-left: 8.547008547008547%;
+    *margin-left: 8.440625568285142%;
+  }
+  input,
+  textarea,
+  .uneditable-input {
+    margin-left: 0;
+  }
+  .controls-row [class*="span"] + [class*="span"] {
+    margin-left: 30px;
+  }
+  input.span12,
+  textarea.span12,
+  .uneditable-input.span12 {
+    width: 1156px;
+  }
+  input.span11,
+  textarea.span11,
+  .uneditable-input.span11 {
+    width: 1056px;
+  }
+  input.span10,
+  textarea.span10,
+  .uneditable-input.span10 {
+    width: 956px;
+  }
+  input.span9,
+  textarea.span9,
+  .uneditable-input.span9 {
+    width: 856px;
+  }
+  input.span8,
+  textarea.span8,
+  .uneditable-input.span8 {
+    width: 756px;
+  }
+  input.span7,
+  textarea.span7,
+  .uneditable-input.span7 {
+    width: 656px;
+  }
+  input.span6,
+  textarea.span6,
+  .uneditable-input.span6 {
+    width: 556px;
+  }
+  input.span5,
+  textarea.span5,
+  .uneditable-input.span5 {
+    width: 456px;
+  }
+  input.span4,
+  textarea.span4,
+  .uneditable-input.span4 {
+    width: 356px;
+  }
+  input.span3,
+  textarea.span3,
+  .uneditable-input.span3 {
+    width: 256px;
+  }
+  input.span2,
+  textarea.span2,
+  .uneditable-input.span2 {
+    width: 156px;
+  }
+  input.span1,
+  textarea.span1,
+  .uneditable-input.span1 {
+    width: 56px;
+  }
+  .thumbnails {
+    margin-left: -30px;
+  }
+  .thumbnails > li {
+    margin-left: 30px;
+  }
+  .row-fluid .thumbnails {
+    margin-left: 0;
+  }
+}
+
+@media (min-width: 768px) and (max-width: 979px) {
+  .row {
+    margin-left: -20px;
+    *zoom: 1;
+  }
+  .row:before,
+  .row:after {
+    display: table;
+    line-height: 0;
+    content: "";
+  }
+  .row:after {
+    clear: both;
+  }
+  [class*="span"] {
+    float: left;
+    min-height: 1px;
+    margin-left: 20px;
+  }
+  .container,
+  .navbar-static-top .container,
+  .navbar-fixed-top .container,
+  .navbar-fixed-bottom .container {
+    width: 724px;
+  }
+  .span12 {
+    width: 724px;
+  }
+  .span11 {
+    width: 662px;
+  }
+  .span10 {
+    width: 600px;
+  }
+  .span9 {
+    width: 538px;
+  }
+  .span8 {
+    width: 476px;
+  }
+  .span7 {
+    width: 414px;
+  }
+  .span6 {
+    width: 352px;
+  }
+  .span5 {
+    width: 290px;
+  }
+  .span4 {
+    width: 228px;
+  }
+  .span3 {
+    width: 166px;
+  }
+  .span2 {
+    width: 104px;
+  }
+  .span1 {
+    width: 42px;
+  }
+  .offset12 {
+    margin-left: 764px;
+  }
+  .offset11 {
+    margin-left: 702px;
+  }
+  .offset10 {
+    margin-left: 640px;
+  }
+  .offset9 {
+    margin-left: 578px;
+  }
+  .offset8 {
+    margin-left: 516px;
+  }
+  .offset7 {
+    margin-left: 454px;
+  }
+  .offset6 {
+    margin-left: 392px;
+  }
+  .offset5 {
+    margin-left: 330px;
+  }
+  .offset4 {
+    margin-left: 268px;
+  }
+  .offset3 {
+    margin-left: 206px;
+  }
+  .offset2 {
+    margin-left: 144px;
+  }
+  .offset1 {
+    margin-left: 82px;
+  }
+  .row-fluid {
+    width: 100%;
+    *zoom: 1;
+  }
+  .row-fluid:before,
+  .row-fluid:after {
+    display: table;
+    line-height: 0;
+    content: "";
+  }
+  .row-fluid:after {
+    clear: both;
+  }
+  .row-fluid [class*="span"] {
+    display: block;
+    float: left;
+    width: 100%;
+    min-height: 30px;
+    margin-left: 2.7624309392265194%;
+    *margin-left: 2.709239449864817%;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+            box-sizing: border-box;
+  }
+  .row-fluid [class*="span"]:first-child {
+    margin-left: 0;
+  }
+  .row-fluid .span12 {
+    width: 100%;
+    *width: 99.94680851063829%;
+  }
+  .row-fluid .span11 {
+    width: 91.43646408839778%;
+    *width: 91.38327259903608%;
+  }
+  .row-fluid .span10 {
+    width: 82.87292817679558%;
+    *width: 82.81973668743387%;
+  }
+  .row-fluid .span9 {
+    width: 74.30939226519337%;
+    *width: 74.25620077583166%;
+  }
+  .row-fluid .span8 {
+    width: 65.74585635359117%;
+    *width: 65.69266486422946%;
+  }
+  .row-fluid .span7 {
+    width: 57.18232044198895%;
+    *width: 57.12912895262725%;
+  }
+  .row-fluid .span6 {
+    width: 48.61878453038674%;
+    *width: 48.56559304102504%;
+  }
+  .row-fluid .span5 {
+    width: 40.05524861878453%;
+    *width: 40.00205712942283%;
+  }
+  .row-fluid .span4 {
+    width: 31.491712707182323%;
+    *width: 31.43852121782062%;
+  }
+  .row-fluid .span3 {
+    width: 22.92817679558011%;
+    *width: 22.87498530621841%;
+  }
+  .row-fluid .span2 {
+    width: 14.3646408839779%;
+    *width: 14.311449394616199%;
+  }
+  .row-fluid .span1 {
+    width: 5.801104972375691%;
+    *width: 5.747913483013988%;
+  }
+  .row-fluid .offset12 {
+    margin-left: 105.52486187845304%;
+    *margin-left: 105.41847889972962%;
+  }
+  .row-fluid .offset12:first-child {
+    margin-left: 102.76243093922652%;
+    *margin-left: 102.6560479605031%;
+  }
+  .row-fluid .offset11 {
+    margin-left: 96.96132596685082%;
+    *margin-left: 96.8549429881274%;
+  }
+  .row-fluid .offset11:first-child {
+    margin-left: 94.1988950276243%;
+    *margin-left: 94.09251204890089%;
+  }
+  .row-fluid .offset10 {
+    margin-left: 88.39779005524862%;
+    *margin-left: 88.2914070765252%;
+  }
+  .row-fluid .offset10:first-child {
+    margin-left: 85.6353591160221%;
+    *margin-left: 85.52897613729868%;
+  }
+  .row-fluid .offset9 {
+    margin-left: 79.8342541436464%;
+    *margin-left: 79.72787116492299%;
+  }
+  .row-fluid .offset9:first-child {
+    margin-left: 77.07182320441989%;
+    *margin-left: 76.96544022569647%;
+  }
+  .row-fluid .offset8 {
+    margin-left: 71.2707182320442%;
+    *margin-left: 71.16433525332079%;
+  }
+  .row-fluid .offset8:first-child {
+    margin-left: 68.50828729281768%;
+    *margin-left: 68.40190431409427%;
+  }
+  .row-fluid .offset7 {
+    margin-left: 62.70718232044199%;
+    *margin-left: 62.600799341718584%;
+  }
+  .row-fluid .offset7:first-child {
+    margin-left: 59.94475138121547%;
+    *margin-left: 59.838368402492065%;
+  }
+  .row-fluid .offset6 {
+    margin-left: 54.14364640883978%;
+    *margin-left: 54.037263430116376%;
+  }
+  .row-fluid .offset6:first-child {
+    margin-left: 51.38121546961326%;
+    *margin-left: 51.27483249088986%;
+  }
+  .row-fluid .offset5 {
+    margin-left: 45.58011049723757%;
+    *margin-left: 45.47372751851417%;
+  }
+  .row-fluid .offset5:first-child {
+    margin-left: 42.81767955801105%;
+    *margin-left: 42.71129657928765%;
+  }
+  .row-fluid .offset4 {
+    margin-left: 37.01657458563536%;
+    *margin-left: 36.91019160691196%;
+  }
+  .row-fluid .offset4:first-child {
+    margin-left: 34.25414364640884%;
+    *margin-left: 34.14776066768544%;
+  }
+  .row-fluid .offset3 {
+    margin-left: 28.45303867403315%;
+    *margin-left: 28.346655695309746%;
+  }
+  .row-fluid .offset3:first-child {
+    margin-left: 25.69060773480663%;
+    *margin-left: 25.584224756083227%;
+  }
+  .row-fluid .offset2 {
+    margin-left: 19.88950276243094%;
+    *margin-left: 19.783119783707537%;
+  }
+  .row-fluid .offset2:first-child {
+    margin-left: 17.12707182320442%;
+    *margin-left: 17.02068884448102%;
+  }
+  .row-fluid .offset1 {
+    margin-left: 11.32596685082873%;
+    *margin-left: 11.219583872105325%;
+  }
+  .row-fluid .offset1:first-child {
+    margin-left: 8.56353591160221%;
+    *margin-left: 8.457152932878806%;
+  }
+  input,
+  textarea,
+  .uneditable-input {
+    margin-left: 0;
+  }
+  .controls-row [class*="span"] + [class*="span"] {
+    margin-left: 20px;
+  }
+  input.span12,
+  textarea.span12,
+  .uneditable-input.span12 {
+    width: 710px;
+  }
+  input.span11,
+  textarea.span11,
+  .uneditable-input.span11 {
+    width: 648px;
+  }
+  input.span10,
+  textarea.span10,
+  .uneditable-input.span10 {
+    width: 586px;
+  }
+  input.span9,
+  textarea.span9,
+  .uneditable-input.span9 {
+    width: 524px;
+  }
+  input.span8,
+  textarea.span8,
+  .uneditable-input.span8 {
+    width: 462px;
+  }
+  input.span7,
+  textarea.span7,
+  .uneditable-input.span7 {
+    width: 400px;
+  }
+  input.span6,
+  textarea.span6,
+  .uneditable-input.span6 {
+    width: 338px;
+  }
+  input.span5,
+  textarea.span5,
+  .uneditable-input.span5 {
+    width: 276px;
+  }
+  input.span4,
+  textarea.span4,
+  .uneditable-input.span4 {
+    width: 214px;
+  }
+  input.span3,
+  textarea.span3,
+  .uneditable-input.span3 {
+    width: 152px;
+  }
+  input.span2,
+  textarea.span2,
+  .uneditable-input.span2 {
+    width: 90px;
+  }
+  input.span1,
+  textarea.span1,
+  .uneditable-input.span1 {
+    width: 28px;
+  }
+}
+
+@media (max-width: 767px) {
+  body {
+    padding-right: 20px;
+    padding-left: 20px;
+  }
+  .navbar-fixed-top,
+  .navbar-fixed-bottom,
+  .navbar-static-top {
+    margin-right: -20px;
+    margin-left: -20px;
+  }
+  .container-fluid {
+    padding: 0;
+  }
+  .dl-horizontal dt {
+    float: none;
+    width: auto;
+    clear: none;
+    text-align: left;
+  }
+  .dl-horizontal dd {
+    margin-left: 0;
+  }
+  .container {
+    width: auto;
+  }
+  .row-fluid {
+    width: 100%;
+  }
+  .row,
+  .thumbnails {
+    margin-left: 0;
+  }
+  .thumbnails > li {
+    float: none;
+    margin-left: 0;
+  }
+  [class*="span"],
+  .row-fluid [class*="span"] {
+    display: block;
+    float: none;
+    width: auto;
+    margin-left: 0;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+            box-sizing: border-box;
+  }
+  .span12,
+  .row-fluid .span12 {
+    width: 100%;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+            box-sizing: border-box;
+  }
+  .input-large,
+  .input-xlarge,
+  .input-xxlarge,
+  input[class*="span"],
+  select[class*="span"],
+  textarea[class*="span"],
+  .uneditable-input {
+    display: block;
+    width: 100%;
+    min-height: 30px;
+    -webkit-box-sizing: border-box;
+       -moz-box-sizing: border-box;
+            box-sizing: border-box;
+  }
+  .input-prepend input,
+  .input-append input,
+  .input-prepend input[class*="span"],
+  .input-append input[class*="span"] {
+    display: inline-block;
+    width: auto;
+  }
+  .controls-row [class*="span"] + [class*="span"] {
+    margin-left: 0;
+  }
+  .modal {
+    position: fixed;
+    top: 20px;
+    right: 20px;
+    left: 20px;
+    width: auto;
+    margin: 0;
+  }
+  .modal.fade.in {
+    top: auto;
+  }
+}
+
+@media (max-width: 480px) {
+  .nav-collapse {
+    -webkit-transform: translate3d(0, 0, 0);
+  }
+  .page-header h1 small {
+    display: block;
+    line-height: 20px;
+  }
+  input[type="checkbox"],
+  input[type="radio"] {
+    border: 1px solid #ccc;
+  }
+  .form-horizontal .control-label {
+    float: none;
+    width: auto;
+    padding-top: 0;
+    text-align: left;
+  }
+  .form-horizontal .controls {
+    margin-left: 0;
+  }
+  .form-horizontal .control-list {
+    padding-top: 0;
+  }
+  .form-horizontal .form-actions {
+    padding-right: 10px;
+    padding-left: 10px;
+  }
+  .modal {
+    top: 10px;
+    right: 10px;
+    left: 10px;
+  }
+  .modal-header .close {
+    padding: 10px;
+    margin: -10px;
+  }
+  .carousel-caption {
+    position: static;
+  }
+}
+
+@media (max-width: 979px) {
+  body {
+    padding-top: 0;
+  }
+  .navbar-fixed-top,
+  .navbar-fixed-bottom {
+    position: static;
+  }
+  .navbar-fixed-top {
+    margin-bottom: 20px;
+  }
+  .navbar-fixed-bottom {
+    margin-top: 20px;
+  }
+  .navbar-fixed-top .navbar-inner,
+  .navbar-fixed-bottom .navbar-inner {
+    padding: 5px;
+  }
+  .navbar .container {
+    width: auto;
+    padding: 0;
+  }
+  .navbar .brand {
+    padding-right: 10px;
+    padding-left: 10px;
+    margin: 0 0 0 -5px;
+  }
+  .nav-collapse {
+    clear: both;
+  }
+  .nav-collapse .nav {
+    float: none;
+    margin: 0 0 10px;
+  }
+  .nav-collapse .nav > li {
+    float: none;
+  }
+  .nav-collapse .nav > li > a {
+    margin-bottom: 2px;
+  }
+  .nav-collapse .nav > .divider-vertical {
+    display: none;
+  }
+  .nav-collapse .nav .nav-header {
+    color: #777777;
+    text-shadow: none;
+  }
+  .nav-collapse .nav > li > a,
+  .nav-collapse .dropdown-menu a {
+    padding: 9px 15px;
+    font-weight: bold;
+    color: #777777;
+    -webkit-border-radius: 3px;
+       -moz-border-radius: 3px;
+            border-radius: 3px;
+  }
+  .nav-collapse .btn {
+    padding: 4px 10px 4px;
+    font-weight: normal;
+    -webkit-border-radius: 4px;
+       -moz-border-radius: 4px;
+            border-radius: 4px;
+  }
+  .nav-collapse .dropdown-menu li + li a {
+    margin-bottom: 2px;
+  }
+  .nav-collapse .nav > li > a:hover,
+  .nav-collapse .dropdown-menu a:hover {
+    background-color: #f2f2f2;
+  }
+  .navbar-inverse .nav-collapse .nav > li > a:hover,
+  .navbar-inverse .nav-collapse .dropdown-menu a:hover {
+    background-color: #111111;
+  }
+  .nav-collapse.in .btn-group {
+    padding: 0;
+    margin-top: 5px;
+  }
+  .nav-collapse .dropdown-menu {
+    position: static;
+    top: auto;
+    left: auto;
+    display: block;
+    float: none;
+    max-width: none;
+    padding: 0;
+    margin: 0 15px;
+    background-color: transparent;
+    border: none;
+    -webkit-border-radius: 0;
+       -moz-border-radius: 0;
+            border-radius: 0;
+    -webkit-box-shadow: none;
+       -moz-box-shadow: none;
+            box-shadow: none;
+  }
+  .nav-collapse .dropdown-menu:before,
+  .nav-collapse .dropdown-menu:after {
+    display: none;
+  }
+  .nav-collapse .dropdown-menu .divider {
+    display: none;
+  }
+  .nav-collapse .nav > li > .dropdown-menu:before,
+  .nav-collapse .nav > li > .dropdown-menu:after {
+    display: none;
+  }
+  .nav-collapse .navbar-form,
+  .nav-collapse .navbar-search {
+    float: none;
+    padding: 10px 15px;
+    margin: 10px 0;
+    border-top: 1px solid #f2f2f2;
+    border-bottom: 1px solid #f2f2f2;
+    -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+       -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+            box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1);
+  }
+  .navbar-inverse .nav-collapse .navbar-form,
+  .navbar-inverse .nav-collapse .navbar-search {
+    border-top-color: #111111;
+    border-bottom-color: #111111;
+  }
+  .navbar .nav-collapse .nav.pull-right {
+    float: none;
+    margin-left: 0;
+  }
+  .nav-collapse,
+  .nav-collapse.collapse {
+    height: 0;
+    overflow: hidden;
+  }
+  .navbar .btn-navbar {
+    display: block;
+  }
+  .navbar-static .navbar-inner {
+    padding-right: 10px;
+    padding-left: 10px;
+  }
+}
+
+@media (min-width: 980px) {
+  .nav-collapse.collapse {
+    height: auto !important;
+    overflow: visible !important;
+  }
+}
diff --git a/docs/css/bootstrap-responsive.min.css b/docs/css/bootstrap-responsive.min.css
new file mode 100644
index 00000000000..8eb78d8e67c
--- /dev/null
+++ b/docs/css/bootstrap-responsive.min.css
@@ -0,0 +1,9 @@
+/*!
+ * Bootstrap Responsive v2.1.1
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.hidden{display:none;visibility:hidden}.visible-phone{display:none!important}.visible-tablet{display:none!important}.hidden-desktop{display:none!important}.visible-desktop{display:inherit!important}@media(min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-tablet{display:inherit!important}.hidden-tablet{display:none!important}}@media(max-width:767px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-phone{display:inherit!important}.hidden-phone{display:none!important}}@media(min-width:1200px){.row{margin-left:-30px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:30px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px}.span12{width:1170px}.span11{width:1070px}.span10{width:970px}.span9{width:870px}.span8{width:770px}.span7{width:670px}.span6{width:570px}.span5{width:470px}.span4{width:370px}.span3{width:270px}.span2{width:170px}.span1{width:70px}.offset12{margin-left:1230px}.offset11{margin-left:1130px}.offset10{margin-left:1030px}.offset9{margin-left:930px}.offset8{margin-left:830px}.offset7{margin-left:730px}.offset6{margin-left:630px}.offset5{margin-left:530px}.offset4{margin-left:430px}.offset3{margin-left:330px}.offset2{margin-left:230px}.offset1{margin-left:130px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%}.row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%}.row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%}.row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%}.row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%}.row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%}.row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%}.row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%}.row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%}.row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%}.row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%}.row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%}.row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%}.row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%}.row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%}.row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%}.row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%}.row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%}.row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%}.row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%}.row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%}.row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%}.row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%}.row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%}.row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%}.row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%}.row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%}.row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%}.row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%}.row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%}.row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%}.row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%}.row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%}.row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%}.row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:30px}input.span12,textarea.span12,.uneditable-input.span12{width:1156px}input.span11,textarea.span11,.uneditable-input.span11{width:1056px}input.span10,textarea.span10,.uneditable-input.span10{width:956px}input.span9,textarea.span9,.uneditable-input.span9{width:856px}input.span8,textarea.span8,.uneditable-input.span8{width:756px}input.span7,textarea.span7,.uneditable-input.span7{width:656px}input.span6,textarea.span6,.uneditable-input.span6{width:556px}input.span5,textarea.span5,.uneditable-input.span5{width:456px}input.span4,textarea.span4,.uneditable-input.span4{width:356px}input.span3,textarea.span3,.uneditable-input.span3{width:256px}input.span2,textarea.span2,.uneditable-input.span2{width:156px}input.span1,textarea.span1,.uneditable-input.span1{width:56px}.thumbnails{margin-left:-30px}.thumbnails>li{margin-left:30px}.row-fluid .thumbnails{margin-left:0}}@media(min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px}.span12{width:724px}.span11{width:662px}.span10{width:600px}.span9{width:538px}.span8{width:476px}.span7{width:414px}.span6{width:352px}.span5{width:290px}.span4{width:228px}.span3{width:166px}.span2{width:104px}.span1{width:42px}.offset12{margin-left:764px}.offset11{margin-left:702px}.offset10{margin-left:640px}.offset9{margin-left:578px}.offset8{margin-left:516px}.offset7{margin-left:454px}.offset6{margin-left:392px}.offset5{margin-left:330px}.offset4{margin-left:268px}.offset3{margin-left:206px}.offset2{margin-left:144px}.offset1{margin-left:82px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%}.row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%}.row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%}.row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%}.row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%}.row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%}.row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%}.row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%}.row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%}.row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%}.row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%}.row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%}.row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%}.row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%}.row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%}.row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%}.row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%}.row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%}.row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%}.row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%}.row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%}.row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%}.row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%}.row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%}.row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%}.row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%}.row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%}.row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%}.row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%}.row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%}.row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%}.row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%}.row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%}.row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%}.row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:710px}input.span11,textarea.span11,.uneditable-input.span11{width:648px}input.span10,textarea.span10,.uneditable-input.span10{width:586px}input.span9,textarea.span9,.uneditable-input.span9{width:524px}input.span8,textarea.span8,.uneditable-input.span8{width:462px}input.span7,textarea.span7,.uneditable-input.span7{width:400px}input.span6,textarea.span6,.uneditable-input.span6{width:338px}input.span5,textarea.span5,.uneditable-input.span5{width:276px}input.span4,textarea.span4,.uneditable-input.span4{width:214px}input.span3,textarea.span3,.uneditable-input.span3{width:152px}input.span2,textarea.span2,.uneditable-input.span2{width:90px}input.span1,textarea.span1,.uneditable-input.span1{width:28px}}@media(max-width:767px){body{padding-right:20px;padding-left:20px}.navbar-fixed-top,.navbar-fixed-bottom,.navbar-static-top{margin-right:-20px;margin-left:-20px}.container-fluid{padding:0}.dl-horizontal dt{float:none;width:auto;clear:none;text-align:left}.dl-horizontal dd{margin-left:0}.container{width:auto}.row-fluid{width:100%}.row,.thumbnails{margin-left:0}.thumbnails>li{float:none;margin-left:0}[class*="span"],.row-fluid [class*="span"]{display:block;float:none;width:100%;margin-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto}.controls-row [class*="span"]+[class*="span"]{margin-left:0}.modal{position:fixed;top:20px;right:20px;left:20px;width:auto;margin:0}.modal.fade.in{top:auto}}@media(max-width:480px){.nav-collapse{-webkit-transform:translate3d(0,0,0)}.page-header h1 small{display:block;line-height:20px}input[type="checkbox"],input[type="radio"]{border:1px solid #ccc}.form-horizontal .control-label{float:none;width:auto;padding-top:0;text-align:left}.form-horizontal .controls{margin-left:0}.form-horizontal .control-list{padding-top:0}.form-horizontal .form-actions{padding-right:10px;padding-left:10px}.modal{top:10px;right:10px;left:10px}.modal-header .close{padding:10px;margin:-10px}.carousel-caption{position:static}}@media(max-width:979px){body{padding-top:0}.navbar-fixed-top,.navbar-fixed-bottom{position:static}.navbar-fixed-top{margin-bottom:20px}.navbar-fixed-bottom{margin-top:20px}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px}.navbar .container{width:auto;padding:0}.navbar .brand{padding-right:10px;padding-left:10px;margin:0 0 0 -5px}.nav-collapse{clear:both}.nav-collapse .nav{float:none;margin:0 0 10px}.nav-collapse .nav>li{float:none}.nav-collapse .nav>li>a{margin-bottom:2px}.nav-collapse .nav>.divider-vertical{display:none}.nav-collapse .nav .nav-header{color:#777;text-shadow:none}.nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#777;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-collapse .dropdown-menu li+li a{margin-bottom:2px}.nav-collapse .nav>li>a:hover,.nav-collapse .dropdown-menu a:hover{background-color:#f2f2f2}.navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:hover{background-color:#111}.nav-collapse.in .btn-group{padding:0;margin-top:5px}.nav-collapse .dropdown-menu{position:static;top:auto;left:auto;display:block;float:none;max-width:none;padding:0;margin:0 15px;background-color:transparent;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none}.nav-collapse .dropdown-menu .divider{display:none}.nav-collapse .nav>li>.dropdown-menu:before,.nav-collapse .nav>li>.dropdown-menu:after{display:none}.nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1)}.navbar-inverse .nav-collapse .navbar-form,.navbar-inverse .nav-collapse .navbar-search{border-top-color:#111;border-bottom-color:#111}.navbar .nav-collapse .nav.pull-right{float:none;margin-left:0}.nav-collapse,.nav-collapse.collapse{height:0;overflow:hidden}.navbar .btn-navbar{display:block}.navbar-static .navbar-inner{padding-right:10px;padding-left:10px}}@media(min-width:980px){.nav-collapse.collapse{height:auto!important;overflow:visible!important}}
diff --git a/docs/css/bootstrap.css b/docs/css/bootstrap.css
new file mode 100644
index 00000000000..8dca0b71366
--- /dev/null
+++ b/docs/css/bootstrap.css
@@ -0,0 +1,5774 @@
+/*!
+ * Bootstrap v2.1.1
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */
+
+article,
+aside,
+details,
+figcaption,
+figure,
+footer,
+header,
+hgroup,
+nav,
+section {
+  display: block;
+}
+
+audio,
+canvas,
+video {
+  display: inline-block;
+  *display: inline;
+  *zoom: 1;
+}
+
+audio:not([controls]) {
+  display: none;
+}
+
+html {
+  font-size: 100%;
+  -webkit-text-size-adjust: 100%;
+      -ms-text-size-adjust: 100%;
+}
+
+a:focus {
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
+}
+
+a:hover,
+a:active {
+  outline: 0;
+}
+
+sub,
+sup {
+  position: relative;
+  font-size: 75%;
+  line-height: 0;
+  vertical-align: baseline;
+}
+
+sup {
+  top: -0.5em;
+}
+
+sub {
+  bottom: -0.25em;
+}
+
+img {
+  width: auto\9;
+  height: auto;
+  max-width: 100%;
+  vertical-align: middle;
+  border: 0;
+  -ms-interpolation-mode: bicubic;
+}
+
+#map_canvas img {
+  max-width: none;
+}
+
+button,
+input,
+select,
+textarea {
+  margin: 0;
+  font-size: 100%;
+  vertical-align: middle;
+}
+
+button,
+input {
+  *overflow: visible;
+  line-height: normal;
+}
+
+button::-moz-focus-inner,
+input::-moz-focus-inner {
+  padding: 0;
+  border: 0;
+}
+
+button,
+input[type="button"],
+input[type="reset"],
+input[type="submit"] {
+  cursor: pointer;
+  -webkit-appearance: button;
+}
+
+input[type="search"] {
+  -webkit-box-sizing: content-box;
+     -moz-box-sizing: content-box;
+          box-sizing: content-box;
+  -webkit-appearance: textfield;
+}
+
+input[type="search"]::-webkit-search-decoration,
+input[type="search"]::-webkit-search-cancel-button {
+  -webkit-appearance: none;
+}
+
+textarea {
+  overflow: auto;
+  vertical-align: top;
+}
+
+.clearfix {
+  *zoom: 1;
+}
+
+.clearfix:before,
+.clearfix:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.clearfix:after {
+  clear: both;
+}
+
+.hide-text {
+  font: 0/0 a;
+  color: transparent;
+  text-shadow: none;
+  background-color: transparent;
+  border: 0;
+}
+
+.input-block-level {
+  display: block;
+  width: 100%;
+  min-height: 30px;
+  -webkit-box-sizing: border-box;
+     -moz-box-sizing: border-box;
+          box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  font-size: 14px;
+  line-height: 20px;
+  color: #333333;
+  background-color: #ffffff;
+}
+
+a {
+  color: #0088cc;
+  text-decoration: none;
+}
+
+a:hover {
+  color: #005580;
+  text-decoration: underline;
+}
+
+.img-rounded {
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+}
+
+.img-polaroid {
+  padding: 4px;
+  background-color: #fff;
+  border: 1px solid #ccc;
+  border: 1px solid rgba(0, 0, 0, 0.2);
+  -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+     -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+          box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+}
+
+.img-circle {
+  -webkit-border-radius: 500px;
+     -moz-border-radius: 500px;
+          border-radius: 500px;
+}
+
+.row {
+  margin-left: -20px;
+  *zoom: 1;
+}
+
+.row:before,
+.row:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.row:after {
+  clear: both;
+}
+
+[class*="span"] {
+  float: left;
+  min-height: 1px;
+  margin-left: 20px;
+}
+
+.container,
+.navbar-static-top .container,
+.navbar-fixed-top .container,
+.navbar-fixed-bottom .container {
+  width: 940px;
+}
+
+.span12 {
+  width: 940px;
+}
+
+.span11 {
+  width: 860px;
+}
+
+.span10 {
+  width: 780px;
+}
+
+.span9 {
+  width: 700px;
+}
+
+.span8 {
+  width: 620px;
+}
+
+.span7 {
+  width: 540px;
+}
+
+.span6 {
+  width: 460px;
+}
+
+.span5 {
+  width: 380px;
+}
+
+.span4 {
+  width: 300px;
+}
+
+.span3 {
+  width: 220px;
+}
+
+.span2 {
+  width: 140px;
+}
+
+.span1 {
+  width: 60px;
+}
+
+.offset12 {
+  margin-left: 980px;
+}
+
+.offset11 {
+  margin-left: 900px;
+}
+
+.offset10 {
+  margin-left: 820px;
+}
+
+.offset9 {
+  margin-left: 740px;
+}
+
+.offset8 {
+  margin-left: 660px;
+}
+
+.offset7 {
+  margin-left: 580px;
+}
+
+.offset6 {
+  margin-left: 500px;
+}
+
+.offset5 {
+  margin-left: 420px;
+}
+
+.offset4 {
+  margin-left: 340px;
+}
+
+.offset3 {
+  margin-left: 260px;
+}
+
+.offset2 {
+  margin-left: 180px;
+}
+
+.offset1 {
+  margin-left: 100px;
+}
+
+.row-fluid {
+  width: 100%;
+  *zoom: 1;
+}
+
+.row-fluid:before,
+.row-fluid:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.row-fluid:after {
+  clear: both;
+}
+
+.row-fluid [class*="span"] {
+  display: block;
+  float: left;
+  width: 100%;
+  min-height: 30px;
+  margin-left: 2.127659574468085%;
+  *margin-left: 2.074468085106383%;
+  -webkit-box-sizing: border-box;
+     -moz-box-sizing: border-box;
+          box-sizing: border-box;
+}
+
+.row-fluid [class*="span"]:first-child {
+  margin-left: 0;
+}
+
+.row-fluid .span12 {
+  width: 100%;
+  *width: 99.94680851063829%;
+}
+
+.row-fluid .span11 {
+  width: 91.48936170212765%;
+  *width: 91.43617021276594%;
+}
+
+.row-fluid .span10 {
+  width: 82.97872340425532%;
+  *width: 82.92553191489361%;
+}
+
+.row-fluid .span9 {
+  width: 74.46808510638297%;
+  *width: 74.41489361702126%;
+}
+
+.row-fluid .span8 {
+  width: 65.95744680851064%;
+  *width: 65.90425531914893%;
+}
+
+.row-fluid .span7 {
+  width: 57.44680851063829%;
+  *width: 57.39361702127659%;
+}
+
+.row-fluid .span6 {
+  width: 48.93617021276595%;
+  *width: 48.88297872340425%;
+}
+
+.row-fluid .span5 {
+  width: 40.42553191489362%;
+  *width: 40.37234042553192%;
+}
+
+.row-fluid .span4 {
+  width: 31.914893617021278%;
+  *width: 31.861702127659576%;
+}
+
+.row-fluid .span3 {
+  width: 23.404255319148934%;
+  *width: 23.351063829787233%;
+}
+
+.row-fluid .span2 {
+  width: 14.893617021276595%;
+  *width: 14.840425531914894%;
+}
+
+.row-fluid .span1 {
+  width: 6.382978723404255%;
+  *width: 6.329787234042553%;
+}
+
+.row-fluid .offset12 {
+  margin-left: 104.25531914893617%;
+  *margin-left: 104.14893617021275%;
+}
+
+.row-fluid .offset12:first-child {
+  margin-left: 102.12765957446808%;
+  *margin-left: 102.02127659574467%;
+}
+
+.row-fluid .offset11 {
+  margin-left: 95.74468085106382%;
+  *margin-left: 95.6382978723404%;
+}
+
+.row-fluid .offset11:first-child {
+  margin-left: 93.61702127659574%;
+  *margin-left: 93.51063829787232%;
+}
+
+.row-fluid .offset10 {
+  margin-left: 87.23404255319149%;
+  *margin-left: 87.12765957446807%;
+}
+
+.row-fluid .offset10:first-child {
+  margin-left: 85.1063829787234%;
+  *margin-left: 84.99999999999999%;
+}
+
+.row-fluid .offset9 {
+  margin-left: 78.72340425531914%;
+  *margin-left: 78.61702127659572%;
+}
+
+.row-fluid .offset9:first-child {
+  margin-left: 76.59574468085106%;
+  *margin-left: 76.48936170212764%;
+}
+
+.row-fluid .offset8 {
+  margin-left: 70.2127659574468%;
+  *margin-left: 70.10638297872339%;
+}
+
+.row-fluid .offset8:first-child {
+  margin-left: 68.08510638297872%;
+  *margin-left: 67.9787234042553%;
+}
+
+.row-fluid .offset7 {
+  margin-left: 61.70212765957446%;
+  *margin-left: 61.59574468085106%;
+}
+
+.row-fluid .offset7:first-child {
+  margin-left: 59.574468085106375%;
+  *margin-left: 59.46808510638297%;
+}
+
+.row-fluid .offset6 {
+  margin-left: 53.191489361702125%;
+  *margin-left: 53.085106382978715%;
+}
+
+.row-fluid .offset6:first-child {
+  margin-left: 51.063829787234035%;
+  *margin-left: 50.95744680851063%;
+}
+
+.row-fluid .offset5 {
+  margin-left: 44.68085106382979%;
+  *margin-left: 44.57446808510638%;
+}
+
+.row-fluid .offset5:first-child {
+  margin-left: 42.5531914893617%;
+  *margin-left: 42.4468085106383%;
+}
+
+.row-fluid .offset4 {
+  margin-left: 36.170212765957444%;
+  *margin-left: 36.06382978723405%;
+}
+
+.row-fluid .offset4:first-child {
+  margin-left: 34.04255319148936%;
+  *margin-left: 33.93617021276596%;
+}
+
+.row-fluid .offset3 {
+  margin-left: 27.659574468085104%;
+  *margin-left: 27.5531914893617%;
+}
+
+.row-fluid .offset3:first-child {
+  margin-left: 25.53191489361702%;
+  *margin-left: 25.425531914893618%;
+}
+
+.row-fluid .offset2 {
+  margin-left: 19.148936170212764%;
+  *margin-left: 19.04255319148936%;
+}
+
+.row-fluid .offset2:first-child {
+  margin-left: 17.02127659574468%;
+  *margin-left: 16.914893617021278%;
+}
+
+.row-fluid .offset1 {
+  margin-left: 10.638297872340425%;
+  *margin-left: 10.53191489361702%;
+}
+
+.row-fluid .offset1:first-child {
+  margin-left: 8.51063829787234%;
+  *margin-left: 8.404255319148938%;
+}
+
+[class*="span"].hide,
+.row-fluid [class*="span"].hide {
+  display: none;
+}
+
+[class*="span"].pull-right,
+.row-fluid [class*="span"].pull-right {
+  float: right;
+}
+
+.container {
+  margin-right: auto;
+  margin-left: auto;
+  *zoom: 1;
+}
+
+.container:before,
+.container:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.container:after {
+  clear: both;
+}
+
+.container-fluid {
+  padding-right: 20px;
+  padding-left: 20px;
+  *zoom: 1;
+}
+
+.container-fluid:before,
+.container-fluid:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.container-fluid:after {
+  clear: both;
+}
+
+p {
+  margin: 0 0 10px;
+}
+
+.lead {
+  margin-bottom: 20px;
+  font-size: 21px;
+  font-weight: 200;
+  line-height: 30px;
+}
+
+small {
+  font-size: 85%;
+}
+
+strong {
+  font-weight: bold;
+}
+
+em {
+  font-style: italic;
+}
+
+cite {
+  font-style: normal;
+}
+
+.muted {
+  color: #999999;
+}
+
+.text-warning {
+  color: #c09853;
+}
+
+.text-error {
+  color: #b94a48;
+}
+
+.text-info {
+  color: #3a87ad;
+}
+
+.text-success {
+  color: #468847;
+}
+
+h1,
+h2,
+h3,
+h4,
+h5,
+h6 {
+  margin: 10px 0;
+  font-family: inherit;
+  font-weight: bold;
+  line-height: 1;
+  color: inherit;
+  text-rendering: optimizelegibility;
+}
+
+h1 small,
+h2 small,
+h3 small,
+h4 small,
+h5 small,
+h6 small {
+  font-weight: normal;
+  line-height: 1;
+  color: #999999;
+}
+
+h1 {
+  font-size: 36px;
+  line-height: 40px;
+}
+
+h2 {
+  font-size: 30px;
+  line-height: 40px;
+}
+
+h3 {
+  font-size: 24px;
+  line-height: 40px;
+}
+
+h4 {
+  font-size: 18px;
+  line-height: 20px;
+}
+
+h5 {
+  font-size: 14px;
+  line-height: 20px;
+}
+
+h6 {
+  font-size: 12px;
+  line-height: 20px;
+}
+
+h1 small {
+  font-size: 24px;
+}
+
+h2 small {
+  font-size: 18px;
+}
+
+h3 small {
+  font-size: 14px;
+}
+
+h4 small {
+  font-size: 14px;
+}
+
+.page-header {
+  padding-bottom: 9px;
+  margin: 20px 0 30px;
+  border-bottom: 1px solid #eeeeee;
+}
+
+ul,
+ol {
+  padding: 0;
+  margin: 0 0 10px 25px;
+}
+
+ul ul,
+ul ol,
+ol ol,
+ol ul {
+  margin-bottom: 0;
+}
+
+li {
+  line-height: 20px;
+}
+
+ul.unstyled,
+ol.unstyled {
+  margin-left: 0;
+  list-style: none;
+}
+
+dl {
+  margin-bottom: 20px;
+}
+
+dt,
+dd {
+  line-height: 20px;
+}
+
+dt {
+  font-weight: bold;
+}
+
+dd {
+  margin-left: 10px;
+}
+
+.dl-horizontal {
+  *zoom: 1;
+}
+
+.dl-horizontal:before,
+.dl-horizontal:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.dl-horizontal:after {
+  clear: both;
+}
+
+.dl-horizontal dt {
+  float: left;
+  width: 160px;
+  overflow: hidden;
+  clear: left;
+  text-align: right;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+
+.dl-horizontal dd {
+  margin-left: 180px;
+}
+
+hr {
+  margin: 20px 0;
+  border: 0;
+  border-top: 1px solid #eeeeee;
+  border-bottom: 1px solid #ffffff;
+}
+
+abbr[title] {
+  cursor: help;
+  border-bottom: 1px dotted #999999;
+}
+
+abbr.initialism {
+  font-size: 90%;
+  text-transform: uppercase;
+}
+
+blockquote {
+  padding: 0 0 0 15px;
+  margin: 0 0 20px;
+  border-left: 5px solid #eeeeee;
+}
+
+blockquote p {
+  margin-bottom: 0;
+  font-size: 16px;
+  font-weight: 300;
+  line-height: 25px;
+}
+
+blockquote small {
+  display: block;
+  line-height: 20px;
+  color: #999999;
+}
+
+blockquote small:before {
+  content: '\2014 \00A0';
+}
+
+blockquote.pull-right {
+  float: right;
+  padding-right: 15px;
+  padding-left: 0;
+  border-right: 5px solid #eeeeee;
+  border-left: 0;
+}
+
+blockquote.pull-right p,
+blockquote.pull-right small {
+  text-align: right;
+}
+
+blockquote.pull-right small:before {
+  content: '';
+}
+
+blockquote.pull-right small:after {
+  content: '\00A0 \2014';
+}
+
+q:before,
+q:after,
+blockquote:before,
+blockquote:after {
+  content: "";
+}
+
+address {
+  display: block;
+  margin-bottom: 20px;
+  font-style: normal;
+  line-height: 20px;
+}
+
+code,
+pre {
+  padding: 0 3px 2px;
+  font-family: Monaco, Menlo, Consolas, "Courier New", monospace;
+  font-size: 12px;
+  color: #333333;
+  -webkit-border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+}
+
+code {
+  padding: 2px 4px;
+  color: #d14;
+  background-color: #f7f7f9;
+  border: 1px solid #e1e1e8;
+}
+
+pre {
+  display: block;
+  padding: 9.5px;
+  margin: 0 0 10px;
+  font-size: 13px;
+  line-height: 20px;
+  word-break: break-all;
+  word-wrap: break-word;
+  white-space: pre;
+  white-space: pre-wrap;
+  background-color: #f5f5f5;
+  border: 1px solid #ccc;
+  border: 1px solid rgba(0, 0, 0, 0.15);
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+pre.prettyprint {
+  margin-bottom: 20px;
+}
+
+pre code {
+  padding: 0;
+  color: inherit;
+  background-color: transparent;
+  border: 0;
+}
+
+.pre-scrollable {
+  max-height: 340px;
+  overflow-y: scroll;
+}
+
+form {
+  margin: 0 0 20px;
+}
+
+fieldset {
+  padding: 0;
+  margin: 0;
+  border: 0;
+}
+
+legend {
+  display: block;
+  width: 100%;
+  padding: 0;
+  margin-bottom: 20px;
+  font-size: 21px;
+  line-height: 40px;
+  color: #333333;
+  border: 0;
+  border-bottom: 1px solid #e5e5e5;
+}
+
+legend small {
+  font-size: 15px;
+  color: #999999;
+}
+
+label,
+input,
+button,
+select,
+textarea {
+  font-size: 14px;
+  font-weight: normal;
+  line-height: 20px;
+}
+
+input,
+button,
+select,
+textarea {
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+}
+
+label {
+  display: block;
+  margin-bottom: 5px;
+}
+
+select,
+textarea,
+input[type="text"],
+input[type="password"],
+input[type="datetime"],
+input[type="datetime-local"],
+input[type="date"],
+input[type="month"],
+input[type="time"],
+input[type="week"],
+input[type="number"],
+input[type="email"],
+input[type="url"],
+input[type="search"],
+input[type="tel"],
+input[type="color"],
+.uneditable-input {
+  display: inline-block;
+  height: 20px;
+  padding: 4px 6px;
+  margin-bottom: 9px;
+  font-size: 14px;
+  line-height: 20px;
+  color: #555555;
+  -webkit-border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+}
+
+input,
+textarea,
+.uneditable-input {
+  width: 206px;
+}
+
+textarea {
+  height: auto;
+}
+
+textarea,
+input[type="text"],
+input[type="password"],
+input[type="datetime"],
+input[type="datetime-local"],
+input[type="date"],
+input[type="month"],
+input[type="time"],
+input[type="week"],
+input[type="number"],
+input[type="email"],
+input[type="url"],
+input[type="search"],
+input[type="tel"],
+input[type="color"],
+.uneditable-input {
+  background-color: #ffffff;
+  border: 1px solid #cccccc;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+  -webkit-transition: border linear 0.2s, box-shadow linear 0.2s;
+     -moz-transition: border linear 0.2s, box-shadow linear 0.2s;
+       -o-transition: border linear 0.2s, box-shadow linear 0.2s;
+          transition: border linear 0.2s, box-shadow linear 0.2s;
+}
+
+textarea:focus,
+input[type="text"]:focus,
+input[type="password"]:focus,
+input[type="datetime"]:focus,
+input[type="datetime-local"]:focus,
+input[type="date"]:focus,
+input[type="month"]:focus,
+input[type="time"]:focus,
+input[type="week"]:focus,
+input[type="number"]:focus,
+input[type="email"]:focus,
+input[type="url"]:focus,
+input[type="search"]:focus,
+input[type="tel"]:focus,
+input[type="color"]:focus,
+.uneditable-input:focus {
+  border-color: rgba(82, 168, 236, 0.8);
+  outline: 0;
+  outline: thin dotted \9;
+  /* IE6-9 */
+
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6);
+}
+
+input[type="radio"],
+input[type="checkbox"] {
+  margin: 4px 0 0;
+  margin-top: 1px \9;
+  *margin-top: 0;
+  line-height: normal;
+  cursor: pointer;
+}
+
+input[type="file"],
+input[type="image"],
+input[type="submit"],
+input[type="reset"],
+input[type="button"],
+input[type="radio"],
+input[type="checkbox"] {
+  width: auto;
+}
+
+select,
+input[type="file"] {
+  height: 30px;
+  /* In IE7, the height of the select element cannot be changed by height, only font-size */
+
+  *margin-top: 4px;
+  /* For IE7, add top margin to align select with labels */
+
+  line-height: 30px;
+}
+
+select {
+  width: 220px;
+  background-color: #ffffff;
+  border: 1px solid #cccccc;
+}
+
+select[multiple],
+select[size] {
+  height: auto;
+}
+
+select:focus,
+input[type="file"]:focus,
+input[type="radio"]:focus,
+input[type="checkbox"]:focus {
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
+}
+
+.uneditable-input,
+.uneditable-textarea {
+  color: #999999;
+  cursor: not-allowed;
+  background-color: #fcfcfc;
+  border-color: #cccccc;
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025);
+}
+
+.uneditable-input {
+  overflow: hidden;
+  white-space: nowrap;
+}
+
+.uneditable-textarea {
+  width: auto;
+  height: auto;
+}
+
+input:-moz-placeholder,
+textarea:-moz-placeholder {
+  color: #999999;
+}
+
+input:-ms-input-placeholder,
+textarea:-ms-input-placeholder {
+  color: #999999;
+}
+
+input::-webkit-input-placeholder,
+textarea::-webkit-input-placeholder {
+  color: #999999;
+}
+
+.radio,
+.checkbox {
+  min-height: 18px;
+  padding-left: 18px;
+}
+
+.radio input[type="radio"],
+.checkbox input[type="checkbox"] {
+  float: left;
+  margin-left: -18px;
+}
+
+.controls > .radio:first-child,
+.controls > .checkbox:first-child {
+  padding-top: 5px;
+}
+
+.radio.inline,
+.checkbox.inline {
+  display: inline-block;
+  padding-top: 5px;
+  margin-bottom: 0;
+  vertical-align: middle;
+}
+
+.radio.inline + .radio.inline,
+.checkbox.inline + .checkbox.inline {
+  margin-left: 10px;
+}
+
+.input-mini {
+  width: 60px;
+}
+
+.input-small {
+  width: 90px;
+}
+
+.input-medium {
+  width: 150px;
+}
+
+.input-large {
+  width: 210px;
+}
+
+.input-xlarge {
+  width: 270px;
+}
+
+.input-xxlarge {
+  width: 530px;
+}
+
+input[class*="span"],
+select[class*="span"],
+textarea[class*="span"],
+.uneditable-input[class*="span"],
+.row-fluid input[class*="span"],
+.row-fluid select[class*="span"],
+.row-fluid textarea[class*="span"],
+.row-fluid .uneditable-input[class*="span"] {
+  float: none;
+  margin-left: 0;
+}
+
+.input-append input[class*="span"],
+.input-append .uneditable-input[class*="span"],
+.input-prepend input[class*="span"],
+.input-prepend .uneditable-input[class*="span"],
+.row-fluid input[class*="span"],
+.row-fluid select[class*="span"],
+.row-fluid textarea[class*="span"],
+.row-fluid .uneditable-input[class*="span"],
+.row-fluid .input-prepend [class*="span"],
+.row-fluid .input-append [class*="span"] {
+  display: inline-block;
+}
+
+input,
+textarea,
+.uneditable-input {
+  margin-left: 0;
+}
+
+.controls-row [class*="span"] + [class*="span"] {
+  margin-left: 20px;
+}
+
+input.span12,
+textarea.span12,
+.uneditable-input.span12 {
+  width: 926px;
+}
+
+input.span11,
+textarea.span11,
+.uneditable-input.span11 {
+  width: 846px;
+}
+
+input.span10,
+textarea.span10,
+.uneditable-input.span10 {
+  width: 766px;
+}
+
+input.span9,
+textarea.span9,
+.uneditable-input.span9 {
+  width: 686px;
+}
+
+input.span8,
+textarea.span8,
+.uneditable-input.span8 {
+  width: 606px;
+}
+
+input.span7,
+textarea.span7,
+.uneditable-input.span7 {
+  width: 526px;
+}
+
+input.span6,
+textarea.span6,
+.uneditable-input.span6 {
+  width: 446px;
+}
+
+input.span5,
+textarea.span5,
+.uneditable-input.span5 {
+  width: 366px;
+}
+
+input.span4,
+textarea.span4,
+.uneditable-input.span4 {
+  width: 286px;
+}
+
+input.span3,
+textarea.span3,
+.uneditable-input.span3 {
+  width: 206px;
+}
+
+input.span2,
+textarea.span2,
+.uneditable-input.span2 {
+  width: 126px;
+}
+
+input.span1,
+textarea.span1,
+.uneditable-input.span1 {
+  width: 46px;
+}
+
+.controls-row {
+  *zoom: 1;
+}
+
+.controls-row:before,
+.controls-row:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.controls-row:after {
+  clear: both;
+}
+
+.controls-row [class*="span"] {
+  float: left;
+}
+
+input[disabled],
+select[disabled],
+textarea[disabled],
+input[readonly],
+select[readonly],
+textarea[readonly] {
+  cursor: not-allowed;
+  background-color: #eeeeee;
+}
+
+input[type="radio"][disabled],
+input[type="checkbox"][disabled],
+input[type="radio"][readonly],
+input[type="checkbox"][readonly] {
+  background-color: transparent;
+}
+
+.control-group.warning > label,
+.control-group.warning .help-block,
+.control-group.warning .help-inline {
+  color: #c09853;
+}
+
+.control-group.warning .checkbox,
+.control-group.warning .radio,
+.control-group.warning input,
+.control-group.warning select,
+.control-group.warning textarea {
+  color: #c09853;
+}
+
+.control-group.warning input,
+.control-group.warning select,
+.control-group.warning textarea {
+  border-color: #c09853;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+
+.control-group.warning input:focus,
+.control-group.warning select:focus,
+.control-group.warning textarea:focus {
+  border-color: #a47e3c;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e;
+}
+
+.control-group.warning .input-prepend .add-on,
+.control-group.warning .input-append .add-on {
+  color: #c09853;
+  background-color: #fcf8e3;
+  border-color: #c09853;
+}
+
+.control-group.error > label,
+.control-group.error .help-block,
+.control-group.error .help-inline {
+  color: #b94a48;
+}
+
+.control-group.error .checkbox,
+.control-group.error .radio,
+.control-group.error input,
+.control-group.error select,
+.control-group.error textarea {
+  color: #b94a48;
+}
+
+.control-group.error input,
+.control-group.error select,
+.control-group.error textarea {
+  border-color: #b94a48;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+
+.control-group.error input:focus,
+.control-group.error select:focus,
+.control-group.error textarea:focus {
+  border-color: #953b39;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392;
+}
+
+.control-group.error .input-prepend .add-on,
+.control-group.error .input-append .add-on {
+  color: #b94a48;
+  background-color: #f2dede;
+  border-color: #b94a48;
+}
+
+.control-group.success > label,
+.control-group.success .help-block,
+.control-group.success .help-inline {
+  color: #468847;
+}
+
+.control-group.success .checkbox,
+.control-group.success .radio,
+.control-group.success input,
+.control-group.success select,
+.control-group.success textarea {
+  color: #468847;
+}
+
+.control-group.success input,
+.control-group.success select,
+.control-group.success textarea {
+  border-color: #468847;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+
+.control-group.success input:focus,
+.control-group.success select:focus,
+.control-group.success textarea:focus {
+  border-color: #356635;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b;
+}
+
+.control-group.success .input-prepend .add-on,
+.control-group.success .input-append .add-on {
+  color: #468847;
+  background-color: #dff0d8;
+  border-color: #468847;
+}
+
+.control-group.info > label,
+.control-group.info .help-block,
+.control-group.info .help-inline {
+  color: #3a87ad;
+}
+
+.control-group.info .checkbox,
+.control-group.info .radio,
+.control-group.info input,
+.control-group.info select,
+.control-group.info textarea {
+  color: #3a87ad;
+}
+
+.control-group.info input,
+.control-group.info select,
+.control-group.info textarea {
+  border-color: #3a87ad;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
+}
+
+.control-group.info input:focus,
+.control-group.info select:focus,
+.control-group.info textarea:focus {
+  border-color: #2d6987;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3;
+}
+
+.control-group.info .input-prepend .add-on,
+.control-group.info .input-append .add-on {
+  color: #3a87ad;
+  background-color: #d9edf7;
+  border-color: #3a87ad;
+}
+
+input:focus:required:invalid,
+textarea:focus:required:invalid,
+select:focus:required:invalid {
+  color: #b94a48;
+  border-color: #ee5f5b;
+}
+
+input:focus:required:invalid:focus,
+textarea:focus:required:invalid:focus,
+select:focus:required:invalid:focus {
+  border-color: #e9322d;
+  -webkit-box-shadow: 0 0 6px #f8b9b7;
+     -moz-box-shadow: 0 0 6px #f8b9b7;
+          box-shadow: 0 0 6px #f8b9b7;
+}
+
+.form-actions {
+  padding: 19px 20px 20px;
+  margin-top: 20px;
+  margin-bottom: 20px;
+  background-color: #f5f5f5;
+  border-top: 1px solid #e5e5e5;
+  *zoom: 1;
+}
+
+.form-actions:before,
+.form-actions:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.form-actions:after {
+  clear: both;
+}
+
+.help-block,
+.help-inline {
+  color: #595959;
+}
+
+.help-block {
+  display: block;
+  margin-bottom: 10px;
+}
+
+.help-inline {
+  display: inline-block;
+  *display: inline;
+  padding-left: 5px;
+  vertical-align: middle;
+  *zoom: 1;
+}
+
+.input-append,
+.input-prepend {
+  margin-bottom: 5px;
+  font-size: 0;
+  white-space: nowrap;
+}
+
+.input-append input,
+.input-prepend input,
+.input-append select,
+.input-prepend select,
+.input-append .uneditable-input,
+.input-prepend .uneditable-input {
+  position: relative;
+  margin-bottom: 0;
+  *margin-left: 0;
+  font-size: 14px;
+  vertical-align: top;
+  -webkit-border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
+}
+
+.input-append input:focus,
+.input-prepend input:focus,
+.input-append select:focus,
+.input-prepend select:focus,
+.input-append .uneditable-input:focus,
+.input-prepend .uneditable-input:focus {
+  z-index: 2;
+}
+
+.input-append .add-on,
+.input-prepend .add-on {
+  display: inline-block;
+  width: auto;
+  height: 20px;
+  min-width: 16px;
+  padding: 4px 5px;
+  font-size: 14px;
+  font-weight: normal;
+  line-height: 20px;
+  text-align: center;
+  text-shadow: 0 1px 0 #ffffff;
+  background-color: #eeeeee;
+  border: 1px solid #ccc;
+}
+
+.input-append .add-on,
+.input-prepend .add-on,
+.input-append .btn,
+.input-prepend .btn {
+  vertical-align: top;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.input-append .active,
+.input-prepend .active {
+  background-color: #a9dba9;
+  border-color: #46a546;
+}
+
+.input-prepend .add-on,
+.input-prepend .btn {
+  margin-right: -1px;
+}
+
+.input-prepend .add-on:first-child,
+.input-prepend .btn:first-child {
+  -webkit-border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
+}
+
+.input-append input,
+.input-append select,
+.input-append .uneditable-input {
+  -webkit-border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
+}
+
+.input-append .add-on,
+.input-append .btn {
+  margin-left: -1px;
+}
+
+.input-append .add-on:last-child,
+.input-append .btn:last-child {
+  -webkit-border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
+}
+
+.input-prepend.input-append input,
+.input-prepend.input-append select,
+.input-prepend.input-append .uneditable-input {
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.input-prepend.input-append .add-on:first-child,
+.input-prepend.input-append .btn:first-child {
+  margin-right: -1px;
+  -webkit-border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
+}
+
+.input-prepend.input-append .add-on:last-child,
+.input-prepend.input-append .btn:last-child {
+  margin-left: -1px;
+  -webkit-border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
+}
+
+input.search-query {
+  padding-right: 14px;
+  padding-right: 4px \9;
+  padding-left: 14px;
+  padding-left: 4px \9;
+  /* IE7-8 doesn't have border-radius, so don't indent the padding */
+
+  margin-bottom: 0;
+  -webkit-border-radius: 15px;
+     -moz-border-radius: 15px;
+          border-radius: 15px;
+}
+
+/* Allow for input prepend/append in search forms */
+
+.form-search .input-append .search-query,
+.form-search .input-prepend .search-query {
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.form-search .input-append .search-query {
+  -webkit-border-radius: 14px 0 0 14px;
+     -moz-border-radius: 14px 0 0 14px;
+          border-radius: 14px 0 0 14px;
+}
+
+.form-search .input-append .btn {
+  -webkit-border-radius: 0 14px 14px 0;
+     -moz-border-radius: 0 14px 14px 0;
+          border-radius: 0 14px 14px 0;
+}
+
+.form-search .input-prepend .search-query {
+  -webkit-border-radius: 0 14px 14px 0;
+     -moz-border-radius: 0 14px 14px 0;
+          border-radius: 0 14px 14px 0;
+}
+
+.form-search .input-prepend .btn {
+  -webkit-border-radius: 14px 0 0 14px;
+     -moz-border-radius: 14px 0 0 14px;
+          border-radius: 14px 0 0 14px;
+}
+
+.form-search input,
+.form-inline input,
+.form-horizontal input,
+.form-search textarea,
+.form-inline textarea,
+.form-horizontal textarea,
+.form-search select,
+.form-inline select,
+.form-horizontal select,
+.form-search .help-inline,
+.form-inline .help-inline,
+.form-horizontal .help-inline,
+.form-search .uneditable-input,
+.form-inline .uneditable-input,
+.form-horizontal .uneditable-input,
+.form-search .input-prepend,
+.form-inline .input-prepend,
+.form-horizontal .input-prepend,
+.form-search .input-append,
+.form-inline .input-append,
+.form-horizontal .input-append {
+  display: inline-block;
+  *display: inline;
+  margin-bottom: 0;
+  vertical-align: middle;
+  *zoom: 1;
+}
+
+.form-search .hide,
+.form-inline .hide,
+.form-horizontal .hide {
+  display: none;
+}
+
+.form-search label,
+.form-inline label,
+.form-search .btn-group,
+.form-inline .btn-group {
+  display: inline-block;
+}
+
+.form-search .input-append,
+.form-inline .input-append,
+.form-search .input-prepend,
+.form-inline .input-prepend {
+  margin-bottom: 0;
+}
+
+.form-search .radio,
+.form-search .checkbox,
+.form-inline .radio,
+.form-inline .checkbox {
+  padding-left: 0;
+  margin-bottom: 0;
+  vertical-align: middle;
+}
+
+.form-search .radio input[type="radio"],
+.form-search .checkbox input[type="checkbox"],
+.form-inline .radio input[type="radio"],
+.form-inline .checkbox input[type="checkbox"] {
+  float: left;
+  margin-right: 3px;
+  margin-left: 0;
+}
+
+.control-group {
+  margin-bottom: 10px;
+}
+
+legend + .control-group {
+  margin-top: 20px;
+  -webkit-margin-top-collapse: separate;
+}
+
+.form-horizontal .control-group {
+  margin-bottom: 20px;
+  *zoom: 1;
+}
+
+.form-horizontal .control-group:before,
+.form-horizontal .control-group:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.form-horizontal .control-group:after {
+  clear: both;
+}
+
+.form-horizontal .control-label {
+  float: left;
+  width: 160px;
+  padding-top: 5px;
+  text-align: right;
+}
+
+.form-horizontal .controls {
+  *display: inline-block;
+  *padding-left: 20px;
+  margin-left: 180px;
+  *margin-left: 0;
+}
+
+.form-horizontal .controls:first-child {
+  *padding-left: 180px;
+}
+
+.form-horizontal .help-block {
+  margin-bottom: 0;
+}
+
+.form-horizontal input + .help-block,
+.form-horizontal select + .help-block,
+.form-horizontal textarea + .help-block {
+  margin-top: 10px;
+}
+
+.form-horizontal .form-actions {
+  padding-left: 180px;
+}
+
+table {
+  max-width: 100%;
+  background-color: transparent;
+  border-collapse: collapse;
+  border-spacing: 0;
+}
+
+.table {
+  width: 100%;
+  margin-bottom: 20px;
+}
+
+.table th,
+.table td {
+  padding: 8px;
+  line-height: 20px;
+  text-align: left;
+  vertical-align: top;
+  border-top: 1px solid #dddddd;
+}
+
+.table th {
+  font-weight: bold;
+}
+
+.table thead th {
+  vertical-align: bottom;
+}
+
+.table caption + thead tr:first-child th,
+.table caption + thead tr:first-child td,
+.table colgroup + thead tr:first-child th,
+.table colgroup + thead tr:first-child td,
+.table thead:first-child tr:first-child th,
+.table thead:first-child tr:first-child td {
+  border-top: 0;
+}
+
+.table tbody + tbody {
+  border-top: 2px solid #dddddd;
+}
+
+.table-condensed th,
+.table-condensed td {
+  padding: 4px 5px;
+}
+
+.table-bordered {
+  border: 1px solid #dddddd;
+  border-collapse: separate;
+  *border-collapse: collapse;
+  border-left: 0;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+.table-bordered th,
+.table-bordered td {
+  border-left: 1px solid #dddddd;
+}
+
+.table-bordered caption + thead tr:first-child th,
+.table-bordered caption + tbody tr:first-child th,
+.table-bordered caption + tbody tr:first-child td,
+.table-bordered colgroup + thead tr:first-child th,
+.table-bordered colgroup + tbody tr:first-child th,
+.table-bordered colgroup + tbody tr:first-child td,
+.table-bordered thead:first-child tr:first-child th,
+.table-bordered tbody:first-child tr:first-child th,
+.table-bordered tbody:first-child tr:first-child td {
+  border-top: 0;
+}
+
+.table-bordered thead:first-child tr:first-child th:first-child,
+.table-bordered tbody:first-child tr:first-child td:first-child {
+  -webkit-border-top-left-radius: 4px;
+          border-top-left-radius: 4px;
+  -moz-border-radius-topleft: 4px;
+}
+
+.table-bordered thead:first-child tr:first-child th:last-child,
+.table-bordered tbody:first-child tr:first-child td:last-child {
+  -webkit-border-top-right-radius: 4px;
+          border-top-right-radius: 4px;
+  -moz-border-radius-topright: 4px;
+}
+
+.table-bordered thead:last-child tr:last-child th:first-child,
+.table-bordered tbody:last-child tr:last-child td:first-child,
+.table-bordered tfoot:last-child tr:last-child td:first-child {
+  -webkit-border-radius: 0 0 0 4px;
+     -moz-border-radius: 0 0 0 4px;
+          border-radius: 0 0 0 4px;
+  -webkit-border-bottom-left-radius: 4px;
+          border-bottom-left-radius: 4px;
+  -moz-border-radius-bottomleft: 4px;
+}
+
+.table-bordered thead:last-child tr:last-child th:last-child,
+.table-bordered tbody:last-child tr:last-child td:last-child,
+.table-bordered tfoot:last-child tr:last-child td:last-child {
+  -webkit-border-bottom-right-radius: 4px;
+          border-bottom-right-radius: 4px;
+  -moz-border-radius-bottomright: 4px;
+}
+
+.table-bordered caption + thead tr:first-child th:first-child,
+.table-bordered caption + tbody tr:first-child td:first-child,
+.table-bordered colgroup + thead tr:first-child th:first-child,
+.table-bordered colgroup + tbody tr:first-child td:first-child {
+  -webkit-border-top-left-radius: 4px;
+          border-top-left-radius: 4px;
+  -moz-border-radius-topleft: 4px;
+}
+
+.table-bordered caption + thead tr:first-child th:last-child,
+.table-bordered caption + tbody tr:first-child td:last-child,
+.table-bordered colgroup + thead tr:first-child th:last-child,
+.table-bordered colgroup + tbody tr:first-child td:last-child {
+  -webkit-border-top-right-radius: 4px;
+          border-top-right-radius: 4px;
+  -moz-border-radius-topleft: 4px;
+}
+
+.table-striped tbody tr:nth-child(odd) td,
+.table-striped tbody tr:nth-child(odd) th {
+  background-color: #f9f9f9;
+}
+
+.table-hover tbody tr:hover td,
+.table-hover tbody tr:hover th {
+  background-color: #f5f5f5;
+}
+
+table [class*=span],
+.row-fluid table [class*=span] {
+  display: table-cell;
+  float: none;
+  margin-left: 0;
+}
+
+.table .span1 {
+  float: none;
+  width: 44px;
+  margin-left: 0;
+}
+
+.table .span2 {
+  float: none;
+  width: 124px;
+  margin-left: 0;
+}
+
+.table .span3 {
+  float: none;
+  width: 204px;
+  margin-left: 0;
+}
+
+.table .span4 {
+  float: none;
+  width: 284px;
+  margin-left: 0;
+}
+
+.table .span5 {
+  float: none;
+  width: 364px;
+  margin-left: 0;
+}
+
+.table .span6 {
+  float: none;
+  width: 444px;
+  margin-left: 0;
+}
+
+.table .span7 {
+  float: none;
+  width: 524px;
+  margin-left: 0;
+}
+
+.table .span8 {
+  float: none;
+  width: 604px;
+  margin-left: 0;
+}
+
+.table .span9 {
+  float: none;
+  width: 684px;
+  margin-left: 0;
+}
+
+.table .span10 {
+  float: none;
+  width: 764px;
+  margin-left: 0;
+}
+
+.table .span11 {
+  float: none;
+  width: 844px;
+  margin-left: 0;
+}
+
+.table .span12 {
+  float: none;
+  width: 924px;
+  margin-left: 0;
+}
+
+.table .span13 {
+  float: none;
+  width: 1004px;
+  margin-left: 0;
+}
+
+.table .span14 {
+  float: none;
+  width: 1084px;
+  margin-left: 0;
+}
+
+.table .span15 {
+  float: none;
+  width: 1164px;
+  margin-left: 0;
+}
+
+.table .span16 {
+  float: none;
+  width: 1244px;
+  margin-left: 0;
+}
+
+.table .span17 {
+  float: none;
+  width: 1324px;
+  margin-left: 0;
+}
+
+.table .span18 {
+  float: none;
+  width: 1404px;
+  margin-left: 0;
+}
+
+.table .span19 {
+  float: none;
+  width: 1484px;
+  margin-left: 0;
+}
+
+.table .span20 {
+  float: none;
+  width: 1564px;
+  margin-left: 0;
+}
+
+.table .span21 {
+  float: none;
+  width: 1644px;
+  margin-left: 0;
+}
+
+.table .span22 {
+  float: none;
+  width: 1724px;
+  margin-left: 0;
+}
+
+.table .span23 {
+  float: none;
+  width: 1804px;
+  margin-left: 0;
+}
+
+.table .span24 {
+  float: none;
+  width: 1884px;
+  margin-left: 0;
+}
+
+.table tbody tr.success td {
+  background-color: #dff0d8;
+}
+
+.table tbody tr.error td {
+  background-color: #f2dede;
+}
+
+.table tbody tr.warning td {
+  background-color: #fcf8e3;
+}
+
+.table tbody tr.info td {
+  background-color: #d9edf7;
+}
+
+.table-hover tbody tr.success:hover td {
+  background-color: #d0e9c6;
+}
+
+.table-hover tbody tr.error:hover td {
+  background-color: #ebcccc;
+}
+
+.table-hover tbody tr.warning:hover td {
+  background-color: #faf2cc;
+}
+
+.table-hover tbody tr.info:hover td {
+  background-color: #c4e3f3;
+}
+
+[class^="icon-"],
+[class*=" icon-"] {
+  display: inline-block;
+  width: 14px;
+  height: 14px;
+  margin-top: 1px;
+  *margin-right: .3em;
+  line-height: 14px;
+  vertical-align: text-top;
+  background-image: url("../img/glyphicons-halflings.png");
+  background-position: 14px 14px;
+  background-repeat: no-repeat;
+}
+
+/* White icons with optional class, or on hover/active states of certain elements */
+
+.icon-white,
+.nav-tabs > .active > a > [class^="icon-"],
+.nav-tabs > .active > a > [class*=" icon-"],
+.nav-pills > .active > a > [class^="icon-"],
+.nav-pills > .active > a > [class*=" icon-"],
+.nav-list > .active > a > [class^="icon-"],
+.nav-list > .active > a > [class*=" icon-"],
+.navbar-inverse .nav > .active > a > [class^="icon-"],
+.navbar-inverse .nav > .active > a > [class*=" icon-"],
+.dropdown-menu > li > a:hover > [class^="icon-"],
+.dropdown-menu > li > a:hover > [class*=" icon-"],
+.dropdown-menu > .active > a > [class^="icon-"],
+.dropdown-menu > .active > a > [class*=" icon-"] {
+  background-image: url("../img/glyphicons-halflings-white.png");
+}
+
+.icon-glass {
+  background-position: 0      0;
+}
+
+.icon-music {
+  background-position: -24px 0;
+}
+
+.icon-search {
+  background-position: -48px 0;
+}
+
+.icon-envelope {
+  background-position: -72px 0;
+}
+
+.icon-heart {
+  background-position: -96px 0;
+}
+
+.icon-star {
+  background-position: -120px 0;
+}
+
+.icon-star-empty {
+  background-position: -144px 0;
+}
+
+.icon-user {
+  background-position: -168px 0;
+}
+
+.icon-film {
+  background-position: -192px 0;
+}
+
+.icon-th-large {
+  background-position: -216px 0;
+}
+
+.icon-th {
+  background-position: -240px 0;
+}
+
+.icon-th-list {
+  background-position: -264px 0;
+}
+
+.icon-ok {
+  background-position: -288px 0;
+}
+
+.icon-remove {
+  background-position: -312px 0;
+}
+
+.icon-zoom-in {
+  background-position: -336px 0;
+}
+
+.icon-zoom-out {
+  background-position: -360px 0;
+}
+
+.icon-off {
+  background-position: -384px 0;
+}
+
+.icon-signal {
+  background-position: -408px 0;
+}
+
+.icon-cog {
+  background-position: -432px 0;
+}
+
+.icon-trash {
+  background-position: -456px 0;
+}
+
+.icon-home {
+  background-position: 0 -24px;
+}
+
+.icon-file {
+  background-position: -24px -24px;
+}
+
+.icon-time {
+  background-position: -48px -24px;
+}
+
+.icon-road {
+  background-position: -72px -24px;
+}
+
+.icon-download-alt {
+  background-position: -96px -24px;
+}
+
+.icon-download {
+  background-position: -120px -24px;
+}
+
+.icon-upload {
+  background-position: -144px -24px;
+}
+
+.icon-inbox {
+  background-position: -168px -24px;
+}
+
+.icon-play-circle {
+  background-position: -192px -24px;
+}
+
+.icon-repeat {
+  background-position: -216px -24px;
+}
+
+.icon-refresh {
+  background-position: -240px -24px;
+}
+
+.icon-list-alt {
+  background-position: -264px -24px;
+}
+
+.icon-lock {
+  background-position: -287px -24px;
+}
+
+.icon-flag {
+  background-position: -312px -24px;
+}
+
+.icon-headphones {
+  background-position: -336px -24px;
+}
+
+.icon-volume-off {
+  background-position: -360px -24px;
+}
+
+.icon-volume-down {
+  background-position: -384px -24px;
+}
+
+.icon-volume-up {
+  background-position: -408px -24px;
+}
+
+.icon-qrcode {
+  background-position: -432px -24px;
+}
+
+.icon-barcode {
+  background-position: -456px -24px;
+}
+
+.icon-tag {
+  background-position: 0 -48px;
+}
+
+.icon-tags {
+  background-position: -25px -48px;
+}
+
+.icon-book {
+  background-position: -48px -48px;
+}
+
+.icon-bookmark {
+  background-position: -72px -48px;
+}
+
+.icon-print {
+  background-position: -96px -48px;
+}
+
+.icon-camera {
+  background-position: -120px -48px;
+}
+
+.icon-font {
+  background-position: -144px -48px;
+}
+
+.icon-bold {
+  background-position: -167px -48px;
+}
+
+.icon-italic {
+  background-position: -192px -48px;
+}
+
+.icon-text-height {
+  background-position: -216px -48px;
+}
+
+.icon-text-width {
+  background-position: -240px -48px;
+}
+
+.icon-align-left {
+  background-position: -264px -48px;
+}
+
+.icon-align-center {
+  background-position: -288px -48px;
+}
+
+.icon-align-right {
+  background-position: -312px -48px;
+}
+
+.icon-align-justify {
+  background-position: -336px -48px;
+}
+
+.icon-list {
+  background-position: -360px -48px;
+}
+
+.icon-indent-left {
+  background-position: -384px -48px;
+}
+
+.icon-indent-right {
+  background-position: -408px -48px;
+}
+
+.icon-facetime-video {
+  background-position: -432px -48px;
+}
+
+.icon-picture {
+  background-position: -456px -48px;
+}
+
+.icon-pencil {
+  background-position: 0 -72px;
+}
+
+.icon-map-marker {
+  background-position: -24px -72px;
+}
+
+.icon-adjust {
+  background-position: -48px -72px;
+}
+
+.icon-tint {
+  background-position: -72px -72px;
+}
+
+.icon-edit {
+  background-position: -96px -72px;
+}
+
+.icon-share {
+  background-position: -120px -72px;
+}
+
+.icon-check {
+  background-position: -144px -72px;
+}
+
+.icon-move {
+  background-position: -168px -72px;
+}
+
+.icon-step-backward {
+  background-position: -192px -72px;
+}
+
+.icon-fast-backward {
+  background-position: -216px -72px;
+}
+
+.icon-backward {
+  background-position: -240px -72px;
+}
+
+.icon-play {
+  background-position: -264px -72px;
+}
+
+.icon-pause {
+  background-position: -288px -72px;
+}
+
+.icon-stop {
+  background-position: -312px -72px;
+}
+
+.icon-forward {
+  background-position: -336px -72px;
+}
+
+.icon-fast-forward {
+  background-position: -360px -72px;
+}
+
+.icon-step-forward {
+  background-position: -384px -72px;
+}
+
+.icon-eject {
+  background-position: -408px -72px;
+}
+
+.icon-chevron-left {
+  background-position: -432px -72px;
+}
+
+.icon-chevron-right {
+  background-position: -456px -72px;
+}
+
+.icon-plus-sign {
+  background-position: 0 -96px;
+}
+
+.icon-minus-sign {
+  background-position: -24px -96px;
+}
+
+.icon-remove-sign {
+  background-position: -48px -96px;
+}
+
+.icon-ok-sign {
+  background-position: -72px -96px;
+}
+
+.icon-question-sign {
+  background-position: -96px -96px;
+}
+
+.icon-info-sign {
+  background-position: -120px -96px;
+}
+
+.icon-screenshot {
+  background-position: -144px -96px;
+}
+
+.icon-remove-circle {
+  background-position: -168px -96px;
+}
+
+.icon-ok-circle {
+  background-position: -192px -96px;
+}
+
+.icon-ban-circle {
+  background-position: -216px -96px;
+}
+
+.icon-arrow-left {
+  background-position: -240px -96px;
+}
+
+.icon-arrow-right {
+  background-position: -264px -96px;
+}
+
+.icon-arrow-up {
+  background-position: -289px -96px;
+}
+
+.icon-arrow-down {
+  background-position: -312px -96px;
+}
+
+.icon-share-alt {
+  background-position: -336px -96px;
+}
+
+.icon-resize-full {
+  background-position: -360px -96px;
+}
+
+.icon-resize-small {
+  background-position: -384px -96px;
+}
+
+.icon-plus {
+  background-position: -408px -96px;
+}
+
+.icon-minus {
+  background-position: -433px -96px;
+}
+
+.icon-asterisk {
+  background-position: -456px -96px;
+}
+
+.icon-exclamation-sign {
+  background-position: 0 -120px;
+}
+
+.icon-gift {
+  background-position: -24px -120px;
+}
+
+.icon-leaf {
+  background-position: -48px -120px;
+}
+
+.icon-fire {
+  background-position: -72px -120px;
+}
+
+.icon-eye-open {
+  background-position: -96px -120px;
+}
+
+.icon-eye-close {
+  background-position: -120px -120px;
+}
+
+.icon-warning-sign {
+  background-position: -144px -120px;
+}
+
+.icon-plane {
+  background-position: -168px -120px;
+}
+
+.icon-calendar {
+  background-position: -192px -120px;
+}
+
+.icon-random {
+  width: 16px;
+  background-position: -216px -120px;
+}
+
+.icon-comment {
+  background-position: -240px -120px;
+}
+
+.icon-magnet {
+  background-position: -264px -120px;
+}
+
+.icon-chevron-up {
+  background-position: -288px -120px;
+}
+
+.icon-chevron-down {
+  background-position: -313px -119px;
+}
+
+.icon-retweet {
+  background-position: -336px -120px;
+}
+
+.icon-shopping-cart {
+  background-position: -360px -120px;
+}
+
+.icon-folder-close {
+  background-position: -384px -120px;
+}
+
+.icon-folder-open {
+  width: 16px;
+  background-position: -408px -120px;
+}
+
+.icon-resize-vertical {
+  background-position: -432px -119px;
+}
+
+.icon-resize-horizontal {
+  background-position: -456px -118px;
+}
+
+.icon-hdd {
+  background-position: 0 -144px;
+}
+
+.icon-bullhorn {
+  background-position: -24px -144px;
+}
+
+.icon-bell {
+  background-position: -48px -144px;
+}
+
+.icon-certificate {
+  background-position: -72px -144px;
+}
+
+.icon-thumbs-up {
+  background-position: -96px -144px;
+}
+
+.icon-thumbs-down {
+  background-position: -120px -144px;
+}
+
+.icon-hand-right {
+  background-position: -144px -144px;
+}
+
+.icon-hand-left {
+  background-position: -168px -144px;
+}
+
+.icon-hand-up {
+  background-position: -192px -144px;
+}
+
+.icon-hand-down {
+  background-position: -216px -144px;
+}
+
+.icon-circle-arrow-right {
+  background-position: -240px -144px;
+}
+
+.icon-circle-arrow-left {
+  background-position: -264px -144px;
+}
+
+.icon-circle-arrow-up {
+  background-position: -288px -144px;
+}
+
+.icon-circle-arrow-down {
+  background-position: -312px -144px;
+}
+
+.icon-globe {
+  background-position: -336px -144px;
+}
+
+.icon-wrench {
+  background-position: -360px -144px;
+}
+
+.icon-tasks {
+  background-position: -384px -144px;
+}
+
+.icon-filter {
+  background-position: -408px -144px;
+}
+
+.icon-briefcase {
+  background-position: -432px -144px;
+}
+
+.icon-fullscreen {
+  background-position: -456px -144px;
+}
+
+.dropup,
+.dropdown {
+  position: relative;
+}
+
+.dropdown-toggle {
+  *margin-bottom: -3px;
+}
+
+.dropdown-toggle:active,
+.open .dropdown-toggle {
+  outline: 0;
+}
+
+.caret {
+  display: inline-block;
+  width: 0;
+  height: 0;
+  vertical-align: top;
+  border-top: 4px solid #000000;
+  border-right: 4px solid transparent;
+  border-left: 4px solid transparent;
+  content: "";
+}
+
+.dropdown .caret {
+  margin-top: 8px;
+  margin-left: 2px;
+}
+
+.dropdown-menu {
+  position: absolute;
+  top: 100%;
+  left: 0;
+  z-index: 1000;
+  display: none;
+  float: left;
+  min-width: 160px;
+  padding: 5px 0;
+  margin: 2px 0 0;
+  list-style: none;
+  background-color: #ffffff;
+  border: 1px solid #ccc;
+  border: 1px solid rgba(0, 0, 0, 0.2);
+  *border-right-width: 2px;
+  *border-bottom-width: 2px;
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+     -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+          box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+  -webkit-background-clip: padding-box;
+     -moz-background-clip: padding;
+          background-clip: padding-box;
+}
+
+.dropdown-menu.pull-right {
+  right: 0;
+  left: auto;
+}
+
+.dropdown-menu .divider {
+  *width: 100%;
+  height: 1px;
+  margin: 9px 1px;
+  *margin: -5px 0 5px;
+  overflow: hidden;
+  background-color: #e5e5e5;
+  border-bottom: 1px solid #ffffff;
+}
+
+.dropdown-menu a {
+  display: block;
+  padding: 3px 20px;
+  clear: both;
+  font-weight: normal;
+  line-height: 20px;
+  color: #333333;
+  white-space: nowrap;
+}
+
+.dropdown-menu li > a:hover,
+.dropdown-menu li > a:focus,
+.dropdown-submenu:hover > a {
+  color: #ffffff;
+  text-decoration: none;
+  background-color: #0088cc;
+  background-color: #0081c2;
+  background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
+  background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
+  background-image: -o-linear-gradient(top, #0088cc, #0077b3);
+  background-image: linear-gradient(to bottom, #0088cc, #0077b3);
+  background-repeat: repeat-x;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
+}
+
+.dropdown-menu .active > a,
+.dropdown-menu .active > a:hover {
+  color: #ffffff;
+  text-decoration: none;
+  background-color: #0088cc;
+  background-color: #0081c2;
+  background-image: linear-gradient(to bottom, #0088cc, #0077b3);
+  background-image: -moz-linear-gradient(top, #0088cc, #0077b3);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3));
+  background-image: -webkit-linear-gradient(top, #0088cc, #0077b3);
+  background-image: -o-linear-gradient(top, #0088cc, #0077b3);
+  background-repeat: repeat-x;
+  outline: 0;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0);
+}
+
+.dropdown-menu .disabled > a,
+.dropdown-menu .disabled > a:hover {
+  color: #999999;
+}
+
+.dropdown-menu .disabled > a:hover {
+  text-decoration: none;
+  cursor: default;
+  background-color: transparent;
+}
+
+.open {
+  *z-index: 1000;
+}
+
+.open > .dropdown-menu {
+  display: block;
+}
+
+.pull-right > .dropdown-menu {
+  right: 0;
+  left: auto;
+}
+
+.dropup .caret,
+.navbar-fixed-bottom .dropdown .caret {
+  border-top: 0;
+  border-bottom: 4px solid #000000;
+  content: "";
+}
+
+.dropup .dropdown-menu,
+.navbar-fixed-bottom .dropdown .dropdown-menu {
+  top: auto;
+  bottom: 100%;
+  margin-bottom: 1px;
+}
+
+.dropdown-submenu {
+  position: relative;
+}
+
+.dropdown-submenu > .dropdown-menu {
+  top: 0;
+  left: 100%;
+  margin-top: -6px;
+  margin-left: -1px;
+  -webkit-border-radius: 0 6px 6px 6px;
+     -moz-border-radius: 0 6px 6px 6px;
+          border-radius: 0 6px 6px 6px;
+}
+
+.dropdown-submenu:hover > .dropdown-menu {
+  display: block;
+}
+
+.dropdown-submenu > a:after {
+  display: block;
+  float: right;
+  width: 0;
+  height: 0;
+  margin-top: 5px;
+  margin-right: -10px;
+  border-color: transparent;
+  border-left-color: #cccccc;
+  border-style: solid;
+  border-width: 5px 0 5px 5px;
+  content: " ";
+}
+
+.dropdown-submenu:hover > a:after {
+  border-left-color: #ffffff;
+}
+
+.dropdown .dropdown-menu .nav-header {
+  padding-right: 20px;
+  padding-left: 20px;
+}
+
+.typeahead {
+  margin-top: 2px;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+.well {
+  min-height: 20px;
+  padding: 19px;
+  margin-bottom: 20px;
+  background-color: #f5f5f5;
+  border: 1px solid #e3e3e3;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05);
+}
+
+.well blockquote {
+  border-color: #ddd;
+  border-color: rgba(0, 0, 0, 0.15);
+}
+
+.well-large {
+  padding: 24px;
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+}
+
+.well-small {
+  padding: 9px;
+  -webkit-border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+}
+
+.fade {
+  opacity: 0;
+  -webkit-transition: opacity 0.15s linear;
+     -moz-transition: opacity 0.15s linear;
+       -o-transition: opacity 0.15s linear;
+          transition: opacity 0.15s linear;
+}
+
+.fade.in {
+  opacity: 1;
+}
+
+.collapse {
+  position: relative;
+  height: 0;
+  overflow: hidden;
+  -webkit-transition: height 0.35s ease;
+     -moz-transition: height 0.35s ease;
+       -o-transition: height 0.35s ease;
+          transition: height 0.35s ease;
+}
+
+.collapse.in {
+  height: auto;
+}
+
+.close {
+  float: right;
+  font-size: 20px;
+  font-weight: bold;
+  line-height: 20px;
+  color: #000000;
+  text-shadow: 0 1px 0 #ffffff;
+  opacity: 0.2;
+  filter: alpha(opacity=20);
+}
+
+.close:hover {
+  color: #000000;
+  text-decoration: none;
+  cursor: pointer;
+  opacity: 0.4;
+  filter: alpha(opacity=40);
+}
+
+button.close {
+  padding: 0;
+  cursor: pointer;
+  background: transparent;
+  border: 0;
+  -webkit-appearance: none;
+}
+
+.btn {
+  display: inline-block;
+  *display: inline;
+  padding: 4px 14px;
+  margin-bottom: 0;
+  *margin-left: .3em;
+  font-size: 14px;
+  line-height: 20px;
+  *line-height: 20px;
+  color: #333333;
+  text-align: center;
+  text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75);
+  vertical-align: middle;
+  cursor: pointer;
+  background-color: #f5f5f5;
+  *background-color: #e6e6e6;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6));
+  background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: -o-linear-gradient(top, #ffffff, #e6e6e6);
+  background-image: linear-gradient(to bottom, #ffffff, #e6e6e6);
+  background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6);
+  background-repeat: repeat-x;
+  border: 1px solid #bbbbbb;
+  *border: 0;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  border-color: #e6e6e6 #e6e6e6 #bfbfbf;
+  border-bottom-color: #a2a2a2;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+  *zoom: 1;
+  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+
+.btn:hover,
+.btn:active,
+.btn.active,
+.btn.disabled,
+.btn[disabled] {
+  color: #333333;
+  background-color: #e6e6e6;
+  *background-color: #d9d9d9;
+}
+
+.btn:active,
+.btn.active {
+  background-color: #cccccc \9;
+}
+
+.btn:first-child {
+  *margin-left: 0;
+}
+
+.btn:hover {
+  color: #333333;
+  text-decoration: none;
+  background-color: #e6e6e6;
+  *background-color: #d9d9d9;
+  /* Buttons in IE7 don't get borders, so darken on hover */
+
+  background-position: 0 -15px;
+  -webkit-transition: background-position 0.1s linear;
+     -moz-transition: background-position 0.1s linear;
+       -o-transition: background-position 0.1s linear;
+          transition: background-position 0.1s linear;
+}
+
+.btn:focus {
+  outline: thin dotted #333;
+  outline: 5px auto -webkit-focus-ring-color;
+  outline-offset: -2px;
+}
+
+.btn.active,
+.btn:active {
+  background-color: #e6e6e6;
+  background-color: #d9d9d9 \9;
+  background-image: none;
+  outline: 0;
+  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+
+.btn.disabled,
+.btn[disabled] {
+  cursor: default;
+  background-color: #e6e6e6;
+  background-image: none;
+  opacity: 0.65;
+  filter: alpha(opacity=65);
+  -webkit-box-shadow: none;
+     -moz-box-shadow: none;
+          box-shadow: none;
+}
+
+.btn-large {
+  padding: 9px 14px;
+  font-size: 16px;
+  line-height: normal;
+  -webkit-border-radius: 5px;
+     -moz-border-radius: 5px;
+          border-radius: 5px;
+}
+
+.btn-large [class^="icon-"] {
+  margin-top: 2px;
+}
+
+.btn-small {
+  padding: 3px 9px;
+  font-size: 12px;
+  line-height: 18px;
+}
+
+.btn-small [class^="icon-"] {
+  margin-top: 0;
+}
+
+.btn-mini {
+  padding: 2px 6px;
+  font-size: 11px;
+  line-height: 17px;
+}
+
+.btn-block {
+  display: block;
+  width: 100%;
+  padding-right: 0;
+  padding-left: 0;
+  -webkit-box-sizing: border-box;
+     -moz-box-sizing: border-box;
+          box-sizing: border-box;
+}
+
+.btn-block + .btn-block {
+  margin-top: 5px;
+}
+
+input[type="submit"].btn-block,
+input[type="reset"].btn-block,
+input[type="button"].btn-block {
+  width: 100%;
+}
+
+.btn-primary.active,
+.btn-warning.active,
+.btn-danger.active,
+.btn-success.active,
+.btn-info.active,
+.btn-inverse.active {
+  color: rgba(255, 255, 255, 0.75);
+}
+
+.btn {
+  border-color: #c5c5c5;
+  border-color: rgba(0, 0, 0, 0.15) rgba(0, 0, 0, 0.15) rgba(0, 0, 0, 0.25);
+}
+
+.btn-primary {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #006dcc;
+  *background-color: #0044cc;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc));
+  background-image: -webkit-linear-gradient(top, #0088cc, #0044cc);
+  background-image: -o-linear-gradient(top, #0088cc, #0044cc);
+  background-image: linear-gradient(to bottom, #0088cc, #0044cc);
+  background-image: -moz-linear-gradient(top, #0088cc, #0044cc);
+  background-repeat: repeat-x;
+  border-color: #0044cc #0044cc #002a80;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.btn-primary:hover,
+.btn-primary:active,
+.btn-primary.active,
+.btn-primary.disabled,
+.btn-primary[disabled] {
+  color: #ffffff;
+  background-color: #0044cc;
+  *background-color: #003bb3;
+}
+
+.btn-primary:active,
+.btn-primary.active {
+  background-color: #003399 \9;
+}
+
+.btn-warning {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #faa732;
+  *background-color: #f89406;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
+  background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
+  background-image: -o-linear-gradient(top, #fbb450, #f89406);
+  background-image: linear-gradient(to bottom, #fbb450, #f89406);
+  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
+  background-repeat: repeat-x;
+  border-color: #f89406 #f89406 #ad6704;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.btn-warning:hover,
+.btn-warning:active,
+.btn-warning.active,
+.btn-warning.disabled,
+.btn-warning[disabled] {
+  color: #ffffff;
+  background-color: #f89406;
+  *background-color: #df8505;
+}
+
+.btn-warning:active,
+.btn-warning.active {
+  background-color: #c67605 \9;
+}
+
+.btn-danger {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #da4f49;
+  *background-color: #bd362f;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f));
+  background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: -o-linear-gradient(top, #ee5f5b, #bd362f);
+  background-image: linear-gradient(to bottom, #ee5f5b, #bd362f);
+  background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f);
+  background-repeat: repeat-x;
+  border-color: #bd362f #bd362f #802420;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.btn-danger:hover,
+.btn-danger:active,
+.btn-danger.active,
+.btn-danger.disabled,
+.btn-danger[disabled] {
+  color: #ffffff;
+  background-color: #bd362f;
+  *background-color: #a9302a;
+}
+
+.btn-danger:active,
+.btn-danger.active {
+  background-color: #942a25 \9;
+}
+
+.btn-success {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #5bb75b;
+  *background-color: #51a351;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351));
+  background-image: -webkit-linear-gradient(top, #62c462, #51a351);
+  background-image: -o-linear-gradient(top, #62c462, #51a351);
+  background-image: linear-gradient(to bottom, #62c462, #51a351);
+  background-image: -moz-linear-gradient(top, #62c462, #51a351);
+  background-repeat: repeat-x;
+  border-color: #51a351 #51a351 #387038;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.btn-success:hover,
+.btn-success:active,
+.btn-success.active,
+.btn-success.disabled,
+.btn-success[disabled] {
+  color: #ffffff;
+  background-color: #51a351;
+  *background-color: #499249;
+}
+
+.btn-success:active,
+.btn-success.active {
+  background-color: #408140 \9;
+}
+
+.btn-info {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #49afcd;
+  *background-color: #2f96b4;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4));
+  background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: -o-linear-gradient(top, #5bc0de, #2f96b4);
+  background-image: linear-gradient(to bottom, #5bc0de, #2f96b4);
+  background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4);
+  background-repeat: repeat-x;
+  border-color: #2f96b4 #2f96b4 #1f6377;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.btn-info:hover,
+.btn-info:active,
+.btn-info.active,
+.btn-info.disabled,
+.btn-info[disabled] {
+  color: #ffffff;
+  background-color: #2f96b4;
+  *background-color: #2a85a0;
+}
+
+.btn-info:active,
+.btn-info.active {
+  background-color: #24748c \9;
+}
+
+.btn-inverse {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #363636;
+  *background-color: #222222;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222));
+  background-image: -webkit-linear-gradient(top, #444444, #222222);
+  background-image: -o-linear-gradient(top, #444444, #222222);
+  background-image: linear-gradient(to bottom, #444444, #222222);
+  background-image: -moz-linear-gradient(top, #444444, #222222);
+  background-repeat: repeat-x;
+  border-color: #222222 #222222 #000000;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.btn-inverse:hover,
+.btn-inverse:active,
+.btn-inverse.active,
+.btn-inverse.disabled,
+.btn-inverse[disabled] {
+  color: #ffffff;
+  background-color: #222222;
+  *background-color: #151515;
+}
+
+.btn-inverse:active,
+.btn-inverse.active {
+  background-color: #080808 \9;
+}
+
+button.btn,
+input[type="submit"].btn {
+  *padding-top: 3px;
+  *padding-bottom: 3px;
+}
+
+button.btn::-moz-focus-inner,
+input[type="submit"].btn::-moz-focus-inner {
+  padding: 0;
+  border: 0;
+}
+
+button.btn.btn-large,
+input[type="submit"].btn.btn-large {
+  *padding-top: 7px;
+  *padding-bottom: 7px;
+}
+
+button.btn.btn-small,
+input[type="submit"].btn.btn-small {
+  *padding-top: 3px;
+  *padding-bottom: 3px;
+}
+
+button.btn.btn-mini,
+input[type="submit"].btn.btn-mini {
+  *padding-top: 1px;
+  *padding-bottom: 1px;
+}
+
+.btn-link,
+.btn-link:active,
+.btn-link[disabled] {
+  background-color: transparent;
+  background-image: none;
+  -webkit-box-shadow: none;
+     -moz-box-shadow: none;
+          box-shadow: none;
+}
+
+.btn-link {
+  color: #0088cc;
+  cursor: pointer;
+  border-color: transparent;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.btn-link:hover {
+  color: #005580;
+  text-decoration: underline;
+  background-color: transparent;
+}
+
+.btn-link[disabled]:hover {
+  color: #333333;
+  text-decoration: none;
+}
+
+.btn-group {
+  position: relative;
+  *margin-left: .3em;
+  font-size: 0;
+  white-space: nowrap;
+  vertical-align: middle;
+}
+
+.btn-group:first-child {
+  *margin-left: 0;
+}
+
+.btn-group + .btn-group {
+  margin-left: 5px;
+}
+
+.btn-toolbar {
+  margin-top: 10px;
+  margin-bottom: 10px;
+  font-size: 0;
+}
+
+.btn-toolbar .btn-group {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+}
+
+.btn-toolbar .btn + .btn,
+.btn-toolbar .btn-group + .btn,
+.btn-toolbar .btn + .btn-group {
+  margin-left: 5px;
+}
+
+.btn-group > .btn {
+  position: relative;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.btn-group > .btn + .btn {
+  margin-left: -1px;
+}
+
+.btn-group > .btn,
+.btn-group > .dropdown-menu {
+  font-size: 14px;
+}
+
+.btn-group > .btn-mini {
+  font-size: 11px;
+}
+
+.btn-group > .btn-small {
+  font-size: 12px;
+}
+
+.btn-group > .btn-large {
+  font-size: 16px;
+}
+
+.btn-group > .btn:first-child {
+  margin-left: 0;
+  -webkit-border-bottom-left-radius: 4px;
+          border-bottom-left-radius: 4px;
+  -webkit-border-top-left-radius: 4px;
+          border-top-left-radius: 4px;
+  -moz-border-radius-bottomleft: 4px;
+  -moz-border-radius-topleft: 4px;
+}
+
+.btn-group > .btn:last-child,
+.btn-group > .dropdown-toggle {
+  -webkit-border-top-right-radius: 4px;
+          border-top-right-radius: 4px;
+  -webkit-border-bottom-right-radius: 4px;
+          border-bottom-right-radius: 4px;
+  -moz-border-radius-topright: 4px;
+  -moz-border-radius-bottomright: 4px;
+}
+
+.btn-group > .btn.large:first-child {
+  margin-left: 0;
+  -webkit-border-bottom-left-radius: 6px;
+          border-bottom-left-radius: 6px;
+  -webkit-border-top-left-radius: 6px;
+          border-top-left-radius: 6px;
+  -moz-border-radius-bottomleft: 6px;
+  -moz-border-radius-topleft: 6px;
+}
+
+.btn-group > .btn.large:last-child,
+.btn-group > .large.dropdown-toggle {
+  -webkit-border-top-right-radius: 6px;
+          border-top-right-radius: 6px;
+  -webkit-border-bottom-right-radius: 6px;
+          border-bottom-right-radius: 6px;
+  -moz-border-radius-topright: 6px;
+  -moz-border-radius-bottomright: 6px;
+}
+
+.btn-group > .btn:hover,
+.btn-group > .btn:focus,
+.btn-group > .btn:active,
+.btn-group > .btn.active {
+  z-index: 2;
+}
+
+.btn-group .dropdown-toggle:active,
+.btn-group.open .dropdown-toggle {
+  outline: 0;
+}
+
+.btn-group > .btn + .dropdown-toggle {
+  *padding-top: 5px;
+  padding-right: 8px;
+  *padding-bottom: 5px;
+  padding-left: 8px;
+  -webkit-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+
+.btn-group > .btn-mini + .dropdown-toggle {
+  *padding-top: 2px;
+  padding-right: 5px;
+  *padding-bottom: 2px;
+  padding-left: 5px;
+}
+
+.btn-group > .btn-small + .dropdown-toggle {
+  *padding-top: 5px;
+  *padding-bottom: 4px;
+}
+
+.btn-group > .btn-large + .dropdown-toggle {
+  *padding-top: 7px;
+  padding-right: 12px;
+  *padding-bottom: 7px;
+  padding-left: 12px;
+}
+
+.btn-group.open .dropdown-toggle {
+  background-image: none;
+  -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+
+.btn-group.open .btn.dropdown-toggle {
+  background-color: #e6e6e6;
+}
+
+.btn-group.open .btn-primary.dropdown-toggle {
+  background-color: #0044cc;
+}
+
+.btn-group.open .btn-warning.dropdown-toggle {
+  background-color: #f89406;
+}
+
+.btn-group.open .btn-danger.dropdown-toggle {
+  background-color: #bd362f;
+}
+
+.btn-group.open .btn-success.dropdown-toggle {
+  background-color: #51a351;
+}
+
+.btn-group.open .btn-info.dropdown-toggle {
+  background-color: #2f96b4;
+}
+
+.btn-group.open .btn-inverse.dropdown-toggle {
+  background-color: #222222;
+}
+
+.btn .caret {
+  margin-top: 8px;
+  margin-left: 0;
+}
+
+.btn-mini .caret,
+.btn-small .caret,
+.btn-large .caret {
+  margin-top: 6px;
+}
+
+.btn-large .caret {
+  border-top-width: 5px;
+  border-right-width: 5px;
+  border-left-width: 5px;
+}
+
+.dropup .btn-large .caret {
+  border-top: 0;
+  border-bottom: 5px solid #000000;
+}
+
+.btn-primary .caret,
+.btn-warning .caret,
+.btn-danger .caret,
+.btn-info .caret,
+.btn-success .caret,
+.btn-inverse .caret {
+  border-top-color: #ffffff;
+  border-bottom-color: #ffffff;
+}
+
+.btn-group-vertical {
+  display: inline-block;
+  *display: inline;
+  /* IE7 inline-block hack */
+
+  *zoom: 1;
+}
+
+.btn-group-vertical .btn {
+  display: block;
+  float: none;
+  width: 100%;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.btn-group-vertical .btn + .btn {
+  margin-top: -1px;
+  margin-left: 0;
+}
+
+.btn-group-vertical .btn:first-child {
+  -webkit-border-radius: 4px 4px 0 0;
+     -moz-border-radius: 4px 4px 0 0;
+          border-radius: 4px 4px 0 0;
+}
+
+.btn-group-vertical .btn:last-child {
+  -webkit-border-radius: 0 0 4px 4px;
+     -moz-border-radius: 0 0 4px 4px;
+          border-radius: 0 0 4px 4px;
+}
+
+.btn-group-vertical .btn-large:first-child {
+  -webkit-border-radius: 6px 6px 0 0;
+     -moz-border-radius: 6px 6px 0 0;
+          border-radius: 6px 6px 0 0;
+}
+
+.btn-group-vertical .btn-large:last-child {
+  -webkit-border-radius: 0 0 6px 6px;
+     -moz-border-radius: 0 0 6px 6px;
+          border-radius: 0 0 6px 6px;
+}
+
+.alert {
+  padding: 8px 35px 8px 14px;
+  margin-bottom: 20px;
+  color: #c09853;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+  background-color: #fcf8e3;
+  border: 1px solid #fbeed5;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+.alert h4 {
+  margin: 0;
+}
+
+.alert .close {
+  position: relative;
+  top: -2px;
+  right: -21px;
+  line-height: 20px;
+}
+
+.alert-success {
+  color: #468847;
+  background-color: #dff0d8;
+  border-color: #d6e9c6;
+}
+
+.alert-danger,
+.alert-error {
+  color: #b94a48;
+  background-color: #f2dede;
+  border-color: #eed3d7;
+}
+
+.alert-info {
+  color: #3a87ad;
+  background-color: #d9edf7;
+  border-color: #bce8f1;
+}
+
+.alert-block {
+  padding-top: 14px;
+  padding-bottom: 14px;
+}
+
+.alert-block > p,
+.alert-block > ul {
+  margin-bottom: 0;
+}
+
+.alert-block p + p {
+  margin-top: 5px;
+}
+
+.nav {
+  margin-bottom: 20px;
+  margin-left: 0;
+  list-style: none;
+}
+
+.nav > li > a {
+  display: block;
+}
+
+.nav > li > a:hover {
+  text-decoration: none;
+  background-color: #eeeeee;
+}
+
+.nav > .pull-right {
+  float: right;
+}
+
+.nav-header {
+  display: block;
+  padding: 3px 15px;
+  font-size: 11px;
+  font-weight: bold;
+  line-height: 20px;
+  color: #999999;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+  text-transform: uppercase;
+}
+
+.nav li + .nav-header {
+  margin-top: 9px;
+}
+
+.nav-list {
+  padding-right: 15px;
+  padding-left: 15px;
+  margin-bottom: 0;
+}
+
+.nav-list > li > a,
+.nav-list .nav-header {
+  margin-right: -15px;
+  margin-left: -15px;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5);
+}
+
+.nav-list > li > a {
+  padding: 3px 15px;
+}
+
+.nav-list > .active > a,
+.nav-list > .active > a:hover {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
+  background-color: #0088cc;
+}
+
+.nav-list [class^="icon-"] {
+  margin-right: 2px;
+}
+
+.nav-list .divider {
+  *width: 100%;
+  height: 1px;
+  margin: 9px 1px;
+  *margin: -5px 0 5px;
+  overflow: hidden;
+  background-color: #e5e5e5;
+  border-bottom: 1px solid #ffffff;
+}
+
+.nav-tabs,
+.nav-pills {
+  *zoom: 1;
+}
+
+.nav-tabs:before,
+.nav-pills:before,
+.nav-tabs:after,
+.nav-pills:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.nav-tabs:after,
+.nav-pills:after {
+  clear: both;
+}
+
+.nav-tabs > li,
+.nav-pills > li {
+  float: left;
+}
+
+.nav-tabs > li > a,
+.nav-pills > li > a {
+  padding-right: 12px;
+  padding-left: 12px;
+  margin-right: 2px;
+  line-height: 14px;
+}
+
+.nav-tabs {
+  border-bottom: 1px solid #ddd;
+}
+
+.nav-tabs > li {
+  margin-bottom: -1px;
+}
+
+.nav-tabs > li > a {
+  padding-top: 8px;
+  padding-bottom: 8px;
+  line-height: 20px;
+  border: 1px solid transparent;
+  -webkit-border-radius: 4px 4px 0 0;
+     -moz-border-radius: 4px 4px 0 0;
+          border-radius: 4px 4px 0 0;
+}
+
+.nav-tabs > li > a:hover {
+  border-color: #eeeeee #eeeeee #dddddd;
+}
+
+.nav-tabs > .active > a,
+.nav-tabs > .active > a:hover {
+  color: #555555;
+  cursor: default;
+  background-color: #ffffff;
+  border: 1px solid #ddd;
+  border-bottom-color: transparent;
+}
+
+.nav-pills > li > a {
+  padding-top: 8px;
+  padding-bottom: 8px;
+  margin-top: 2px;
+  margin-bottom: 2px;
+  -webkit-border-radius: 5px;
+     -moz-border-radius: 5px;
+          border-radius: 5px;
+}
+
+.nav-pills > .active > a,
+.nav-pills > .active > a:hover {
+  color: #ffffff;
+  background-color: #0088cc;
+}
+
+.nav-stacked > li {
+  float: none;
+}
+
+.nav-stacked > li > a {
+  margin-right: 0;
+}
+
+.nav-tabs.nav-stacked {
+  border-bottom: 0;
+}
+
+.nav-tabs.nav-stacked > li > a {
+  border: 1px solid #ddd;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.nav-tabs.nav-stacked > li:first-child > a {
+  -webkit-border-top-right-radius: 4px;
+          border-top-right-radius: 4px;
+  -webkit-border-top-left-radius: 4px;
+          border-top-left-radius: 4px;
+  -moz-border-radius-topright: 4px;
+  -moz-border-radius-topleft: 4px;
+}
+
+.nav-tabs.nav-stacked > li:last-child > a {
+  -webkit-border-bottom-right-radius: 4px;
+          border-bottom-right-radius: 4px;
+  -webkit-border-bottom-left-radius: 4px;
+          border-bottom-left-radius: 4px;
+  -moz-border-radius-bottomright: 4px;
+  -moz-border-radius-bottomleft: 4px;
+}
+
+.nav-tabs.nav-stacked > li > a:hover {
+  z-index: 2;
+  border-color: #ddd;
+}
+
+.nav-pills.nav-stacked > li > a {
+  margin-bottom: 3px;
+}
+
+.nav-pills.nav-stacked > li:last-child > a {
+  margin-bottom: 1px;
+}
+
+.nav-tabs .dropdown-menu {
+  -webkit-border-radius: 0 0 6px 6px;
+     -moz-border-radius: 0 0 6px 6px;
+          border-radius: 0 0 6px 6px;
+}
+
+.nav-pills .dropdown-menu {
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+}
+
+.nav .dropdown-toggle .caret {
+  margin-top: 6px;
+  border-top-color: #0088cc;
+  border-bottom-color: #0088cc;
+}
+
+.nav .dropdown-toggle:hover .caret {
+  border-top-color: #005580;
+  border-bottom-color: #005580;
+}
+
+/* move down carets for tabs */
+
+.nav-tabs .dropdown-toggle .caret {
+  margin-top: 8px;
+}
+
+.nav .active .dropdown-toggle .caret {
+  border-top-color: #fff;
+  border-bottom-color: #fff;
+}
+
+.nav-tabs .active .dropdown-toggle .caret {
+  border-top-color: #555555;
+  border-bottom-color: #555555;
+}
+
+.nav > .dropdown.active > a:hover {
+  cursor: pointer;
+}
+
+.nav-tabs .open .dropdown-toggle,
+.nav-pills .open .dropdown-toggle,
+.nav > li.dropdown.open.active > a:hover {
+  color: #ffffff;
+  background-color: #999999;
+  border-color: #999999;
+}
+
+.nav li.dropdown.open .caret,
+.nav li.dropdown.open.active .caret,
+.nav li.dropdown.open a:hover .caret {
+  border-top-color: #ffffff;
+  border-bottom-color: #ffffff;
+  opacity: 1;
+  filter: alpha(opacity=100);
+}
+
+.tabs-stacked .open > a:hover {
+  border-color: #999999;
+}
+
+.tabbable {
+  *zoom: 1;
+}
+
+.tabbable:before,
+.tabbable:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.tabbable:after {
+  clear: both;
+}
+
+.tab-content {
+  overflow: auto;
+}
+
+.tabs-below > .nav-tabs,
+.tabs-right > .nav-tabs,
+.tabs-left > .nav-tabs {
+  border-bottom: 0;
+}
+
+.tab-content > .tab-pane,
+.pill-content > .pill-pane {
+  display: none;
+}
+
+.tab-content > .active,
+.pill-content > .active {
+  display: block;
+}
+
+.tabs-below > .nav-tabs {
+  border-top: 1px solid #ddd;
+}
+
+.tabs-below > .nav-tabs > li {
+  margin-top: -1px;
+  margin-bottom: 0;
+}
+
+.tabs-below > .nav-tabs > li > a {
+  -webkit-border-radius: 0 0 4px 4px;
+     -moz-border-radius: 0 0 4px 4px;
+          border-radius: 0 0 4px 4px;
+}
+
+.tabs-below > .nav-tabs > li > a:hover {
+  border-top-color: #ddd;
+  border-bottom-color: transparent;
+}
+
+.tabs-below > .nav-tabs > .active > a,
+.tabs-below > .nav-tabs > .active > a:hover {
+  border-color: transparent #ddd #ddd #ddd;
+}
+
+.tabs-left > .nav-tabs > li,
+.tabs-right > .nav-tabs > li {
+  float: none;
+}
+
+.tabs-left > .nav-tabs > li > a,
+.tabs-right > .nav-tabs > li > a {
+  min-width: 74px;
+  margin-right: 0;
+  margin-bottom: 3px;
+}
+
+.tabs-left > .nav-tabs {
+  float: left;
+  margin-right: 19px;
+  border-right: 1px solid #ddd;
+}
+
+.tabs-left > .nav-tabs > li > a {
+  margin-right: -1px;
+  -webkit-border-radius: 4px 0 0 4px;
+     -moz-border-radius: 4px 0 0 4px;
+          border-radius: 4px 0 0 4px;
+}
+
+.tabs-left > .nav-tabs > li > a:hover {
+  border-color: #eeeeee #dddddd #eeeeee #eeeeee;
+}
+
+.tabs-left > .nav-tabs .active > a,
+.tabs-left > .nav-tabs .active > a:hover {
+  border-color: #ddd transparent #ddd #ddd;
+  *border-right-color: #ffffff;
+}
+
+.tabs-right > .nav-tabs {
+  float: right;
+  margin-left: 19px;
+  border-left: 1px solid #ddd;
+}
+
+.tabs-right > .nav-tabs > li > a {
+  margin-left: -1px;
+  -webkit-border-radius: 0 4px 4px 0;
+     -moz-border-radius: 0 4px 4px 0;
+          border-radius: 0 4px 4px 0;
+}
+
+.tabs-right > .nav-tabs > li > a:hover {
+  border-color: #eeeeee #eeeeee #eeeeee #dddddd;
+}
+
+.tabs-right > .nav-tabs .active > a,
+.tabs-right > .nav-tabs .active > a:hover {
+  border-color: #ddd #ddd #ddd transparent;
+  *border-left-color: #ffffff;
+}
+
+.nav > .disabled > a {
+  color: #999999;
+}
+
+.nav > .disabled > a:hover {
+  text-decoration: none;
+  cursor: default;
+  background-color: transparent;
+}
+
+.navbar {
+  *position: relative;
+  *z-index: 2;
+  margin-bottom: 20px;
+  overflow: visible;
+  color: #777777;
+}
+
+.navbar-inner {
+  min-height: 40px;
+  padding-right: 20px;
+  padding-left: 20px;
+  background-color: #fafafa;
+  background-image: -moz-linear-gradient(top, #ffffff, #f2f2f2);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f2f2f2));
+  background-image: -webkit-linear-gradient(top, #ffffff, #f2f2f2);
+  background-image: -o-linear-gradient(top, #ffffff, #f2f2f2);
+  background-image: linear-gradient(to bottom, #ffffff, #f2f2f2);
+  background-repeat: repeat-x;
+  border: 1px solid #d4d4d4;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff2f2f2', GradientType=0);
+  *zoom: 1;
+  -webkit-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
+     -moz-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
+          box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065);
+}
+
+.navbar-inner:before,
+.navbar-inner:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.navbar-inner:after {
+  clear: both;
+}
+
+.navbar .container {
+  width: auto;
+}
+
+.nav-collapse.collapse {
+  height: auto;
+}
+
+.navbar .brand {
+  display: block;
+  float: left;
+  padding: 10px 20px 10px;
+  margin-left: -20px;
+  font-size: 20px;
+  font-weight: 200;
+  color: #777777;
+  text-shadow: 0 1px 0 #ffffff;
+}
+
+.navbar .brand:hover {
+  text-decoration: none;
+}
+
+.navbar-text {
+  margin-bottom: 0;
+  line-height: 40px;
+}
+
+.navbar-link {
+  color: #777777;
+}
+
+.navbar-link:hover {
+  color: #333333;
+}
+
+.navbar .divider-vertical {
+  height: 40px;
+  margin: 0 9px;
+  border-right: 1px solid #ffffff;
+  border-left: 1px solid #f2f2f2;
+}
+
+.navbar .btn,
+.navbar .btn-group {
+  margin-top: 5px;
+}
+
+.navbar .btn-group .btn,
+.navbar .input-prepend .btn,
+.navbar .input-append .btn {
+  margin-top: 0;
+}
+
+.navbar-form {
+  margin-bottom: 0;
+  *zoom: 1;
+}
+
+.navbar-form:before,
+.navbar-form:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.navbar-form:after {
+  clear: both;
+}
+
+.navbar-form input,
+.navbar-form select,
+.navbar-form .radio,
+.navbar-form .checkbox {
+  margin-top: 5px;
+}
+
+.navbar-form input,
+.navbar-form select,
+.navbar-form .btn {
+  display: inline-block;
+  margin-bottom: 0;
+}
+
+.navbar-form input[type="image"],
+.navbar-form input[type="checkbox"],
+.navbar-form input[type="radio"] {
+  margin-top: 3px;
+}
+
+.navbar-form .input-append,
+.navbar-form .input-prepend {
+  margin-top: 6px;
+  white-space: nowrap;
+}
+
+.navbar-form .input-append input,
+.navbar-form .input-prepend input {
+  margin-top: 0;
+}
+
+.navbar-search {
+  position: relative;
+  float: left;
+  margin-top: 5px;
+  margin-bottom: 0;
+}
+
+.navbar-search .search-query {
+  padding: 4px 14px;
+  margin-bottom: 0;
+  font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+  font-size: 13px;
+  font-weight: normal;
+  line-height: 1;
+  -webkit-border-radius: 15px;
+     -moz-border-radius: 15px;
+          border-radius: 15px;
+}
+
+.navbar-static-top {
+  position: static;
+  width: 100%;
+  margin-bottom: 0;
+}
+
+.navbar-static-top .navbar-inner {
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.navbar-fixed-top,
+.navbar-fixed-bottom {
+  position: fixed;
+  right: 0;
+  left: 0;
+  z-index: 1030;
+  margin-bottom: 0;
+}
+
+.navbar-fixed-top .navbar-inner,
+.navbar-static-top .navbar-inner {
+  border-width: 0 0 1px;
+}
+
+.navbar-fixed-bottom .navbar-inner {
+  border-width: 1px 0 0;
+}
+
+.navbar-fixed-top .navbar-inner,
+.navbar-fixed-bottom .navbar-inner {
+  padding-right: 0;
+  padding-left: 0;
+  -webkit-border-radius: 0;
+     -moz-border-radius: 0;
+          border-radius: 0;
+}
+
+.navbar-static-top .container,
+.navbar-fixed-top .container,
+.navbar-fixed-bottom .container {
+  width: 940px;
+}
+
+.navbar-fixed-top {
+  top: 0;
+}
+
+.navbar-fixed-top .navbar-inner,
+.navbar-static-top .navbar-inner {
+  -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1);
+     -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1);
+          box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1);
+}
+
+.navbar-fixed-bottom {
+  bottom: 0;
+}
+
+.navbar-fixed-bottom .navbar-inner {
+  -webkit-box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1);
+     -moz-box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1);
+          box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1);
+}
+
+.navbar .nav {
+  position: relative;
+  left: 0;
+  display: block;
+  float: left;
+  margin: 0 10px 0 0;
+}
+
+.navbar .nav.pull-right {
+  float: right;
+  margin-right: 0;
+}
+
+.navbar .nav > li {
+  float: left;
+}
+
+.navbar .nav > li > a {
+  float: none;
+  padding: 10px 15px 10px;
+  color: #777777;
+  text-decoration: none;
+  text-shadow: 0 1px 0 #ffffff;
+}
+
+.navbar .nav .dropdown-toggle .caret {
+  margin-top: 8px;
+}
+
+.navbar .nav > li > a:focus,
+.navbar .nav > li > a:hover {
+  color: #333333;
+  text-decoration: none;
+  background-color: transparent;
+}
+
+.navbar .nav > .active > a,
+.navbar .nav > .active > a:hover,
+.navbar .nav > .active > a:focus {
+  color: #555555;
+  text-decoration: none;
+  background-color: #e5e5e5;
+  -webkit-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
+     -moz-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
+          box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125);
+}
+
+.navbar .btn-navbar {
+  display: none;
+  float: right;
+  padding: 7px 10px;
+  margin-right: 5px;
+  margin-left: 5px;
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #ededed;
+  *background-color: #e5e5e5;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5));
+  background-image: -webkit-linear-gradient(top, #f2f2f2, #e5e5e5);
+  background-image: -o-linear-gradient(top, #f2f2f2, #e5e5e5);
+  background-image: linear-gradient(to bottom, #f2f2f2, #e5e5e5);
+  background-image: -moz-linear-gradient(top, #f2f2f2, #e5e5e5);
+  background-repeat: repeat-x;
+  border-color: #e5e5e5 #e5e5e5 #bfbfbf;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+     -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075);
+}
+
+.navbar .btn-navbar:hover,
+.navbar .btn-navbar:active,
+.navbar .btn-navbar.active,
+.navbar .btn-navbar.disabled,
+.navbar .btn-navbar[disabled] {
+  color: #ffffff;
+  background-color: #e5e5e5;
+  *background-color: #d9d9d9;
+}
+
+.navbar .btn-navbar:active,
+.navbar .btn-navbar.active {
+  background-color: #cccccc \9;
+}
+
+.navbar .btn-navbar .icon-bar {
+  display: block;
+  width: 18px;
+  height: 2px;
+  background-color: #f5f5f5;
+  -webkit-border-radius: 1px;
+     -moz-border-radius: 1px;
+          border-radius: 1px;
+  -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+     -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+          box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25);
+}
+
+.btn-navbar .icon-bar + .icon-bar {
+  margin-top: 3px;
+}
+
+.navbar .nav > li > .dropdown-menu:before {
+  position: absolute;
+  top: -7px;
+  left: 9px;
+  display: inline-block;
+  border-right: 7px solid transparent;
+  border-bottom: 7px solid #ccc;
+  border-left: 7px solid transparent;
+  border-bottom-color: rgba(0, 0, 0, 0.2);
+  content: '';
+}
+
+.navbar .nav > li > .dropdown-menu:after {
+  position: absolute;
+  top: -6px;
+  left: 10px;
+  display: inline-block;
+  border-right: 6px solid transparent;
+  border-bottom: 6px solid #ffffff;
+  border-left: 6px solid transparent;
+  content: '';
+}
+
+.navbar-fixed-bottom .nav > li > .dropdown-menu:before {
+  top: auto;
+  bottom: -7px;
+  border-top: 7px solid #ccc;
+  border-bottom: 0;
+  border-top-color: rgba(0, 0, 0, 0.2);
+}
+
+.navbar-fixed-bottom .nav > li > .dropdown-menu:after {
+  top: auto;
+  bottom: -6px;
+  border-top: 6px solid #ffffff;
+  border-bottom: 0;
+}
+
+.navbar .nav li.dropdown.open > .dropdown-toggle,
+.navbar .nav li.dropdown.active > .dropdown-toggle,
+.navbar .nav li.dropdown.open.active > .dropdown-toggle {
+  color: #555555;
+  background-color: #e5e5e5;
+}
+
+.navbar .nav li.dropdown > .dropdown-toggle .caret {
+  border-top-color: #777777;
+  border-bottom-color: #777777;
+}
+
+.navbar .nav li.dropdown.open > .dropdown-toggle .caret,
+.navbar .nav li.dropdown.active > .dropdown-toggle .caret,
+.navbar .nav li.dropdown.open.active > .dropdown-toggle .caret {
+  border-top-color: #555555;
+  border-bottom-color: #555555;
+}
+
+.navbar .pull-right > li > .dropdown-menu,
+.navbar .nav > li > .dropdown-menu.pull-right {
+  right: 0;
+  left: auto;
+}
+
+.navbar .pull-right > li > .dropdown-menu:before,
+.navbar .nav > li > .dropdown-menu.pull-right:before {
+  right: 12px;
+  left: auto;
+}
+
+.navbar .pull-right > li > .dropdown-menu:after,
+.navbar .nav > li > .dropdown-menu.pull-right:after {
+  right: 13px;
+  left: auto;
+}
+
+.navbar .pull-right > li > .dropdown-menu .dropdown-menu,
+.navbar .nav > li > .dropdown-menu.pull-right .dropdown-menu {
+  right: 100%;
+  left: auto;
+  margin-right: -1px;
+  margin-left: 0;
+  -webkit-border-radius: 6px 0 6px 6px;
+     -moz-border-radius: 6px 0 6px 6px;
+          border-radius: 6px 0 6px 6px;
+}
+
+.navbar-inverse {
+  color: #999999;
+}
+
+.navbar-inverse .navbar-inner {
+  background-color: #1b1b1b;
+  background-image: -moz-linear-gradient(top, #222222, #111111);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111));
+  background-image: -webkit-linear-gradient(top, #222222, #111111);
+  background-image: -o-linear-gradient(top, #222222, #111111);
+  background-image: linear-gradient(to bottom, #222222, #111111);
+  background-repeat: repeat-x;
+  border-color: #252525;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0);
+}
+
+.navbar-inverse .brand,
+.navbar-inverse .nav > li > a {
+  color: #999999;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+}
+
+.navbar-inverse .brand:hover,
+.navbar-inverse .nav > li > a:hover {
+  color: #ffffff;
+}
+
+.navbar-inverse .nav > li > a:focus,
+.navbar-inverse .nav > li > a:hover {
+  color: #ffffff;
+  background-color: transparent;
+}
+
+.navbar-inverse .nav .active > a,
+.navbar-inverse .nav .active > a:hover,
+.navbar-inverse .nav .active > a:focus {
+  color: #ffffff;
+  background-color: #111111;
+}
+
+.navbar-inverse .navbar-link {
+  color: #999999;
+}
+
+.navbar-inverse .navbar-link:hover {
+  color: #ffffff;
+}
+
+.navbar-inverse .divider-vertical {
+  border-right-color: #222222;
+  border-left-color: #111111;
+}
+
+.navbar-inverse .nav li.dropdown.open > .dropdown-toggle,
+.navbar-inverse .nav li.dropdown.active > .dropdown-toggle,
+.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle {
+  color: #ffffff;
+  background-color: #111111;
+}
+
+.navbar-inverse .nav li.dropdown > .dropdown-toggle .caret {
+  border-top-color: #999999;
+  border-bottom-color: #999999;
+}
+
+.navbar-inverse .nav li.dropdown.open > .dropdown-toggle .caret,
+.navbar-inverse .nav li.dropdown.active > .dropdown-toggle .caret,
+.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle .caret {
+  border-top-color: #ffffff;
+  border-bottom-color: #ffffff;
+}
+
+.navbar-inverse .navbar-search .search-query {
+  color: #ffffff;
+  background-color: #515151;
+  border-color: #111111;
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
+     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
+          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15);
+  -webkit-transition: none;
+     -moz-transition: none;
+       -o-transition: none;
+          transition: none;
+}
+
+.navbar-inverse .navbar-search .search-query:-moz-placeholder {
+  color: #cccccc;
+}
+
+.navbar-inverse .navbar-search .search-query:-ms-input-placeholder {
+  color: #cccccc;
+}
+
+.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder {
+  color: #cccccc;
+}
+
+.navbar-inverse .navbar-search .search-query:focus,
+.navbar-inverse .navbar-search .search-query.focused {
+  padding: 5px 15px;
+  color: #333333;
+  text-shadow: 0 1px 0 #ffffff;
+  background-color: #ffffff;
+  border: 0;
+  outline: 0;
+  -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+     -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+          box-shadow: 0 0 3px rgba(0, 0, 0, 0.15);
+}
+
+.navbar-inverse .btn-navbar {
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #0e0e0e;
+  *background-color: #040404;
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404));
+  background-image: -webkit-linear-gradient(top, #151515, #040404);
+  background-image: -o-linear-gradient(top, #151515, #040404);
+  background-image: linear-gradient(to bottom, #151515, #040404);
+  background-image: -moz-linear-gradient(top, #151515, #040404);
+  background-repeat: repeat-x;
+  border-color: #040404 #040404 #000000;
+  border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25);
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0);
+  filter: progid:dximagetransform.microsoft.gradient(enabled=false);
+}
+
+.navbar-inverse .btn-navbar:hover,
+.navbar-inverse .btn-navbar:active,
+.navbar-inverse .btn-navbar.active,
+.navbar-inverse .btn-navbar.disabled,
+.navbar-inverse .btn-navbar[disabled] {
+  color: #ffffff;
+  background-color: #040404;
+  *background-color: #000000;
+}
+
+.navbar-inverse .btn-navbar:active,
+.navbar-inverse .btn-navbar.active {
+  background-color: #000000 \9;
+}
+
+.breadcrumb {
+  padding: 8px 15px;
+  margin: 0 0 20px;
+  list-style: none;
+  background-color: #f5f5f5;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+.breadcrumb li {
+  display: inline-block;
+  *display: inline;
+  text-shadow: 0 1px 0 #ffffff;
+  *zoom: 1;
+}
+
+.breadcrumb .divider {
+  padding: 0 5px;
+  color: #ccc;
+}
+
+.breadcrumb .active {
+  color: #999999;
+}
+
+.pagination {
+  height: 40px;
+  margin: 20px 0;
+}
+
+.pagination ul {
+  display: inline-block;
+  *display: inline;
+  margin-bottom: 0;
+  margin-left: 0;
+  -webkit-border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+  *zoom: 1;
+  -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+     -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+          box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
+}
+
+.pagination ul > li {
+  display: inline;
+}
+
+.pagination ul > li > a,
+.pagination ul > li > span {
+  float: left;
+  padding: 0 14px;
+  line-height: 38px;
+  text-decoration: none;
+  background-color: #ffffff;
+  border: 1px solid #dddddd;
+  border-left-width: 0;
+}
+
+.pagination ul > li > a:hover,
+.pagination ul > .active > a,
+.pagination ul > .active > span {
+  background-color: #f5f5f5;
+}
+
+.pagination ul > .active > a,
+.pagination ul > .active > span {
+  color: #999999;
+  cursor: default;
+}
+
+.pagination ul > .disabled > span,
+.pagination ul > .disabled > a,
+.pagination ul > .disabled > a:hover {
+  color: #999999;
+  cursor: default;
+  background-color: transparent;
+}
+
+.pagination ul > li:first-child > a,
+.pagination ul > li:first-child > span {
+  border-left-width: 1px;
+  -webkit-border-radius: 3px 0 0 3px;
+     -moz-border-radius: 3px 0 0 3px;
+          border-radius: 3px 0 0 3px;
+}
+
+.pagination ul > li:last-child > a,
+.pagination ul > li:last-child > span {
+  -webkit-border-radius: 0 3px 3px 0;
+     -moz-border-radius: 0 3px 3px 0;
+          border-radius: 0 3px 3px 0;
+}
+
+.pagination-centered {
+  text-align: center;
+}
+
+.pagination-right {
+  text-align: right;
+}
+
+.pager {
+  margin: 20px 0;
+  text-align: center;
+  list-style: none;
+  *zoom: 1;
+}
+
+.pager:before,
+.pager:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.pager:after {
+  clear: both;
+}
+
+.pager li {
+  display: inline;
+}
+
+.pager a,
+.pager span {
+  display: inline-block;
+  padding: 5px 14px;
+  background-color: #fff;
+  border: 1px solid #ddd;
+  -webkit-border-radius: 15px;
+     -moz-border-radius: 15px;
+          border-radius: 15px;
+}
+
+.pager a:hover {
+  text-decoration: none;
+  background-color: #f5f5f5;
+}
+
+.pager .next a,
+.pager .next span {
+  float: right;
+}
+
+.pager .previous a {
+  float: left;
+}
+
+.pager .disabled a,
+.pager .disabled a:hover,
+.pager .disabled span {
+  color: #999999;
+  cursor: default;
+  background-color: #fff;
+}
+
+.modal-open .modal .dropdown-menu {
+  z-index: 2050;
+}
+
+.modal-open .modal .dropdown.open {
+  *z-index: 2050;
+}
+
+.modal-open .modal .popover {
+  z-index: 2060;
+}
+
+.modal-open .modal .tooltip {
+  z-index: 2080;
+}
+
+.modal-backdrop {
+  position: fixed;
+  top: 0;
+  right: 0;
+  bottom: 0;
+  left: 0;
+  z-index: 1040;
+  background-color: #000000;
+}
+
+.modal-backdrop.fade {
+  opacity: 0;
+}
+
+.modal-backdrop,
+.modal-backdrop.fade.in {
+  opacity: 0.8;
+  filter: alpha(opacity=80);
+}
+
+.modal {
+  position: fixed;
+  top: 50%;
+  left: 50%;
+  z-index: 1050;
+  width: 560px;
+  margin: -250px 0 0 -280px;
+  overflow: auto;
+  background-color: #ffffff;
+  border: 1px solid #999;
+  border: 1px solid rgba(0, 0, 0, 0.3);
+  *border: 1px solid #999;
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+  -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+     -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+          box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3);
+  -webkit-background-clip: padding-box;
+     -moz-background-clip: padding-box;
+          background-clip: padding-box;
+}
+
+.modal.fade {
+  top: -25%;
+  -webkit-transition: opacity 0.3s linear, top 0.3s ease-out;
+     -moz-transition: opacity 0.3s linear, top 0.3s ease-out;
+       -o-transition: opacity 0.3s linear, top 0.3s ease-out;
+          transition: opacity 0.3s linear, top 0.3s ease-out;
+}
+
+.modal.fade.in {
+  top: 50%;
+}
+
+.modal-header {
+  padding: 9px 15px;
+  border-bottom: 1px solid #eee;
+}
+
+.modal-header .close {
+  margin-top: 2px;
+}
+
+.modal-header h3 {
+  margin: 0;
+  line-height: 30px;
+}
+
+.modal-body {
+  max-height: 400px;
+  padding: 15px;
+  overflow-y: auto;
+}
+
+.modal-form {
+  margin-bottom: 0;
+}
+
+.modal-footer {
+  padding: 14px 15px 15px;
+  margin-bottom: 0;
+  text-align: right;
+  background-color: #f5f5f5;
+  border-top: 1px solid #ddd;
+  -webkit-border-radius: 0 0 6px 6px;
+     -moz-border-radius: 0 0 6px 6px;
+          border-radius: 0 0 6px 6px;
+  *zoom: 1;
+  -webkit-box-shadow: inset 0 1px 0 #ffffff;
+     -moz-box-shadow: inset 0 1px 0 #ffffff;
+          box-shadow: inset 0 1px 0 #ffffff;
+}
+
+.modal-footer:before,
+.modal-footer:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.modal-footer:after {
+  clear: both;
+}
+
+.modal-footer .btn + .btn {
+  margin-bottom: 0;
+  margin-left: 5px;
+}
+
+.modal-footer .btn-group .btn + .btn {
+  margin-left: -1px;
+}
+
+.tooltip {
+  position: absolute;
+  z-index: 1030;
+  display: block;
+  padding: 5px;
+  font-size: 11px;
+  opacity: 0;
+  filter: alpha(opacity=0);
+  visibility: visible;
+}
+
+.tooltip.in {
+  opacity: 0.8;
+  filter: alpha(opacity=80);
+}
+
+.tooltip.top {
+  margin-top: -3px;
+}
+
+.tooltip.right {
+  margin-left: 3px;
+}
+
+.tooltip.bottom {
+  margin-top: 3px;
+}
+
+.tooltip.left {
+  margin-left: -3px;
+}
+
+.tooltip-inner {
+  max-width: 200px;
+  padding: 3px 8px;
+  color: #ffffff;
+  text-align: center;
+  text-decoration: none;
+  background-color: #000000;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+.tooltip-arrow {
+  position: absolute;
+  width: 0;
+  height: 0;
+  border-color: transparent;
+  border-style: solid;
+}
+
+.tooltip.top .tooltip-arrow {
+  bottom: 0;
+  left: 50%;
+  margin-left: -5px;
+  border-top-color: #000000;
+  border-width: 5px 5px 0;
+}
+
+.tooltip.right .tooltip-arrow {
+  top: 50%;
+  left: 0;
+  margin-top: -5px;
+  border-right-color: #000000;
+  border-width: 5px 5px 5px 0;
+}
+
+.tooltip.left .tooltip-arrow {
+  top: 50%;
+  right: 0;
+  margin-top: -5px;
+  border-left-color: #000000;
+  border-width: 5px 0 5px 5px;
+}
+
+.tooltip.bottom .tooltip-arrow {
+  top: 0;
+  left: 50%;
+  margin-left: -5px;
+  border-bottom-color: #000000;
+  border-width: 0 5px 5px;
+}
+
+.popover {
+  position: absolute;
+  top: 0;
+  left: 0;
+  z-index: 1010;
+  display: none;
+  width: 236px;
+  padding: 1px;
+  background-color: #ffffff;
+  border: 1px solid #ccc;
+  border: 1px solid rgba(0, 0, 0, 0.2);
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+  -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+     -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+          box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2);
+  -webkit-background-clip: padding-box;
+     -moz-background-clip: padding;
+          background-clip: padding-box;
+}
+
+.popover.top {
+  margin-bottom: 10px;
+}
+
+.popover.right {
+  margin-left: 10px;
+}
+
+.popover.bottom {
+  margin-top: 10px;
+}
+
+.popover.left {
+  margin-right: 10px;
+}
+
+.popover-title {
+  padding: 8px 14px;
+  margin: 0;
+  font-size: 14px;
+  font-weight: normal;
+  line-height: 18px;
+  background-color: #f7f7f7;
+  border-bottom: 1px solid #ebebeb;
+  -webkit-border-radius: 5px 5px 0 0;
+     -moz-border-radius: 5px 5px 0 0;
+          border-radius: 5px 5px 0 0;
+}
+
+.popover-content {
+  padding: 9px 14px;
+}
+
+.popover-content p,
+.popover-content ul,
+.popover-content ol {
+  margin-bottom: 0;
+}
+
+.popover .arrow,
+.popover .arrow:after {
+  position: absolute;
+  display: inline-block;
+  width: 0;
+  height: 0;
+  border-color: transparent;
+  border-style: solid;
+}
+
+.popover .arrow:after {
+  z-index: -1;
+  content: "";
+}
+
+.popover.top .arrow {
+  bottom: -10px;
+  left: 50%;
+  margin-left: -10px;
+  border-top-color: #ffffff;
+  border-width: 10px 10px 0;
+}
+
+.popover.top .arrow:after {
+  bottom: -1px;
+  left: -11px;
+  border-top-color: rgba(0, 0, 0, 0.25);
+  border-width: 11px 11px 0;
+}
+
+.popover.right .arrow {
+  top: 50%;
+  left: -10px;
+  margin-top: -10px;
+  border-right-color: #ffffff;
+  border-width: 10px 10px 10px 0;
+}
+
+.popover.right .arrow:after {
+  bottom: -11px;
+  left: -1px;
+  border-right-color: rgba(0, 0, 0, 0.25);
+  border-width: 11px 11px 11px 0;
+}
+
+.popover.bottom .arrow {
+  top: -10px;
+  left: 50%;
+  margin-left: -10px;
+  border-bottom-color: #ffffff;
+  border-width: 0 10px 10px;
+}
+
+.popover.bottom .arrow:after {
+  top: -1px;
+  left: -11px;
+  border-bottom-color: rgba(0, 0, 0, 0.25);
+  border-width: 0 11px 11px;
+}
+
+.popover.left .arrow {
+  top: 50%;
+  right: -10px;
+  margin-top: -10px;
+  border-left-color: #ffffff;
+  border-width: 10px 0 10px 10px;
+}
+
+.popover.left .arrow:after {
+  right: -1px;
+  bottom: -11px;
+  border-left-color: rgba(0, 0, 0, 0.25);
+  border-width: 11px 0 11px 11px;
+}
+
+.thumbnails {
+  margin-left: -20px;
+  list-style: none;
+  *zoom: 1;
+}
+
+.thumbnails:before,
+.thumbnails:after {
+  display: table;
+  line-height: 0;
+  content: "";
+}
+
+.thumbnails:after {
+  clear: both;
+}
+
+.row-fluid .thumbnails {
+  margin-left: 0;
+}
+
+.thumbnails > li {
+  float: left;
+  margin-bottom: 20px;
+  margin-left: 20px;
+}
+
+.thumbnail {
+  display: block;
+  padding: 4px;
+  line-height: 20px;
+  border: 1px solid #ddd;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055);
+     -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055);
+          box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055);
+  -webkit-transition: all 0.2s ease-in-out;
+     -moz-transition: all 0.2s ease-in-out;
+       -o-transition: all 0.2s ease-in-out;
+          transition: all 0.2s ease-in-out;
+}
+
+a.thumbnail:hover {
+  border-color: #0088cc;
+  -webkit-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+     -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+          box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25);
+}
+
+.thumbnail > img {
+  display: block;
+  max-width: 100%;
+  margin-right: auto;
+  margin-left: auto;
+}
+
+.thumbnail .caption {
+  padding: 9px;
+  color: #555555;
+}
+
+.label,
+.badge {
+  font-size: 11.844px;
+  font-weight: bold;
+  line-height: 14px;
+  color: #ffffff;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  white-space: nowrap;
+  vertical-align: baseline;
+  background-color: #999999;
+}
+
+.label {
+  padding: 1px 4px 2px;
+  -webkit-border-radius: 3px;
+     -moz-border-radius: 3px;
+          border-radius: 3px;
+}
+
+.badge {
+  padding: 1px 9px 2px;
+  -webkit-border-radius: 9px;
+     -moz-border-radius: 9px;
+          border-radius: 9px;
+}
+
+a.label:hover,
+a.badge:hover {
+  color: #ffffff;
+  text-decoration: none;
+  cursor: pointer;
+}
+
+.label-important,
+.badge-important {
+  background-color: #b94a48;
+}
+
+.label-important[href],
+.badge-important[href] {
+  background-color: #953b39;
+}
+
+.label-warning,
+.badge-warning {
+  background-color: #f89406;
+}
+
+.label-warning[href],
+.badge-warning[href] {
+  background-color: #c67605;
+}
+
+.label-success,
+.badge-success {
+  background-color: #468847;
+}
+
+.label-success[href],
+.badge-success[href] {
+  background-color: #356635;
+}
+
+.label-info,
+.badge-info {
+  background-color: #3a87ad;
+}
+
+.label-info[href],
+.badge-info[href] {
+  background-color: #2d6987;
+}
+
+.label-inverse,
+.badge-inverse {
+  background-color: #333333;
+}
+
+.label-inverse[href],
+.badge-inverse[href] {
+  background-color: #1a1a1a;
+}
+
+.btn .label,
+.btn .badge {
+  position: relative;
+  top: -1px;
+}
+
+.btn-mini .label,
+.btn-mini .badge {
+  top: 0;
+}
+
+@-webkit-keyframes progress-bar-stripes {
+  from {
+    background-position: 40px 0;
+  }
+  to {
+    background-position: 0 0;
+  }
+}
+
+@-moz-keyframes progress-bar-stripes {
+  from {
+    background-position: 40px 0;
+  }
+  to {
+    background-position: 0 0;
+  }
+}
+
+@-ms-keyframes progress-bar-stripes {
+  from {
+    background-position: 40px 0;
+  }
+  to {
+    background-position: 0 0;
+  }
+}
+
+@-o-keyframes progress-bar-stripes {
+  from {
+    background-position: 0 0;
+  }
+  to {
+    background-position: 40px 0;
+  }
+}
+
+@keyframes progress-bar-stripes {
+  from {
+    background-position: 40px 0;
+  }
+  to {
+    background-position: 0 0;
+  }
+}
+
+.progress {
+  height: 20px;
+  margin-bottom: 20px;
+  overflow: hidden;
+  background-color: #f7f7f7;
+  background-image: -moz-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f5f5f5), to(#f9f9f9));
+  background-image: -webkit-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: -o-linear-gradient(top, #f5f5f5, #f9f9f9);
+  background-image: linear-gradient(to bottom, #f5f5f5, #f9f9f9);
+  background-repeat: repeat-x;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#fff9f9f9', GradientType=0);
+  -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+     -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+          box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1);
+}
+
+.progress .bar {
+  float: left;
+  width: 0;
+  height: 100%;
+  font-size: 12px;
+  color: #ffffff;
+  text-align: center;
+  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
+  background-color: #0e90d2;
+  background-image: -moz-linear-gradient(top, #149bdf, #0480be);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be));
+  background-image: -webkit-linear-gradient(top, #149bdf, #0480be);
+  background-image: -o-linear-gradient(top, #149bdf, #0480be);
+  background-image: linear-gradient(to bottom, #149bdf, #0480be);
+  background-repeat: repeat-x;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff149bdf', endColorstr='#ff0480be', GradientType=0);
+  -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+     -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+          box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+  -webkit-box-sizing: border-box;
+     -moz-box-sizing: border-box;
+          box-sizing: border-box;
+  -webkit-transition: width 0.6s ease;
+     -moz-transition: width 0.6s ease;
+       -o-transition: width 0.6s ease;
+          transition: width 0.6s ease;
+}
+
+.progress .bar + .bar {
+  -webkit-box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+     -moz-box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+          box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15);
+}
+
+.progress-striped .bar {
+  background-color: #149bdf;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  -webkit-background-size: 40px 40px;
+     -moz-background-size: 40px 40px;
+       -o-background-size: 40px 40px;
+          background-size: 40px 40px;
+}
+
+.progress.active .bar {
+  -webkit-animation: progress-bar-stripes 2s linear infinite;
+     -moz-animation: progress-bar-stripes 2s linear infinite;
+      -ms-animation: progress-bar-stripes 2s linear infinite;
+       -o-animation: progress-bar-stripes 2s linear infinite;
+          animation: progress-bar-stripes 2s linear infinite;
+}
+
+.progress-danger .bar,
+.progress .bar-danger {
+  background-color: #dd514c;
+  background-image: -moz-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#c43c35));
+  background-image: -webkit-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: -o-linear-gradient(top, #ee5f5b, #c43c35);
+  background-image: linear-gradient(to bottom, #ee5f5b, #c43c35);
+  background-repeat: repeat-x;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffc43c35', GradientType=0);
+}
+
+.progress-danger.progress-striped .bar,
+.progress-striped .bar-danger {
+  background-color: #ee5f5b;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+
+.progress-success .bar,
+.progress .bar-success {
+  background-color: #5eb95e;
+  background-image: -moz-linear-gradient(top, #62c462, #57a957);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#57a957));
+  background-image: -webkit-linear-gradient(top, #62c462, #57a957);
+  background-image: -o-linear-gradient(top, #62c462, #57a957);
+  background-image: linear-gradient(to bottom, #62c462, #57a957);
+  background-repeat: repeat-x;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff57a957', GradientType=0);
+}
+
+.progress-success.progress-striped .bar,
+.progress-striped .bar-success {
+  background-color: #62c462;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+
+.progress-info .bar,
+.progress .bar-info {
+  background-color: #4bb1cf;
+  background-image: -moz-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#339bb9));
+  background-image: -webkit-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: -o-linear-gradient(top, #5bc0de, #339bb9);
+  background-image: linear-gradient(to bottom, #5bc0de, #339bb9);
+  background-repeat: repeat-x;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff339bb9', GradientType=0);
+}
+
+.progress-info.progress-striped .bar,
+.progress-striped .bar-info {
+  background-color: #5bc0de;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+
+.progress-warning .bar,
+.progress .bar-warning {
+  background-color: #faa732;
+  background-image: -moz-linear-gradient(top, #fbb450, #f89406);
+  background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406));
+  background-image: -webkit-linear-gradient(top, #fbb450, #f89406);
+  background-image: -o-linear-gradient(top, #fbb450, #f89406);
+  background-image: linear-gradient(to bottom, #fbb450, #f89406);
+  background-repeat: repeat-x;
+  filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0);
+}
+
+.progress-warning.progress-striped .bar,
+.progress-striped .bar-warning {
+  background-color: #fbb450;
+  background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent));
+  background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+  background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent);
+}
+
+.accordion {
+  margin-bottom: 20px;
+}
+
+.accordion-group {
+  margin-bottom: 2px;
+  border: 1px solid #e5e5e5;
+  -webkit-border-radius: 4px;
+     -moz-border-radius: 4px;
+          border-radius: 4px;
+}
+
+.accordion-heading {
+  border-bottom: 0;
+}
+
+.accordion-heading .accordion-toggle {
+  display: block;
+  padding: 8px 15px;
+}
+
+.accordion-toggle {
+  cursor: pointer;
+}
+
+.accordion-inner {
+  padding: 9px 15px;
+  border-top: 1px solid #e5e5e5;
+}
+
+.carousel {
+  position: relative;
+  margin-bottom: 20px;
+  line-height: 1;
+}
+
+.carousel-inner {
+  position: relative;
+  width: 100%;
+  overflow: hidden;
+}
+
+.carousel .item {
+  position: relative;
+  display: none;
+  -webkit-transition: 0.6s ease-in-out left;
+     -moz-transition: 0.6s ease-in-out left;
+       -o-transition: 0.6s ease-in-out left;
+          transition: 0.6s ease-in-out left;
+}
+
+.carousel .item > img {
+  display: block;
+  line-height: 1;
+}
+
+.carousel .active,
+.carousel .next,
+.carousel .prev {
+  display: block;
+}
+
+.carousel .active {
+  left: 0;
+}
+
+.carousel .next,
+.carousel .prev {
+  position: absolute;
+  top: 0;
+  width: 100%;
+}
+
+.carousel .next {
+  left: 100%;
+}
+
+.carousel .prev {
+  left: -100%;
+}
+
+.carousel .next.left,
+.carousel .prev.right {
+  left: 0;
+}
+
+.carousel .active.left {
+  left: -100%;
+}
+
+.carousel .active.right {
+  left: 100%;
+}
+
+.carousel-control {
+  position: absolute;
+  top: 40%;
+  left: 15px;
+  width: 40px;
+  height: 40px;
+  margin-top: -20px;
+  font-size: 60px;
+  font-weight: 100;
+  line-height: 30px;
+  color: #ffffff;
+  text-align: center;
+  background: #222222;
+  border: 3px solid #ffffff;
+  -webkit-border-radius: 23px;
+     -moz-border-radius: 23px;
+          border-radius: 23px;
+  opacity: 0.5;
+  filter: alpha(opacity=50);
+}
+
+.carousel-control.right {
+  right: 15px;
+  left: auto;
+}
+
+.carousel-control:hover {
+  color: #ffffff;
+  text-decoration: none;
+  opacity: 0.9;
+  filter: alpha(opacity=90);
+}
+
+.carousel-caption {
+  position: absolute;
+  right: 0;
+  bottom: 0;
+  left: 0;
+  padding: 15px;
+  background: #333333;
+  background: rgba(0, 0, 0, 0.75);
+}
+
+.carousel-caption h4,
+.carousel-caption p {
+  line-height: 20px;
+  color: #ffffff;
+}
+
+.carousel-caption h4 {
+  margin: 0 0 5px;
+}
+
+.carousel-caption p {
+  margin-bottom: 0;
+}
+
+.hero-unit {
+  padding: 60px;
+  margin-bottom: 30px;
+  background-color: #eeeeee;
+  -webkit-border-radius: 6px;
+     -moz-border-radius: 6px;
+          border-radius: 6px;
+}
+
+.hero-unit h1 {
+  margin-bottom: 0;
+  font-size: 60px;
+  line-height: 1;
+  letter-spacing: -1px;
+  color: inherit;
+}
+
+.hero-unit p {
+  font-size: 18px;
+  font-weight: 200;
+  line-height: 30px;
+  color: inherit;
+}
+
+.pull-right {
+  float: right;
+}
+
+.pull-left {
+  float: left;
+}
+
+.hide {
+  display: none;
+}
+
+.show {
+  display: block;
+}
+
+.invisible {
+  visibility: hidden;
+}
+
+.affix {
+  position: fixed;
+}
diff --git a/docs/css/bootstrap.min.css b/docs/css/bootstrap.min.css
new file mode 100644
index 00000000000..ef013cba233
--- /dev/null
+++ b/docs/css/bootstrap.min.css
@@ -0,0 +1,9 @@
+/*!
+ * Bootstrap v2.1.1
+ *
+ * Copyright 2012 Twitter, Inc
+ * Licensed under the Apache License v2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Designed and built with all the love in the world @twitter by @mdo and @fat.
+ */article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{width:auto\9;height:auto;max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}#map_canvas img{max-width:none}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:20px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.img-rounded{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.img-polaroid{padding:4px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.1);box-shadow:0 1px 3px rgba(0,0,0,0.1)}.img-circle{-webkit-border-radius:500px;-moz-border-radius:500px;border-radius:500px}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.span12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.127659574468085%;*margin-left:2.074468085106383%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.48936170212765%;*width:91.43617021276594%}.row-fluid .span10{width:82.97872340425532%;*width:82.92553191489361%}.row-fluid .span9{width:74.46808510638297%;*width:74.41489361702126%}.row-fluid .span8{width:65.95744680851064%;*width:65.90425531914893%}.row-fluid .span7{width:57.44680851063829%;*width:57.39361702127659%}.row-fluid .span6{width:48.93617021276595%;*width:48.88297872340425%}.row-fluid .span5{width:40.42553191489362%;*width:40.37234042553192%}.row-fluid .span4{width:31.914893617021278%;*width:31.861702127659576%}.row-fluid .span3{width:23.404255319148934%;*width:23.351063829787233%}.row-fluid .span2{width:14.893617021276595%;*width:14.840425531914894%}.row-fluid .span1{width:6.382978723404255%;*width:6.329787234042553%}.row-fluid .offset12{margin-left:104.25531914893617%;*margin-left:104.14893617021275%}.row-fluid .offset12:first-child{margin-left:102.12765957446808%;*margin-left:102.02127659574467%}.row-fluid .offset11{margin-left:95.74468085106382%;*margin-left:95.6382978723404%}.row-fluid .offset11:first-child{margin-left:93.61702127659574%;*margin-left:93.51063829787232%}.row-fluid .offset10{margin-left:87.23404255319149%;*margin-left:87.12765957446807%}.row-fluid .offset10:first-child{margin-left:85.1063829787234%;*margin-left:84.99999999999999%}.row-fluid .offset9{margin-left:78.72340425531914%;*margin-left:78.61702127659572%}.row-fluid .offset9:first-child{margin-left:76.59574468085106%;*margin-left:76.48936170212764%}.row-fluid .offset8{margin-left:70.2127659574468%;*margin-left:70.10638297872339%}.row-fluid .offset8:first-child{margin-left:68.08510638297872%;*margin-left:67.9787234042553%}.row-fluid .offset7{margin-left:61.70212765957446%;*margin-left:61.59574468085106%}.row-fluid .offset7:first-child{margin-left:59.574468085106375%;*margin-left:59.46808510638297%}.row-fluid .offset6{margin-left:53.191489361702125%;*margin-left:53.085106382978715%}.row-fluid .offset6:first-child{margin-left:51.063829787234035%;*margin-left:50.95744680851063%}.row-fluid .offset5{margin-left:44.68085106382979%;*margin-left:44.57446808510638%}.row-fluid .offset5:first-child{margin-left:42.5531914893617%;*margin-left:42.4468085106383%}.row-fluid .offset4{margin-left:36.170212765957444%;*margin-left:36.06382978723405%}.row-fluid .offset4:first-child{margin-left:34.04255319148936%;*margin-left:33.93617021276596%}.row-fluid .offset3{margin-left:27.659574468085104%;*margin-left:27.5531914893617%}.row-fluid .offset3:first-child{margin-left:25.53191489361702%;*margin-left:25.425531914893618%}.row-fluid .offset2{margin-left:19.148936170212764%;*margin-left:19.04255319148936%}.row-fluid .offset2:first-child{margin-left:17.02127659574468%;*margin-left:16.914893617021278%}.row-fluid .offset1{margin-left:10.638297872340425%;*margin-left:10.53191489361702%}.row-fluid .offset1:first-child{margin-left:8.51063829787234%;*margin-left:8.404255319148938%}[class*="span"].hide,.row-fluid [class*="span"].hide{display:none}[class*="span"].pull-right,.row-fluid [class*="span"].pull-right{float:right}.container{margin-right:auto;margin-left:auto;*zoom:1}.container:before,.container:after{display:table;line-height:0;content:""}.container:after{clear:both}.container-fluid{padding-right:20px;padding-left:20px;*zoom:1}.container-fluid:before,.container-fluid:after{display:table;line-height:0;content:""}.container-fluid:after{clear:both}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:21px;font-weight:200;line-height:30px}small{font-size:85%}strong{font-weight:bold}em{font-style:italic}cite{font-style:normal}.muted{color:#999}.text-warning{color:#c09853}.text-error{color:#b94a48}.text-info{color:#3a87ad}.text-success{color:#468847}h1,h2,h3,h4,h5,h6{margin:10px 0;font-family:inherit;font-weight:bold;line-height:1;color:inherit;text-rendering:optimizelegibility}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small{font-weight:normal;line-height:1;color:#999}h1{font-size:36px;line-height:40px}h2{font-size:30px;line-height:40px}h3{font-size:24px;line-height:40px}h4{font-size:18px;line-height:20px}h5{font-size:14px;line-height:20px}h6{font-size:12px;line-height:20px}h1 small{font-size:24px}h2 small{font-size:18px}h3 small{font-size:14px}h4 small{font-size:14px}.page-header{padding-bottom:9px;margin:20px 0 30px;border-bottom:1px solid #eee}ul,ol{padding:0;margin:0 0 10px 25px}ul ul,ul ol,ol ol,ol ul{margin-bottom:0}li{line-height:20px}ul.unstyled,ol.unstyled{margin-left:0;list-style:none}dl{margin-bottom:20px}dt,dd{line-height:20px}dt{font-weight:bold}dd{margin-left:10px}.dl-horizontal{*zoom:1}.dl-horizontal:before,.dl-horizontal:after{display:table;line-height:0;content:""}.dl-horizontal:after{clear:both}.dl-horizontal dt{float:left;width:160px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}hr{margin:20px 0;border:0;border-top:1px solid #eee;border-bottom:1px solid #fff}abbr[title]{cursor:help;border-bottom:1px dotted #999}abbr.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:0 0 0 15px;margin:0 0 20px;border-left:5px solid #eee}blockquote p{margin-bottom:0;font-size:16px;font-weight:300;line-height:25px}blockquote small{display:block;line-height:20px;color:#999}blockquote small:before{content:'\2014 \00A0'}blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0}blockquote.pull-right p,blockquote.pull-right small{text-align:right}blockquote.pull-right small:before{content:''}blockquote.pull-right small:after{content:'\00A0 \2014'}q:before,q:after,blockquote:before,blockquote:after{content:""}address{display:block;margin-bottom:20px;font-style:normal;line-height:20px}code,pre{padding:0 3px 2px;font-family:Monaco,Menlo,Consolas,"Courier New",monospace;font-size:12px;color:#333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:20px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}pre.prettyprint{margin-bottom:20px}pre code{padding:0;color:inherit;background-color:transparent;border:0}.pre-scrollable{max-height:340px;overflow-y:scroll}form{margin:0 0 20px}fieldset{padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:40px;color:#333;border:0;border-bottom:1px solid #e5e5e5}legend small{font-size:15px;color:#999}label,input,button,select,textarea{font-size:14px;font-weight:normal;line-height:20px}input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}label{display:block;margin-bottom:5px}select,textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{display:inline-block;height:20px;padding:4px 6px;margin-bottom:9px;font-size:14px;line-height:20px;color:#555;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}input,textarea,.uneditable-input{width:206px}textarea{height:auto}textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{background-color:#fff;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border linear .2s,box-shadow linear .2s;-moz-transition:border linear .2s,box-shadow linear .2s;-o-transition:border linear .2s,box-shadow linear .2s;transition:border linear .2s,box-shadow linear .2s}textarea:focus,input[type="text"]:focus,input[type="password"]:focus,input[type="datetime"]:focus,input[type="datetime-local"]:focus,input[type="date"]:focus,input[type="month"]:focus,input[type="time"]:focus,input[type="week"]:focus,input[type="number"]:focus,input[type="email"]:focus,input[type="url"]:focus,input[type="search"]:focus,input[type="tel"]:focus,input[type="color"]:focus,.uneditable-input:focus{border-color:rgba(82,168,236,0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6)}input[type="radio"],input[type="checkbox"]{margin:4px 0 0;margin-top:1px \9;*margin-top:0;line-height:normal;cursor:pointer}input[type="file"],input[type="image"],input[type="submit"],input[type="reset"],input[type="button"],input[type="radio"],input[type="checkbox"]{width:auto}select,input[type="file"]{height:30px;*margin-top:4px;line-height:30px}select{width:220px;background-color:#fff;border:1px solid #ccc}select[multiple],select[size]{height:auto}select:focus,input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.uneditable-input,.uneditable-textarea{color:#999;cursor:not-allowed;background-color:#fcfcfc;border-color:#ccc;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);box-shadow:inset 0 1px 2px rgba(0,0,0,0.025)}.uneditable-input{overflow:hidden;white-space:nowrap}.uneditable-textarea{width:auto;height:auto}input:-moz-placeholder,textarea:-moz-placeholder{color:#999}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#999}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#999}.radio,.checkbox{min-height:18px;padding-left:18px}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-18px}.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px}.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:middle}.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px}.input-mini{width:60px}.input-small{width:90px}.input-medium{width:150px}.input-large{width:210px}.input-xlarge{width:270px}.input-xxlarge{width:530px}input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0}.input-append input[class*="span"],.input-append .uneditable-input[class*="span"],.input-prepend input[class*="span"],.input-prepend .uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"],.row-fluid .input-prepend [class*="span"],.row-fluid .input-append [class*="span"]{display:inline-block}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:926px}input.span11,textarea.span11,.uneditable-input.span11{width:846px}input.span10,textarea.span10,.uneditable-input.span10{width:766px}input.span9,textarea.span9,.uneditable-input.span9{width:686px}input.span8,textarea.span8,.uneditable-input.span8{width:606px}input.span7,textarea.span7,.uneditable-input.span7{width:526px}input.span6,textarea.span6,.uneditable-input.span6{width:446px}input.span5,textarea.span5,.uneditable-input.span5{width:366px}input.span4,textarea.span4,.uneditable-input.span4{width:286px}input.span3,textarea.span3,.uneditable-input.span3{width:206px}input.span2,textarea.span2,.uneditable-input.span2{width:126px}input.span1,textarea.span1,.uneditable-input.span1{width:46px}.controls-row{*zoom:1}.controls-row:before,.controls-row:after{display:table;line-height:0;content:""}.controls-row:after{clear:both}.controls-row [class*="span"]{float:left}input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eee}input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent}.control-group.warning>label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853}.control-group.warning .checkbox,.control-group.warning .radio,.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853}.control-group.warning input,.control-group.warning select,.control-group.warning textarea{border-color:#c09853;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus{border-color:#a47e3c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e}.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853}.control-group.error>label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48}.control-group.error .checkbox,.control-group.error .radio,.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48}.control-group.error input,.control-group.error select,.control-group.error textarea{border-color:#b94a48;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392}.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48}.control-group.success>label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847}.control-group.success .checkbox,.control-group.success .radio,.control-group.success input,.control-group.success select,.control-group.success textarea{color:#468847}.control-group.success input,.control-group.success select,.control-group.success textarea{border-color:#468847;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b}.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847}.control-group.info>label,.control-group.info .help-block,.control-group.info .help-inline{color:#3a87ad}.control-group.info .checkbox,.control-group.info .radio,.control-group.info input,.control-group.info select,.control-group.info textarea{color:#3a87ad}.control-group.info input,.control-group.info select,.control-group.info textarea{border-color:#3a87ad;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.info input:focus,.control-group.info select:focus,.control-group.info textarea:focus{border-color:#2d6987;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7ab5d3;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7ab5d3;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7ab5d3}.control-group.info .input-prepend .add-on,.control-group.info .input-append .add-on{color:#3a87ad;background-color:#d9edf7;border-color:#3a87ad}input:focus:required:invalid,textarea:focus:required:invalid,select:focus:required:invalid{color:#b94a48;border-color:#ee5f5b}input:focus:required:invalid:focus,textarea:focus:required:invalid:focus,select:focus:required:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7}.form-actions{padding:19px 20px 20px;margin-top:20px;margin-bottom:20px;background-color:#f5f5f5;border-top:1px solid #e5e5e5;*zoom:1}.form-actions:before,.form-actions:after{display:table;line-height:0;content:""}.form-actions:after{clear:both}.help-block,.help-inline{color:#595959}.help-block{display:block;margin-bottom:10px}.help-inline{display:inline-block;*display:inline;padding-left:5px;vertical-align:middle;*zoom:1}.input-append,.input-prepend{margin-bottom:5px;font-size:0;white-space:nowrap}.input-append input,.input-prepend input,.input-append select,.input-prepend select,.input-append .uneditable-input,.input-prepend .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;font-size:14px;vertical-align:top;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-append input:focus,.input-prepend input:focus,.input-append select:focus,.input-prepend select:focus,.input-append .uneditable-input:focus,.input-prepend .uneditable-input:focus{z-index:2}.input-append .add-on,.input-prepend .add-on{display:inline-block;width:auto;height:20px;min-width:16px;padding:4px 5px;font-size:14px;font-weight:normal;line-height:20px;text-align:center;text-shadow:0 1px 0 #fff;background-color:#eee;border:1px solid #ccc}.input-append .add-on,.input-prepend .add-on,.input-append .btn,.input-prepend .btn{vertical-align:top;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-append .active,.input-prepend .active{background-color:#a9dba9;border-color:#46a546}.input-prepend .add-on,.input-prepend .btn{margin-right:-1px}.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append input,.input-append select,.input-append .uneditable-input{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append .add-on,.input-append .btn{margin-left:-1px}.input-append .add-on:last-child,.input-append .btn:last-child{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}input.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;padding-left:4px \9;margin-bottom:0;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.form-search .input-append .search-query,.form-search .input-prepend .search-query{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.form-search .input-append .search-query{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px}.form-search .input-append .btn{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0}.form-search .input-prepend .search-query{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0}.form-search .input-prepend .btn{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px}.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;margin-bottom:0;vertical-align:middle;*zoom:1}.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none}.form-search label,.form-inline label,.form-search .btn-group,.form-inline .btn-group{display:inline-block}.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0}.form-search .radio,.form-search .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle}.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0}.control-group{margin-bottom:10px}legend+.control-group{margin-top:20px;-webkit-margin-top-collapse:separate}.form-horizontal .control-group{margin-bottom:20px;*zoom:1}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;line-height:0;content:""}.form-horizontal .control-group:after{clear:both}.form-horizontal .control-label{float:left;width:160px;padding-top:5px;text-align:right}.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:180px;*margin-left:0}.form-horizontal .controls:first-child{*padding-left:180px}.form-horizontal .help-block{margin-bottom:0}.form-horizontal input+.help-block,.form-horizontal select+.help-block,.form-horizontal textarea+.help-block{margin-top:10px}.form-horizontal .form-actions{padding-left:180px}table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0}.table{width:100%;margin-bottom:20px}.table th,.table td{padding:8px;line-height:20px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th{font-weight:bold}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #ddd}.table-condensed th,.table-condensed td{padding:4px 5px}.table-bordered{border:1px solid #ddd;border-collapse:separate;*border-collapse:collapse;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child th,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topright:4px}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child,.table-bordered tfoot:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;border-radius:0 0 0 4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child,.table-bordered tfoot:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px}.table-bordered caption+thead tr:first-child th:first-child,.table-bordered caption+tbody tr:first-child td:first-child,.table-bordered colgroup+thead tr:first-child th:first-child,.table-bordered colgroup+tbody tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered caption+thead tr:first-child th:last-child,.table-bordered caption+tbody tr:first-child td:last-child,.table-bordered colgroup+thead tr:first-child th:last-child,.table-bordered colgroup+tbody tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topleft:4px}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.table-hover tbody tr:hover td,.table-hover tbody tr:hover th{background-color:#f5f5f5}table [class*=span],.row-fluid table [class*=span]{display:table-cell;float:none;margin-left:0}.table .span1{float:none;width:44px;margin-left:0}.table .span2{float:none;width:124px;margin-left:0}.table .span3{float:none;width:204px;margin-left:0}.table .span4{float:none;width:284px;margin-left:0}.table .span5{float:none;width:364px;margin-left:0}.table .span6{float:none;width:444px;margin-left:0}.table .span7{float:none;width:524px;margin-left:0}.table .span8{float:none;width:604px;margin-left:0}.table .span9{float:none;width:684px;margin-left:0}.table .span10{float:none;width:764px;margin-left:0}.table .span11{float:none;width:844px;margin-left:0}.table .span12{float:none;width:924px;margin-left:0}.table .span13{float:none;width:1004px;margin-left:0}.table .span14{float:none;width:1084px;margin-left:0}.table .span15{float:none;width:1164px;margin-left:0}.table .span16{float:none;width:1244px;margin-left:0}.table .span17{float:none;width:1324px;margin-left:0}.table .span18{float:none;width:1404px;margin-left:0}.table .span19{float:none;width:1484px;margin-left:0}.table .span20{float:none;width:1564px;margin-left:0}.table .span21{float:none;width:1644px;margin-left:0}.table .span22{float:none;width:1724px;margin-left:0}.table .span23{float:none;width:1804px;margin-left:0}.table .span24{float:none;width:1884px;margin-left:0}.table tbody tr.success td{background-color:#dff0d8}.table tbody tr.error td{background-color:#f2dede}.table tbody tr.warning td{background-color:#fcf8e3}.table tbody tr.info td{background-color:#d9edf7}.table-hover tbody tr.success:hover td{background-color:#d0e9c6}.table-hover tbody tr.error:hover td{background-color:#ebcccc}.table-hover tbody tr.warning:hover td{background-color:#faf2cc}.table-hover tbody tr.info:hover td{background-color:#c4e3f3}[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;margin-top:1px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;background-repeat:no-repeat}.icon-white,.nav-tabs>.active>a>[class^="icon-"],.nav-tabs>.active>a>[class*=" icon-"],.nav-pills>.active>a>[class^="icon-"],.nav-pills>.active>a>[class*=" icon-"],.nav-list>.active>a>[class^="icon-"],.nav-list>.active>a>[class*=" icon-"],.navbar-inverse .nav>.active>a>[class^="icon-"],.navbar-inverse .nav>.active>a>[class*=" icon-"],.dropdown-menu>li>a:hover>[class^="icon-"],.dropdown-menu>li>a:hover>[class*=" icon-"],.dropdown-menu>.active>a>[class^="icon-"],.dropdown-menu>.active>a>[class*=" icon-"]{background-image:url("../img/glyphicons-halflings-white.png")}.icon-glass{background-position:0 0}.icon-music{background-position:-24px 0}.icon-search{background-position:-48px 0}.icon-envelope{background-position:-72px 0}.icon-heart{background-position:-96px 0}.icon-star{background-position:-120px 0}.icon-star-empty{background-position:-144px 0}.icon-user{background-position:-168px 0}.icon-film{background-position:-192px 0}.icon-th-large{background-position:-216px 0}.icon-th{background-position:-240px 0}.icon-th-list{background-position:-264px 0}.icon-ok{background-position:-288px 0}.icon-remove{background-position:-312px 0}.icon-zoom-in{background-position:-336px 0}.icon-zoom-out{background-position:-360px 0}.icon-off{background-position:-384px 0}.icon-signal{background-position:-408px 0}.icon-cog{background-position:-432px 0}.icon-trash{background-position:-456px 0}.icon-home{background-position:0 -24px}.icon-file{background-position:-24px -24px}.icon-time{background-position:-48px -24px}.icon-road{background-position:-72px -24px}.icon-download-alt{background-position:-96px -24px}.icon-download{background-position:-120px -24px}.icon-upload{background-position:-144px -24px}.icon-inbox{background-position:-168px -24px}.icon-play-circle{background-position:-192px -24px}.icon-repeat{background-position:-216px -24px}.icon-refresh{background-position:-240px -24px}.icon-list-alt{background-position:-264px -24px}.icon-lock{background-position:-287px -24px}.icon-flag{background-position:-312px -24px}.icon-headphones{background-position:-336px -24px}.icon-volume-off{background-position:-360px -24px}.icon-volume-down{background-position:-384px -24px}.icon-volume-up{background-position:-408px -24px}.icon-qrcode{background-position:-432px -24px}.icon-barcode{background-position:-456px -24px}.icon-tag{background-position:0 -48px}.icon-tags{background-position:-25px -48px}.icon-book{background-position:-48px -48px}.icon-bookmark{background-position:-72px -48px}.icon-print{background-position:-96px -48px}.icon-camera{background-position:-120px -48px}.icon-font{background-position:-144px -48px}.icon-bold{background-position:-167px -48px}.icon-italic{background-position:-192px -48px}.icon-text-height{background-position:-216px -48px}.icon-text-width{background-position:-240px -48px}.icon-align-left{background-position:-264px -48px}.icon-align-center{background-position:-288px -48px}.icon-align-right{background-position:-312px -48px}.icon-align-justify{background-position:-336px -48px}.icon-list{background-position:-360px -48px}.icon-indent-left{background-position:-384px -48px}.icon-indent-right{background-position:-408px -48px}.icon-facetime-video{background-position:-432px -48px}.icon-picture{background-position:-456px -48px}.icon-pencil{background-position:0 -72px}.icon-map-marker{background-position:-24px -72px}.icon-adjust{background-position:-48px -72px}.icon-tint{background-position:-72px -72px}.icon-edit{background-position:-96px -72px}.icon-share{background-position:-120px -72px}.icon-check{background-position:-144px -72px}.icon-move{background-position:-168px -72px}.icon-step-backward{background-position:-192px -72px}.icon-fast-backward{background-position:-216px -72px}.icon-backward{background-position:-240px -72px}.icon-play{background-position:-264px -72px}.icon-pause{background-position:-288px -72px}.icon-stop{background-position:-312px -72px}.icon-forward{background-position:-336px -72px}.icon-fast-forward{background-position:-360px -72px}.icon-step-forward{background-position:-384px -72px}.icon-eject{background-position:-408px -72px}.icon-chevron-left{background-position:-432px -72px}.icon-chevron-right{background-position:-456px -72px}.icon-plus-sign{background-position:0 -96px}.icon-minus-sign{background-position:-24px -96px}.icon-remove-sign{background-position:-48px -96px}.icon-ok-sign{background-position:-72px -96px}.icon-question-sign{background-position:-96px -96px}.icon-info-sign{background-position:-120px -96px}.icon-screenshot{background-position:-144px -96px}.icon-remove-circle{background-position:-168px -96px}.icon-ok-circle{background-position:-192px -96px}.icon-ban-circle{background-position:-216px -96px}.icon-arrow-left{background-position:-240px -96px}.icon-arrow-right{background-position:-264px -96px}.icon-arrow-up{background-position:-289px -96px}.icon-arrow-down{background-position:-312px -96px}.icon-share-alt{background-position:-336px -96px}.icon-resize-full{background-position:-360px -96px}.icon-resize-small{background-position:-384px -96px}.icon-plus{background-position:-408px -96px}.icon-minus{background-position:-433px -96px}.icon-asterisk{background-position:-456px -96px}.icon-exclamation-sign{background-position:0 -120px}.icon-gift{background-position:-24px -120px}.icon-leaf{background-position:-48px -120px}.icon-fire{background-position:-72px -120px}.icon-eye-open{background-position:-96px -120px}.icon-eye-close{background-position:-120px -120px}.icon-warning-sign{background-position:-144px -120px}.icon-plane{background-position:-168px -120px}.icon-calendar{background-position:-192px -120px}.icon-random{width:16px;background-position:-216px -120px}.icon-comment{background-position:-240px -120px}.icon-magnet{background-position:-264px -120px}.icon-chevron-up{background-position:-288px -120px}.icon-chevron-down{background-position:-313px -119px}.icon-retweet{background-position:-336px -120px}.icon-shopping-cart{background-position:-360px -120px}.icon-folder-close{background-position:-384px -120px}.icon-folder-open{width:16px;background-position:-408px -120px}.icon-resize-vertical{background-position:-432px -119px}.icon-resize-horizontal{background-position:-456px -118px}.icon-hdd{background-position:0 -144px}.icon-bullhorn{background-position:-24px -144px}.icon-bell{background-position:-48px -144px}.icon-certificate{background-position:-72px -144px}.icon-thumbs-up{background-position:-96px -144px}.icon-thumbs-down{background-position:-120px -144px}.icon-hand-right{background-position:-144px -144px}.icon-hand-left{background-position:-168px -144px}.icon-hand-up{background-position:-192px -144px}.icon-hand-down{background-position:-216px -144px}.icon-circle-arrow-right{background-position:-240px -144px}.icon-circle-arrow-left{background-position:-264px -144px}.icon-circle-arrow-up{background-position:-288px -144px}.icon-circle-arrow-down{background-position:-312px -144px}.icon-globe{background-position:-336px -144px}.icon-wrench{background-position:-360px -144px}.icon-tasks{background-position:-384px -144px}.icon-filter{background-position:-408px -144px}.icon-briefcase{background-position:-432px -144px}.icon-fullscreen{background-position:-456px -144px}.dropup,.dropdown{position:relative}.dropdown-toggle{*margin-bottom:-3px}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.caret{display:inline-block;width:0;height:0;vertical-align:top;border-top:4px solid #000;border-right:4px solid transparent;border-left:4px solid transparent;content:""}.dropdown .caret{margin-top:8px;margin-left:2px}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.dropdown-menu a{display:block;padding:3px 20px;clear:both;font-weight:normal;line-height:20px;color:#333;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu li>a:focus,.dropdown-submenu:hover>a{color:#fff;text-decoration:none;background-color:#08c;background-color:#0081c2;background-image:-moz-linear-gradient(top,#08c,#0077b3);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#0077b3));background-image:-webkit-linear-gradient(top,#08c,#0077b3);background-image:-o-linear-gradient(top,#08c,#0077b3);background-image:linear-gradient(to bottom,#08c,#0077b3);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0077b3',GradientType=0)}.dropdown-menu .active>a,.dropdown-menu .active>a:hover{color:#fff;text-decoration:none;background-color:#08c;background-color:#0081c2;background-image:linear-gradient(to bottom,#08c,#0077b3);background-image:-moz-linear-gradient(top,#08c,#0077b3);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#0077b3));background-image:-webkit-linear-gradient(top,#08c,#0077b3);background-image:-o-linear-gradient(top,#08c,#0077b3);background-repeat:repeat-x;outline:0;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0077b3',GradientType=0)}.dropdown-menu .disabled>a,.dropdown-menu .disabled>a:hover{color:#999}.dropdown-menu .disabled>a:hover{text-decoration:none;cursor:default;background-color:transparent}.open{*z-index:1000}.open>.dropdown-menu{display:block}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000;content:""}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}.dropdown-submenu{position:relative}.dropdown-submenu>.dropdown-menu{top:0;left:100%;margin-top:-6px;margin-left:-1px;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px}.dropdown-submenu:hover>.dropdown-menu{display:block}.dropdown-submenu>a:after{display:block;float:right;width:0;height:0;margin-top:5px;margin-right:-10px;border-color:transparent;border-left-color:#ccc;border-style:solid;border-width:5px 0 5px 5px;content:" "}.dropdown-submenu:hover>a:after{border-left-color:#fff}.dropdown .dropdown-menu .nav-header{padding-right:20px;padding-left:20px}.typeahead{margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);box-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.fade{opacity:0;-webkit-transition:opacity .15s linear;-moz-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-moz-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.collapse.in{height:auto}.close{float:right;font-size:20px;font-weight:bold;line-height:20px;color:#000;text-shadow:0 1px 0 #fff;opacity:.2;filter:alpha(opacity=20)}.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.4;filter:alpha(opacity=40)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.btn{display:inline-block;*display:inline;padding:4px 14px;margin-bottom:0;*margin-left:.3em;font-size:14px;line-height:20px;*line-height:20px;color:#333;text-align:center;text-shadow:0 1px 1px rgba(255,255,255,0.75);vertical-align:middle;cursor:pointer;background-color:#f5f5f5;*background-color:#e6e6e6;background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));background-image:-webkit-linear-gradient(top,#fff,#e6e6e6);background-image:-o-linear-gradient(top,#fff,#e6e6e6);background-image:linear-gradient(to bottom,#fff,#e6e6e6);background-image:-moz-linear-gradient(top,#fff,#e6e6e6);background-repeat:repeat-x;border:1px solid #bbb;*border:0;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-bottom-color:#a2a2a2;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);*zoom:1;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn:hover,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{color:#333;background-color:#e6e6e6;*background-color:#d9d9d9}.btn:active,.btn.active{background-color:#ccc \9}.btn:first-child{*margin-left:0}.btn:hover{color:#333;text-decoration:none;background-color:#e6e6e6;*background-color:#d9d9d9;background-position:0 -15px;-webkit-transition:background-position .1s linear;-moz-transition:background-position .1s linear;-o-transition:background-position .1s linear;transition:background-position .1s linear}.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.active,.btn:active{background-color:#e6e6e6;background-color:#d9d9d9 \9;background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn.disabled,.btn[disabled]{cursor:default;background-color:#e6e6e6;background-image:none;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-large{padding:9px 14px;font-size:16px;line-height:normal;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.btn-large [class^="icon-"]{margin-top:2px}.btn-small{padding:3px 9px;font-size:12px;line-height:18px}.btn-small [class^="icon-"]{margin-top:0}.btn-mini{padding:2px 6px;font-size:11px;line-height:17px}.btn-block{display:block;width:100%;padding-right:0;padding-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.btn-block+.btn-block{margin-top:5px}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255,255,255,0.75)}.btn{border-color:#c5c5c5;border-color:rgba(0,0,0,0.15) rgba(0,0,0,0.15) rgba(0,0,0,0.25)}.btn-primary{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#006dcc;*background-color:#04c;background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#04c));background-image:-webkit-linear-gradient(top,#08c,#04c);background-image:-o-linear-gradient(top,#08c,#04c);background-image:linear-gradient(to bottom,#08c,#04c);background-image:-moz-linear-gradient(top,#08c,#04c);background-repeat:repeat-x;border-color:#04c #04c #002a80;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0044cc',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-primary:hover,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{color:#fff;background-color:#04c;*background-color:#003bb3}.btn-primary:active,.btn-primary.active{background-color:#039 \9}.btn-warning{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#faa732;*background-color:#f89406;background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(to bottom,#fbb450,#f89406);background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;border-color:#f89406 #f89406 #ad6704;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450',endColorstr='#fff89406',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-warning:hover,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{color:#fff;background-color:#f89406;*background-color:#df8505}.btn-warning:active,.btn-warning.active{background-color:#c67605 \9}.btn-danger{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#da4f49;*background-color:#bd362f;background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#bd362f));background-image:-webkit-linear-gradient(top,#ee5f5b,#bd362f);background-image:-o-linear-gradient(top,#ee5f5b,#bd362f);background-image:linear-gradient(to bottom,#ee5f5b,#bd362f);background-image:-moz-linear-gradient(top,#ee5f5b,#bd362f);background-repeat:repeat-x;border-color:#bd362f #bd362f #802420;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b',endColorstr='#ffbd362f',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-danger:hover,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{color:#fff;background-color:#bd362f;*background-color:#a9302a}.btn-danger:active,.btn-danger.active{background-color:#942a25 \9}.btn-success{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#5bb75b;*background-color:#51a351;background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#51a351));background-image:-webkit-linear-gradient(top,#62c462,#51a351);background-image:-o-linear-gradient(top,#62c462,#51a351);background-image:linear-gradient(to bottom,#62c462,#51a351);background-image:-moz-linear-gradient(top,#62c462,#51a351);background-repeat:repeat-x;border-color:#51a351 #51a351 #387038;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462',endColorstr='#ff51a351',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-success:hover,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{color:#fff;background-color:#51a351;*background-color:#499249}.btn-success:active,.btn-success.active{background-color:#408140 \9}.btn-info{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#49afcd;*background-color:#2f96b4;background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#2f96b4));background-image:-webkit-linear-gradient(top,#5bc0de,#2f96b4);background-image:-o-linear-gradient(top,#5bc0de,#2f96b4);background-image:linear-gradient(to bottom,#5bc0de,#2f96b4);background-image:-moz-linear-gradient(top,#5bc0de,#2f96b4);background-repeat:repeat-x;border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de',endColorstr='#ff2f96b4',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-info:hover,.btn-info:active,.btn-info.active,.btn-info.disabled,.btn-info[disabled]{color:#fff;background-color:#2f96b4;*background-color:#2a85a0}.btn-info:active,.btn-info.active{background-color:#24748c \9}.btn-inverse{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#363636;*background-color:#222;background-image:-webkit-gradient(linear,0 0,0 100%,from(#444),to(#222));background-image:-webkit-linear-gradient(top,#444,#222);background-image:-o-linear-gradient(top,#444,#222);background-image:linear-gradient(to bottom,#444,#222);background-image:-moz-linear-gradient(top,#444,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff444444',endColorstr='#ff222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-inverse:hover,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{color:#fff;background-color:#222;*background-color:#151515}.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9}button.btn,input[type="submit"].btn{*padding-top:3px;*padding-bottom:3px}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0}button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px}button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px}button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px}.btn-link,.btn-link:active,.btn-link[disabled]{background-color:transparent;background-image:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-link{color:#08c;cursor:pointer;border-color:transparent;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-link:hover{color:#005580;text-decoration:underline;background-color:transparent}.btn-link[disabled]:hover{color:#333;text-decoration:none}.btn-group{position:relative;*margin-left:.3em;font-size:0;white-space:nowrap;vertical-align:middle}.btn-group:first-child{*margin-left:0}.btn-group+.btn-group{margin-left:5px}.btn-toolbar{margin-top:10px;margin-bottom:10px;font-size:0}.btn-toolbar .btn-group{display:inline-block;*display:inline;*zoom:1}.btn-toolbar .btn+.btn,.btn-toolbar .btn-group+.btn,.btn-toolbar .btn+.btn-group{margin-left:5px}.btn-group>.btn{position:relative;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group>.btn+.btn{margin-left:-1px}.btn-group>.btn,.btn-group>.dropdown-menu{font-size:14px}.btn-group>.btn-mini{font-size:11px}.btn-group>.btn-small{font-size:12px}.btn-group>.btn-large{font-size:16px}.btn-group>.btn:first-child{margin-left:0;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-bottomleft:4px;-moz-border-radius-topleft:4px}.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-bottomright:4px}.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-bottom-left-radius:6px;border-bottom-left-radius:6px;-webkit-border-top-left-radius:6px;border-top-left-radius:6px;-moz-border-radius-bottomleft:6px;-moz-border-radius-topleft:6px}.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;border-bottom-right-radius:6px;-moz-border-radius-topright:6px;-moz-border-radius-bottomright:6px}.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{*padding-top:5px;padding-right:8px;*padding-bottom:5px;padding-left:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn-group>.btn-mini+.dropdown-toggle{*padding-top:2px;padding-right:5px;*padding-bottom:2px;padding-left:5px}.btn-group>.btn-small+.dropdown-toggle{*padding-top:5px;*padding-bottom:4px}.btn-group>.btn-large+.dropdown-toggle{*padding-top:7px;padding-right:12px;*padding-bottom:7px;padding-left:12px}.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6}.btn-group.open .btn-primary.dropdown-toggle{background-color:#04c}.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406}.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f}.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351}.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4}.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222}.btn .caret{margin-top:8px;margin-left:0}.btn-mini .caret,.btn-small .caret,.btn-large .caret{margin-top:6px}.btn-large .caret{border-top-width:5px;border-right-width:5px;border-left-width:5px}.dropup .btn-large .caret{border-top:0;border-bottom:5px solid #000}.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#fff;border-bottom-color:#fff}.btn-group-vertical{display:inline-block;*display:inline;*zoom:1}.btn-group-vertical .btn{display:block;float:none;width:100%;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group-vertical .btn+.btn{margin-top:-1px;margin-left:0}.btn-group-vertical .btn:first-child{-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.btn-group-vertical .btn:last-child{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.btn-group-vertical .btn-large:first-child{-webkit-border-radius:6px 6px 0 0;-moz-border-radius:6px 6px 0 0;border-radius:6px 6px 0 0}.btn-group-vertical .btn-large:last-child{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px}.alert{padding:8px 35px 8px 14px;margin-bottom:20px;color:#c09853;text-shadow:0 1px 0 rgba(255,255,255,0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.alert h4{margin:0}.alert .close{position:relative;top:-2px;right:-21px;line-height:20px}.alert-success{color:#468847;background-color:#dff0d8;border-color:#d6e9c6}.alert-danger,.alert-error{color:#b94a48;background-color:#f2dede;border-color:#eed3d7}.alert-info{color:#3a87ad;background-color:#d9edf7;border-color:#bce8f1}.alert-block{padding-top:14px;padding-bottom:14px}.alert-block>p,.alert-block>ul{margin-bottom:0}.alert-block p+p{margin-top:5px}.nav{margin-bottom:20px;margin-left:0;list-style:none}.nav>li>a{display:block}.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>.pull-right{float:right}.nav-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:20px;color:#999;text-shadow:0 1px 0 rgba(255,255,255,0.5);text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-right:15px;padding-left:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-right:-15px;margin-left:-15px;text-shadow:0 1px 0 rgba(255,255,255,0.5)}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.2);background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.nav-tabs,.nav-pills{*zoom:1}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;line-height:0;content:""}.nav-tabs:after,.nav-pills:after{clear:both}.nav-tabs>li,.nav-pills>li{float:left}.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margin-right:2px;line-height:14px}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{margin-bottom:-1px}.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:20px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>.active>a,.nav-tabs>.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nav-pills>.active>a,.nav-pills>.active>a:hover{color:#fff;background-color:#08c}.nav-stacked>li{float:none}.nav-stacked>li>a{margin-right:0}.nav-tabs.nav-stacked{border-bottom:0}.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-topleft:4px}.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomright:4px;-moz-border-radius-bottomleft:4px}.nav-tabs.nav-stacked>li>a:hover{z-index:2;border-color:#ddd}.nav-pills.nav-stacked>li>a{margin-bottom:3px}.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px}.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px}.nav-pills .dropdown-menu{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.nav .dropdown-toggle .caret{margin-top:6px;border-top-color:#08c;border-bottom-color:#08c}.nav .dropdown-toggle:hover .caret{border-top-color:#005580;border-bottom-color:#005580}.nav-tabs .dropdown-toggle .caret{margin-top:8px}.nav .active .dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff}.nav-tabs .active .dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.nav>.dropdown.active>a:hover{cursor:pointer}.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover{color:#fff;background-color:#999;border-color:#999}.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:1;filter:alpha(opacity=100)}.tabs-stacked .open>a:hover{border-color:#999}.tabbable{*zoom:1}.tabbable:before,.tabbable:after{display:table;line-height:0;content:""}.tabbable:after{clear:both}.tab-content{overflow:auto}.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0}.tab-content>.tab-pane,.pill-content>.pill-pane{display:none}.tab-content>.active,.pill-content>.active{display:block}.tabs-below>.nav-tabs{border-top:1px solid #ddd}.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0}.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.tabs-below>.nav-tabs>li>a:hover{border-top-color:#ddd;border-bottom-color:transparent}.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover{border-color:transparent #ddd #ddd #ddd}.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none}.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px}.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd}.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px}.tabs-left>.nav-tabs>li>a:hover{border-color:#eee #ddd #eee #eee}.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover{border-color:#ddd transparent #ddd #ddd;*border-right-color:#fff}.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd}.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0}.tabs-right>.nav-tabs>li>a:hover{border-color:#eee #eee #eee #ddd}.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover{border-color:#ddd #ddd #ddd transparent;*border-left-color:#fff}.nav>.disabled>a{color:#999}.nav>.disabled>a:hover{text-decoration:none;cursor:default;background-color:transparent}.navbar{*position:relative;*z-index:2;margin-bottom:20px;overflow:visible;color:#777}.navbar-inner{min-height:40px;padding-right:20px;padding-left:20px;background-color:#fafafa;background-image:-moz-linear-gradient(top,#fff,#f2f2f2);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#f2f2f2));background-image:-webkit-linear-gradient(top,#fff,#f2f2f2);background-image:-o-linear-gradient(top,#fff,#f2f2f2);background-image:linear-gradient(to bottom,#fff,#f2f2f2);background-repeat:repeat-x;border:1px solid #d4d4d4;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff',endColorstr='#fff2f2f2',GradientType=0);*zoom:1;-webkit-box-shadow:0 1px 4px rgba(0,0,0,0.065);-moz-box-shadow:0 1px 4px rgba(0,0,0,0.065);box-shadow:0 1px 4px rgba(0,0,0,0.065)}.navbar-inner:before,.navbar-inner:after{display:table;line-height:0;content:""}.navbar-inner:after{clear:both}.navbar .container{width:auto}.nav-collapse.collapse{height:auto}.navbar .brand{display:block;float:left;padding:10px 20px 10px;margin-left:-20px;font-size:20px;font-weight:200;color:#777;text-shadow:0 1px 0 #fff}.navbar .brand:hover{text-decoration:none}.navbar-text{margin-bottom:0;line-height:40px}.navbar-link{color:#777}.navbar-link:hover{color:#333}.navbar .divider-vertical{height:40px;margin:0 9px;border-right:1px solid #fff;border-left:1px solid #f2f2f2}.navbar .btn,.navbar .btn-group{margin-top:5px}.navbar .btn-group .btn,.navbar .input-prepend .btn,.navbar .input-append .btn{margin-top:0}.navbar-form{margin-bottom:0;*zoom:1}.navbar-form:before,.navbar-form:after{display:table;line-height:0;content:""}.navbar-form:after{clear:both}.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px}.navbar-form input,.navbar-form select,.navbar-form .btn{display:inline-block;margin-bottom:0}.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px}.navbar-form .input-append,.navbar-form .input-prepend{margin-top:6px;white-space:nowrap}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0}.navbar-search{position:relative;float:left;margin-top:5px;margin-bottom:0}.navbar-search .search-query{padding:4px 14px;margin-bottom:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.navbar-static-top{position:static;width:100%;margin-bottom:0}.navbar-static-top .navbar-inner{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0}.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{border-width:0 0 1px}.navbar-fixed-bottom .navbar-inner{border-width:1px 0 0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-right:0;padding-left:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.navbar-fixed-top{top:0}.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1)}.navbar-fixed-bottom{bottom:0}.navbar-fixed-bottom .navbar-inner{-webkit-box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1);box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1)}.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0}.navbar .nav.pull-right{float:right;margin-right:0}.navbar .nav>li{float:left}.navbar .nav>li>a{float:none;padding:10px 15px 10px;color:#777;text-decoration:none;text-shadow:0 1px 0 #fff}.navbar .nav .dropdown-toggle .caret{margin-top:8px}.navbar .nav>li>a:focus,.navbar .nav>li>a:hover{color:#333;text-decoration:none;background-color:transparent}.navbar .nav>.active>a,.navbar .nav>.active>a:hover,.navbar .nav>.active>a:focus{color:#555;text-decoration:none;background-color:#e5e5e5;-webkit-box-shadow:inset 0 3px 8px rgba(0,0,0,0.125);-moz-box-shadow:inset 0 3px 8px rgba(0,0,0,0.125);box-shadow:inset 0 3px 8px rgba(0,0,0,0.125)}.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-right:5px;margin-left:5px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#ededed;*background-color:#e5e5e5;background-image:-webkit-gradient(linear,0 0,0 100%,from(#f2f2f2),to(#e5e5e5));background-image:-webkit-linear-gradient(top,#f2f2f2,#e5e5e5);background-image:-o-linear-gradient(top,#f2f2f2,#e5e5e5);background-image:linear-gradient(to bottom,#f2f2f2,#e5e5e5);background-image:-moz-linear-gradient(top,#f2f2f2,#e5e5e5);background-repeat:repeat-x;border-color:#e5e5e5 #e5e5e5 #bfbfbf;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fff2f2f2',endColorstr='#ffe5e5e5',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075)}.navbar .btn-navbar:hover,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{color:#fff;background-color:#e5e5e5;*background-color:#d9d9d9}.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#ccc \9}.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,0.25);-moz-box-shadow:0 1px 0 rgba(0,0,0,0.25);box-shadow:0 1px 0 rgba(0,0,0,0.25)}.btn-navbar .icon-bar+.icon-bar{margin-top:3px}.navbar .nav>li>.dropdown-menu:before{position:absolute;top:-7px;left:9px;display:inline-block;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-left:7px solid transparent;border-bottom-color:rgba(0,0,0,0.2);content:''}.navbar .nav>li>.dropdown-menu:after{position:absolute;top:-6px;left:10px;display:inline-block;border-right:6px solid transparent;border-bottom:6px solid #fff;border-left:6px solid transparent;content:''}.navbar-fixed-bottom .nav>li>.dropdown-menu:before{top:auto;bottom:-7px;border-top:7px solid #ccc;border-bottom:0;border-top-color:rgba(0,0,0,0.2)}.navbar-fixed-bottom .nav>li>.dropdown-menu:after{top:auto;bottom:-6px;border-top:6px solid #fff;border-bottom:0}.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.dropdown-toggle{color:#555;background-color:#e5e5e5}.navbar .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#777;border-bottom-color:#777}.navbar .nav li.dropdown.open>.dropdown-toggle .caret,.navbar .nav li.dropdown.active>.dropdown-toggle .caret,.navbar .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.navbar .pull-right>li>.dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right{right:0;left:auto}.navbar .pull-right>li>.dropdown-menu:before,.navbar .nav>li>.dropdown-menu.pull-right:before{right:12px;left:auto}.navbar .pull-right>li>.dropdown-menu:after,.navbar .nav>li>.dropdown-menu.pull-right:after{right:13px;left:auto}.navbar .pull-right>li>.dropdown-menu .dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right .dropdown-menu{right:100%;left:auto;margin-right:-1px;margin-left:0;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px}.navbar-inverse{color:#999}.navbar-inverse .navbar-inner{background-color:#1b1b1b;background-image:-moz-linear-gradient(top,#222,#111);background-image:-webkit-gradient(linear,0 0,0 100%,from(#222),to(#111));background-image:-webkit-linear-gradient(top,#222,#111);background-image:-o-linear-gradient(top,#222,#111);background-image:linear-gradient(to bottom,#222,#111);background-repeat:repeat-x;border-color:#252525;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff222222',endColorstr='#ff111111',GradientType=0)}.navbar-inverse .brand,.navbar-inverse .nav>li>a{color:#999;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.navbar-inverse .brand:hover,.navbar-inverse .nav>li>a:hover{color:#fff}.navbar-inverse .nav>li>a:focus,.navbar-inverse .nav>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .nav .active>a,.navbar-inverse .nav .active>a:hover,.navbar-inverse .nav .active>a:focus{color:#fff;background-color:#111}.navbar-inverse .navbar-link{color:#999}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .divider-vertical{border-right-color:#222;border-left-color:#111}.navbar-inverse .nav li.dropdown.open>.dropdown-toggle,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle{color:#fff;background-color:#111}.navbar-inverse .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#999;border-bottom-color:#999}.navbar-inverse .nav li.dropdown.open>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff}.navbar-inverse .navbar-search .search-query{color:#fff;background-color:#515151;border-color:#111;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-webkit-transition:none;-moz-transition:none;-o-transition:none;transition:none}.navbar-inverse .navbar-search .search-query:-moz-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query:-ms-input-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query:focus,.navbar-inverse .navbar-search .search-query.focused{padding:5px 15px;color:#333;text-shadow:0 1px 0 #fff;background-color:#fff;border:0;outline:0;-webkit-box-shadow:0 0 3px rgba(0,0,0,0.15);-moz-box-shadow:0 0 3px rgba(0,0,0,0.15);box-shadow:0 0 3px rgba(0,0,0,0.15)}.navbar-inverse .btn-navbar{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e0e0e;*background-color:#040404;background-image:-webkit-gradient(linear,0 0,0 100%,from(#151515),to(#040404));background-image:-webkit-linear-gradient(top,#151515,#040404);background-image:-o-linear-gradient(top,#151515,#040404);background-image:linear-gradient(to bottom,#151515,#040404);background-image:-moz-linear-gradient(top,#151515,#040404);background-repeat:repeat-x;border-color:#040404 #040404 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff151515',endColorstr='#ff040404',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.navbar-inverse .btn-navbar:hover,.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active,.navbar-inverse .btn-navbar.disabled,.navbar-inverse .btn-navbar[disabled]{color:#fff;background-color:#040404;*background-color:#000}.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active{background-color:#000 \9}.breadcrumb{padding:8px 15px;margin:0 0 20px;list-style:none;background-color:#f5f5f5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.breadcrumb li{display:inline-block;*display:inline;text-shadow:0 1px 0 #fff;*zoom:1}.breadcrumb .divider{padding:0 5px;color:#ccc}.breadcrumb .active{color:#999}.pagination{height:40px;margin:20px 0}.pagination ul{display:inline-block;*display:inline;margin-bottom:0;margin-left:0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;*zoom:1;-webkit-box-shadow:0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:0 1px 2px rgba(0,0,0,0.05);box-shadow:0 1px 2px rgba(0,0,0,0.05)}.pagination ul>li{display:inline}.pagination ul>li>a,.pagination ul>li>span{float:left;padding:0 14px;line-height:38px;text-decoration:none;background-color:#fff;border:1px solid #ddd;border-left-width:0}.pagination ul>li>a:hover,.pagination ul>.active>a,.pagination ul>.active>span{background-color:#f5f5f5}.pagination ul>.active>a,.pagination ul>.active>span{color:#999;cursor:default}.pagination ul>.disabled>span,.pagination ul>.disabled>a,.pagination ul>.disabled>a:hover{color:#999;cursor:default;background-color:transparent}.pagination ul>li:first-child>a,.pagination ul>li:first-child>span{border-left-width:1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.pagination ul>li:last-child>a,.pagination ul>li:last-child>span{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.pagination-centered{text-align:center}.pagination-right{text-align:right}.pager{margin:20px 0;text-align:center;list-style:none;*zoom:1}.pager:before,.pager:after{display:table;line-height:0;content:""}.pager:after{clear:both}.pager li{display:inline}.pager a,.pager span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.pager a:hover{text-decoration:none;background-color:#f5f5f5}.pager .next a,.pager .next span{float:right}.pager .previous a{float:left}.pager .disabled a,.pager .disabled a:hover,.pager .disabled span{color:#999;cursor:default;background-color:#fff}.modal-open .modal .dropdown-menu{z-index:2050}.modal-open .modal .dropdown.open{*z-index:2050}.modal-open .modal .popover{z-index:2060}.modal-open .modal .tooltip{z-index:2080}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop,.modal-backdrop.fade.in{opacity:.8;filter:alpha(opacity=80)}.modal{position:fixed;top:50%;left:50%;z-index:1050;width:560px;margin:-250px 0 0 -280px;overflow:auto;background-color:#fff;border:1px solid #999;border:1px solid rgba(0,0,0,0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.modal.fade{top:-25%;-webkit-transition:opacity .3s linear,top .3s ease-out;-moz-transition:opacity .3s linear,top .3s ease-out;-o-transition:opacity .3s linear,top .3s ease-out;transition:opacity .3s linear,top .3s ease-out}.modal.fade.in{top:50%}.modal-header{padding:9px 15px;border-bottom:1px solid #eee}.modal-header .close{margin-top:2px}.modal-header h3{margin:0;line-height:30px}.modal-body{max-height:400px;padding:15px;overflow-y:auto}.modal-form{margin-bottom:0}.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;*zoom:1;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.modal-footer:before,.modal-footer:after{display:table;line-height:0;content:""}.modal-footer:after{clear:both}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.tooltip{position:absolute;z-index:1030;display:block;padding:5px;font-size:11px;opacity:0;filter:alpha(opacity=0);visibility:visible}.tooltip.in{opacity:.8;filter:alpha(opacity=80)}.tooltip.top{margin-top:-3px}.tooltip.right{margin-left:3px}.tooltip.bottom{margin-top:3px}.tooltip.left{margin-left:-3px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;text-decoration:none;background-color:#000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-top-color:#000;border-width:5px 5px 0}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-right-color:#000;border-width:5px 5px 5px 0}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-left-color:#000;border-width:5px 0 5px 5px}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-bottom-color:#000;border-width:0 5px 5px}.popover{position:absolute;top:0;left:0;z-index:1010;display:none;width:236px;padding:1px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.popover.top{margin-bottom:10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-right:10px}.popover-title{padding:8px 14px;margin:0;font-size:14px;font-weight:normal;line-height:18px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;-webkit-border-radius:5px 5px 0 0;-moz-border-radius:5px 5px 0 0;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover-content p,.popover-content ul,.popover-content ol{margin-bottom:0}.popover .arrow,.popover .arrow:after{position:absolute;display:inline-block;width:0;height:0;border-color:transparent;border-style:solid}.popover .arrow:after{z-index:-1;content:""}.popover.top .arrow{bottom:-10px;left:50%;margin-left:-10px;border-top-color:#fff;border-width:10px 10px 0}.popover.top .arrow:after{bottom:-1px;left:-11px;border-top-color:rgba(0,0,0,0.25);border-width:11px 11px 0}.popover.right .arrow{top:50%;left:-10px;margin-top:-10px;border-right-color:#fff;border-width:10px 10px 10px 0}.popover.right .arrow:after{bottom:-11px;left:-1px;border-right-color:rgba(0,0,0,0.25);border-width:11px 11px 11px 0}.popover.bottom .arrow{top:-10px;left:50%;margin-left:-10px;border-bottom-color:#fff;border-width:0 10px 10px}.popover.bottom .arrow:after{top:-1px;left:-11px;border-bottom-color:rgba(0,0,0,0.25);border-width:0 11px 11px}.popover.left .arrow{top:50%;right:-10px;margin-top:-10px;border-left-color:#fff;border-width:10px 0 10px 10px}.popover.left .arrow:after{right:-1px;bottom:-11px;border-left-color:rgba(0,0,0,0.25);border-width:11px 0 11px 11px}.thumbnails{margin-left:-20px;list-style:none;*zoom:1}.thumbnails:before,.thumbnails:after{display:table;line-height:0;content:""}.thumbnails:after{clear:both}.row-fluid .thumbnails{margin-left:0}.thumbnails>li{float:left;margin-bottom:20px;margin-left:20px}.thumbnail{display:block;padding:4px;line-height:20px;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.055);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.055);box-shadow:0 1px 3px rgba(0,0,0,0.055);-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}a.thumbnail:hover{border-color:#08c;-webkit-box-shadow:0 1px 4px rgba(0,105,214,0.25);-moz-box-shadow:0 1px 4px rgba(0,105,214,0.25);box-shadow:0 1px 4px rgba(0,105,214,0.25)}.thumbnail>img{display:block;max-width:100%;margin-right:auto;margin-left:auto}.thumbnail .caption{padding:9px;color:#555}.label,.badge{font-size:11.844px;font-weight:bold;line-height:14px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);white-space:nowrap;vertical-align:baseline;background-color:#999}.label{padding:1px 4px 2px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.badge{padding:1px 9px 2px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px}a.label:hover,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.label-important,.badge-important{background-color:#b94a48}.label-important[href],.badge-important[href]{background-color:#953b39}.label-warning,.badge-warning{background-color:#f89406}.label-warning[href],.badge-warning[href]{background-color:#c67605}.label-success,.badge-success{background-color:#468847}.label-success[href],.badge-success[href]{background-color:#356635}.label-info,.badge-info{background-color:#3a87ad}.label-info[href],.badge-info[href]{background-color:#2d6987}.label-inverse,.badge-inverse{background-color:#333}.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a}.btn .label,.btn .badge{position:relative;top:-1px}.btn-mini .label,.btn-mini .badge{top:0}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:0 0}to{background-position:40px 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f7f7f7;background-image:-moz-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#f5f5f5),to(#f9f9f9));background-image:-webkit-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-o-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:linear-gradient(to bottom,#f5f5f5,#f9f9f9);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fff5f5f5',endColorstr='#fff9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress .bar{float:left;width:0;height:100%;font-size:12px;color:#fff;text-align:center;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top,#149bdf,#0480be);background-image:-webkit-gradient(linear,0 0,0 100%,from(#149bdf),to(#0480be));background-image:-webkit-linear-gradient(top,#149bdf,#0480be);background-image:-o-linear-gradient(top,#149bdf,#0480be);background-image:linear-gradient(to bottom,#149bdf,#0480be);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff149bdf',endColorstr='#ff0480be',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width .6s ease;-moz-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress .bar+.bar{-webkit-box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15)}.progress-striped .bar{background-color:#149bdf;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px}.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-danger .bar,.progress .bar-danger{background-color:#dd514c;background-image:-moz-linear-gradient(top,#ee5f5b,#c43c35);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#c43c35));background-image:-webkit-linear-gradient(top,#ee5f5b,#c43c35);background-image:-o-linear-gradient(top,#ee5f5b,#c43c35);background-image:linear-gradient(to bottom,#ee5f5b,#c43c35);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b',endColorstr='#ffc43c35',GradientType=0)}.progress-danger.progress-striped .bar,.progress-striped .bar-danger{background-color:#ee5f5b;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-success .bar,.progress .bar-success{background-color:#5eb95e;background-image:-moz-linear-gradient(top,#62c462,#57a957);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#57a957));background-image:-webkit-linear-gradient(top,#62c462,#57a957);background-image:-o-linear-gradient(top,#62c462,#57a957);background-image:linear-gradient(to bottom,#62c462,#57a957);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462',endColorstr='#ff57a957',GradientType=0)}.progress-success.progress-striped .bar,.progress-striped .bar-success{background-color:#62c462;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-info .bar,.progress .bar-info{background-color:#4bb1cf;background-image:-moz-linear-gradient(top,#5bc0de,#339bb9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#339bb9));background-image:-webkit-linear-gradient(top,#5bc0de,#339bb9);background-image:-o-linear-gradient(top,#5bc0de,#339bb9);background-image:linear-gradient(to bottom,#5bc0de,#339bb9);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de',endColorstr='#ff339bb9',GradientType=0)}.progress-info.progress-striped .bar,.progress-striped .bar-info{background-color:#5bc0de;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-warning .bar,.progress .bar-warning{background-color:#faa732;background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(to bottom,#fbb450,#f89406);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450',endColorstr='#fff89406',GradientType=0)}.progress-warning.progress-striped .bar,.progress-striped .bar-warning{background-color:#fbb450;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.accordion{margin-bottom:20px}.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.accordion-heading{border-bottom:0}.accordion-heading .accordion-toggle{display:block;padding:8px 15px}.accordion-toggle{cursor:pointer}.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5}.carousel{position:relative;margin-bottom:20px;line-height:1}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel .item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-moz-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel .item>img{display:block;line-height:1}.carousel .active,.carousel .next,.carousel .prev{display:block}.carousel .active{left:0}.carousel .next,.carousel .prev{position:absolute;top:0;width:100%}.carousel .next{left:100%}.carousel .prev{left:-100%}.carousel .next.left,.carousel .prev.right{left:0}.carousel .active.left{left:-100%}.carousel .active.right{left:100%}.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20px;font-size:60px;font-weight:100;line-height:30px;color:#fff;text-align:center;background:#222;border:3px solid #fff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:.5;filter:alpha(opacity=50)}.carousel-control.right{right:15px;left:auto}.carousel-control:hover{color:#fff;text-decoration:none;opacity:.9;filter:alpha(opacity=90)}.carousel-caption{position:absolute;right:0;bottom:0;left:0;padding:15px;background:#333;background:rgba(0,0,0,0.75)}.carousel-caption h4,.carousel-caption p{line-height:20px;color:#fff}.carousel-caption h4{margin:0 0 5px}.carousel-caption p{margin-bottom:0}.hero-unit{padding:60px;margin-bottom:30px;background-color:#eee;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;letter-spacing:-1px;color:inherit}.hero-unit p{font-size:18px;font-weight:200;line-height:30px;color:inherit}.pull-right{float:right}.pull-left{float:left}.hide{display:none}.show{display:block}.invisible{visibility:hidden}.affix{position:fixed}
diff --git a/docs/css/custom.css b/docs/css/custom.css
new file mode 100644
index 00000000000..ce15235e889
--- /dev/null
+++ b/docs/css/custom.css
@@ -0,0 +1,592 @@
+@font-face {
+    font-family: 'Conv_framd';
+    src: url('../fonts/framd.eot');
+    src: url('../fonts/framd.eot?#iefix') format('embedded-opentype'),
+         url('../fonts/framd.woff') format('woff'),
+         url('../fonts/framd.ttf') format('truetype'),
+         url('../fonts/framd.svg#heroregular') format('svg');
+    font-weight: normal;
+    font-style: normal;
+}
+
+
+html, body {
+	position:relative;
+	height:100%;
+	min-height:100%;
+	height:100%;	
+	color:#252525;
+	font:400 18px/26px 'Open Sans', Arial, Helvetica, sans-serif;
+	padding:0;
+	margin:0;
+	word-wrap:break-word;
+}
+a {
+	color:#6ab6dd;
+	position:relative;
+}
+a:hover {
+	text-decoration:underline;
+	color:#2c79a1;
+}
+.wrapper {
+	min-height:100%;
+}
+header {
+	margin:0 0 100px;
+}
+header .span12 {
+	padding:0 0 7px 0;
+}
+.logo.custom {
+	display:inline-block;
+	margin:0;
+	vertical-align:25px;
+}
+.logo.custom a {
+	background:url(../img/logo.png) no-repeat;
+	width: 110px;
+	height: 49px;
+	display:block;
+	text-indent:-9999px;
+}
+.custom.navbar {
+	margin:10px 0;
+}
+.custom.navbar .nav li  {
+	padding:0 !important;
+}
+.custom.navbar .nav li a, .navbar .brand, .custom.navbar .nav li.pull-right span {
+	font:300 14px/20px 'Open Sans', Arial, Helvetica, sans-serif;
+	padding:10px 10px;
+}
+.navbar .brand.active {
+	color:#fff;
+}
+.custom.navbar .nav li.pull-right span {
+	padding-right:0;	
+	color:#e76d4c !important;
+	display:block;
+}
+.custom.navbar .nav li a.doc-link {
+	padding:5px 10px 0;
+}
+.custom.navbar .nav li a.doc-link span {
+	display:inline-block;
+	background:url(../img/icon-git.png) no-repeat;
+	width: 28px;
+	height: 28px;
+	vertical-align:-7px;
+	margin-left:5px;
+}
+.custom.navbar .nav li a.doc-link:hover span, .custom.navbar .nav li.active a.doc-link span {
+	background-position:0 -28px;
+}
+.custom.navbar .nav {
+	float:none;
+}
+.navbar .nav > li.pull-right {
+	float:right;
+	padding:10px 0;
+}
+h1 {
+	font:300 48px/48px 'Open Sans', Arial, Helvetica, sans-serif;
+	margin:0 0 45px;
+}
+h1.index {
+	text-align:center;
+}
+h1 span {
+	display:block;
+	font:400 14px/28px 'Open Sans', Arial, Helvetica, sans-serif;
+}
+h2 {
+	font:30px/30px 'Conv_framd', Arial, Helvetica, sans-serif;
+	margin:0 0 20px;
+	color:#0f1e35;
+}
+h3 {
+	font:300 30px/36px 'Open Sans', Arial, Helvetica, sans-serif;
+	margin:0 0 33px;
+	text-align:center;
+}
+.btn {
+	display:block;
+	margin:0 auto 65px;
+	background:#6ab6dd;
+	border:none;
+	box-shadow:inset -3px -3px 3px #5592b1;
+	height:53px;
+	width:205px;
+	font:30px/53px 'Conv_framd', Arial, Helvetica, sans-serif;
+	color:#252424;
+	text-shadow:none;
+	padding:0;
+	z-index:100;
+	position:relative;
+}
+.btn a {
+	color:#252424;
+}
+.btn:hover {
+	background:#83c6e9;
+}
+.index-content {
+	margin:0 auto 60px;
+	text-align:center;
+}
+.third {
+	margin:0 auto 35px; 
+}
+.third-item {
+	text-align:left;
+}
+.third-item:first-child {
+	margin:0;
+}
+.third-item a {
+	display:block;
+	font-family:'Open Sans', Arial, Helvetica, sans-serif;
+	font-weight:700;
+	font-size:30px;
+	margin:0 auto 20px;
+	color:#252424;
+	text-align:center;
+}
+.container.custom {
+	padding:0;
+	margin:0 auto;
+}
+.container.custom.main-cont {
+	padding-bottom:230px;	
+}
+.text-part {
+	padding-top:70px;
+}
+.row-fluid.index-page {
+	padding-top:100px;
+}
+.index-page .content {
+	padding:15px 0 0;
+}
+.index-page h3 {
+	text-align:left;
+}
+.index-page .sidebar {
+	padding:65px 0 30px;
+}
+.container.custom p {
+	margin:0 0 17px;
+}
+.homepage .index-page .content h2 {
+	margin:0 0 20px;
+}
+.container.custom .unstyled {
+	margin:0;
+	color:#353535;
+}
+.container.custom .unstyled li {
+	margin:0 0 17px;
+}
+.container.custom .unstyled li span {
+	font-family:'Open Sans', Arial, Helvetica, sans-serif;
+	font-weight:700;
+	display:block;
+}
+.container.custom .unstyled li a {
+	display:inline;
+}
+.homepage h4 {
+	font:24px/24px 'Conv_framd', Arial, Helvetica, sans-serif;
+	margin:0 0 15px;
+}
+.container.custom .sidebar .unstyled {
+	margin:0 0 100px;
+}
+.container.custom .sidebar .unstyled li {
+	margin:0 0 12px;
+	border-bottom:1px solid #adadad;
+	padding: 5px 7px;
+}
+.grey-box {
+	background:#e5e4e3;
+	border-radius:3px;
+	-moz-border-radius:3px;
+	-webkit-border-radius:3px;
+	position:relative;
+	padding:20px 10px 130px;
+	color:#000;
+}
+footer {
+	text-align:center;
+	font-size:14px;
+	color:#000;
+	margin:-135px 0 0;
+}
+footer .container.custom {
+	border-top:1px solid #e1e1e1;
+	padding:20px 0 25px;
+}
+footer .span9 {
+	text-align:left;
+}
+footer .container.custom ul.unstyled {
+	display:inline-block;
+	margin:0 120px 30px 30px;
+	text-align:left;
+	vertical-align:top;
+}
+footer .container.custom ul.unstyled li {
+	font:300 14px/26px 'Open Sans', Arial, Helvetica, sans-serif;
+	margin:0;
+}
+footer .container.custom .unstyled li a {
+	color:#000;
+	font-weight:300;
+}
+footer .container.custom .unstyled li:first-child a {
+	font-weight:400;
+}
+footer ul li a:hover {
+	text-decoration:underline;
+	color:#fff;
+}
+footer .logo-block {
+	text-align:right;
+}
+footer .container.custom p {
+	display:inline-block;
+	margin:28px 0 0 10px;
+	text-align:left;
+}
+.contact-item {
+	margin:0 0 30px 30px;
+	text-align:left;
+	font-weight:300;
+}
+.contact-item a {
+	color:#000;
+}
+footer .contact-item span {
+	font-weight:400;
+	display:block;
+}
+footer .contact-item:first-child span {
+	text-transform:uppercase;
+}
+footer .span4 {
+	text-align:left;
+}
+footer .span5 {
+	padding-top: 75px;
+}
+.soc {
+	text-align:left;
+	margin:5px 0 0 0;
+}
+.soc a {
+	display:inline-block;
+	width:35px;
+	height:34px;
+	background:url(../img/icons-soc.png) no-repeat;
+}
+.soc a.github {
+	background-position:0 -34px;
+}
+.soc a.meet {
+	background-position:0 -68px;
+}
+.soc a.rss {
+        background-position:0 -102px;
+}
+
+.text-item {
+	margin:0 0 75px;
+}
+.container.custom p.note {
+	text-align:center;
+	padding:30px 0 0;
+}
+.text-item strong {
+	font-weight:normal;
+	font-family:'Open Sans', Arial, Helvetica, sans-serif;
+	font-weight:700;
+}
+h2.date {
+	font-family:'Open Sans', Arial, Helvetica, sans-serif;
+	font-weight:400;
+}
+.blog h2.date {	
+	margin:0 0 25px;
+}
+h2.date span {
+	display:block;
+	margin:0 0 5px;
+	padding:0 0 15px;
+	font-size:20px;
+	border-bottom:1px solid #ccc;
+}
+.blog h2.date a {
+	font-weight:700;
+}
+.blog.inner h2.date span:first-child {
+	display:block;
+	font-size:30px;
+	font-weight:700;
+	padding:0;
+	border:none;
+}
+.blog.inner h3 {
+	text-align:left;
+	font-size:25px;
+	font-weight:700;
+	margin:0 0 15px;
+}
+.blog.inner ul li {
+	margin-left: 50px;
+	line-height:26px;
+}
+.recent h3 {
+	font-size: 25px;
+    font-weight: 700;
+    margin: 0 0 15px;
+    text-align: left;
+}
+.recent ul li.active a {
+	color:#252525;
+}
+.border {
+	width:130px;
+	margin: 45px auto;
+	border-top:1px solid #dfdfdf;
+	border-top:1px solid #81807f;
+}
+.text-img {
+	display:block;
+	margin:0 auto 17px;
+}
+.indent p, .indent ul {
+	padding:0 0 0 50px;
+}
+.span3 {
+	margin-left:0;
+}
+.nav.nav-list.bs-docs-sidenav {
+	border:1px solid #e5e5e5;
+	border-radius:5px;
+	box-shadow:0 0 3px #f9f9f9;
+	padding:0;
+	width:auto;
+}
+.nav.nav-list.bs-docs-sidenav li {
+	border-top:1px solid #e5e5e5;
+}
+.nav.nav-list.bs-docs-sidenav li:first-child {
+	border:none;
+}
+.nav.nav-list.bs-docs-sidenav li:first-child a {
+	border-radius:5px 5px 0 0;
+	-moz-border-radius:5px 5px 0 0;
+	-webkit-border-radius:5px 5px 0 0;
+}
+.nav.nav-list.bs-docs-sidenav li:last-child, .nav.nav-list.bs-docs-sidenav li:last-child a {
+	border-radius:0 0 5px 5px;
+	-moz-border-radius:0 0 5px 5px;
+	-webkit-border-radius:0 0 5px 5px;
+}
+.nav.nav-list.bs-docs-sidenav li a {
+	padding:10px;
+	margin:0;
+	font-weight:400;
+	font-size:14px;
+	line-height:18px;
+}
+.icon-chevron-right {
+	float: right;
+    margin-right: -6px;
+    margin-top: 2px;
+    opacity: 0.25;
+}
+.indent ul li {
+	line-height:26px;
+}
+.span8 h3 {
+	text-align:left;
+	margin:0 0 50px;
+}
+.span8 h3 a {
+	font-weight:800;
+}
+.span8 h4 {
+	font:700 18px/26px 'Open Sans', Arial, Helvetica, sans-serif;
+	margin:0 0 20px;
+}
+.span8 p span {
+	font-weight:700;
+}
+header.index-head {
+	background:#f9f9f9;
+	margin:0 0 30px;
+}
+header.index-head .span12 {
+	margin-bottom:80px;
+}
+.index-content h2 {
+	text-align:center;
+}
+.third-item img {
+	display:block;
+	margin:0 auto 70px;
+}
+.container.custom .third-item p {
+	margin:0 0 0 20px;
+} 
+.row-fluid {
+	margin:0;
+	padding:0;
+}
+.nav-list [class^="icon-"] {
+	margin-right:-2px;
+}
+@media (min-width: 1200px) {
+	.custom.navbar .nav li a, .navbar .brand, .custom.navbar .nav li.pull-right span {
+		padding: 10px 20px;
+		font-size:16px;
+	}
+	.nav.nav-list.bs-docs-sidenav {
+		width:258px;
+	}
+	.container.custom .recent ul.unstyled {
+		margin-right:100px;
+	}
+}
+@media (max-width: 980px) {
+	.container.custom {
+		width:95%;
+	}
+	.bs-docs-sidenav.affix {
+		position: static;
+		top: 0;
+		width: 100%;
+	}	
+	.nav.nav-list.bs-docs-sidenav {
+		width:100%;
+		margin-bottom:20px;
+	}
+}
+@media only screen 
+and (min-device-width : 710px) 
+and (max-device-width : 770px) {
+	.container.custom {
+		width:700px;
+		position:relative;
+	}	
+	.custom.navbar .nav li {
+		font-size:22px;
+		padding:0 10px;
+	}
+	.nav.nav-list.bs-docs-sidenav.affix {
+		position:fixed;
+		top:175px;
+		width:218px;
+	}
+}
+@media only screen and (min-device-width : 770px) 
+and (max-device-width : 810px) {
+	.container.custom {
+		width:700px;
+	}
+	.custom.navbar .nav li {
+		font-size:15px;
+		padding:0 15px;
+	}
+	.custom.navbar .nav {
+		margin-left:30px;
+	}
+}
+@media only screen 
+and (min-device-width : 320px) 
+and (max-device-width : 480px) {
+	.container.custom {
+		width:100%;
+		margin:0;
+	}
+	footer .logo-block {
+		text-align: left;
+		padding-left:30px;
+	}
+	.offset1, .row-fluid .offset1:first-child {
+		margin:0;
+	}
+	.indent p, .indent ul {
+		padding:0;
+	}
+	.indent, .blog, .recent h3, .recent ul, .text-part {
+		padding:0 20px;
+	}
+	.index-head h3 {
+		font-size:20px;
+	}
+	.index-content {
+		padding:0 20px;
+	}
+	.content h2, .content h3, .content ul.unstyled, .sidebar h3 {
+		padding:0 20px;
+	}
+	h1 {
+		padding:0 20px;
+	}
+}
+@media (max-width: 320px) {
+	.container.custom {
+		width:100%;
+	}
+	footer .logo-block {
+		text-align: left;
+		padding-left:30px;
+	}
+	footer .container.custom p {
+		margin-top:10px;
+	}
+	.offset1, .row-fluid .offset1:first-child {
+		margin:0;
+	}
+	.indent p, .indent ul {
+		padding:0;
+	}
+	.indent, .blog, .recent h3, .recent ul, .text-part {
+		padding:0 20px;
+	}
+	.index-head h3 {
+		font-size:25px;
+		line-height:30px;
+		padding:0 20px;
+	}
+	.index-content {
+		padding:0 20px;
+	}
+	.content h2, .content h3, .content ul.unstyled, .sidebar h3 {
+		padding:0 20px;
+	}
+	h1 {
+		padding:0 20px;
+	}
+}
+.container.custom .faq-page p {
+	margin-bottom:10px;
+}
+.index-head h3 {
+	margin-bottom:50px;
+}
+h1.center {
+	text-align:center;
+}
+.btn.btn-navbar {
+	height:auto;
+	width:auto;
+	margin:10px 0 0;
+}
+.navbar-inner {
+	z-index:100000;
+	position:relative;
+}
diff --git a/docs/css/default.html b/docs/css/default.html
new file mode 100644
index 00000000000..12106274e0f
--- /dev/null
+++ b/docs/css/default.html
@@ -0,0 +1,147 @@
+
+
+  	
+    	
+    Druid | {{page.title}}	
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  
+  	
+    

+      
+      

+        

+          

+            

+              

+                
+                Home
+                

+                  
+                

+              

+            

+          

+        

+        {% if page.id == 'home' %}
+        
Druid is open-source infrastructure for real²time exploratory analytics on large datasets.

+        
+        {% endif %}
+      

+      
+      

+
+        {{ content }}
+
+      

+    

+    
+    
+    
+  
+
+
diff --git a/docs/css/docs.html b/docs/css/docs.html
new file mode 100644
index 00000000000..cb38f8c336e
--- /dev/null
+++ b/docs/css/docs.html
@@ -0,0 +1,8 @@
+---
+layout: default
+---
+  

+
+    {{ content }}
+      
+  

diff --git a/docs/css/main.css b/docs/css/main.css
new file mode 100755
index 00000000000..50a818048ad
--- /dev/null
+++ b/docs/css/main.css
@@ -0,0 +1,160 @@
+/*****************************************************************************/
+/*
+/* Common
+/*
+/*****************************************************************************/
+
+/* Global Reset */
+* {
+  margin: 0;
+  padding: 0;
+}
+
+html, body { height: 100%; }
+
+body {
+  background-color: #FFF;
+  font: 13.34px Helvetica, Arial, sans-serif;
+  font-size: small;
+  text-align: center;
+}
+
+h1, h2, h3, h4, h5, h6 {
+  font-size: 100%; }
+
+h1 { margin-bottom: 1em; }
+p { margin: 1em 0; }
+
+a         { color: #00a; }
+a:hover   { color: #000; }
+a:visited { color: #a0a; }
+
+/*****************************************************************************/
+/*
+/* Home
+/*
+/*****************************************************************************/
+ul.posts {
+  list-style-type: none;
+  margin-bottom: 2em;
+}
+
+ul.posts li {
+  line-height: 1.75em;
+}
+
+ul.posts span {
+  color: #aaa;
+  font-family: Monaco, "Courier New", monospace;
+  font-size: 80%;
+}
+
+/*****************************************************************************/
+/*
+/* Site
+/*
+/*****************************************************************************/
+
+.site {
+  font-size: 115%;
+  text-align: justify;
+  width: 42em;
+  margin: 3em auto 2em;
+  line-height: 1.5em;
+}
+
+.site .header a {
+  font-weight: bold;
+  text-decoration: none;
+}
+
+.site .header h1.title {
+  display: inline-block;
+  margin-bottom: 2em;
+}
+
+.site .header h1.title a {
+  color: #a00;
+}
+
+.site .header h1.title a:hover {
+  color: #000;
+}
+
+.site .header a.extra {
+  color: #aaa;
+  margin-left: 1em;
+}
+
+.site .header a.extra:hover {
+  color: #000;
+}
+
+.site .meta {
+  color: #aaa;
+}
+
+.site .footer {
+  font-size: 80%;
+  color: #666;
+  border-top: 4px solid #eee;
+  margin-top: 2em;
+  overflow: hidden;
+}
+
+.site .footer .contact {
+  float: left;
+  margin-right: 3em;
+}
+
+.site .footer .contact a {
+  color: #8085C1;
+}
+
+.site .footer .rss {
+  margin-top: 1.1em;
+  margin-right: -.2em;
+  float: right;
+}
+
+.site .footer .rss img {
+  border: 0;
+}
+
+/*****************************************************************************/
+/*
+/* Posts
+/*
+/*****************************************************************************/
+
+/* standard */
+.post pre {
+  border: 1px solid #ddd;
+  background-color: #eef;
+  padding: 0 .4em;
+}
+
+.post ul, .post ol {
+  margin-left: 1.35em;
+}
+
+.post code {
+  border: 1px solid #ddd;
+  background-color: #eef;
+  padding: 0 .2em;
+}
+
+.post pre code {
+  border: none;
+}
+
+/* terminal */
+.post pre.terminal {
+  border: 1px solid #000;
+  background-color: #333;
+  color: #FFF;
+}
+
+.post pre.terminal code {
+  background-color: #333;
+}
diff --git a/docs/css/page.html b/docs/css/page.html
new file mode 100644
index 00000000000..b5ebfb17ace
--- /dev/null
+++ b/docs/css/page.html
@@ -0,0 +1,11 @@
+---
+layout: default
+---
+
+

+  

+
+    {{ content }}
+    
+  

+

diff --git a/docs/css/pie.htc b/docs/css/pie.htc
new file mode 100644
index 00000000000..ca3b54700cc
--- /dev/null
+++ b/docs/css/pie.htc
@@ -0,0 +1,96 @@
+
+
+
+
+
+
+
+
+
diff --git a/docs/css/post.html b/docs/css/post.html
new file mode 100644
index 00000000000..affb233fd23
--- /dev/null
+++ b/docs/css/post.html
@@ -0,0 +1,44 @@
+---
+layout: default
+sectionid: blog
+---
+
+

+  

+    
Recent posts

+    
+  

+
+  

+    

+      

+        {{ page.title }}
+        {{ page.date | date: "%B %e, %Y" }} · {{ page.author | upcase }}
+      

+
+      {% if page.image %}{{ page.title }}{% endif %}
+
+      {{ content }}
+
+    

+    
+    
+    comments powered by Disqus
+    
+    

+  

+

diff --git a/docs/css/syntax.css b/docs/css/syntax.css
new file mode 100644
index 00000000000..1e651cf79db
--- /dev/null
+++ b/docs/css/syntax.css
@@ -0,0 +1,60 @@
+.highlight  { background: #ffffff; }
+.highlight .c { color: #999988; font-style: italic } /* Comment */
+.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */
+.highlight .k { font-weight: bold } /* Keyword */
+.highlight .o { font-weight: bold } /* Operator */
+.highlight .cm { color: #999988; font-style: italic } /* Comment.Multiline */
+.highlight .cp { color: #999999; font-weight: bold } /* Comment.Preproc */
+.highlight .c1 { color: #999988; font-style: italic } /* Comment.Single */
+.highlight .cs { color: #999999; font-weight: bold; font-style: italic } /* Comment.Special */
+.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */
+.highlight .gd .x { color: #000000; background-color: #ffaaaa } /* Generic.Deleted.Specific */
+.highlight .ge { font-style: italic } /* Generic.Emph */
+.highlight .gr { color: #aa0000 } /* Generic.Error */
+.highlight .gh { color: #999999 } /* Generic.Heading */
+.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */
+.highlight .gi .x { color: #000000; background-color: #aaffaa } /* Generic.Inserted.Specific */
+.highlight .go { color: #888888 } /* Generic.Output */
+.highlight .gp { color: #555555 } /* Generic.Prompt */
+.highlight .gs { font-weight: bold } /* Generic.Strong */
+.highlight .gu { color: #aaaaaa } /* Generic.Subheading */
+.highlight .gt { color: #aa0000 } /* Generic.Traceback */
+.highlight .kc { font-weight: bold } /* Keyword.Constant */
+.highlight .kd { font-weight: bold } /* Keyword.Declaration */
+.highlight .kp { font-weight: bold } /* Keyword.Pseudo */
+.highlight .kr { font-weight: bold } /* Keyword.Reserved */
+.highlight .kt { color: #445588; font-weight: bold } /* Keyword.Type */
+.highlight .m { color: #009999 } /* Literal.Number */
+.highlight .s { color: #d14 } /* Literal.String */
+.highlight .na { color: #008080 } /* Name.Attribute */
+.highlight .nb { color: #0086B3 } /* Name.Builtin */
+.highlight .nc { color: #445588; font-weight: bold } /* Name.Class */
+.highlight .no { color: #008080 } /* Name.Constant */
+.highlight .ni { color: #800080 } /* Name.Entity */
+.highlight .ne { color: #990000; font-weight: bold } /* Name.Exception */
+.highlight .nf { color: #990000; font-weight: bold } /* Name.Function */
+.highlight .nn { color: #555555 } /* Name.Namespace */
+.highlight .nt { color: #000080 } /* Name.Tag */
+.highlight .nv { color: #008080 } /* Name.Variable */
+.highlight .ow { font-weight: bold } /* Operator.Word */
+.highlight .w { color: #bbbbbb } /* Text.Whitespace */
+.highlight .mf { color: #009999 } /* Literal.Number.Float */
+.highlight .mh { color: #009999 } /* Literal.Number.Hex */
+.highlight .mi { color: #009999 } /* Literal.Number.Integer */
+.highlight .mo { color: #009999 } /* Literal.Number.Oct */
+.highlight .sb { color: #d14 } /* Literal.String.Backtick */
+.highlight .sc { color: #d14 } /* Literal.String.Char */
+.highlight .sd { color: #d14 } /* Literal.String.Doc */
+.highlight .s2 { color: #d14 } /* Literal.String.Double */
+.highlight .se { color: #d14 } /* Literal.String.Escape */
+.highlight .sh { color: #d14 } /* Literal.String.Heredoc */
+.highlight .si { color: #d14 } /* Literal.String.Interpol */
+.highlight .sx { color: #d14 } /* Literal.String.Other */
+.highlight .sr { color: #009926 } /* Literal.String.Regex */
+.highlight .s1 { color: #d14 } /* Literal.String.Single */
+.highlight .ss { color: #990073 } /* Literal.String.Symbol */
+.highlight .bp { color: #999999 } /* Name.Builtin.Pseudo */
+.highlight .vc { color: #008080 } /* Name.Variable.Class */
+.highlight .vg { color: #008080 } /* Name.Variable.Global */
+.highlight .vi { color: #008080 } /* Name.Variable.Instance */
+.highlight .il { color: #009999 } /* Literal.Number.Integer.Long */
\ No newline at end of file
diff --git a/docs/index.html b/docs/index.html
new file mode 100644
index 00000000000..c7268192374
--- /dev/null
+++ b/docs/index.html
@@ -0,0 +1,13 @@
+---
+layout: default
+title: Your New Jekyll Site
+---
+
+

+  
Blog Posts

+  
    
+    {% for post in site.posts %}
+      
  • {{ post.date | date_to_string }} » {{ post.title }}
    • 
+    {% endfor %}
+  

+

\ No newline at end of file
diff --git a/examples/bin/ec2/run.sh b/examples/bin/ec2/run.sh
index 933de0a2b42..df7438ccf5b 100644
--- a/examples/bin/ec2/run.sh
+++ b/examples/bin/ec2/run.sh
@@ -7,16 +7,16 @@ cd druid-services-* 2>&1 > /dev/null
 mkdir logs 2>&1 > /dev/null
 
 # Now start a realtime node
-nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=config/realtime/realtime.spec -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/realtime com.metamx.druid.realtime.RealtimeMain 2>&1 > logs/realtime.log &
+nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=config/realtime/realtime.spec -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/realtime io.druid.cli.Main server realtime 2>&1 > logs/realtime.log &
 
 # And a master node
-nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/master com.metamx.druid.http.MasterMain 2>&1 > logs/master.log &
+nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/master io.druid.cli.Main server coordinator 2>&1 > logs/master.log &
 
 # And a compute node
-nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/compute com.metamx.druid.http.ComputeMain 2>&1 > logs/compute.log &
+nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/compute io.druid.cli.Main server historical 2>&1 > logs/compute.log &
 
 # And a broker node
-nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/broker com.metamx.druid.http.BrokerMain 2>&1 > logs/broker.log &
+nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/broker io.druid.cli.Main server broker 2>&1 > logs/broker.log &
 
 echo "Hit CTRL-C to continue..."
 exit 0
diff --git a/examples/bin/run_example_server.sh b/examples/bin/run_example_server.sh
index 38e372e3e3c..dc0483b932a 100755
--- a/examples/bin/run_example_server.sh
+++ b/examples/bin/run_example_server.sh
@@ -63,4 +63,4 @@ DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/config/realtime
 
 echo "Running command:"
 
-(set -x; java ${JAVA_ARGS} -classpath ${DRUID_CP}  druid.examples.RealtimeStandaloneMain)
+(set -x; java ${JAVA_ARGS} -classpath ${DRUID_CP}  io.druid.cli.Main druid server realtimeStandalone)
diff --git a/examples/pom.xml b/examples/pom.xml
index 83de1896523..2a8310e92ba 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -1,4 +1,23 @@
 
+
+
 
     4.0.0
     com.metamx.druid
@@ -9,7 +28,7 @@
     
         com.metamx
         druid
-        0.5.59-SNAPSHOT
+        0.6.0-SNAPSHOT
     
 
     
@@ -29,99 +48,6 @@
             ${project.parent.version}
         
 
-        
-            com.metamx
-            emitter
-        
-        
-            com.metamx
-            http-client
-        
-        
-            com.metamx
-            java-util
-        
-        
-            com.metamx
-            server-metrics
-        
-
-        
-            com.davekoelle
-            alphanum
-        
-        
-            commons-codec
-            commons-codec
-        
-        
-            org.skife.config
-            config-magic
-        
-        
-            com.google.guava
-            guava
-        
-        
-            com.google.inject
-            guice
-        
-        
-            com.google.inject.extensions
-            guice-servlet
-        
-        
-            com.ibm.icu
-            icu4j
-        
-        
-            com.fasterxml.jackson.core
-            jackson-core
-        
-        
-            com.fasterxml.jackson.jaxrs
-            jackson-jaxrs-json-provider
-        
-        
-            com.fasterxml.jackson.core
-            jackson-databind
-        
-        
-            com.fasterxml.jackson.dataformat
-            jackson-dataformat-smile
-        
-        
-            org.jdbi
-            jdbi
-        
-        
-            com.sun.jersey
-            jersey-server
-        
-        
-            com.sun.jersey
-            jersey-core
-        
-        
-            com.sun.jersey.contribs
-            jersey-guice
-        
-        
-            org.mortbay.jetty
-            jetty
-        
-        
-            joda-time
-            joda-time
-        
-        
-            log4j
-            log4j
-        
-        
-            org.slf4j
-            slf4j-log4j12
-        
         
             org.twitter4j
             twitter4j-core
@@ -137,18 +63,6 @@
             twitter4j-stream
             3.0.3
         
-
-        
-        
-            junit
-            junit
-            test
-        
-        
-            org.easymock
-            easymock
-            test
-        
     
 
     
diff --git a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java b/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java
deleted file mode 100644
index ebdd59d46c1..00000000000
--- a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java
+++ /dev/null
@@ -1,161 +0,0 @@
-package druid.examples;
-
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.google.common.collect.ImmutableList;
-import com.metamx.common.lifecycle.Lifecycle;
-import com.metamx.common.logger.Logger;
-import com.metamx.druid.client.DataSegment;
-import com.metamx.druid.client.DruidServer;
-import com.metamx.druid.client.InventoryView;
-import com.metamx.druid.client.ServerView;
-import com.metamx.druid.coordination.DataSegmentAnnouncer;
-import com.metamx.druid.loading.DataSegmentPusher;
-import com.metamx.druid.log.LogLevelAdjuster;
-import com.metamx.druid.realtime.RealtimeNode;
-import com.metamx.druid.realtime.SegmentPublisher;
-import druid.examples.flights.FlightsFirehoseFactory;
-import druid.examples.rand.RandomFirehoseFactory;
-import druid.examples.twitter.TwitterSpritzerFirehoseFactory;
-import druid.examples.web.WebFirehoseFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.Executor;
-
-/**
- * Standalone Demo Realtime process.
- */
-public class RealtimeStandaloneMain
-{
-  private static final Logger log = new Logger(RealtimeStandaloneMain.class);
-
-  public static void main(String[] args) throws Exception
-  {
-    LogLevelAdjuster.register();
-
-    final Lifecycle lifecycle = new Lifecycle();
-
-    RealtimeNode rn = RealtimeNode.builder().build();
-    lifecycle.addManagedInstance(rn);
-
-    // register the Firehoses
-    rn.registerJacksonSubtype(
-        new NamedType(TwitterSpritzerFirehoseFactory.class, "twitzer"),
-        new NamedType(FlightsFirehoseFactory.class, "flights"),
-        new NamedType(RandomFirehoseFactory.class, "rand"),
-        new NamedType(WebFirehoseFactory.class, "webstream")
-
-    );
-
-    // Create dummy objects for the various interfaces that interact with the DB, ZK and deep storage
-    rn.setSegmentPublisher(new NoopSegmentPublisher());
-    rn.setAnnouncer(new NoopDataSegmentAnnouncer());
-    rn.setDataSegmentPusher(new NoopDataSegmentPusher());
-    rn.setServerView(new NoopServerView());
-    rn.setInventoryView(new NoopInventoryView());
-
-    Runtime.getRuntime().addShutdownHook(
-        new Thread(
-            new Runnable()
-            {
-              @Override
-              public void run()
-              {
-                log.info("Running shutdown hook");
-                lifecycle.stop();
-              }
-            }
-        )
-    );
-
-    try {
-      lifecycle.start();
-    }
-    catch (Throwable t) {
-      log.info(t, "Throwable caught at startup, committing seppuku");
-      t.printStackTrace();
-      System.exit(2);
-    }
-
-    lifecycle.join();
-  }
-
-  private static class NoopServerView implements ServerView
-  {
-    @Override
-    public void registerServerCallback(
-        Executor exec, ServerCallback callback
-    )
-    {
-
-    }
-
-    @Override
-    public void registerSegmentCallback(
-        Executor exec, SegmentCallback callback
-    )
-    {
-
-    }
-  }
-
-  private static class NoopInventoryView implements InventoryView
-  {
-    @Override
-    public DruidServer getInventoryValue(String string)
-    {
-      return null;
-    }
-
-    @Override
-    public Iterable getInventory()
-    {
-      return ImmutableList.of();
-    }
-  }
-
-  private static class NoopDataSegmentPusher implements DataSegmentPusher
-  {
-    @Override
-    public DataSegment push(File file, DataSegment segment) throws IOException
-    {
-      return segment;
-    }
-  }
-
-  private static class NoopSegmentPublisher implements SegmentPublisher
-  {
-    @Override
-    public void publishSegment(DataSegment segment) throws IOException
-    {
-      // do nothing
-    }
-  }
-
-  private static class NoopDataSegmentAnnouncer implements DataSegmentAnnouncer
-  {
-    @Override
-    public void announceSegment(DataSegment segment) throws IOException
-    {
-      // do nothing
-    }
-
-    @Override
-    public void unannounceSegment(DataSegment segment) throws IOException
-    {
-      // do nothing
-    }
-
-    @Override
-    public void announceSegments(Iterable segments) throws IOException
-    {
-      // do nothing
-    }
-
-    @Override
-    public void unannounceSegments(Iterable segments) throws IOException
-    {
-      // do nothing
-    }
-  }
-}
\ No newline at end of file
diff --git a/examples/src/main/java/druid/examples/flights/FlightsConverter.java b/examples/src/main/java/druid/examples/flights/FlightsConverter.java
index 53ea6638e60..2edb012b62b 100644
--- a/examples/src/main/java/druid/examples/flights/FlightsConverter.java
+++ b/examples/src/main/java/druid/examples/flights/FlightsConverter.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -22,7 +22,7 @@ package druid.examples.flights;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.io.Closeables;
 import com.metamx.common.parsers.CSVParser;
-import com.metamx.druid.jackson.DefaultObjectMapper;
+import io.druid.jackson.DefaultObjectMapper;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.joda.time.Period;
diff --git a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java
index 66b8de4450c..42f62440736 100644
--- a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java
+++ b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -25,10 +25,10 @@ import com.google.common.base.Charsets;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Iterators;
 import com.google.common.io.Closeables;
-import com.metamx.druid.indexer.data.StringInputRowParser;
-import com.metamx.druid.input.InputRow;
-import com.metamx.druid.realtime.firehose.Firehose;
-import com.metamx.druid.realtime.firehose.FirehoseFactory;
+import io.druid.data.input.Firehose;
+import io.druid.data.input.FirehoseFactory;
+import io.druid.data.input.InputRow;
+import io.druid.data.input.StringInputRowParser;
 
 import java.io.BufferedReader;
 import java.io.File;
diff --git a/examples/src/main/java/druid/examples/guice/RealtimeExampleModule.java b/examples/src/main/java/druid/examples/guice/RealtimeExampleModule.java
new file mode 100644
index 00000000000..816a8d367da
--- /dev/null
+++ b/examples/src/main/java/druid/examples/guice/RealtimeExampleModule.java
@@ -0,0 +1,163 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package druid.examples.guice;
+
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import com.google.inject.TypeLiteral;
+import com.metamx.common.guava.LazySequence;
+import com.metamx.common.logger.Logger;
+import druid.examples.flights.FlightsFirehoseFactory;
+import druid.examples.rand.RandomFirehoseFactory;
+import druid.examples.twitter.TwitterSpritzerFirehoseFactory;
+import druid.examples.web.WebFirehoseFactory;
+import io.druid.client.DruidServer;
+import io.druid.client.InventoryView;
+import io.druid.client.ServerView;
+import io.druid.guice.FireDepartmentsProvider;
+import io.druid.guice.JsonConfigProvider;
+import io.druid.guice.LazySingleton;
+import io.druid.guice.ManageLifecycle;
+import io.druid.guice.NoopSegmentPublisherProvider;
+import io.druid.guice.RealtimeManagerConfig;
+import io.druid.initialization.DruidModule;
+import io.druid.segment.loading.DataSegmentPusher;
+import io.druid.segment.realtime.FireDepartment;
+import io.druid.segment.realtime.RealtimeManager;
+import io.druid.segment.realtime.SegmentPublisher;
+import io.druid.server.coordination.DataSegmentAnnouncer;
+import io.druid.timeline.DataSegment;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+/**
+ */
+public class RealtimeExampleModule implements DruidModule
+{
+  private static final Logger log = new Logger(RealtimeExampleModule.class);
+
+  @Override
+  public void configure(Binder binder)
+  {
+    binder.bind(SegmentPublisher.class).toProvider(NoopSegmentPublisherProvider.class);
+    binder.bind(DataSegmentPusher.class).to(NoopDataSegmentPusher.class);
+    binder.bind(DataSegmentAnnouncer.class).to(NoopDataSegmentAnnouncer.class);
+    binder.bind(InventoryView.class).to(NoopInventoryView.class);
+    binder.bind(ServerView.class).to(NoopServerView.class);
+
+    JsonConfigProvider.bind(binder, "druid.realtime", RealtimeManagerConfig.class);
+    binder.bind(
+        new TypeLiteral>()
+        {
+        }
+    ).toProvider(FireDepartmentsProvider.class).in(LazySingleton.class);
+    binder.bind(RealtimeManager.class).in(ManageLifecycle.class);
+  }
+
+  @Override
+  public List getJacksonModules()
+  {
+    return Arrays.asList(
+        new SimpleModule("RealtimeExampleModule")
+            .registerSubtypes(
+                new NamedType(TwitterSpritzerFirehoseFactory.class, "twitzer"),
+                new NamedType(FlightsFirehoseFactory.class, "flights"),
+                new NamedType(RandomFirehoseFactory.class, "rand"),
+                new NamedType(WebFirehoseFactory.class, "webstream")
+            )
+    );
+  }
+
+  private static class NoopServerView implements ServerView
+  {
+    @Override
+    public void registerServerCallback(
+        Executor exec, ServerCallback callback
+    )
+    {
+      // do nothing
+    }
+
+    @Override
+    public void registerSegmentCallback(
+        Executor exec, SegmentCallback callback
+    )
+    {
+      // do nothing
+    }
+  }
+
+  private static class NoopInventoryView implements InventoryView
+  {
+    @Override
+    public DruidServer getInventoryValue(String string)
+    {
+      return null;
+    }
+
+    @Override
+    public Iterable getInventory()
+    {
+      return ImmutableList.of();
+    }
+  }
+
+  private static class NoopDataSegmentPusher implements DataSegmentPusher
+  {
+    @Override
+    public DataSegment push(File file, DataSegment segment) throws IOException
+    {
+      return segment;
+    }
+  }
+
+  private static class NoopDataSegmentAnnouncer implements DataSegmentAnnouncer
+  {
+    @Override
+    public void announceSegment(DataSegment segment) throws IOException
+    {
+      // do nothing
+    }
+
+    @Override
+    public void unannounceSegment(DataSegment segment) throws IOException
+    {
+      // do nothing
+    }
+
+    @Override
+    public void announceSegments(Iterable segments) throws IOException
+    {
+      // do nothing
+    }
+
+    @Override
+    public void unannounceSegments(Iterable segments) throws IOException
+    {
+      // do nothing
+    }
+  }
+}
diff --git a/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java b/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java
index a070bfff929..0e9bd2be4dd 100644
--- a/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java
+++ b/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java
@@ -3,10 +3,21 @@ package druid.examples.rabbitmq;
 import com.rabbitmq.client.Channel;
 import com.rabbitmq.client.Connection;
 import com.rabbitmq.client.ConnectionFactory;
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+import java.util.Random;
 
 /**
  *
diff --git a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java
index 71a0c1fbb92..6c02600650c 100644
--- a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java
+++ b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java
@@ -1,3 +1,22 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
 package druid.examples.rand;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -5,10 +24,10 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Maps;
 import com.metamx.common.logger.Logger;
-import com.metamx.druid.input.InputRow;
-import com.metamx.druid.input.MapBasedInputRow;
-import com.metamx.druid.realtime.firehose.Firehose;
-import com.metamx.druid.realtime.firehose.FirehoseFactory;
+import io.druid.data.input.Firehose;
+import io.druid.data.input.FirehoseFactory;
+import io.druid.data.input.InputRow;
+import io.druid.data.input.MapBasedInputRow;
 
 import java.io.IOException;
 import java.util.LinkedList;
diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
index e87b5e44de2..dcb5915b8a0 100644
--- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
+++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java
@@ -1,3 +1,22 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
 package druid.examples.twitter;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -5,19 +24,19 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.collect.Lists;
 import com.metamx.common.logger.Logger;
-import com.metamx.druid.input.InputRow;
-import com.metamx.druid.input.MapBasedInputRow;
-import com.metamx.druid.realtime.firehose.Firehose;
-import com.metamx.druid.realtime.firehose.FirehoseFactory;
+import io.druid.data.input.Firehose;
+import io.druid.data.input.FirehoseFactory;
+import io.druid.data.input.InputRow;
+import io.druid.data.input.MapBasedInputRow;
 import twitter4j.ConnectionLifeCycleListener;
 import twitter4j.HashtagEntity;
+import twitter4j.StallWarning;
 import twitter4j.Status;
 import twitter4j.StatusDeletionNotice;
 import twitter4j.StatusListener;
 import twitter4j.TwitterStream;
 import twitter4j.TwitterStreamFactory;
 import twitter4j.User;
-import twitter4j.StallWarning;
 
 import java.io.IOException;
 import java.util.Arrays;
diff --git a/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java
index 40d7cf71616..4546d8cb2a9 100644
--- a/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java
+++ b/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -24,8 +24,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.io.InputSupplier;
-import com.metamx.druid.jackson.DefaultObjectMapper;
 import com.metamx.emitter.EmittingLogger;
+import io.druid.jackson.DefaultObjectMapper;
 
 import java.io.BufferedReader;
 import java.io.IOException;
diff --git a/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java
index 885089a2001..2c7ccfa09a5 100644
--- a/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java
+++ b/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -24,12 +24,12 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonTypeName;
 import com.google.common.base.Throwables;
 import com.metamx.common.parsers.TimestampParser;
-import com.metamx.druid.guava.Runnables;
-import com.metamx.druid.input.InputRow;
-import com.metamx.druid.input.MapBasedInputRow;
-import com.metamx.druid.realtime.firehose.Firehose;
-import com.metamx.druid.realtime.firehose.FirehoseFactory;
 import com.metamx.emitter.EmittingLogger;
+import io.druid.common.guava.Runnables;
+import io.druid.data.input.Firehose;
+import io.druid.data.input.FirehoseFactory;
+import io.druid.data.input.InputRow;
+import io.druid.data.input.MapBasedInputRow;
 import org.joda.time.DateTime;
 
 import java.io.IOException;
diff --git a/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java b/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java
index 755bcfd197a..bb7f6b25a24 100644
--- a/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java
+++ b/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -21,8 +21,8 @@ package druid.examples.web;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
-import com.metamx.druid.input.InputRow;
-import com.metamx.druid.realtime.firehose.Firehose;
+import io.druid.data.input.Firehose;
+import io.druid.data.input.InputRow;
 import org.joda.time.DateTime;
 import org.junit.Assert;
 import org.junit.Before;
diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ValueType.java b/indexing-common/src/main/java/com/metamx/druid/index/column/ValueType.java
deleted file mode 100644
index 245ff682bd7..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/index/column/ValueType.java
+++ /dev/null
@@ -1,11 +0,0 @@
-package com.metamx.druid.index.column;
-
-/**
-*/
-public enum ValueType
-{
-  FLOAT,
-  LONG,
-  STRING,
-  COMPLEX
-}
diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/Index.java b/indexing-common/src/main/java/com/metamx/druid/index/v1/Index.java
deleted file mode 100644
index fcec6135b86..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/index/v1/Index.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-package com.metamx.druid.index.v1;
-
-import com.metamx.collections.spatial.ImmutableRTree;
-import com.metamx.common.logger.Logger;
-import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet;
-import org.joda.time.Interval;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * In-memory representation of a segment
- */
-public class Index
-{
-  private static final Logger log = new Logger(Index.class);
-  private static final ImmutableConciseSet emptySet = new ImmutableConciseSet();
-
-  final Map dimToPositionMap = new HashMap();
-  final Map metricToPositionMap = new HashMap();
-
-  final String[] dimensions;
-  final String[] metrics;
-  final Interval dataInterval;
-  final long[] timeOffsets;
-  final Map metricVals;
-  final Map> dimIdLookup;
-  final Map reverseDimLookup;
-  final Map indexes;
-  final Map spatialIndexes;
-  final Map dimensionValues;
-
-  /*
-   * If we name the various occurrences of String and int then the types are more informative:
-   *
-   *   dimToPositionMap         : Dim       -> DimColumn
-   *   dimensions               : DimColumn -> Dim
-   *   metricToPositionMap      : Met       -> MetColumn
-   *   metrics                  : MetColumn -> Met
-   *
-   *   dataInterval             : Interval
-   *
-   *   timeOffsets              : Milli[]
-   *   metricVals               : float[] (size() == timeOffsets.size() * metrics.size())
-   *
-   *   dimIdLookup              : Dim -> Value   -> ValueID
-   *   reverseDimLookup         : Dim -> ValueID -> Value
-   *   indexes                  : Dim -> ValueID -> Row[]
-   *
-   *   dimensionValues          : Dim ->
-   *     getRowValues           :        Row         -> ExpansionID
-   *     getDimensionExpansions :        ExpansionID -> ValueID[]
-   *     getDimValues           :        Row         -> ValueID[]
-   */
-
-  public Index(
-      String[] dimensions,
-      String[] metrics,
-      Interval dataInterval,
-      long[] timeOffsets,
-      Map metricVals,
-      Map> dimIdLookup,
-      Map reverseDimLookup,
-      Map indexes,
-      Map spatialIndexes,
-      Map dimensionValues
-  )
-  {
-    this.dimensions = dimensions;
-    this.metrics = metrics;
-    this.dataInterval = dataInterval;
-    this.timeOffsets = timeOffsets;
-    this.metricVals = metricVals;
-    this.dimIdLookup = dimIdLookup;
-    this.reverseDimLookup = reverseDimLookup;
-    this.indexes = indexes;
-    this.spatialIndexes = spatialIndexes;
-    this.dimensionValues = dimensionValues;
-
-    for (int i = 0; i < dimensions.length; i++) {
-      dimToPositionMap.put(dimensions[i].toLowerCase(), i);
-    }
-    for (int i = 0; i < metrics.length; i++) {
-      metricToPositionMap.put(metrics[i].toLowerCase(), i);
-    }
-  }
-
-  public ImmutableConciseSet getInvertedIndex(String dimension, String value)
-  {
-    final Map lookup = dimIdLookup.get(dimension);
-    if (lookup == null) {
-      return emptySet;
-    }
-
-    final Integer integer = lookup.get(value);
-    if (integer == null) {
-      return emptySet;
-    }
-
-    try {
-      return indexes.get(dimension)[integer];
-    }
-    catch (NullPointerException e) {
-      log.warn(
-          e,
-          "NPE on dimension[%s], value[%s], with index over interval[%s]",
-          dimension,
-          value,
-          dataInterval
-      );
-      return emptySet;
-    }
-  }
-
-  public ImmutableConciseSet getInvertedIndex(String dimension, int valueIndex)
-  {
-    try {
-      return indexes.get(dimension)[valueIndex];
-    }
-    catch (NullPointerException e) {
-      log.warn(
-          e,
-          "NPE on dimension[%s], valueIndex[%d], with index over interval[%s]",
-          dimension,
-          valueIndex,
-          dataInterval
-      );
-      return emptySet;
-    }
-  }
-
-  public ImmutableRTree getSpatialIndex(String dimension)
-  {
-    try {
-      return spatialIndexes.get(dimension);
-    }
-    catch (NullPointerException e) {
-      log.warn(
-          e,
-          "NPE on dimension[%s] over interval[%s]",
-          dimension,
-          dataInterval
-      );
-      return new ImmutableRTree();
-    }
-  }
-}
diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java b/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java
deleted file mode 100644
index d792d80de01..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-package com.metamx.druid.index.v1;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.primitives.Ints;
-import com.metamx.collections.spatial.ImmutableRTree;
-import com.metamx.common.io.smoosh.SmooshedFileMapper;
-import com.metamx.common.logger.Logger;
-import com.metamx.druid.kv.ConciseCompressedIndexedInts;
-import com.metamx.druid.kv.GenericIndexed;
-import com.metamx.druid.kv.Indexed;
-import com.metamx.druid.kv.IndexedList;
-import com.metamx.druid.kv.IndexedLongs;
-import com.metamx.druid.kv.VSizeIndexed;
-import com.metamx.druid.kv.VSizeIndexedInts;
-import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet;
-import org.joda.time.Interval;
-
-import java.io.IOException;
-import java.nio.ByteOrder;
-import java.nio.LongBuffer;
-import java.util.Arrays;
-import java.util.Map;
-
-/**
- */
-public class MMappedIndex
-{
-  private static final Logger log = new Logger(MMappedIndex.class);
-  private static final ImmutableConciseSet emptySet = new ImmutableConciseSet();
-
-  final GenericIndexed availableDimensions;
-  final GenericIndexed availableMetrics;
-  final Interval dataInterval;
-  final CompressedLongsIndexedSupplier timestamps;
-  final Map metrics;
-  final Map> dimValueLookups;
-  final Map dimColumns;
-  final Map> invertedIndexes;
-  final Map spatialIndexes;
-  final SmooshedFileMapper fileMapper;
-
-  private final Map metricIndexes = Maps.newHashMap();
-
-  public MMappedIndex(
-      GenericIndexed availableDimensions,
-      GenericIndexed availableMetrics,
-      Interval dataInterval,
-      CompressedLongsIndexedSupplier timestamps,
-      Map metrics,
-      Map> dimValueLookups,
-      Map dimColumns,
-      Map> invertedIndexes,
-      Map spatialIndexes,
-      SmooshedFileMapper fileMapper
-  )
-  {
-    this.availableDimensions = availableDimensions;
-    this.availableMetrics = availableMetrics;
-    this.dataInterval = dataInterval;
-    this.timestamps = timestamps;
-    this.metrics = metrics;
-    this.dimValueLookups = dimValueLookups;
-    this.dimColumns = dimColumns;
-    this.invertedIndexes = invertedIndexes;
-    this.spatialIndexes = spatialIndexes;
-    this.fileMapper = fileMapper;
-
-    for (int i = 0; i < availableMetrics.size(); i++) {
-      metricIndexes.put(availableMetrics.get(i), i);
-    }
-  }
-
-  public CompressedLongsIndexedSupplier getTimestamps()
-  {
-    return timestamps;
-  }
-
-  public GenericIndexed getAvailableDimensions()
-  {
-    return availableDimensions;
-  }
-
-  public GenericIndexed getAvailableMetrics()
-  {
-    return availableMetrics;
-  }
-
-  public Map getMetrics()
-  {
-    return metrics;
-  }
-
-  public Integer getMetricIndex(String metricName)
-  {
-    return metricIndexes.get(metricName);
-  }
-
-  public Interval getDataInterval()
-  {
-    return dataInterval;
-  }
-
-  public IndexedLongs getReadOnlyTimestamps()
-  {
-    return timestamps.get();
-  }
-
-  public MetricHolder getMetricHolder(String metric)
-  {
-    final MetricHolder retVal = metrics.get(metric);
-
-    if (retVal == null) {
-      return null;
-    }
-
-    return retVal;
-  }
-
-  public GenericIndexed getDimValueLookup(String dimension)
-  {
-    return dimValueLookups.get(dimension);
-  }
-
-  public VSizeIndexed getDimColumn(String dimension)
-  {
-    return dimColumns.get(dimension);
-  }
-
-  public Map> getInvertedIndexes()
-  {
-    return invertedIndexes;
-  }
-
-  public Map getSpatialIndexes()
-  {
-    return spatialIndexes;
-  }
-
-  public ImmutableConciseSet getInvertedIndex(String dimension, String value)
-  {
-    final GenericIndexed lookup = dimValueLookups.get(dimension);
-    if (lookup == null) {
-      return emptySet;
-    }
-
-    int indexOf = lookup.indexOf(value);
-    if (indexOf < 0) {
-      return emptySet;
-    }
-
-    ImmutableConciseSet retVal = invertedIndexes.get(dimension).get(indexOf);
-    return (retVal == null) ? emptySet : retVal;
-  }
-
-  public SmooshedFileMapper getFileMapper()
-  {
-    return fileMapper;
-  }
-
-  public void close() throws IOException
-  {
-    if (fileMapper != null) {
-      fileMapper.close();
-    }
-  }
-
-  public static MMappedIndex fromIndex(Index index)
-  {
-    log.info("Converting timestamps");
-    CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromLongBuffer(
-        LongBuffer.wrap(index.timeOffsets), ByteOrder.nativeOrder()
-    );
-
-    log.info("Converting dimValueLookups");
-    Map> dimValueLookups = Maps.newHashMap();
-    for (Map.Entry entry : index.reverseDimLookup.entrySet()) {
-      String[] theValues = Arrays.copyOf(entry.getValue(), entry.getValue().length);
-      Arrays.sort(theValues);
-
-      dimValueLookups.put(entry.getKey(), GenericIndexed.fromArray(theValues, GenericIndexed.stringStrategy));
-    }
-
-    Map dimColumns = Maps.newHashMap();
-    Map> invertedIndexes = Maps.newLinkedHashMap();
-    Map spatialIndexes = Maps.newLinkedHashMap();
-
-    for (String dimension : Arrays.asList(index.dimensions)) {
-      final String[] dimVals = index.reverseDimLookup.get(dimension);
-      final DimensionColumn dimColumn = index.dimensionValues.get(dimension);
-
-      log.info(
-          "Converting dim[%s] with valueCardinality[%,d] and expansionCardinality[%,d]",
-          dimension,
-          dimColumn.getDimensionExpansions().length,
-          dimVals.length
-      );
-
-      final Indexed lookup = dimValueLookups.get(dimension);
-      final int[] expansionConversion = new int[dimVals.length];
-      final int[] reverseExpansionConversion = new int[dimVals.length];
-      for (int i = 0; i < expansionConversion.length; i++) {
-        expansionConversion[i] = lookup.indexOf(dimVals[i]);
-        reverseExpansionConversion[expansionConversion[i]] = i;
-      }
-
-      int[][] originalDimExpansions = dimColumn.getDimensionExpansions();
-      final int[][] dimensionExpansions = new int[originalDimExpansions.length][];
-      for (int i = 0; i < originalDimExpansions.length; i++) {
-        int[] originalDimExpansion = originalDimExpansions[i];
-        int[] mappedValues = new int[originalDimExpansion.length];
-
-        for (int j = 0; j < originalDimExpansion.length; j++) {
-          mappedValues[j] = expansionConversion[originalDimExpansion[j]];
-        }
-        Arrays.sort(mappedValues);
-
-        dimensionExpansions[i] = mappedValues;
-      }
-
-      final int[] originalRows = dimColumn.getDimensionRowValues();
-      final ImmutableConciseSet[] origInvertedIndexes = index.indexes.get(dimension);
-
-      dimColumns.put(
-          dimension,
-          VSizeIndexed.fromIterable(
-              Iterables.transform(
-                  Ints.asList(originalRows),
-                  new Function()
-                  {
-                    @Override
-                    public VSizeIndexedInts apply(Integer input)
-                    {
-                      return VSizeIndexedInts.fromArray(dimensionExpansions[input], dimVals.length - 1);
-                    }
-                  }
-              )
-          )
-      );
-
-      invertedIndexes.put(
-          dimension,
-          GenericIndexed.fromIterable(
-              Iterables.transform(
-                  new IndexedList(lookup),
-                  new Function()
-                  {
-                    @Override
-                    public ImmutableConciseSet apply(String input)
-                    {
-                      return origInvertedIndexes[reverseExpansionConversion[lookup.indexOf(input)]];
-                    }
-                  }
-              ),
-              ConciseCompressedIndexedInts.objectStrategy
-          )
-      );
-
-      spatialIndexes.put(dimension, index.getSpatialIndex(dimension));
-    }
-
-    log.info("Making MMappedIndex");
-    return new MMappedIndex(
-        GenericIndexed.fromArray(index.dimensions, GenericIndexed.stringStrategy),
-        GenericIndexed.fromArray(index.metrics, GenericIndexed.stringStrategy),
-        index.dataInterval,
-        timestamps,
-        index.metricVals,
-        dimValueLookups,
-        dimColumns,
-        invertedIndexes,
-        spatialIndexes,
-        null
-    );
-  }
-}
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ByteBufferInputRowParser.java b/indexing-common/src/main/java/com/metamx/druid/indexer/data/ByteBufferInputRowParser.java
deleted file mode 100644
index bc3721bf719..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ByteBufferInputRowParser.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package com.metamx.druid.indexer.data;
-
-import java.nio.ByteBuffer;
-
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class)
-@JsonSubTypes(value = {
-        @JsonSubTypes.Type(name = "protobuf", value = ProtoBufInputRowParser.class),
-        @JsonSubTypes.Type(name = "string", value = StringInputRowParser.class)
-})
-public interface ByteBufferInputRowParser extends InputRowParser {
-}
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java b/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java
deleted file mode 100644
index 231cbd44102..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java
+++ /dev/null
@@ -1,10 +0,0 @@
-package com.metamx.druid.indexer.data;
-
-import com.metamx.common.exception.FormattedException;
-import com.metamx.druid.input.InputRow;
-
-public interface InputRowParser
-{
-  public InputRow parse(T input) throws FormattedException;
-  public void addDimensionExclusion(String dimension);
-}
diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferSerializer.java b/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferSerializer.java
deleted file mode 100644
index 93f49451941..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferSerializer.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.metamx.druid.kv;
-
-import com.google.common.primitives.Ints;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.WritableByteChannel;
-
-/**
- */
-public class ByteBufferSerializer
-{
-  public static  T read(ByteBuffer buffer, ObjectStrategy strategy)
-  {
-    int size = buffer.getInt();
-    ByteBuffer bufferToUse = buffer.asReadOnlyBuffer();
-    bufferToUse.limit(bufferToUse.position() + size);
-    buffer.position(bufferToUse.limit());
-
-    return strategy.fromByteBuffer(bufferToUse, size);
-  }
-
-  public static  void writeToChannel(T obj, ObjectStrategy strategy, WritableByteChannel channel)
-      throws IOException
-  {
-    byte[] toWrite = strategy.toBytes(obj);
-    channel.write(ByteBuffer.allocate(Ints.BYTES).putInt(0, toWrite.length));
-    channel.write(ByteBuffer.wrap(toWrite));
-  }
-}
diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/SingleIndexedInts.java b/indexing-common/src/main/java/com/metamx/druid/kv/SingleIndexedInts.java
deleted file mode 100644
index cba9195e0f6..00000000000
--- a/indexing-common/src/main/java/com/metamx/druid/kv/SingleIndexedInts.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package com.metamx.druid.kv;
-
-import com.google.common.collect.Iterators;
-
-import java.util.Iterator;
-
-/**
-*/
-public class SingleIndexedInts implements IndexedInts
-{
-  private final int value;
-
-  public SingleIndexedInts(int value) {
-    this.value = value;
-  }
-
-  @Override
-  public int size()
-  {
-    return 1;
-  }
-
-  @Override
-  public int get(int index)
-  {
-    return value;
-  }
-
-  @Override
-  public Iterator iterator()
-  {
-    return Iterators.singletonIterator(value);
-  }
-}
diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml
index 460e9ab6e42..569a61db92b 100644
--- a/indexing-hadoop/pom.xml
+++ b/indexing-hadoop/pom.xml
@@ -1,7 +1,7 @@
 
 
             
                 com.metamx
                 emitter
-                0.2.3
+                0.2.5
             
             
                 com.metamx
@@ -80,7 +85,7 @@
             
                 com.metamx
                 server-metrics
-                0.0.3
+                0.0.4
             
 
             
@@ -113,11 +118,35 @@
                 commons-lang
                 2.6
             
+            
+                com.amazonaws
+                aws-java-sdk
+                1.3.27
+                
+                    
+                        javax.mail
+                        mail
+                    
+                    
+                        org.codehaus.jackson
+                        jackson-core-asl
+                    
+                    
+                        org.codehaus.jackson
+                        jackson-mapper-asl
+                    
+                
+            
             
                 com.ning
                 compress-lzf
                 0.8.4
             
+            
+                io.airlift
+                airline
+                0.5
+            
             
                 org.skife.config
                 config-magic
@@ -167,12 +196,17 @@
             
                 com.google.inject
                 guice
-                3.0
+                4.0-beta
             
             
                 com.google.inject.extensions
                 guice-servlet
-                3.0
+                4.0-beta
+            
+            
+                com.google.inject.extensions
+                guice-multibindings
+                4.0-beta
             
             
                 com.ibm.icu
@@ -214,6 +248,26 @@
                 jackson-jaxrs-json-provider
                 2.2.2
             
+            
+                org.codehaus.jackson
+                jackson-core-asl
+                1.9.11
+            
+            
+                org.codehaus.jackson
+                jackson-mapper-asl
+                1.9.11
+            
+            
+                org.hibernate
+                hibernate-validator
+                5.0.1.Final
+            
+            
+                javax.validation
+                validation-api
+                1.1.0.Final
+            
             
                 javax.inject
                 javax.inject
@@ -232,17 +286,17 @@
             
                 com.sun.jersey
                 jersey-core
-                1.9.1
+                1.17.1
             
             
                 com.sun.jersey.contribs
                 jersey-guice
-                1.9.1
+                1.17.1
             
             
                 com.sun.jersey
                 jersey-server
-                1.9.1
+                1.17.1
             
             
                 net.java.dev.jets3t
@@ -250,20 +304,19 @@
                 0.8.1
             
             
-                org.mortbay.jetty
-                jetty
-                6.1.26
-                
-                    
-                        org.mortbay.jetty
-                        servlet-api
-                    
-                
+                org.eclipse.jetty
+                jetty-server
+                8.1.11.v20130520
             
             
-                org.mortbay.jetty
-                jetty-util
-                6.1.26
+                org.eclipse.jetty
+                jetty-servlet
+                8.1.11.v20130520
+            
+            
+                org.eclipse.jetty
+                jetty-servlets
+                8.1.11.v20130520
             
             
                 joda-time
@@ -282,8 +335,8 @@
             
             
                 javax.servlet
-                servlet-api
-                2.5
+                javax.servlet-api
+                3.0.1
             
             
                 org.slf4j
@@ -330,6 +383,16 @@
                 protobuf-java
                 2.4.0a
             
+            
+                io.tesla.aether
+                tesla-aether
+                0.0.2
+            
+            
+                org.eclipse.aether
+                aether-api
+                0.9.0.M2
+            
 
 
 
@@ -399,8 +462,8 @@
                     maven-compiler-plugin
                     2.5.1
                     
-                        1.6
-                        1.6
+                        1.7
+                        1.7
                     
                 
                 
@@ -429,7 +492,7 @@
                 
                 
                     maven-shade-plugin
-                    1.7.1
+                    2.1
                 
                 
                     maven-site-plugin
diff --git a/indexing-common/pom.xml b/processing/pom.xml
similarity index 70%
rename from indexing-common/pom.xml
rename to processing/pom.xml
index aa63c90ded0..405c8997078 100644
--- a/indexing-common/pom.xml
+++ b/processing/pom.xml
@@ -1,7 +1,7 @@
 
 
@@ -91,6 +115,11 @@
             junit
             test
         
+        
+            org.easymock
+            easymock
+            test
+        
     
 
     
diff --git a/processing/src/main/java/io/druid/data/input/ByteBufferInputRowParser.java b/processing/src/main/java/io/druid/data/input/ByteBufferInputRowParser.java
new file mode 100644
index 00000000000..d033013bc6d
--- /dev/null
+++ b/processing/src/main/java/io/druid/data/input/ByteBufferInputRowParser.java
@@ -0,0 +1,33 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.data.input;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+import java.nio.ByteBuffer;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class)
+@JsonSubTypes(value = {
+        @JsonSubTypes.Type(name = "protobuf", value = ProtoBufInputRowParser.class),
+        @JsonSubTypes.Type(name = "string", value = StringInputRowParser.class)
+})
+public interface ByteBufferInputRowParser extends InputRowParser {
+}
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/CSVDataSpec.java b/processing/src/main/java/io/druid/data/input/CSVDataSpec.java
similarity index 95%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/CSVDataSpec.java
rename to processing/src/main/java/io/druid/data/input/CSVDataSpec.java
index 60303703e18..c0cedfe2095 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/CSVDataSpec.java
+++ b/processing/src/main/java/io/druid/data/input/CSVDataSpec.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -25,7 +25,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.metamx.common.parsers.CSVParser;
 import com.metamx.common.parsers.Parser;
-import com.metamx.druid.index.v1.SpatialDimensionSchema;
+import io.druid.segment.incremental.SpatialDimensionSchema;
 
 import java.util.List;
 
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/DataSpec.java b/processing/src/main/java/io/druid/data/input/DataSpec.java
similarity index 91%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/DataSpec.java
rename to processing/src/main/java/io/druid/data/input/DataSpec.java
index 099d0c8d535..b45faec2a9a 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/DataSpec.java
+++ b/processing/src/main/java/io/druid/data/input/DataSpec.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,12 +17,12 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.metamx.common.parsers.Parser;
-import com.metamx.druid.index.v1.SpatialDimensionSchema;
+import io.druid.segment.incremental.SpatialDimensionSchema;
 
 import java.util.List;
 
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/DelimitedDataSpec.java b/processing/src/main/java/io/druid/data/input/DelimitedDataSpec.java
similarity index 95%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/DelimitedDataSpec.java
rename to processing/src/main/java/io/druid/data/input/DelimitedDataSpec.java
index c9aa2d253ce..f9e819f07ec 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/DelimitedDataSpec.java
+++ b/processing/src/main/java/io/druid/data/input/DelimitedDataSpec.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -25,7 +25,7 @@ import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.metamx.common.parsers.DelimitedParser;
 import com.metamx.common.parsers.Parser;
-import com.metamx.druid.index.v1.SpatialDimensionSchema;
+import io.druid.segment.incremental.SpatialDimensionSchema;
 
 import java.util.List;
 
diff --git a/processing/src/main/java/io/druid/data/input/InputRowParser.java b/processing/src/main/java/io/druid/data/input/InputRowParser.java
new file mode 100644
index 00000000000..ea1a0dfa8a0
--- /dev/null
+++ b/processing/src/main/java/io/druid/data/input/InputRowParser.java
@@ -0,0 +1,28 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.data.input;
+
+import com.metamx.common.exception.FormattedException;
+
+public interface InputRowParser
+{
+  public InputRow parse(T input) throws FormattedException;
+  public void addDimensionExclusion(String dimension);
+}
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/JSONDataSpec.java b/processing/src/main/java/io/druid/data/input/JSONDataSpec.java
similarity index 93%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/JSONDataSpec.java
rename to processing/src/main/java/io/druid/data/input/JSONDataSpec.java
index 0f07161b4bf..9badb3e2527 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/JSONDataSpec.java
+++ b/processing/src/main/java/io/druid/data/input/JSONDataSpec.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,14 +17,14 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.collect.Lists;
 import com.metamx.common.parsers.JSONParser;
 import com.metamx.common.parsers.Parser;
-import com.metamx.druid.index.v1.SpatialDimensionSchema;
+import io.druid.segment.incremental.SpatialDimensionSchema;
 
 import java.util.List;
 
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/MapInputRowParser.java b/processing/src/main/java/io/druid/data/input/MapInputRowParser.java
similarity index 76%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/MapInputRowParser.java
rename to processing/src/main/java/io/druid/data/input/MapInputRowParser.java
index b39b70471fa..adb4cc534e5 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/MapInputRowParser.java
+++ b/processing/src/main/java/io/druid/data/input/MapInputRowParser.java
@@ -1,4 +1,23 @@
-package com.metamx.druid.indexer.data;
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -6,8 +25,6 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.metamx.common.exception.FormattedException;
-import com.metamx.druid.input.InputRow;
-import com.metamx.druid.input.MapBasedInputRow;
 import org.joda.time.DateTime;
 
 import java.util.List;
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ProtoBufInputRowParser.java b/processing/src/main/java/io/druid/data/input/ProtoBufInputRowParser.java
similarity index 71%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/ProtoBufInputRowParser.java
rename to processing/src/main/java/io/druid/data/input/ProtoBufInputRowParser.java
index b5cb4e2e0c9..f1f8ea875f5 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ProtoBufInputRowParser.java
+++ b/processing/src/main/java/io/druid/data/input/ProtoBufInputRowParser.java
@@ -1,13 +1,23 @@
-package com.metamx.druid.indexer.data;
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
 
-import static com.google.protobuf.DescriptorProtos.FileDescriptorSet;
-import static com.google.protobuf.Descriptors.Descriptor;
-import static com.google.protobuf.Descriptors.FileDescriptor;
-
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Map;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -18,10 +28,20 @@ import com.google.protobuf.Descriptors;
 import com.google.protobuf.DynamicMessage;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.metamx.common.exception.FormattedException;
-import com.metamx.druid.input.InputRow;
+import com.metamx.common.logger.Logger;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.protobuf.DescriptorProtos.FileDescriptorSet;
+import static com.google.protobuf.Descriptors.Descriptor;
+import static com.google.protobuf.Descriptors.FileDescriptor;
 
 public class ProtoBufInputRowParser implements ByteBufferInputRowParser
 {
+  private static final Logger log = new Logger(ProtoBufInputRowParser.class);
 
 	private final MapInputRowParser inputRowCreator;
 	private final Descriptor descriptor;
@@ -42,8 +62,8 @@ public class ProtoBufInputRowParser implements ByteBufferInputRowParser
 	@Override
 	public InputRow parse(ByteBuffer input) throws FormattedException
 	{
-        // TODO there should be a ProtoBufBasedInputRow that does not need an intermediate map but accesses
-        // the DynamicMessage directly
+        // We should really create a ProtoBufBasedInputRow that does not need an intermediate map but accesses
+        // the DynamicMessage directly...
 		Map theMap = buildStringKeyMap(input);
 
 		return inputRowCreator.parse(theMap);
@@ -64,9 +84,8 @@ public class ProtoBufInputRowParser implements ByteBufferInputRowParser
 				if (theMap.containsKey(name))
 				{
 					continue;
-					// TODO
-					// throw new RuntimeException("dupicate key " + name + " in " +
-					// message);
+					// Perhaps throw an exception here?
+					// throw new RuntimeException("dupicate key " + name + " in " + message);
 				}
         Object value = entry.getValue();
         if(value instanceof Descriptors.EnumValueDescriptor) {
@@ -79,8 +98,7 @@ public class ProtoBufInputRowParser implements ByteBufferInputRowParser
 
 		} catch (InvalidProtocolBufferException e)
 		{
-			// TODO
-			e.printStackTrace();
+			log.warn(e, "Problem with protobuf something");
 		}
 		return theMap;
 	}
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java b/processing/src/main/java/io/druid/data/input/StringInputRowParser.java
similarity index 96%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java
rename to processing/src/main/java/io/druid/data/input/StringInputRowParser.java
index 3d23d5d1dde..f2d9b10df07 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/StringInputRowParser.java
+++ b/processing/src/main/java/io/druid/data/input/StringInputRowParser.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -26,7 +26,6 @@ import com.google.common.collect.ImmutableList;
 import com.metamx.common.exception.FormattedException;
 import com.metamx.common.parsers.Parser;
 import com.metamx.common.parsers.ToLowerCaseParser;
-import com.metamx.druid.input.InputRow;
 
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/TimestampSpec.java b/processing/src/main/java/io/druid/data/input/TimestampSpec.java
similarity index 95%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/TimestampSpec.java
rename to processing/src/main/java/io/druid/data/input/TimestampSpec.java
index ab216a2b415..4052cb63537 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/TimestampSpec.java
+++ b/processing/src/main/java/io/druid/data/input/TimestampSpec.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,13 +17,12 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Function;
 import com.metamx.common.parsers.ParserUtils;
-
 import org.joda.time.DateTime;
 
 import java.util.Map;
diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ToLowercaseDataSpec.java b/processing/src/main/java/io/druid/data/input/ToLowercaseDataSpec.java
similarity index 92%
rename from indexing-common/src/main/java/com/metamx/druid/indexer/data/ToLowercaseDataSpec.java
rename to processing/src/main/java/io/druid/data/input/ToLowercaseDataSpec.java
index c4eda7ffc55..cba57080af6 100644
--- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ToLowercaseDataSpec.java
+++ b/processing/src/main/java/io/druid/data/input/ToLowercaseDataSpec.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,12 +17,12 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.indexer.data;
+package io.druid.data.input;
 
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.metamx.common.parsers.Parser;
 import com.metamx.common.parsers.ToLowerCaseParser;
-import com.metamx.druid.index.v1.SpatialDimensionSchema;
+import io.druid.segment.incremental.SpatialDimensionSchema;
 
 import java.util.List;
 
diff --git a/common/src/main/java/com/metamx/druid/AllGranularity.java b/processing/src/main/java/io/druid/granularity/AllGranularity.java
similarity index 95%
rename from common/src/main/java/com/metamx/druid/AllGranularity.java
rename to processing/src/main/java/io/druid/granularity/AllGranularity.java
index a35b632f5c2..eebe4c43ab1 100644
--- a/common/src/main/java/com/metamx/druid/AllGranularity.java
+++ b/processing/src/main/java/io/druid/granularity/AllGranularity.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.granularity;
 
 import com.google.common.collect.ImmutableList;
 
diff --git a/common/src/main/java/com/metamx/druid/BaseQueryGranularity.java b/processing/src/main/java/io/druid/granularity/BaseQueryGranularity.java
similarity index 96%
rename from common/src/main/java/com/metamx/druid/BaseQueryGranularity.java
rename to processing/src/main/java/io/druid/granularity/BaseQueryGranularity.java
index bf9b8ce1f21..b9755a4ed03 100644
--- a/common/src/main/java/com/metamx/druid/BaseQueryGranularity.java
+++ b/processing/src/main/java/io/druid/granularity/BaseQueryGranularity.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.granularity;
 
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
diff --git a/common/src/main/java/com/metamx/druid/DurationGranularity.java b/processing/src/main/java/io/druid/granularity/DurationGranularity.java
similarity index 97%
rename from common/src/main/java/com/metamx/druid/DurationGranularity.java
rename to processing/src/main/java/io/druid/granularity/DurationGranularity.java
index 5849f834103..a4e8393eabd 100644
--- a/common/src/main/java/com/metamx/druid/DurationGranularity.java
+++ b/processing/src/main/java/io/druid/granularity/DurationGranularity.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,12 +17,11 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.granularity;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.primitives.Longs;
-
 import org.joda.time.DateTime;
 
 import java.nio.ByteBuffer;
diff --git a/common/src/main/java/com/metamx/druid/NoneGranularity.java b/processing/src/main/java/io/druid/granularity/NoneGranularity.java
similarity index 94%
rename from common/src/main/java/com/metamx/druid/NoneGranularity.java
rename to processing/src/main/java/io/druid/granularity/NoneGranularity.java
index d3a0afeb087..3cf9398733a 100644
--- a/common/src/main/java/com/metamx/druid/NoneGranularity.java
+++ b/processing/src/main/java/io/druid/granularity/NoneGranularity.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.granularity;
 
 public final class NoneGranularity extends BaseQueryGranularity
 {
diff --git a/common/src/main/java/com/metamx/druid/PeriodGranularity.java b/processing/src/main/java/io/druid/granularity/PeriodGranularity.java
similarity index 99%
rename from common/src/main/java/com/metamx/druid/PeriodGranularity.java
rename to processing/src/main/java/io/druid/granularity/PeriodGranularity.java
index 5c8e594013a..cc8faf66d41 100644
--- a/common/src/main/java/com/metamx/druid/PeriodGranularity.java
+++ b/processing/src/main/java/io/druid/granularity/PeriodGranularity.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.granularity;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
diff --git a/common/src/main/java/com/metamx/druid/QueryGranularity.java b/processing/src/main/java/io/druid/granularity/QueryGranularity.java
similarity index 81%
rename from common/src/main/java/com/metamx/druid/QueryGranularity.java
rename to processing/src/main/java/io/druid/granularity/QueryGranularity.java
index ee745d3dadb..16b86ac397d 100644
--- a/common/src/main/java/com/metamx/druid/QueryGranularity.java
+++ b/processing/src/main/java/io/druid/granularity/QueryGranularity.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,23 +17,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.granularity;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.metamx.common.IAE;
-
 import org.joda.time.DateTime;
 import org.joda.time.ReadableDuration;
 
-@JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property = "type", defaultImpl = QueryGranularity.class)
-@JsonSubTypes(value = {
-    @JsonSubTypes.Type(name = "period",   value = PeriodGranularity.class),
-    @JsonSubTypes.Type(name = "duration", value = DurationGranularity.class),
-    @JsonSubTypes.Type(name = "all",      value = AllGranularity.class),
-    @JsonSubTypes.Type(name = "none",     value = NoneGranularity.class)
-})
 public abstract class QueryGranularity
 {
   public abstract long next(long offset);
diff --git a/processing/src/main/java/io/druid/jackson/AggregatorsModule.java b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java
new file mode 100644
index 00000000000..8ab41053c30
--- /dev/null
+++ b/processing/src/main/java/io/druid/jackson/AggregatorsModule.java
@@ -0,0 +1,69 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.jackson;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import io.druid.query.aggregation.AggregatorFactory;
+import io.druid.query.aggregation.CountAggregatorFactory;
+import io.druid.query.aggregation.DoubleSumAggregatorFactory;
+import io.druid.query.aggregation.HistogramAggregatorFactory;
+import io.druid.query.aggregation.JavaScriptAggregatorFactory;
+import io.druid.query.aggregation.LongSumAggregatorFactory;
+import io.druid.query.aggregation.MaxAggregatorFactory;
+import io.druid.query.aggregation.MinAggregatorFactory;
+import io.druid.query.aggregation.PostAggregator;
+import io.druid.query.aggregation.post.ArithmeticPostAggregator;
+import io.druid.query.aggregation.post.ConstantPostAggregator;
+import io.druid.query.aggregation.post.FieldAccessPostAggregator;
+
+/**
+ */
+public class AggregatorsModule extends SimpleModule
+{
+  public AggregatorsModule()
+  {
+    super("AggregatorFactories");
+
+    setMixInAnnotation(AggregatorFactory.class, AggregatorFactoryMixin.class);
+    setMixInAnnotation(PostAggregator.class, PostAggregatorMixin.class);
+  }
+
+  @JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property="type")
+  @JsonSubTypes(value={
+      @JsonSubTypes.Type(name="count", value=CountAggregatorFactory.class),
+      @JsonSubTypes.Type(name="longSum", value=LongSumAggregatorFactory.class),
+      @JsonSubTypes.Type(name="doubleSum", value=DoubleSumAggregatorFactory.class),
+      @JsonSubTypes.Type(name="max", value=MaxAggregatorFactory.class),
+      @JsonSubTypes.Type(name="min", value=MinAggregatorFactory.class),
+      @JsonSubTypes.Type(name="javascript", value=JavaScriptAggregatorFactory.class),
+      @JsonSubTypes.Type(name="histogram", value=HistogramAggregatorFactory.class)
+  })
+  public static interface AggregatorFactoryMixin {}
+
+  @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
+  @JsonSubTypes(value = {
+      @JsonSubTypes.Type(name = "arithmetic", value = ArithmeticPostAggregator.class),
+      @JsonSubTypes.Type(name = "fieldAccess", value = FieldAccessPostAggregator.class),
+      @JsonSubTypes.Type(name = "constant", value = ConstantPostAggregator.class)
+  })
+  public static interface PostAggregatorMixin {}
+}
diff --git a/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java b/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java
new file mode 100644
index 00000000000..952b506bd95
--- /dev/null
+++ b/processing/src/main/java/io/druid/jackson/DefaultObjectMapper.java
@@ -0,0 +1,55 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.jackson;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.MapperFeature;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.guava.GuavaModule;
+
+/**
+ */
+public class DefaultObjectMapper extends ObjectMapper
+{
+  public DefaultObjectMapper()
+  {
+    this(null);
+  }
+
+  public DefaultObjectMapper(JsonFactory factory)
+  {
+    super(factory);
+    registerModule(new DruidDefaultSerializersModule());
+    registerModule(new GuavaModule());
+    registerModule(new QueryGranularityModule());
+    registerModule(new AggregatorsModule());
+    registerModule(new SegmentsModule());
+
+    configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    configure(MapperFeature.AUTO_DETECT_GETTERS, false);
+//    configure(MapperFeature.AUTO_DETECT_CREATORS, false); https://github.com/FasterXML/jackson-databind/issues/170
+    configure(MapperFeature.AUTO_DETECT_FIELDS, false);
+    configure(MapperFeature.AUTO_DETECT_IS_GETTERS, false);
+    configure(MapperFeature.AUTO_DETECT_SETTERS, false);
+    configure(SerializationFeature.INDENT_OUTPUT, false);
+  }
+}
diff --git a/common/src/main/java/com/metamx/druid/jackson/DefaultObjectMapper.java b/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java
similarity index 72%
rename from common/src/main/java/com/metamx/druid/jackson/DefaultObjectMapper.java
rename to processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java
index c8e7ab0ab40..38c3bc135ef 100644
--- a/common/src/main/java/com/metamx/druid/jackson/DefaultObjectMapper.java
+++ b/processing/src/main/java/io/druid/jackson/DruidDefaultSerializersModule.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,29 +17,21 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.jackson;
+package io.druid.jackson;
 
-import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.Version;
 import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonDeserializer;
 import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.MapperFeature;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.ser.std.ToStringSerializer;
-import com.fasterxml.jackson.datatype.guava.GuavaModule;
 import com.google.common.base.Throwables;
 import com.metamx.common.Granularity;
 import com.metamx.common.guava.Accumulator;
 import com.metamx.common.guava.Sequence;
-
 import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
@@ -48,19 +40,14 @@ import java.util.TimeZone;
 
 /**
  */
-public class DefaultObjectMapper extends ObjectMapper
+public class DruidDefaultSerializersModule extends SimpleModule
 {
-  public DefaultObjectMapper()
+  public DruidDefaultSerializersModule()
   {
-    this(null);
-  }
+    super("Druid default serializers");
 
-  public DefaultObjectMapper(JsonFactory factory)
-  {
-    super(factory);
-    SimpleModule serializerModule = new SimpleModule("Druid default serializers", new Version(1, 0, 0, null));
-    JodaStuff.register(serializerModule);
-    serializerModule.addDeserializer(
+    JodaStuff.register(this);
+    addDeserializer(
         Granularity.class,
         new JsonDeserializer()
         {
@@ -72,7 +59,7 @@ public class DefaultObjectMapper extends ObjectMapper
           }
         }
     );
-    serializerModule.addDeserializer(
+    addDeserializer(
         DateTimeZone.class,
         new JsonDeserializer()
         {
@@ -90,7 +77,7 @@ public class DefaultObjectMapper extends ObjectMapper
           }
         }
     );
-    serializerModule.addSerializer(
+    addSerializer(
         DateTimeZone.class,
         new JsonSerializer()
         {
@@ -106,7 +93,7 @@ public class DefaultObjectMapper extends ObjectMapper
           }
         }
     );
-    serializerModule.addSerializer(
+    addSerializer(
         Sequence.class,
         new JsonSerializer()
         {
@@ -136,8 +123,8 @@ public class DefaultObjectMapper extends ObjectMapper
           }
         }
     );
-    serializerModule.addSerializer(ByteOrder.class, ToStringSerializer.instance);
-    serializerModule.addDeserializer(
+    addSerializer(ByteOrder.class, ToStringSerializer.instance);
+    addDeserializer(
         ByteOrder.class,
         new JsonDeserializer()
         {
@@ -153,15 +140,5 @@ public class DefaultObjectMapper extends ObjectMapper
           }
         }
     );
-    registerModule(serializerModule);
-    registerModule(new GuavaModule());
-
-    configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-    configure(MapperFeature.AUTO_DETECT_GETTERS, false);
-//    configure(MapperFeature.AUTO_DETECT_CREATORS, false); https://github.com/FasterXML/jackson-databind/issues/170
-    configure(MapperFeature.AUTO_DETECT_FIELDS, false);
-    configure(MapperFeature.AUTO_DETECT_IS_GETTERS, false);
-    configure(MapperFeature.AUTO_DETECT_SETTERS, false);
-    configure(SerializationFeature.INDENT_OUTPUT, false);
   }
 }
diff --git a/processing/src/main/java/io/druid/jackson/JacksonModule.java b/processing/src/main/java/io/druid/jackson/JacksonModule.java
new file mode 100644
index 00000000000..7f6162e7927
--- /dev/null
+++ b/processing/src/main/java/io/druid/jackson/JacksonModule.java
@@ -0,0 +1,55 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.jackson;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+import com.google.inject.Binder;
+import com.google.inject.Key;
+import com.google.inject.Module;
+import com.google.inject.Provides;
+import io.druid.guice.LazySingleton;
+import io.druid.guice.annotations.Json;
+import io.druid.guice.annotations.Smile;
+
+/**
+ */
+public class JacksonModule implements Module
+{
+  @Override
+  public void configure(Binder binder)
+  {
+    binder.bind(ObjectMapper.class).to(Key.get(ObjectMapper.class, Json.class));
+  }
+
+  @Provides @LazySingleton @Json
+  public ObjectMapper jsonMapper()
+  {
+    return new DefaultObjectMapper();
+  }
+
+  @Provides @LazySingleton @Smile
+  public ObjectMapper smileMapper()
+  {
+    ObjectMapper retVal = new DefaultObjectMapper(new SmileFactory());
+    retVal.getJsonFactory().setCodec(retVal);
+    return retVal;
+  }
+}
diff --git a/common/src/main/java/com/metamx/druid/jackson/JodaStuff.java b/processing/src/main/java/io/druid/jackson/JodaStuff.java
similarity index 92%
rename from common/src/main/java/com/metamx/druid/jackson/JodaStuff.java
rename to processing/src/main/java/io/druid/jackson/JodaStuff.java
index 13a5243be1e..30b57d32039 100644
--- a/common/src/main/java/com/metamx/druid/jackson/JodaStuff.java
+++ b/processing/src/main/java/io/druid/jackson/JodaStuff.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.jackson;
+package io.druid.jackson;
 
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
@@ -39,9 +39,9 @@ import java.io.IOException;
 
 /**
  */
-public class JodaStuff
+class JodaStuff
 {
-  public static SimpleModule register(SimpleModule module)
+  static SimpleModule register(SimpleModule module)
   {
     module.addKeyDeserializer(DateTime.class, new DateTimeKeyDeserializer());
     module.addDeserializer(DateTime.class, new DateTimeDeserializer());
@@ -58,7 +58,7 @@ public class JodaStuff
 
   /**
    */
-  public static class IntervalDeserializer extends StdDeserializer
+  private static class IntervalDeserializer extends StdDeserializer
   {
     public IntervalDeserializer()
     {
@@ -82,7 +82,7 @@ public class JodaStuff
     }
   }
 
-  public static class DateTimeDeserializer extends StdDeserializer
+  private static class DateTimeDeserializer extends StdDeserializer
   {
       public DateTimeDeserializer() {
         super(DateTime.class);
@@ -109,5 +109,4 @@ public class JodaStuff
           throw ctxt.mappingException(getValueClass());
       }
   }
-
 }
diff --git a/processing/src/main/java/io/druid/jackson/QueryGranularityModule.java b/processing/src/main/java/io/druid/jackson/QueryGranularityModule.java
new file mode 100644
index 00000000000..4417e6e268f
--- /dev/null
+++ b/processing/src/main/java/io/druid/jackson/QueryGranularityModule.java
@@ -0,0 +1,50 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.jackson;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import io.druid.granularity.AllGranularity;
+import io.druid.granularity.DurationGranularity;
+import io.druid.granularity.NoneGranularity;
+import io.druid.granularity.PeriodGranularity;
+import io.druid.granularity.QueryGranularity;
+
+/**
+ */
+public class QueryGranularityModule extends SimpleModule
+{
+  public QueryGranularityModule()
+  {
+    super("QueryGranularityModule");
+
+    setMixInAnnotation(QueryGranularity.class, QueryGranularityMixin.class);
+    registerSubtypes(
+        new NamedType(PeriodGranularity.class, "period"),
+        new NamedType(DurationGranularity.class, "duration"),
+        new NamedType(AllGranularity.class, "all"),
+        new NamedType(NoneGranularity.class, "none")
+    );
+  }
+
+  @JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property = "type", defaultImpl = QueryGranularity.class)
+  public static interface QueryGranularityMixin {}
+}
diff --git a/processing/src/main/java/io/druid/jackson/SegmentsModule.java b/processing/src/main/java/io/druid/jackson/SegmentsModule.java
new file mode 100644
index 00000000000..45e869d8531
--- /dev/null
+++ b/processing/src/main/java/io/druid/jackson/SegmentsModule.java
@@ -0,0 +1,45 @@
+/*
+ * Druid - a distributed column store.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+package io.druid.jackson;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import io.druid.timeline.partition.NoneShardSpec;
+import io.druid.timeline.partition.ShardSpec;
+
+/**
+ */
+public class SegmentsModule extends SimpleModule
+{
+  public SegmentsModule()
+  {
+    super("SegmentsModule");
+
+    setMixInAnnotation(ShardSpec.class, ShardSpecMixin.class);
+  }
+
+  @JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property="type")
+  @JsonSubTypes({
+      @JsonSubTypes.Type(name="none", value=NoneShardSpec.class),
+  })
+  public static interface ShardSpecMixin
+  {}
+}
diff --git a/client/src/main/java/com/metamx/druid/BaseQuery.java b/processing/src/main/java/io/druid/query/BaseQuery.java
similarity index 93%
rename from client/src/main/java/com/metamx/druid/BaseQuery.java
rename to processing/src/main/java/io/druid/query/BaseQuery.java
index 9f3ae8d30ed..b1e1a587123 100644
--- a/client/src/main/java/com/metamx/druid/BaseQuery.java
+++ b/processing/src/main/java/io/druid/query/BaseQuery.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,16 +17,13 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.query;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import com.metamx.common.guava.Sequence;
-import com.metamx.druid.query.QueryRunner;
-import com.metamx.druid.query.segment.QuerySegmentSpec;
-import com.metamx.druid.query.segment.QuerySegmentWalker;
-
+import io.druid.query.spec.QuerySegmentSpec;
 import org.joda.time.Duration;
 import org.joda.time.Interval;
 
@@ -55,7 +52,6 @@ public abstract class BaseQuery implements Query
     this.dataSource = dataSource.toLowerCase();
     this.context = context;
     this.querySegmentSpec = querySegmentSpec;
-
   }
 
   @JsonProperty
diff --git a/client/src/main/java/com/metamx/druid/query/BySegmentQueryRunner.java b/processing/src/main/java/io/druid/query/BySegmentQueryRunner.java
similarity index 93%
rename from client/src/main/java/com/metamx/druid/query/BySegmentQueryRunner.java
rename to processing/src/main/java/io/druid/query/BySegmentQueryRunner.java
index 4713fc8207a..79c6a4e03a9 100644
--- a/client/src/main/java/com/metamx/druid/query/BySegmentQueryRunner.java
+++ b/processing/src/main/java/io/druid/query/BySegmentQueryRunner.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
 import com.google.common.collect.Lists;
 import com.metamx.common.guava.Accumulator;
@@ -26,9 +26,6 @@ import com.metamx.common.guava.Sequences;
 import com.metamx.common.guava.Yielder;
 import com.metamx.common.guava.Yielders;
 import com.metamx.common.guava.YieldingAccumulator;
-import com.metamx.druid.Query;
-import com.metamx.druid.result.BySegmentResultValueClass;
-import com.metamx.druid.result.Result;
 import org.joda.time.DateTime;
 
 import java.util.List;
diff --git a/client/src/main/java/com/metamx/druid/result/BySegmentResultValue.java b/processing/src/main/java/io/druid/query/BySegmentResultValue.java
similarity index 92%
rename from client/src/main/java/com/metamx/druid/result/BySegmentResultValue.java
rename to processing/src/main/java/io/druid/query/BySegmentResultValue.java
index df92619fc4d..3d2dc0802f3 100644
--- a/client/src/main/java/com/metamx/druid/result/BySegmentResultValue.java
+++ b/processing/src/main/java/io/druid/query/BySegmentResultValue.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.result;
+package io.druid.query;
 
 import java.util.List;
 
diff --git a/client/src/main/java/com/metamx/druid/result/BySegmentResultValueClass.java b/processing/src/main/java/io/druid/query/BySegmentResultValueClass.java
similarity index 95%
rename from client/src/main/java/com/metamx/druid/result/BySegmentResultValueClass.java
rename to processing/src/main/java/io/druid/query/BySegmentResultValueClass.java
index 9bc3ae81e2c..c26bfb35706 100644
--- a/client/src/main/java/com/metamx/druid/result/BySegmentResultValueClass.java
+++ b/processing/src/main/java/io/druid/query/BySegmentResultValueClass.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.result;
+package io.druid.query;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.joda.time.Interval;
diff --git a/client/src/main/java/com/metamx/druid/query/BySegmentSkippingQueryRunner.java b/processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java
similarity index 92%
rename from client/src/main/java/com/metamx/druid/query/BySegmentSkippingQueryRunner.java
rename to processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java
index 3bf8cc2a661..1711fb316f9 100644
--- a/client/src/main/java/com/metamx/druid/query/BySegmentSkippingQueryRunner.java
+++ b/processing/src/main/java/io/druid/query/BySegmentSkippingQueryRunner.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,10 +17,9 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
 import com.metamx.common.guava.Sequence;
-import com.metamx.druid.Query;
 
 /**
  */
diff --git a/client/src/main/java/com/metamx/druid/query/CacheStrategy.java b/processing/src/main/java/io/druid/query/CacheStrategy.java
similarity index 92%
rename from client/src/main/java/com/metamx/druid/query/CacheStrategy.java
rename to processing/src/main/java/io/druid/query/CacheStrategy.java
index 1e73ada3f3b..f77affcb5bf 100644
--- a/client/src/main/java/com/metamx/druid/query/CacheStrategy.java
+++ b/processing/src/main/java/io/druid/query/CacheStrategy.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,12 +17,11 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.google.common.base.Function;
 import com.metamx.common.guava.Sequence;
-import com.metamx.druid.Query;
 
 /**
 */
diff --git a/client/src/main/java/com/metamx/druid/query/ChainedExecutionQueryRunner.java b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java
similarity index 98%
rename from client/src/main/java/com/metamx/druid/query/ChainedExecutionQueryRunner.java
rename to processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java
index 8acc43a7585..8710b627ece 100644
--- a/client/src/main/java/com/metamx/druid/query/ChainedExecutionQueryRunner.java
+++ b/processing/src/main/java/io/druid/query/ChainedExecutionQueryRunner.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
 import com.google.common.base.Function;
 import com.google.common.base.Predicates;
@@ -30,7 +30,6 @@ import com.metamx.common.guava.MergeIterable;
 import com.metamx.common.guava.Sequence;
 import com.metamx.common.guava.Sequences;
 import com.metamx.common.logger.Logger;
-import com.metamx.druid.Query;
 
 import java.util.Arrays;
 import java.util.Iterator;
diff --git a/client/src/main/java/com/metamx/druid/query/ConcatQueryRunner.java b/processing/src/main/java/io/druid/query/ConcatQueryRunner.java
similarity index 93%
rename from client/src/main/java/com/metamx/druid/query/ConcatQueryRunner.java
rename to processing/src/main/java/io/druid/query/ConcatQueryRunner.java
index 0a19840384a..9735d0f5a94 100644
--- a/client/src/main/java/com/metamx/druid/query/ConcatQueryRunner.java
+++ b/processing/src/main/java/io/druid/query/ConcatQueryRunner.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,12 +17,11 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
 import com.google.common.base.Function;
 import com.metamx.common.guava.Sequence;
 import com.metamx.common.guava.Sequences;
-import com.metamx.druid.Query;
 
 /**
 */
diff --git a/server/src/main/java/com/metamx/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/io/druid/query/DefaultQueryRunnerFactoryConglomerate.java
similarity index 91%
rename from server/src/main/java/com/metamx/druid/query/DefaultQueryRunnerFactoryConglomerate.java
rename to processing/src/main/java/io/druid/query/DefaultQueryRunnerFactoryConglomerate.java
index d8db68446b0..63a092df0ea 100644
--- a/server/src/main/java/com/metamx/druid/query/DefaultQueryRunnerFactoryConglomerate.java
+++ b/processing/src/main/java/io/druid/query/DefaultQueryRunnerFactoryConglomerate.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,9 +17,9 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
-import com.metamx.druid.Query;
+import com.google.inject.Inject;
 
 import java.util.Map;
 
@@ -29,6 +29,7 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactory
 {
   private final Map, QueryRunnerFactory> factories;
 
+  @Inject
   public DefaultQueryRunnerFactoryConglomerate(
       Map, QueryRunnerFactory> factories
   )
diff --git a/client/src/main/java/com/metamx/druid/query/DelegatingExecutorService.java b/processing/src/main/java/io/druid/query/DelegatingExecutorService.java
similarity index 97%
rename from client/src/main/java/com/metamx/druid/query/DelegatingExecutorService.java
rename to processing/src/main/java/io/druid/query/DelegatingExecutorService.java
index f9d377694e8..5a6be3f854c 100644
--- a/client/src/main/java/com/metamx/druid/query/DelegatingExecutorService.java
+++ b/processing/src/main/java/io/druid/query/DelegatingExecutorService.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,7 +17,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid.query;
+package io.druid.query;
 
 import java.util.Collection;
 import java.util.List;
diff --git a/client/src/main/java/com/metamx/druid/Druids.java b/processing/src/main/java/io/druid/query/Druids.java
similarity index 93%
rename from client/src/main/java/com/metamx/druid/Druids.java
rename to processing/src/main/java/io/druid/query/Druids.java
index 03401211db7..99d5acdc6f6 100644
--- a/client/src/main/java/com/metamx/druid/Druids.java
+++ b/processing/src/main/java/io/druid/query/Druids.java
@@ -1,6 +1,6 @@
 /*
  * Druid - a distributed column store.
- * Copyright (C) 2012  Metamarkets Group Inc.
+ * Copyright (C) 2012, 2013  Metamarkets Group Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -17,27 +17,27 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  */
 
-package com.metamx.druid;
+package io.druid.query;
 
 import com.google.common.collect.Lists;
-import com.metamx.druid.aggregation.AggregatorFactory;
-import com.metamx.druid.aggregation.post.PostAggregator;
-import com.metamx.druid.query.filter.AndDimFilter;
-import com.metamx.druid.query.filter.DimFilter;
-import com.metamx.druid.query.filter.NoopDimFilter;
-import com.metamx.druid.query.filter.NotDimFilter;
-import com.metamx.druid.query.filter.OrDimFilter;
-import com.metamx.druid.query.filter.SelectorDimFilter;
-import com.metamx.druid.query.search.InsensitiveContainsSearchQuerySpec;
-import com.metamx.druid.query.search.SearchQuery;
-import com.metamx.druid.query.search.SearchQuerySpec;
-import com.metamx.druid.query.segment.LegacySegmentSpec;
-import com.metamx.druid.query.segment.QuerySegmentSpec;
-import com.metamx.druid.query.timeboundary.TimeBoundaryQuery;
-import com.metamx.druid.query.timeseries.TimeseriesQuery;
-import com.metamx.druid.result.Result;
-import com.metamx.druid.result.SearchResultValue;
-import com.metamx.druid.result.TimeBoundaryResultValue;
+import io.druid.granularity.QueryGranularity;
+import io.druid.query.aggregation.AggregatorFactory;
+import io.druid.query.aggregation.PostAggregator;
+import io.druid.query.filter.AndDimFilter;
+import io.druid.query.filter.DimFilter;
+import io.druid.query.filter.NoopDimFilter;
+import io.druid.query.filter.NotDimFilter;
+import io.druid.query.filter.OrDimFilter;
+import io.druid.query.filter.SelectorDimFilter;
+import io.druid.query.search.SearchResultValue;
+import io.druid.query.search.search.InsensitiveContainsSearchQuerySpec;
+import io.druid.query.search.search.SearchQuery;
+import io.druid.query.search.search.SearchQuerySpec;
+import io.druid.query.spec.LegacySegmentSpec;
+import io.druid.query.spec.QuerySegmentSpec;
+import io.druid.query.timeboundary.TimeBoundaryQuery;
+import io.druid.query.timeboundary.TimeBoundaryResultValue;
+import io.druid.query.timeseries.TimeseriesQuery;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
 
@@ -294,7 +294,7 @@ public class Druids
    *                                        .build();
    *
* - * @see com.metamx.druid.query.timeseries.TimeseriesQuery + * @see io.druid.query.timeseries.TimeseriesQuery */ public static class TimeseriesQueryBuilder { @@ -488,7 +488,7 @@ public class Druids * .build(); * * - * @see com.metamx.druid.query.search.SearchQuery + * @see io.druid.query.search.search.SearchQuery */ public static class SearchQueryBuilder { @@ -523,7 +523,7 @@ public class Druids querySegmentSpec, dimensions, querySpec, - querySpec.getSearchSortSpec(), + null, context ); } @@ -638,13 +638,13 @@ public class Druids public SearchQueryBuilder query(String q) { - querySpec = new InsensitiveContainsSearchQuerySpec(q, null); + querySpec = new InsensitiveContainsSearchQuerySpec(q); return this; } public SearchQueryBuilder query(Map q) { - querySpec = new InsensitiveContainsSearchQuerySpec((String) q.get("value"), null); + querySpec = new InsensitiveContainsSearchQuerySpec((String) q.get("value")); return this; } @@ -672,7 +672,7 @@ public class Druids * .build(); * * - * @see com.metamx.druid.query.timeboundary.TimeBoundaryQuery + * @see io.druid.query.timeboundary.TimeBoundaryQuery */ public static class TimeBoundaryQueryBuilder { @@ -753,7 +753,7 @@ public class Druids * .build(); * * - * @see com.metamx.druid.result.Result + * @see Result */ public static class ResultBuilder { diff --git a/client/src/main/java/com/metamx/druid/query/FinalizeResultsQueryRunner.java b/processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java similarity index 92% rename from client/src/main/java/com/metamx/druid/query/FinalizeResultsQueryRunner.java rename to processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java index e54dd9d172a..77823a60730 100644 --- a/client/src/main/java/com/metamx/druid/query/FinalizeResultsQueryRunner.java +++ b/processing/src/main/java/io/druid/query/FinalizeResultsQueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Query; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.result.BySegmentResultValueClass; -import com.metamx.druid.result.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; import javax.annotation.Nullable; diff --git a/client/src/main/java/com/metamx/druid/query/IntervalChunkingQueryRunner.java b/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/IntervalChunkingQueryRunner.java rename to processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java index 9c0a32eac3f..53e889306f5 100644 --- a/client/src/main/java/com/metamx/druid/query/IntervalChunkingQueryRunner.java +++ b/processing/src/main/java/io/druid/query/IntervalChunkingQueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.base.Function; import com.google.common.collect.Lists; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.PeriodGranularity; -import com.metamx.druid.Query; -import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; +import io.druid.granularity.PeriodGranularity; +import io.druid.query.spec.MultipleIntervalSegmentSpec; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/DimensionColumn.java b/processing/src/main/java/io/druid/query/MapQueryToolChestWarehouse.java similarity index 56% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/DimensionColumn.java rename to processing/src/main/java/io/druid/query/MapQueryToolChestWarehouse.java index f2975600d23..c39af79654d 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/DimensionColumn.java +++ b/processing/src/main/java/io/druid/query/MapQueryToolChestWarehouse.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,36 +17,30 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.query; + +import com.google.inject.Inject; + +import java.util.Map; /** */ -public class DimensionColumn +public class MapQueryToolChestWarehouse implements QueryToolChestWarehouse { - final int[][] dimensionLookups; - final int[] dimensionValues; + private final Map, QueryToolChest> toolchests; - public DimensionColumn( - int[][] dimensionLookups, - int[] dimensionValues + @Inject + public MapQueryToolChestWarehouse( + Map, QueryToolChest> toolchests ) { - this.dimensionLookups = dimensionLookups; - this.dimensionValues = dimensionValues; + this.toolchests = toolchests; } - public int[] getDimValues(int rowId) + @Override + @SuppressWarnings("unchecked") + public > QueryToolChest getToolChest(QueryType query) { - return dimensionLookups[dimensionValues[rowId]]; - } - - public int[][] getDimensionExpansions() - { - return dimensionLookups; - } - - public int[] getDimensionRowValues() - { - return dimensionValues; + return toolchests.get(query.getClass()); } } diff --git a/client/src/main/java/com/metamx/druid/result/MetricValueExtractor.java b/processing/src/main/java/io/druid/query/MetricValueExtractor.java similarity index 89% rename from client/src/main/java/com/metamx/druid/result/MetricValueExtractor.java rename to processing/src/main/java/io/druid/query/MetricValueExtractor.java index f330a12400d..a364200f9d9 100644 --- a/client/src/main/java/com/metamx/druid/result/MetricValueExtractor.java +++ b/processing/src/main/java/io/druid/query/MetricValueExtractor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.result; +package io.druid.query; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; -import java.lang.Double;import java.lang.Float;import java.lang.Long;import java.lang.Number;import java.lang.Object;import java.lang.Override;import java.lang.String;import java.util.Map; +import java.util.Map; /** */ diff --git a/client/src/main/java/com/metamx/druid/query/MetricsEmittingExecutorService.java b/processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java similarity index 80% rename from client/src/main/java/com/metamx/druid/query/MetricsEmittingExecutorService.java rename to processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java index f109d2f783a..4893ebd8adf 100644 --- a/client/src/main/java/com/metamx/druid/query/MetricsEmittingExecutorService.java +++ b/processing/src/main/java/io/druid/query/MetricsEmittingExecutorService.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,22 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.AbstractExecutorService; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.RunnableFuture; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; public class MetricsEmittingExecutorService extends DelegatingExecutorService { diff --git a/client/src/main/java/com/metamx/druid/query/MetricsEmittingQueryRunner.java b/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java similarity index 97% rename from client/src/main/java/com/metamx/druid/query/MetricsEmittingQueryRunner.java rename to processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java index 9a7111526de..9da3101003c 100644 --- a/client/src/main/java/com/metamx/druid/query/MetricsEmittingQueryRunner.java +++ b/processing/src/main/java/io/druid/query/MetricsEmittingQueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.base.Function; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Yielder; import com.metamx.common.guava.YieldingAccumulator; -import com.metamx.druid.Query; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; diff --git a/client/src/main/java/com/metamx/druid/query/NoopQueryRunner.java b/processing/src/main/java/io/druid/query/NoopQueryRunner.java similarity index 90% rename from client/src/main/java/com/metamx/druid/query/NoopQueryRunner.java rename to processing/src/main/java/io/druid/query/NoopQueryRunner.java index 57b2f27e46c..355ee4f7a20 100644 --- a/client/src/main/java/com/metamx/druid/query/NoopQueryRunner.java +++ b/processing/src/main/java/io/druid/query/NoopQueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Query; /** */ diff --git a/client/src/main/java/com/metamx/druid/query/PrioritizedCallable.java b/processing/src/main/java/io/druid/query/PrioritizedCallable.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/PrioritizedCallable.java rename to processing/src/main/java/io/druid/query/PrioritizedCallable.java index 3771b8a03c2..e664a8d682b 100644 --- a/client/src/main/java/com/metamx/druid/query/PrioritizedCallable.java +++ b/processing/src/main/java/io/druid/query/PrioritizedCallable.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import java.util.concurrent.Callable; diff --git a/client/src/main/java/com/metamx/druid/query/PrioritizedExecutorService.java b/processing/src/main/java/io/druid/query/PrioritizedExecutorService.java similarity index 97% rename from client/src/main/java/com/metamx/druid/query/PrioritizedExecutorService.java rename to processing/src/main/java/io/druid/query/PrioritizedExecutorService.java index f943a0c112f..f390fe1b7a4 100644 --- a/client/src/main/java/com/metamx/druid/query/PrioritizedExecutorService.java +++ b/processing/src/main/java/io/druid/query/PrioritizedExecutorService.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.primitives.Ints; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -27,7 +27,6 @@ import com.metamx.common.lifecycle.Lifecycle; import java.util.List; import java.util.concurrent.AbstractExecutorService; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; import java.util.concurrent.PriorityBlockingQueue; import java.util.concurrent.RunnableFuture; diff --git a/client/src/main/java/com/metamx/druid/query/Queries.java b/processing/src/main/java/io/druid/query/Queries.java similarity index 92% rename from client/src/main/java/com/metamx/druid/query/Queries.java rename to processing/src/main/java/io/druid/query/Queries.java index 0ebc18e64fd..3fd1f802189 100644 --- a/client/src/main/java/com/metamx/druid/query/Queries.java +++ b/processing/src/main/java/io/druid/query/Queries.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; import javax.annotation.Nullable; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/Query.java b/processing/src/main/java/io/druid/query/Query.java similarity index 82% rename from client/src/main/java/com/metamx/druid/Query.java rename to processing/src/main/java/io/druid/query/Query.java index 02b81bc6586..daf1410308e 100644 --- a/client/src/main/java/com/metamx/druid/Query.java +++ b/processing/src/main/java/io/druid/query/Query.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.metamx.common.guava.Sequence; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.group.GroupByQuery; -import com.metamx.druid.query.metadata.SegmentMetadataQuery; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; -import com.metamx.druid.query.timeseries.TimeseriesQuery; - +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.spec.QuerySegmentSpec; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.query.timeseries.TimeseriesQuery; import org.joda.time.Duration; import org.joda.time.Interval; diff --git a/client/src/main/java/com/metamx/druid/query/QueryCacheHelper.java b/processing/src/main/java/io/druid/query/QueryCacheHelper.java similarity index 91% rename from client/src/main/java/com/metamx/druid/query/QueryCacheHelper.java rename to processing/src/main/java/io/druid/query/QueryCacheHelper.java index dd293fbdf5c..d61a5778d03 100644 --- a/client/src/main/java/com/metamx/druid/query/QueryCacheHelper.java +++ b/processing/src/main/java/io/druid/query/QueryCacheHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.collect.Lists; -import com.metamx.druid.aggregation.AggregatorFactory; +import io.druid.query.aggregation.AggregatorFactory; import java.nio.ByteBuffer; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/QueryRunner.java b/processing/src/main/java/io/druid/query/QueryRunner.java similarity index 89% rename from client/src/main/java/com/metamx/druid/query/QueryRunner.java rename to processing/src/main/java/io/druid/query/QueryRunner.java index 5dd3cb9db15..62c44ad5163 100644 --- a/client/src/main/java/com/metamx/druid/query/QueryRunner.java +++ b/processing/src/main/java/io/druid/query/QueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.metamx.common.guava.Sequence; -import com.metamx.druid.Query; /** */ diff --git a/server/src/main/java/com/metamx/druid/query/QueryRunnerFactory.java b/processing/src/main/java/io/druid/query/QueryRunnerFactory.java similarity index 88% rename from server/src/main/java/com/metamx/druid/query/QueryRunnerFactory.java rename to processing/src/main/java/io/druid/query/QueryRunnerFactory.java index 8dade405e49..3cbe11d6241 100644 --- a/server/src/main/java/com/metamx/druid/query/QueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/QueryRunnerFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; -import com.metamx.druid.Query; -import com.metamx.druid.index.Segment; +import io.druid.segment.Segment; import java.util.concurrent.ExecutorService; diff --git a/server/src/main/java/com/metamx/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/io/druid/query/QueryRunnerFactoryConglomerate.java similarity index 89% rename from server/src/main/java/com/metamx/druid/query/QueryRunnerFactoryConglomerate.java rename to processing/src/main/java/io/druid/query/QueryRunnerFactoryConglomerate.java index f1e7621fd77..1e8b61d812a 100644 --- a/server/src/main/java/com/metamx/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/io/druid/query/QueryRunnerFactoryConglomerate.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; - -import com.metamx.druid.Query; +package io.druid.query; /** */ diff --git a/server/src/main/java/com/metamx/druid/query/QueryRunnerHelper.java b/processing/src/main/java/io/druid/query/QueryRunnerHelper.java similarity index 84% rename from server/src/main/java/com/metamx/druid/query/QueryRunnerHelper.java rename to processing/src/main/java/io/druid/query/QueryRunnerHelper.java index 411542ffe4d..533ccca6039 100644 --- a/server/src/main/java/com/metamx/druid/query/QueryRunnerHelper.java +++ b/processing/src/main/java/io/druid/query/QueryRunnerHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,18 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.logger.Logger; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.aggregation.Aggregator; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.result.Result; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.filter.Filter; +import io.druid.segment.Cursor; +import io.druid.segment.StorageAdapter; import org.joda.time.Interval; import javax.annotation.Nullable; diff --git a/client/src/main/java/com/metamx/druid/query/segment/QuerySegmentWalker.java b/processing/src/main/java/io/druid/query/QuerySegmentWalker.java similarity index 90% rename from client/src/main/java/com/metamx/druid/query/segment/QuerySegmentWalker.java rename to processing/src/main/java/io/druid/query/QuerySegmentWalker.java index efb7667a370..8e8c6b8c0fe 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/QuerySegmentWalker.java +++ b/processing/src/main/java/io/druid/query/QuerySegmentWalker.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query; -import com.metamx.druid.Query; -import com.metamx.druid.query.QueryRunner; import org.joda.time.Interval; /** diff --git a/client/src/main/java/com/metamx/druid/query/QueryToolChest.java b/processing/src/main/java/io/druid/query/QueryToolChest.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/QueryToolChest.java rename to processing/src/main/java/io/druid/query/QueryToolChest.java index e2d227729d3..304d3e1eb14 100644 --- a/client/src/main/java/com/metamx/druid/query/QueryToolChest.java +++ b/processing/src/main/java/io/druid/query/QueryToolChest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.metamx.common.guava.Sequence; -import com.metamx.druid.LogicalSegment; -import com.metamx.druid.Query; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.timeline.LogicalSegment; + import java.util.List; /** diff --git a/client/src/main/java/com/metamx/druid/query/QueryToolChestWarehouse.java b/processing/src/main/java/io/druid/query/QueryToolChestWarehouse.java similarity index 89% rename from client/src/main/java/com/metamx/druid/query/QueryToolChestWarehouse.java rename to processing/src/main/java/io/druid/query/QueryToolChestWarehouse.java index 0eec4114eee..c501eec5885 100644 --- a/client/src/main/java/com/metamx/druid/query/QueryToolChestWarehouse.java +++ b/processing/src/main/java/io/druid/query/QueryToolChestWarehouse.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; - -import com.metamx.druid.Query; +package io.druid.query; /** */ diff --git a/server/src/main/java/com/metamx/druid/query/ReferenceCountingSegmentQueryRunner.java b/processing/src/main/java/io/druid/query/ReferenceCountingSegmentQueryRunner.java similarity index 51% rename from server/src/main/java/com/metamx/druid/query/ReferenceCountingSegmentQueryRunner.java rename to processing/src/main/java/io/druid/query/ReferenceCountingSegmentQueryRunner.java index 8b2879dd493..c366c287ac5 100644 --- a/server/src/main/java/com/metamx/druid/query/ReferenceCountingSegmentQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ReferenceCountingSegmentQueryRunner.java @@ -1,10 +1,28 @@ -package com.metamx.druid.query; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query; import com.google.common.io.Closeables; import com.metamx.common.guava.ResourceClosingSequence; import com.metamx.common.guava.Sequence; -import com.metamx.druid.Query; -import com.metamx.druid.index.ReferenceCountingSegment; +import io.druid.segment.ReferenceCountingSegment; import java.io.Closeable; diff --git a/client/src/main/java/com/metamx/druid/query/ReflectionLoaderThingy.java b/processing/src/main/java/io/druid/query/ReflectionLoaderThingy.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/ReflectionLoaderThingy.java rename to processing/src/main/java/io/druid/query/ReflectionLoaderThingy.java index cd9eba24f25..75636554217 100644 --- a/client/src/main/java/com/metamx/druid/query/ReflectionLoaderThingy.java +++ b/processing/src/main/java/io/druid/query/ReflectionLoaderThingy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.collect.Maps; import com.metamx.common.logger.Logger; diff --git a/client/src/main/java/com/metamx/druid/query/ReflectionQueryToolChestWarehouse.java b/processing/src/main/java/io/druid/query/ReflectionQueryToolChestWarehouse.java similarity index 91% rename from client/src/main/java/com/metamx/druid/query/ReflectionQueryToolChestWarehouse.java rename to processing/src/main/java/io/druid/query/ReflectionQueryToolChestWarehouse.java index 89f193bbf52..37ad0e979c6 100644 --- a/client/src/main/java/com/metamx/druid/query/ReflectionQueryToolChestWarehouse.java +++ b/processing/src/main/java/io/druid/query/ReflectionQueryToolChestWarehouse.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; - -import com.metamx.druid.Query; +package io.druid.query; /** */ diff --git a/client/src/main/java/com/metamx/druid/result/Result.java b/processing/src/main/java/io/druid/query/Result.java similarity index 96% rename from client/src/main/java/com/metamx/druid/result/Result.java rename to processing/src/main/java/io/druid/query/Result.java index e807ce1ddfa..bb299d51c78 100644 --- a/client/src/main/java/com/metamx/druid/result/Result.java +++ b/processing/src/main/java/io/druid/query/Result.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.result; +package io.druid.query; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/ResultGranularTimestampComparator.java b/processing/src/main/java/io/druid/query/ResultGranularTimestampComparator.java similarity index 91% rename from client/src/main/java/com/metamx/druid/ResultGranularTimestampComparator.java rename to processing/src/main/java/io/druid/query/ResultGranularTimestampComparator.java index 972d98b3214..a7d5c509346 100644 --- a/client/src/main/java/com/metamx/druid/ResultGranularTimestampComparator.java +++ b/processing/src/main/java/io/druid/query/ResultGranularTimestampComparator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query; import com.google.common.primitives.Longs; -import com.metamx.druid.result.Result; +import io.druid.granularity.QueryGranularity; import java.util.Comparator; diff --git a/client/src/main/java/com/metamx/druid/query/ResultMergeQueryRunner.java b/processing/src/main/java/io/druid/query/ResultMergeQueryRunner.java similarity index 90% rename from client/src/main/java/com/metamx/druid/query/ResultMergeQueryRunner.java rename to processing/src/main/java/io/druid/query/ResultMergeQueryRunner.java index 77d9eeda07d..94b670444e8 100644 --- a/client/src/main/java/com/metamx/druid/query/ResultMergeQueryRunner.java +++ b/processing/src/main/java/io/druid/query/ResultMergeQueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.collect.Ordering; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.Query; -import com.metamx.druid.guava.CombiningSequence; +import io.druid.common.guava.CombiningSequence; /** */ diff --git a/common/src/main/java/com/metamx/druid/aggregation/Aggregator.java b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java similarity index 88% rename from common/src/main/java/com/metamx/druid/aggregation/Aggregator.java rename to processing/src/main/java/io/druid/query/aggregation/Aggregator.java index fd42549971e..a48feb65e55 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/Aggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/Aggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; /** * An Aggregator is an object that can aggregate metrics. Its aggregation-related methods (namely, aggregate() and get()) @@ -26,7 +26,7 @@ package com.metamx.druid.aggregation; * * Thus, an Aggregator can be thought of as a closure over some other thing that is stateful and changes between calls * to aggregate(). This is currently (as of this documentation) implemented through the use of Offset and - * FloatMetricSelector objects. The Aggregator has a handle on a FloatMetricSelector object which has a handle on an Offset. + * FloatColumnSelector objects. The Aggregator has a handle on a FloatColumnSelector object which has a handle on an Offset. * QueryableIndex has both the Aggregators and the Offset object and iterates through the Offset calling the aggregate() * method on the Aggregators for each applicable row. * diff --git a/common/src/main/java/com/metamx/druid/aggregation/AggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java similarity index 82% rename from common/src/main/java/com/metamx/druid/aggregation/AggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java index 400217d0bc4..87077e45b31 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/AggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/AggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.util.Comparator; import java.util.List; @@ -36,16 +34,6 @@ import java.util.List; * provided to the Aggregator through the MetricSelector object, so whatever creates that object gets to choose how * the data is actually stored and accessed. */ -@JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property="type") -@JsonSubTypes(value={ - @JsonSubTypes.Type(name="count", value=CountAggregatorFactory.class), - @JsonSubTypes.Type(name="longSum", value=LongSumAggregatorFactory.class), - @JsonSubTypes.Type(name="doubleSum", value=DoubleSumAggregatorFactory.class), - @JsonSubTypes.Type(name="max", value=MaxAggregatorFactory.class), - @JsonSubTypes.Type(name="min", value=MinAggregatorFactory.class), - @JsonSubTypes.Type(name="javascript", value=JavaScriptAggregatorFactory.class), - @JsonSubTypes.Type(name="histogram", value=HistogramAggregatorFactory.class) -}) public interface AggregatorFactory { public Aggregator factorize(ColumnSelectorFactory metricFactory); diff --git a/common/src/main/java/com/metamx/druid/aggregation/BufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java similarity index 95% rename from common/src/main/java/com/metamx/druid/aggregation/BufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java index 6a010f7a3db..da581ae3064 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/BufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/BufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import java.nio.ByteBuffer; @@ -80,7 +80,7 @@ public interface BufferAggregator * Implementations must not change the position, limit or mark of the given buffer * * Implementations are only required to support this method if they are aggregations which - * have an {@link com.metamx.druid.aggregation.AggregatorFactory#getTypeName()} of "float". + * have an {@link AggregatorFactory#getTypeName()} of "float". * If unimplemented, throwing an {@link UnsupportedOperationException} is common and recommended. * * @param buf byte buffer storing the byte array representation of the aggregate diff --git a/common/src/main/java/com/metamx/druid/aggregation/CountAggregator.java b/processing/src/main/java/io/druid/query/aggregation/CountAggregator.java similarity index 94% rename from common/src/main/java/com/metamx/druid/aggregation/CountAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/CountAggregator.java index e806aa78e1c..ca934ffad05 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/CountAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/CountAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import java.util.Comparator; diff --git a/common/src/main/java/com/metamx/druid/aggregation/CountAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java similarity index 95% rename from common/src/main/java/com/metamx/druid/aggregation/CountAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java index 55900394860..1b02f923996 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/CountAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/CountAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.primitives.Longs; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.util.Comparator; import java.util.List; diff --git a/common/src/main/java/com/metamx/druid/aggregation/CountBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java similarity index 94% rename from common/src/main/java/com/metamx/druid/aggregation/CountBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java index 0e6f06c786f..b4179f475f0 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/CountBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/CountBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import java.nio.ByteBuffer; diff --git a/common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java similarity index 88% rename from common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java index 278cd686a3b..b9dcd56dbac 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.collect.Ordering; import com.google.common.primitives.Doubles; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.util.Comparator; @@ -43,12 +43,12 @@ public class DoubleSumAggregator implements Aggregator return ((Number) lhs).doubleValue() + ((Number) rhs).doubleValue(); } - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; private final String name; private double sum; - public DoubleSumAggregator(String name, FloatMetricSelector selector) + public DoubleSumAggregator(String name, FloatColumnSelector selector) { this.name = name; this.selector = selector; diff --git a/common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java similarity index 93% rename from common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java index ec89a79f39d..f85d0c677f5 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/DoubleSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; import java.util.Arrays; @@ -57,14 +57,14 @@ public class DoubleSumAggregatorFactory implements AggregatorFactory { return new DoubleSumAggregator( name, - metricFactory.makeFloatMetricSelector(fieldName) + metricFactory.makeFloatColumnSelector(fieldName) ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new DoubleSumBufferAggregator(metricFactory.makeFloatMetricSelector(fieldName)); + return new DoubleSumBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override diff --git a/common/src/main/java/com/metamx/druid/aggregation/DoubleSumBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java similarity index 87% rename from common/src/main/java/com/metamx/druid/aggregation/DoubleSumBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java index 62f4e75f553..7f6e449396d 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/DoubleSumBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/DoubleSumBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; @@ -27,10 +27,10 @@ import java.nio.ByteBuffer; */ public class DoubleSumBufferAggregator implements BufferAggregator { - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; public DoubleSumBufferAggregator( - FloatMetricSelector selector + FloatColumnSelector selector ) { this.selector = selector; diff --git a/common/src/main/java/com/metamx/druid/aggregation/Histogram.java b/processing/src/main/java/io/druid/query/aggregation/Histogram.java similarity index 98% rename from common/src/main/java/com/metamx/druid/aggregation/Histogram.java rename to processing/src/main/java/io/druid/query/aggregation/Histogram.java index 5c2c0f5f069..b46c02bdc7d 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/Histogram.java +++ b/processing/src/main/java/io/druid/query/aggregation/Histogram.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.base.Preconditions; diff --git a/common/src/main/java/com/metamx/druid/aggregation/HistogramAggregator.java b/processing/src/main/java/io/druid/query/aggregation/HistogramAggregator.java similarity index 88% rename from common/src/main/java/com/metamx/druid/aggregation/HistogramAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/HistogramAggregator.java index 23ac98f48a9..8f4e9fd576c 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/HistogramAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/HistogramAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.primitives.Longs; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.util.Comparator; @@ -39,12 +39,12 @@ public class HistogramAggregator implements Aggregator return ((Histogram) lhs).fold((Histogram) rhs); } - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; private final String name; private Histogram histogram; - public HistogramAggregator(String name, FloatMetricSelector selector, float[] breaks) { + public HistogramAggregator(String name, FloatColumnSelector selector, float[] breaks) { this.name = name; this.selector = selector; this.histogram = new Histogram(breaks); diff --git a/common/src/main/java/com/metamx/druid/aggregation/HistogramAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java similarity index 94% rename from common/src/main/java/com/metamx/druid/aggregation/HistogramAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java index 6c65cd95f28..2abd19d2330 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/HistogramAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/HistogramAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -26,7 +26,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.primitives.Floats; import com.google.common.primitives.Longs; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import org.apache.commons.codec.binary.Base64; import java.nio.ByteBuffer; @@ -68,7 +68,7 @@ public class HistogramAggregatorFactory implements AggregatorFactory { return new HistogramAggregator( name, - metricFactory.makeFloatMetricSelector(fieldName), + metricFactory.makeFloatColumnSelector(fieldName), breaks ); } @@ -77,7 +77,7 @@ public class HistogramAggregatorFactory implements AggregatorFactory public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new HistogramBufferAggregator( - metricFactory.makeFloatMetricSelector(fieldName), + metricFactory.makeFloatColumnSelector(fieldName), breaks ); } diff --git a/common/src/main/java/com/metamx/druid/aggregation/HistogramBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/HistogramBufferAggregator.java similarity index 92% rename from common/src/main/java/com/metamx/druid/aggregation/HistogramBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/HistogramBufferAggregator.java index 01196bd6298..5913c32b711 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/HistogramBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/HistogramBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,23 +17,23 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.primitives.Floats; import com.google.common.primitives.Longs; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; import java.util.Arrays; public class HistogramBufferAggregator implements BufferAggregator { - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; private final float[] breaks; private final int minOffset; private final int maxOffset; - public HistogramBufferAggregator(FloatMetricSelector selector, float[] breaks) + public HistogramBufferAggregator(FloatColumnSelector selector, float[] breaks) { this.selector = selector; this.breaks = breaks; diff --git a/common/src/main/java/com/metamx/druid/aggregation/HistogramVisual.java b/processing/src/main/java/io/druid/query/aggregation/HistogramVisual.java similarity index 96% rename from common/src/main/java/com/metamx/druid/aggregation/HistogramVisual.java rename to processing/src/main/java/io/druid/query/aggregation/HistogramVisual.java index 8c696638aa9..cfe7e050fe8 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/HistogramVisual.java +++ b/processing/src/main/java/io/druid/query/aggregation/HistogramVisual.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregator.java b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregator.java similarity index 93% rename from common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregator.java index 6ce103e2af4..4f85c1de81a 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.collect.Lists; -import com.metamx.druid.processing.ObjectColumnSelector; +import io.druid.segment.ObjectColumnSelector; import java.util.List; diff --git a/common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java similarity index 97% rename from common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java index a8375f294fa..ff55c9cee7f 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/JavaScriptAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/JavaScriptAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -26,8 +26,8 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.primitives.Doubles; -import com.metamx.druid.processing.ColumnSelectorFactory; -import com.metamx.druid.processing.ObjectColumnSelector; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.ObjectColumnSelector; import org.mozilla.javascript.Context; import org.mozilla.javascript.ContextAction; import org.mozilla.javascript.ContextFactory; diff --git a/common/src/main/java/com/metamx/druid/aggregation/JavaScriptBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/JavaScriptBufferAggregator.java similarity index 88% rename from common/src/main/java/com/metamx/druid/aggregation/JavaScriptBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/JavaScriptBufferAggregator.java index 90cb171d972..ad723c5caa1 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/JavaScriptBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/JavaScriptBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.collect.Lists; -import com.metamx.druid.processing.ComplexMetricSelector; -import com.metamx.druid.processing.FloatMetricSelector; -import com.metamx.druid.processing.ObjectColumnSelector; +import io.druid.segment.ObjectColumnSelector; import java.nio.ByteBuffer; import java.util.List; diff --git a/common/src/main/java/com/metamx/druid/aggregation/LongSumAggregator.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java similarity index 88% rename from common/src/main/java/com/metamx/druid/aggregation/LongSumAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java index 9c7aadfa4f1..203b22f7860 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/LongSumAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.primitives.Longs; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.util.Comparator; @@ -41,12 +41,12 @@ public class LongSumAggregator implements Aggregator return ((Number) lhs).longValue() + ((Number) rhs).longValue(); } - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; private final String name; private long sum; - public LongSumAggregator(String name, FloatMetricSelector selector) + public LongSumAggregator(String name, FloatColumnSelector selector) { this.name = name; this.selector = selector; diff --git a/common/src/main/java/com/metamx/druid/aggregation/LongSumAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java similarity index 91% rename from common/src/main/java/com/metamx/druid/aggregation/LongSumAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java index 07e04254f76..f1372ff024c 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/LongSumAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Longs; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; import java.util.Arrays; @@ -57,14 +57,14 @@ public class LongSumAggregatorFactory implements AggregatorFactory { return new LongSumAggregator( name, - metricFactory.makeFloatMetricSelector(fieldName) + metricFactory.makeFloatColumnSelector(fieldName) ); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new LongSumBufferAggregator(metricFactory.makeFloatMetricSelector(fieldName)); + return new LongSumBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override diff --git a/common/src/main/java/com/metamx/druid/aggregation/LongSumBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java similarity index 87% rename from common/src/main/java/com/metamx/druid/aggregation/LongSumBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java index 747d383fcb7..72f0fbdd62d 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/LongSumBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/LongSumBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; @@ -27,10 +27,10 @@ import java.nio.ByteBuffer; */ public class LongSumBufferAggregator implements BufferAggregator { - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; public LongSumBufferAggregator( - FloatMetricSelector selector + FloatColumnSelector selector ) { this.selector = selector; diff --git a/common/src/main/java/com/metamx/druid/aggregation/MaxAggregator.java b/processing/src/main/java/io/druid/query/aggregation/MaxAggregator.java similarity index 87% rename from common/src/main/java/com/metamx/druid/aggregation/MaxAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/MaxAggregator.java index 8761c001da4..f3e40183ad7 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/MaxAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/MaxAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.util.Comparator; @@ -34,12 +34,12 @@ public class MaxAggregator implements Aggregator return Math.max(((Number) lhs).doubleValue(), ((Number) rhs).doubleValue()); } - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; private final String name; private double max; - public MaxAggregator(String name, FloatMetricSelector selector) + public MaxAggregator(String name, FloatColumnSelector selector) { this.name = name; this.selector = selector; diff --git a/common/src/main/java/com/metamx/druid/aggregation/MaxAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java similarity index 92% rename from common/src/main/java/com/metamx/druid/aggregation/MaxAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java index 45cd85257f7..d66d9c5b2d9 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/MaxAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/MaxAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; import java.util.Arrays; @@ -55,13 +55,13 @@ public class MaxAggregatorFactory implements AggregatorFactory @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new MaxAggregator(name, metricFactory.makeFloatMetricSelector(fieldName)); + return new MaxAggregator(name, metricFactory.makeFloatColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new MaxBufferAggregator(metricFactory.makeFloatMetricSelector(fieldName)); + return new MaxBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override diff --git a/common/src/main/java/com/metamx/druid/aggregation/MaxBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/MaxBufferAggregator.java similarity index 85% rename from common/src/main/java/com/metamx/druid/aggregation/MaxBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/MaxBufferAggregator.java index 4bfcd2f2bbf..2a793a34d45 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/MaxBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/MaxBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; @@ -27,9 +27,9 @@ import java.nio.ByteBuffer; */ public class MaxBufferAggregator implements BufferAggregator { - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; - public MaxBufferAggregator(FloatMetricSelector selector) + public MaxBufferAggregator(FloatColumnSelector selector) { this.selector = selector; } diff --git a/client/src/main/java/com/metamx/druid/query/MetricManipulationFn.java b/processing/src/main/java/io/druid/query/aggregation/MetricManipulationFn.java similarity index 86% rename from client/src/main/java/com/metamx/druid/query/MetricManipulationFn.java rename to processing/src/main/java/io/druid/query/aggregation/MetricManipulationFn.java index 1ae354f8714..0e9bab40e60 100644 --- a/client/src/main/java/com/metamx/druid/query/MetricManipulationFn.java +++ b/processing/src/main/java/io/druid/query/aggregation/MetricManipulationFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; - -import com.metamx.druid.aggregation.AggregatorFactory; +package io.druid.query.aggregation; /** */ diff --git a/common/src/main/java/com/metamx/druid/aggregation/MinAggregator.java b/processing/src/main/java/io/druid/query/aggregation/MinAggregator.java similarity index 87% rename from common/src/main/java/com/metamx/druid/aggregation/MinAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/MinAggregator.java index a42ba758ed0..c4e3dc67a31 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/MinAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/MinAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.util.Comparator; @@ -34,12 +34,12 @@ public class MinAggregator implements Aggregator return Math.min(((Number) lhs).doubleValue(), ((Number) rhs).doubleValue()); } - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; private final String name; private double min; - public MinAggregator(String name, FloatMetricSelector selector) + public MinAggregator(String name, FloatColumnSelector selector) { this.name = name; this.selector = selector; diff --git a/common/src/main/java/com/metamx/druid/aggregation/MinAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java similarity index 92% rename from common/src/main/java/com/metamx/druid/aggregation/MinAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java index a6d19ebd8e5..0a168114358 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/MinAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/MinAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.primitives.Doubles; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.nio.ByteBuffer; import java.util.Arrays; @@ -55,13 +55,13 @@ public class MinAggregatorFactory implements AggregatorFactory @Override public Aggregator factorize(ColumnSelectorFactory metricFactory) { - return new MinAggregator(name, metricFactory.makeFloatMetricSelector(fieldName)); + return new MinAggregator(name, metricFactory.makeFloatColumnSelector(fieldName)); } @Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { - return new MinBufferAggregator(metricFactory.makeFloatMetricSelector(fieldName)); + return new MinBufferAggregator(metricFactory.makeFloatColumnSelector(fieldName)); } @Override diff --git a/common/src/main/java/com/metamx/druid/aggregation/MinBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/MinBufferAggregator.java similarity index 85% rename from common/src/main/java/com/metamx/druid/aggregation/MinBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/MinBufferAggregator.java index 7bd29477b97..f3c5964e6c1 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/MinBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/MinBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; import java.nio.ByteBuffer; @@ -27,9 +27,9 @@ import java.nio.ByteBuffer; */ public class MinBufferAggregator implements BufferAggregator { - private final FloatMetricSelector selector; + private final FloatColumnSelector selector; - public MinBufferAggregator(FloatMetricSelector selector) + public MinBufferAggregator(FloatColumnSelector selector) { this.selector = selector; } diff --git a/common/src/main/java/com/metamx/druid/aggregation/NoopAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NoopAggregator.java similarity index 93% rename from common/src/main/java/com/metamx/druid/aggregation/NoopAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/NoopAggregator.java index 5f7ddb2e78b..7e2fc7ed5e5 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/NoopAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NoopAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; /** */ diff --git a/common/src/main/java/com/metamx/druid/aggregation/NoopBufferAggregator.java b/processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java similarity index 93% rename from common/src/main/java/com/metamx/druid/aggregation/NoopBufferAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java index a486fbcad2e..7ff2d7cab2c 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/NoopBufferAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/NoopBufferAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import java.nio.ByteBuffer; diff --git a/common/src/main/java/com/metamx/druid/aggregation/post/PostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java similarity index 68% rename from common/src/main/java/com/metamx/druid/aggregation/post/PostAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/PostAggregator.java index df78ad3bf7b..37d0ce3e03b 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/post/PostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/PostAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; +package io.druid.query.aggregation; import java.util.Comparator; import java.util.Map; @@ -29,12 +26,6 @@ import java.util.Set; /** * Functionally similar to an Aggregator. See the Aggregator interface for more comments. */ -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "arithmetic", value = ArithmeticPostAggregator.class), - @JsonSubTypes.Type(name = "fieldAccess", value = FieldAccessPostAggregator.class), - @JsonSubTypes.Type(name = "constant", value = ConstantPostAggregator.class) -}) public interface PostAggregator { public Set getDependentFields(); diff --git a/common/src/main/java/com/metamx/druid/aggregation/ToLowerCaseAggregatorFactory.java b/processing/src/main/java/io/druid/query/aggregation/ToLowerCaseAggregatorFactory.java similarity index 94% rename from common/src/main/java/com/metamx/druid/aggregation/ToLowerCaseAggregatorFactory.java rename to processing/src/main/java/io/druid/query/aggregation/ToLowerCaseAggregatorFactory.java index fcd782a9f01..457d22f2cdf 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/ToLowerCaseAggregatorFactory.java +++ b/processing/src/main/java/io/druid/query/aggregation/ToLowerCaseAggregatorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.ColumnSelectorFactory; +import io.druid.segment.ColumnSelectorFactory; import java.util.Comparator; import java.util.List; diff --git a/common/src/main/java/com/metamx/druid/aggregation/post/ArithmeticPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java similarity index 96% rename from common/src/main/java/com/metamx/druid/aggregation/post/ArithmeticPostAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java index 4d768b6420b..cf6881d18ad 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/post/ArithmeticPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ArithmeticPostAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; +package io.druid.query.aggregation.post; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.metamx.common.IAE; +import io.druid.query.aggregation.PostAggregator; import java.util.Comparator; import java.util.Iterator; diff --git a/common/src/main/java/com/metamx/druid/aggregation/post/ConstantPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/ConstantPostAggregator.java similarity index 93% rename from common/src/main/java/com/metamx/druid/aggregation/post/ConstantPostAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/post/ConstantPostAggregator.java index 21c1e70a50b..d5dce4d3140 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/post/ConstantPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/ConstantPostAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; +package io.druid.query.aggregation.post; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Sets; +import io.druid.query.aggregation.PostAggregator; import java.util.Comparator; import java.util.Map; diff --git a/common/src/main/java/com/metamx/druid/aggregation/post/FieldAccessPostAggregator.java b/processing/src/main/java/io/druid/query/aggregation/post/FieldAccessPostAggregator.java similarity index 93% rename from common/src/main/java/com/metamx/druid/aggregation/post/FieldAccessPostAggregator.java rename to processing/src/main/java/io/druid/query/aggregation/post/FieldAccessPostAggregator.java index a06bf0cf891..59962836d2e 100644 --- a/common/src/main/java/com/metamx/druid/aggregation/post/FieldAccessPostAggregator.java +++ b/processing/src/main/java/io/druid/query/aggregation/post/FieldAccessPostAggregator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; +package io.druid.query.aggregation.post; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Sets; +import io.druid.query.aggregation.PostAggregator; import java.util.Comparator; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/query/dimension/DefaultDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/dimension/DefaultDimensionSpec.java rename to processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java index b27b361eda0..3a9137b008e 100644 --- a/client/src/main/java/com/metamx/druid/query/dimension/DefaultDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.dimension; +package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.DimExtractionFn; import java.nio.ByteBuffer; diff --git a/client/src/main/java/com/metamx/druid/query/dimension/DimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java similarity index 90% rename from client/src/main/java/com/metamx/druid/query/dimension/DimensionSpec.java rename to processing/src/main/java/io/druid/query/dimension/DimensionSpec.java index 0773ef6ca06..3c552e20763 100644 --- a/client/src/main/java/com/metamx/druid/query/dimension/DimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/DimensionSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.dimension; +package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.DimExtractionFn; /** */ diff --git a/client/src/main/java/com/metamx/druid/query/dimension/ExtractionDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java similarity index 94% rename from client/src/main/java/com/metamx/druid/query/dimension/ExtractionDimensionSpec.java rename to processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java index c746b7b3108..82fba73d0a1 100644 --- a/client/src/main/java/com/metamx/druid/query/dimension/ExtractionDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.dimension; +package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.DimExtractionFn; import java.nio.ByteBuffer; diff --git a/client/src/main/java/com/metamx/druid/query/dimension/LegacyDimensionSpec.java b/processing/src/main/java/io/druid/query/dimension/LegacyDimensionSpec.java similarity index 94% rename from client/src/main/java/com/metamx/druid/query/dimension/LegacyDimensionSpec.java rename to processing/src/main/java/io/druid/query/dimension/LegacyDimensionSpec.java index 352142c1166..6e1088ed788 100644 --- a/client/src/main/java/com/metamx/druid/query/dimension/LegacyDimensionSpec.java +++ b/processing/src/main/java/io/druid/query/dimension/LegacyDimensionSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.dimension; +package io.druid.query.dimension; import com.fasterxml.jackson.annotation.JsonCreator; import com.metamx.common.IAE; diff --git a/client/src/main/java/com/metamx/druid/query/extraction/DimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/DimExtractionFn.java similarity index 94% rename from client/src/main/java/com/metamx/druid/query/extraction/DimExtractionFn.java rename to processing/src/main/java/io/druid/query/extraction/DimExtractionFn.java index 45b84313542..65b5c2a5d7b 100644 --- a/client/src/main/java/com/metamx/druid/query/extraction/DimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/DimExtractionFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/client/src/main/java/com/metamx/druid/query/extraction/PartialDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/extraction/PartialDimExtractionFn.java rename to processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java index 86b1d1ddbd2..5ae4e2bf019 100644 --- a/client/src/main/java/com/metamx/druid/query/extraction/PartialDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/PartialDimExtractionFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/extraction/RegexDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/extraction/RegexDimExtractionFn.java rename to processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java index f5135a2d1c5..c25d0ba6c7a 100644 --- a/client/src/main/java/com/metamx/druid/query/extraction/RegexDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/RegexDimExtractionFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFn.java rename to processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java index 3b1a323ac1c..02c13eca630 100644 --- a/client/src/main/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/SearchQuerySpecDimExtractionFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.query.search.SearchQuerySpec; +import io.druid.query.search.search.SearchQuerySpec; import java.nio.ByteBuffer; diff --git a/client/src/main/java/com/metamx/druid/query/extraction/TimeDimExtractionFn.java b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/extraction/TimeDimExtractionFn.java rename to processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java index 71ff8bed15f..57b7f6617d8 100644 --- a/client/src/main/java/com/metamx/druid/query/extraction/TimeDimExtractionFn.java +++ b/processing/src/main/java/io/druid/query/extraction/TimeDimExtractionFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/filter/AndDimFilter.java b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/filter/AndDimFilter.java rename to processing/src/main/java/io/druid/query/filter/AndDimFilter.java index 6a1bb9109aa..00b15c42e71 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/AndDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/AndDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java similarity index 91% rename from server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java rename to processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java index fb8837de1f5..c0a2abca40b 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/BitmapIndexSelector.java +++ b/processing/src/main/java/io/druid/query/filter/BitmapIndexSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.query.filter; import com.metamx.collections.spatial.ImmutableRTree; -import com.metamx.druid.kv.Indexed; +import io.druid.segment.data.Indexed; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; /** diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java b/processing/src/main/java/io/druid/query/filter/DimFilter.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/filter/DimFilter.java rename to processing/src/main/java/io/druid/query/filter/DimFilter.java index 26f9bf8519c..7e303f0f253 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java b/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java rename to processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java index 02d20ea528c..5fa3a7386b6 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilterCacheHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import java.nio.ByteBuffer; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilters.java b/processing/src/main/java/io/druid/query/filter/DimFilters.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/filter/DimFilters.java rename to processing/src/main/java/io/druid/query/filter/DimFilters.java index 0b74f443c3e..5147565d7ca 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilters.java +++ b/processing/src/main/java/io/druid/query/filter/DimFilters.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.google.common.base.Function; import com.google.common.collect.Lists; diff --git a/client/src/main/java/com/metamx/druid/query/filter/ExtractionDimFilter.java b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java similarity index 94% rename from client/src/main/java/com/metamx/druid/query/filter/ExtractionDimFilter.java rename to processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java index b6a43c5af04..8ce26f71f99 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/ExtractionDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.metamx.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.DimExtractionFn; import java.nio.ByteBuffer; diff --git a/server/src/main/java/com/metamx/druid/index/brita/Filter.java b/processing/src/main/java/io/druid/query/filter/Filter.java similarity index 92% rename from server/src/main/java/com/metamx/druid/index/brita/Filter.java rename to processing/src/main/java/io/druid/query/filter/Filter.java index fcfde0be18a..73d39074c75 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/Filter.java +++ b/processing/src/main/java/io/druid/query/filter/Filter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.query.filter; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; diff --git a/client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java similarity index 60% rename from client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java rename to processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java index 00db76b4f05..b9cef360238 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/JavaScriptDimFilter.java @@ -1,4 +1,23 @@ -package com.metamx.druid.query.filter; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/filter/NoopDimFilter.java b/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java similarity index 92% rename from client/src/main/java/com/metamx/druid/query/filter/NoopDimFilter.java rename to processing/src/main/java/io/druid/query/filter/NoopDimFilter.java index 6ceb7f565cd..1d518c07171 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/NoopDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NoopDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import java.nio.ByteBuffer; diff --git a/client/src/main/java/com/metamx/druid/query/filter/NotDimFilter.java b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/filter/NotDimFilter.java rename to processing/src/main/java/io/druid/query/filter/NotDimFilter.java index 9f690f1322d..b2037b49ba9 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/NotDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/NotDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/filter/OrDimFilter.java b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/filter/OrDimFilter.java rename to processing/src/main/java/io/druid/query/filter/OrDimFilter.java index 4fdde0f46bb..bb4cb769a33 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/OrDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/OrDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/filter/RegexDimFilter.java b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/filter/RegexDimFilter.java rename to processing/src/main/java/io/druid/query/filter/RegexDimFilter.java index edbd1b7fc5d..0644250f819 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/RegexDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/RegexDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/filter/SearchQueryDimFilter.java b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/filter/SearchQueryDimFilter.java rename to processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java index edb7ce791e0..76c5ecd0148 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/SearchQueryDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SearchQueryDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; -import com.metamx.druid.query.search.SearchQuerySpec; +import io.druid.query.search.search.SearchQuerySpec; import java.nio.ByteBuffer; diff --git a/client/src/main/java/com/metamx/druid/query/filter/SelectorDimFilter.java b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/filter/SelectorDimFilter.java rename to processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java index dd2f39dfe13..b8d1580a169 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/SelectorDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SelectorDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/filter/SpatialDimFilter.java b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/filter/SpatialDimFilter.java rename to processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java index 4a50668b2e5..6899d306602 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/SpatialDimFilter.java +++ b/processing/src/main/java/io/druid/query/filter/SpatialDimFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -16,7 +16,7 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.filter; +package io.druid.query.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/com/metamx/druid/index/brita/ValueMatcher.java b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java similarity index 90% rename from server/src/main/java/com/metamx/druid/index/brita/ValueMatcher.java rename to processing/src/main/java/io/druid/query/filter/ValueMatcher.java index 5d68a203bfa..aac95f44633 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/ValueMatcher.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcher.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.query.filter; /** */ diff --git a/server/src/main/java/com/metamx/druid/index/brita/ValueMatcherFactory.java b/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java similarity index 93% rename from server/src/main/java/com/metamx/druid/index/brita/ValueMatcherFactory.java rename to processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java index cc9971b23b8..45834974639 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/ValueMatcherFactory.java +++ b/processing/src/main/java/io/druid/query/filter/ValueMatcherFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.query.filter; import com.google.common.base.Predicate; import com.metamx.collections.spatial.search.Bound; diff --git a/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java similarity index 92% rename from client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java rename to processing/src/main/java/io/druid/query/groupby/GroupByQuery.java index 3c48c4cecbc..93db02101ba 100644 --- a/client/src/main/java/com/metamx/druid/query/group/GroupByQuery.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group; +package io.druid.query.groupby; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -30,23 +30,22 @@ import com.google.common.collect.Lists; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.BaseQuery; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.Queries; -import com.metamx.druid.query.dimension.DefaultDimensionSpec; -import com.metamx.druid.query.dimension.DimensionSpec; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.query.group.having.HavingSpec; -import com.metamx.druid.query.group.orderby.DefaultLimitSpec; -import com.metamx.druid.query.group.orderby.LimitSpec; -import com.metamx.druid.query.group.orderby.NoopLimitSpec; -import com.metamx.druid.query.group.orderby.OrderByColumnSpec; -import com.metamx.druid.query.segment.LegacySegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentSpec; +import io.druid.data.input.Row; +import io.druid.granularity.QueryGranularity; +import io.druid.query.BaseQuery; +import io.druid.query.Queries; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.DimFilter; +import io.druid.query.groupby.having.HavingSpec; +import io.druid.query.groupby.orderby.DefaultLimitSpec; +import io.druid.query.groupby.orderby.LimitSpec; +import io.druid.query.groupby.orderby.NoopLimitSpec; +import io.druid.query.groupby.orderby.OrderByColumnSpec; +import io.druid.query.spec.LegacySegmentSpec; +import io.druid.query.spec.QuerySegmentSpec; import javax.annotation.Nullable; import java.util.List; @@ -210,7 +209,7 @@ public class GroupByQuery extends BaseQuery @Override public String getType() { - return Query.GROUP_BY; + return GROUP_BY; } public Sequence applyLimit(Sequence results) diff --git a/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java new file mode 100644 index 00000000000..98062a4f4c9 --- /dev/null +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryConfig.java @@ -0,0 +1,56 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class GroupByQueryConfig +{ + @JsonProperty + private boolean singleThreaded = false; + + @JsonProperty + private int maxIntermediateRows = 50000; + + @JsonProperty + private int maxResults = 500000; + + public boolean isSingleThreaded() + { + return singleThreaded; + } + + public int getMaxIntermediateRows() + { + return maxIntermediateRows; + } + + public void setMaxIntermediateRows(int maxIntermediateRows) + { + this.maxIntermediateRows = maxIntermediateRows; + } + + public int getMaxResults() + { + return maxResults; + } +} diff --git a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java similarity index 92% rename from server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java rename to processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java index c2d46e54d45..ea58501635b 100644 --- a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryEngine.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryEngine.java @@ -17,15 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group; +package io.druid.query.groupby; import com.google.common.base.Function; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.io.Closeables; import com.google.common.primitives.Ints; +import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.guava.BaseSequence; @@ -33,19 +35,20 @@ import com.metamx.common.guava.FunctionalIterator; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.parsers.CloseableIterator; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.BufferAggregator; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.index.brita.Filters; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.index.v1.processing.DimensionSelector; -import com.metamx.druid.input.MapBasedRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.query.dimension.DimensionSpec; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidPool; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.guice.annotations.Global; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.BufferAggregator; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.StorageAdapter; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.filter.Filters; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -62,12 +65,13 @@ import java.util.TreeMap; */ public class GroupByQueryEngine { - private final GroupByQueryEngineConfig config; + private final Supplier config; private final StupidPool intermediateResultsBufferPool; + @Inject public GroupByQueryEngine ( - GroupByQueryEngineConfig config, - StupidPool intermediateResultsBufferPool + Supplier config, + @Global StupidPool intermediateResultsBufferPool ) { this.config = config; @@ -110,7 +114,7 @@ public class GroupByQueryEngine @Override public RowIterator make() { - return new RowIterator(query, cursor, bufferHolder.get()); + return new RowIterator(query, cursor, bufferHolder.get(), config.get()); } @Override @@ -286,6 +290,7 @@ public class GroupByQueryEngine private final GroupByQuery query; private final Cursor cursor; private final ByteBuffer metricsBuffer; + private final GroupByQueryConfig config; private final List dimensionSpecs; private final List dimensions; @@ -298,11 +303,12 @@ public class GroupByQueryEngine private List unprocessedKeys; private Iterator delegate; - public RowIterator(GroupByQuery query, Cursor cursor, ByteBuffer metricsBuffer) + public RowIterator(GroupByQuery query, Cursor cursor, ByteBuffer metricsBuffer, GroupByQueryConfig config) { this.query = query; this.cursor = cursor; this.metricsBuffer = metricsBuffer; + this.config = config; unprocessedKeys = null; delegate = Iterators.emptyIterator(); diff --git a/client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java similarity index 85% rename from client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java rename to processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java index db39bc4b513..40a0203de83 100644 --- a/client/src/main/java/com/metamx/druid/query/group/GroupByQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryQueryToolChest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,40 +17,39 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group; +package io.druid.query.groupby; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.base.Joiner; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.ConcatSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.index.v1.IncrementalIndex; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.input.MapBasedRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.input.Rows; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.dimension.DimensionSpec; -import com.metamx.druid.utils.PropUtils; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.data.input.Rows; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.dimension.DimensionSpec; +import io.druid.segment.incremental.IncrementalIndex; import org.joda.time.Interval; import org.joda.time.Minutes; import javax.annotation.Nullable; import java.util.List; import java.util.Map; -import java.util.Properties; /** */ @@ -62,13 +61,14 @@ public class GroupByQueryQueryToolChest extends QueryToolChest NO_MERGE_CONTEXT = ImmutableMap.of(GROUP_BY_MERGE_KEY, "false"); - private static final int maxRows; + private final Supplier configSupplier; - static { - // I dislike this static loading of properies, but it's the only mechanism available right now. - Properties props = Initialization.loadProperties(); - - maxRows = PropUtils.getPropertyAsInt(props, "com.metamx.query.groupBy.maxResults", 500000); + @Inject + public GroupByQueryQueryToolChest( + Supplier configSupplier + ) + { + this.configSupplier = configSupplier; } @Override @@ -90,6 +90,7 @@ public class GroupByQueryQueryToolChest extends QueryToolChest mergeGroupByResults(final GroupByQuery query, QueryRunner runner) { + final GroupByQueryConfig config = configSupplier.get(); final QueryGranularity gran = query.getGranularity(); final long timeStart = query.getIntervals().get(0).getStartMillis(); @@ -133,8 +134,8 @@ public class GroupByQueryQueryToolChest extends QueryToolChest maxRows) { - throw new ISE("Computation exceeds maxRows limit[%s]", maxRows); + if (accumulated.add(Rows.toCaseInsensitiveInputRow(in, dimensions)) > config.getMaxResults()) { + throw new ISE("Computation exceeds maxRows limit[%s]", config.getMaxResults()); } return accumulated; diff --git a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java similarity index 84% rename from server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactory.java rename to processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java index 0ecec11b76c..0cee492a70b 100644 --- a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/groupby/GroupByQueryRunnerFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,25 +17,27 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group; +package io.druid.query.groupby; import com.google.common.base.Function; +import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.Ordering; import com.google.common.primitives.Longs; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.ExecutorExecutingSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Query; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.Segment; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.ChainedExecutionQueryRunner; -import com.metamx.druid.query.ConcatQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryToolChest; +import io.druid.data.input.Row; +import io.druid.query.ChainedExecutionQueryRunner; +import io.druid.query.ConcatQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -46,18 +48,20 @@ import java.util.concurrent.Future; */ public class GroupByQueryRunnerFactory implements QueryRunnerFactory { - private static final GroupByQueryQueryToolChest toolChest = new GroupByQueryQueryToolChest(); - private final GroupByQueryEngine engine; - private final GroupByQueryRunnerFactoryConfig config; + private final Supplier config; + private final GroupByQueryQueryToolChest toolChest; + @Inject public GroupByQueryRunnerFactory( GroupByQueryEngine engine, - GroupByQueryRunnerFactoryConfig config + Supplier config, + GroupByQueryQueryToolChest toolChest ) { this.engine = engine; this.config = config; + this.toolChest = toolChest; } @Override @@ -69,7 +73,7 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory mergeRunners(final ExecutorService queryExecutor, Iterable> queryRunners) { - if (config.isSingleThreaded()) { + if (config.get().isSingleThreaded()) { return new ConcatQueryRunner( Sequences.map( Sequences.simple(queryRunners), @@ -119,7 +123,7 @@ public class GroupByQueryRunnerFactory implements QueryRunnerFactory getToolchest() { return toolChest; } diff --git a/processing/src/main/java/io/druid/query/groupby/having/AlwaysHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/AlwaysHavingSpec.java new file mode 100644 index 00000000000..51b938a9891 --- /dev/null +++ b/processing/src/main/java/io/druid/query/groupby/having/AlwaysHavingSpec.java @@ -0,0 +1,33 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; + +import io.druid.data.input.Row; + +/** + */ +public class AlwaysHavingSpec implements HavingSpec +{ + @Override + public boolean eval(Row row) + { + return true; + } +} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/AndHavingSpec.java similarity index 63% rename from client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/AndHavingSpec.java index f474bc67c95..8e0fdb6a91f 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/AndHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/AndHavingSpec.java @@ -1,9 +1,28 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java similarity index 71% rename from client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java index f7e5060e4e9..34bd7b887f2 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/EqualToHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/EqualToHavingSpec.java @@ -1,8 +1,27 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; /** * The "=" operator in a "having" clause. This is similar to SQL's "having aggregation = value", diff --git a/client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java similarity index 69% rename from client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java index a64f5ee5050..0c2c9e7810b 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/GreaterThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/GreaterThanHavingSpec.java @@ -1,8 +1,27 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; /** * The ">" operator in a "having" clause. This is similar to SQL's "having aggregation > value", diff --git a/client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java similarity index 62% rename from client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java index b40ae662534..00a8389cff0 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/HavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/HavingSpec.java @@ -1,8 +1,27 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; /** * A "having" clause that filters aggregated value. This is similar to SQL's "having" diff --git a/client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java similarity index 69% rename from client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java index 86ff4dfaf9c..ce8dd20b661 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/LessThanHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/LessThanHavingSpec.java @@ -1,7 +1,26 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; /** * The "<" operator in a "having" clause. This is similar to SQL's "having aggregation < value", diff --git a/client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/NotHavingSpec.java similarity index 58% rename from client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/NotHavingSpec.java index 9324268fbda..e963e0154e5 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/NotHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/NotHavingSpec.java @@ -1,8 +1,27 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; /** * The logical "not" operator for the "having" clause. diff --git a/client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java b/processing/src/main/java/io/druid/query/groupby/having/OrHavingSpec.java similarity index 62% rename from client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java rename to processing/src/main/java/io/druid/query/groupby/having/OrHavingSpec.java index 76c8920182f..73f46dfaa24 100644 --- a/client/src/main/java/com/metamx/druid/query/group/having/OrHavingSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/having/OrHavingSpec.java @@ -1,9 +1,28 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import com.metamx.druid.input.Row; +import io.druid.data.input.Row; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/DefaultLimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/group/orderby/DefaultLimitSpec.java rename to processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java index 66049f925f3..4cd052058bf 100644 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/DefaultLimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/DefaultLimitSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group.orderby; +package io.druid.query.groupby.orderby; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -31,10 +31,10 @@ import com.google.common.primitives.Longs; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.dimension.DimensionSpec; +import io.druid.data.input.Row; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DimensionSpec; import javax.annotation.Nullable; import java.util.ArrayList; diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/LimitSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java similarity index 82% rename from client/src/main/java/com/metamx/druid/query/group/orderby/LimitSpec.java rename to processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java index cee3c76e55d..0d07f1f91c9 100644 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/LimitSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/LimitSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,16 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group.orderby; +package io.druid.query.groupby.orderby; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.base.Function; import com.metamx.common.guava.Sequence; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.dimension.DimensionSpec; +import io.druid.data.input.Row; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DimensionSpec; import java.util.List; diff --git a/common/src/main/java/com/metamx/druid/input/MapBasedInputRow.java b/processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java similarity index 56% rename from common/src/main/java/com/metamx/druid/input/MapBasedInputRow.java rename to processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java index 2177bcde776..6fbc063c72d 100644 --- a/common/src/main/java/com/metamx/druid/input/MapBasedInputRow.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/NoopLimitSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,42 +17,33 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.input; +package io.druid.query.groupby.orderby; -import org.joda.time.DateTime; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.metamx.common.guava.Sequence; +import io.druid.data.input.Row; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.dimension.DimensionSpec; import java.util.List; -import java.util.Map; /** */ -public class MapBasedInputRow extends MapBasedRow implements InputRow +public class NoopLimitSpec implements LimitSpec { - private final List dimensions; - - public MapBasedInputRow( - long timestamp, - List dimensions, - Map event + @Override + public Function, Sequence> build( + List dimensions, List aggs, List postAggs ) { - super(timestamp, event); - this.dimensions = dimensions; - } - - @Override - public List getDimensions() - { - return dimensions; + return Functions.identity(); } @Override public String toString() { - return "MapBasedInputRow{" + - "timestamp=" + new DateTime(getTimestampFromEpoch()) + - ", event=" + getEvent() + - ", dimensions=" + dimensions + - '}'; + return "NoopLimitSpec"; } } diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderByColumnSpec.java b/processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java similarity index 82% rename from client/src/main/java/com/metamx/druid/query/group/orderby/OrderByColumnSpec.java rename to processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java index 6a02ec962f8..147f1911816 100644 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderByColumnSpec.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/OrderByColumnSpec.java @@ -1,4 +1,23 @@ -package com.metamx.druid.query.group.orderby; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.orderby; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/test/java/com/metamx/druid/client/RangeIterable.java b/processing/src/main/java/io/druid/query/groupby/orderby/OrderedPriorityQueueItems.java similarity index 53% rename from server/src/test/java/com/metamx/druid/client/RangeIterable.java rename to processing/src/main/java/io/druid/query/groupby/orderby/OrderedPriorityQueueItems.java index 5a566cab1b4..c2ae523345a 100644 --- a/server/src/test/java/com/metamx/druid/client/RangeIterable.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/OrderedPriorityQueueItems.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,63 +17,45 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.query.groupby.orderby; + +import com.google.common.collect.MinMaxPriorityQueue; import java.util.Iterator; /** + * Utility class that supports iterating a priority queue in sorted order. */ -public class RangeIterable implements Iterable +class OrderedPriorityQueueItems implements Iterable { - private final int startValue; - private final int endValue; - private final int increment; + private MinMaxPriorityQueue rows; - public RangeIterable(int endValue) + public OrderedPriorityQueueItems(MinMaxPriorityQueue rows) { - this(0, endValue); - } - - public RangeIterable(int startValue, int endValue) - { - this(startValue, endValue, 1); - } - - public RangeIterable(int startValue, int endValue, int increment) - { - this.startValue = startValue; - this.endValue = endValue; - this.increment = increment; + this.rows = rows; } @Override - public Iterator iterator() + public Iterator iterator() { - return new Iterator() - { - int value = startValue; + return new Iterator() { @Override public boolean hasNext() { - return value < endValue; + return !rows.isEmpty(); } @Override - public Integer next() + public T next() { - try { - return value; - } - finally { - value += increment; - } + return rows.poll(); } @Override public void remove() { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("Can't remove any item from an intermediary heap for orderBy/limit"); } }; } diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java b/processing/src/main/java/io/druid/query/groupby/orderby/TopNSorter.java similarity index 56% rename from client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java rename to processing/src/main/java/io/druid/query/groupby/orderby/TopNSorter.java index 7160ba62074..d612e5784be 100644 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/TopNSorter.java +++ b/processing/src/main/java/io/druid/query/groupby/orderby/TopNSorter.java @@ -1,4 +1,23 @@ -package com.metamx.druid.query.group.orderby; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.orderby; import com.google.common.collect.ImmutableList; import com.google.common.collect.MinMaxPriorityQueue; diff --git a/server/src/main/java/com/metamx/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java similarity index 85% rename from server/src/main/java/com/metamx/druid/query/metadata/SegmentAnalyzer.java rename to processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java index 8d5a6136998..9f5c8d19527 100644 --- a/server/src/main/java/com/metamx/druid/query/metadata/SegmentAnalyzer.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentAnalyzer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata; import com.google.common.base.Charsets; import com.google.common.base.Function; @@ -25,14 +25,15 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.primitives.Longs; import com.metamx.common.logger.Logger; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.column.BitmapIndex; -import com.metamx.druid.index.column.Column; -import com.metamx.druid.index.column.ColumnCapabilities; -import com.metamx.druid.index.column.ComplexColumn; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.index.v1.serde.ComplexMetrics; +import io.druid.query.metadata.metadata.ColumnAnalysis; +import io.druid.segment.QueryableIndex; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ComplexColumn; +import io.druid.segment.column.ValueType; +import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.ComplexMetrics; import java.util.Map; @@ -105,7 +106,7 @@ public class SegmentAnalyzer return ColumnAnalysis.error("multi_value"); } - return new ColumnAnalysis(capabilities.getType(), column.getLength() * numBytes, null, null); + return new ColumnAnalysis(capabilities.getType().name(), column.getLength() * numBytes, null, null); } public ColumnAnalysis analyzeStringColumn(Column column) @@ -125,7 +126,7 @@ public class SegmentAnalyzer } } - return new ColumnAnalysis(capabilities.getType(), size, cardinality, null); + return new ColumnAnalysis(capabilities.getType().name(), size, cardinality, null); } return ColumnAnalysis.error("string_no_bitmap"); @@ -153,6 +154,6 @@ public class SegmentAnalyzer size += inputSizeFn.apply(complexColumn.getRowValue(i)); } - return new ColumnAnalysis(capabilities.getType(), size, null, null); + return new ColumnAnalysis(capabilities.getType().name(), size, null, null); } } diff --git a/client/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java similarity index 92% rename from client/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java rename to processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java index 1cfceac4db8..a5e1a1a8319 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryQueryToolChest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; @@ -31,22 +31,23 @@ import com.metamx.common.ISE; import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.Query; -import com.metamx.druid.collect.OrderedMergeSequence; -import com.metamx.druid.query.CacheStrategy; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.ResultMergeQueryRunner; -import com.metamx.druid.utils.JodaUtils; import com.metamx.emitter.service.ServiceMetricEvent; - +import io.druid.collections.OrderedMergeSequence; +import io.druid.common.utils.JodaUtils; +import io.druid.query.CacheStrategy; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.ResultMergeQueryRunner; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.metadata.metadata.ColumnAnalysis; +import io.druid.query.metadata.metadata.SegmentAnalysis; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; import org.joda.time.Interval; import org.joda.time.Minutes; import javax.annotation.Nullable; import java.nio.ByteBuffer; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java similarity index 89% rename from server/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java rename to processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java index f44110fcf14..d600b2b4499 100644 --- a/server/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/metadata/SegmentMetadataQueryRunnerFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata; import com.google.common.base.Function; import com.google.common.base.Throwables; @@ -25,13 +25,17 @@ import com.google.common.collect.Maps; import com.metamx.common.guava.ExecutorExecutingSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Query; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.Segment; -import com.metamx.druid.query.ConcatQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryToolChest; +import io.druid.query.ConcatQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.query.metadata.metadata.ColumnAnalysis; +import io.druid.query.metadata.metadata.ColumnIncluderator; +import io.druid.query.metadata.metadata.SegmentAnalysis; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; import java.util.Arrays; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/query/metadata/AllColumnIncluderator.java b/processing/src/main/java/io/druid/query/metadata/metadata/AllColumnIncluderator.java similarity index 91% rename from client/src/main/java/com/metamx/druid/query/metadata/AllColumnIncluderator.java rename to processing/src/main/java/io/druid/query/metadata/metadata/AllColumnIncluderator.java index cd96b5d718a..905025db7ca 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/AllColumnIncluderator.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/AllColumnIncluderator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; /** */ diff --git a/client/src/main/java/com/metamx/druid/query/metadata/ColumnAnalysis.java b/processing/src/main/java/io/druid/query/metadata/metadata/ColumnAnalysis.java similarity index 86% rename from client/src/main/java/com/metamx/druid/query/metadata/ColumnAnalysis.java rename to processing/src/main/java/io/druid/query/metadata/metadata/ColumnAnalysis.java index 3c98dd514d6..c41bd629ca4 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/ColumnAnalysis.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/ColumnAnalysis.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.index.column.ValueType; /** */ @@ -31,7 +30,7 @@ public class ColumnAnalysis public static ColumnAnalysis error(String reason) { - return new ColumnAnalysis(ValueType.STRING, -1, null, ERROR_PREFIX + reason); + return new ColumnAnalysis("STRING", -1, null, ERROR_PREFIX + reason); } private final String type; @@ -41,21 +40,11 @@ public class ColumnAnalysis @JsonCreator public ColumnAnalysis( - @JsonProperty("type") ValueType type, + @JsonProperty("type") String type, @JsonProperty("size") long size, @JsonProperty("cardinality") Integer cardinality, @JsonProperty("errorMessage") String errorMessage ) - { - this(type.name(), size, cardinality, errorMessage); - } - - private ColumnAnalysis( - String type, - long size, - Integer cardinality, - String errorMessage - ) { this.type = type; this.size = size; diff --git a/client/src/main/java/com/metamx/druid/query/metadata/ColumnIncluderator.java b/processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java similarity index 94% rename from client/src/main/java/com/metamx/druid/query/metadata/ColumnIncluderator.java rename to processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java index 230c21a2498..5b5b09940a3 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/ColumnIncluderator.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/ColumnIncluderator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/client/src/main/java/com/metamx/druid/query/metadata/ListColumnIncluderator.java b/processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/metadata/ListColumnIncluderator.java rename to processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java index d6f4b99d74b..89b5cf8dd6b 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/ListColumnIncluderator.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/ListColumnIncluderator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/metadata/NoneColumnIncluderator.java b/processing/src/main/java/io/druid/query/metadata/metadata/NoneColumnIncluderator.java similarity index 91% rename from client/src/main/java/com/metamx/druid/query/metadata/NoneColumnIncluderator.java rename to processing/src/main/java/io/druid/query/metadata/metadata/NoneColumnIncluderator.java index d1d66d26778..afe36427be4 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/NoneColumnIncluderator.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/NoneColumnIncluderator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; /** */ diff --git a/client/src/main/java/com/metamx/druid/query/metadata/SegmentAnalysis.java b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentAnalysis.java similarity index 95% rename from client/src/main/java/com/metamx/druid/query/metadata/SegmentAnalysis.java rename to processing/src/main/java/io/druid/query/metadata/metadata/SegmentAnalysis.java index 359e1020273..077fe3c99c6 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/SegmentAnalysis.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentAnalysis.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQuery.java b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java similarity index 91% rename from client/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQuery.java rename to processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java index 4be24fdaf95..37361815968 100644 --- a/client/src/main/java/com/metamx/druid/query/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/io/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata.metadata; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.BaseQuery; -import com.metamx.druid.Query; -import com.metamx.druid.query.segment.QuerySegmentSpec; +import io.druid.query.BaseQuery; +import io.druid.query.Query; +import io.druid.query.spec.QuerySegmentSpec; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/result/BySegmentSearchResultValue.java b/processing/src/main/java/io/druid/query/search/BySegmentSearchResultValue.java similarity index 91% rename from client/src/main/java/com/metamx/druid/result/BySegmentSearchResultValue.java rename to processing/src/main/java/io/druid/query/search/BySegmentSearchResultValue.java index 64d2d6a7ec8..0ecbdf509c2 100644 --- a/client/src/main/java/com/metamx/druid/result/BySegmentSearchResultValue.java +++ b/processing/src/main/java/io/druid/query/search/BySegmentSearchResultValue.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.result; +package io.druid.query.search; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; -import com.metamx.druid.query.search.SearchHit; +import io.druid.query.BySegmentResultValue; +import io.druid.query.Result; +import io.druid.query.search.search.SearchHit; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/SearchBinaryFn.java b/processing/src/main/java/io/druid/query/search/SearchBinaryFn.java similarity index 87% rename from client/src/main/java/com/metamx/druid/SearchBinaryFn.java rename to processing/src/main/java/io/druid/query/search/SearchBinaryFn.java index b9ebe74b705..99093911739 100644 --- a/client/src/main/java/com/metamx/druid/SearchBinaryFn.java +++ b/processing/src/main/java/io/druid/query/search/SearchBinaryFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,16 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query.search; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchSortSpec; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; +import io.druid.granularity.AllGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Result; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.SearchSortSpec; import java.util.TreeSet; diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java similarity index 87% rename from client/src/main/java/com/metamx/druid/query/search/SearchQueryQueryToolChest.java rename to processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java index f664faaa134..0fdefff0175 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/search/SearchQueryQueryToolChest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; @@ -28,30 +28,28 @@ import com.google.common.collect.Lists; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; +import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.Query; -import com.metamx.druid.ResultGranularTimestampComparator; -import com.metamx.druid.SearchBinaryFn; -import com.metamx.druid.collect.OrderedMergeSequence; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.query.CacheStrategy; -import com.metamx.druid.query.IntervalChunkingQueryRunner; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.ResultMergeQueryRunner; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.result.BySegmentSearchResultValue; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; -import com.metamx.druid.utils.PropUtils; import com.metamx.emitter.service.ServiceMetricEvent; - +import io.druid.collections.OrderedMergeSequence; +import io.druid.query.CacheStrategy; +import io.druid.query.IntervalChunkingQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.ResultGranularTimestampComparator; +import io.druid.query.ResultMergeQueryRunner; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.filter.DimFilter; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQueryConfig; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Minutes; @@ -61,7 +59,6 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; import java.util.List; import java.util.Map; -import java.util.Properties; import java.util.Set; /** @@ -71,23 +68,19 @@ public class SearchQueryQueryToolChest extends QueryToolChest> TYPE_REFERENCE = new TypeReference>() + private static final TypeReference> TYPE_REFERENCE = new TypeReference>(){}; + + private static final TypeReference OBJECT_TYPE_REFERENCE = new TypeReference(){}; + private final SearchQueryConfig config; + + @Inject + public SearchQueryQueryToolChest( + SearchQueryConfig config + ) { - }; - - private static final int maxSearchLimit; - - static { - // I dislike this static loading of properies, but it's the only mechanism available right now. - Properties props = Initialization.loadProperties(); - - maxSearchLimit = PropUtils.getPropertyAsInt(props, "com.metamx.query.search.maxSearchLimit", 1000); + this.config = config; } - private static final TypeReference OBJECT_TYPE_REFERENCE = new TypeReference() - { - }; - @Override public QueryRunner> mergeResults(QueryRunner> runner) { @@ -264,15 +257,23 @@ public class SearchQueryQueryToolChest extends QueryToolChest> preMergeQueryDecoration(QueryRunner> runner) { return new SearchThresholdAdjustingQueryRunner( - new IntervalChunkingQueryRunner>(runner, Period.months(1)) + new IntervalChunkingQueryRunner>(runner, Period.months(1)), + config ); } private static class SearchThresholdAdjustingQueryRunner implements QueryRunner> { private final QueryRunner> runner; + private final SearchQueryConfig config; - public SearchThresholdAdjustingQueryRunner(QueryRunner> runner) {this.runner = runner;} + public SearchThresholdAdjustingQueryRunner( + QueryRunner> runner, + SearchQueryConfig config + ) { + this.runner = runner; + this.config = config; + } @Override public Sequence> run(Query> input) @@ -282,14 +283,14 @@ public class SearchQueryQueryToolChest extends QueryToolChest, Result>() { @Override diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java new file mode 100644 index 00000000000..66547623b2a --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunner.java @@ -0,0 +1,194 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.search; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.metamx.common.ISE; +import com.metamx.common.guava.FunctionalIterable; +import com.metamx.common.guava.Sequence; +import com.metamx.common.guava.Sequences; +import com.metamx.emitter.EmittingLogger; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.Result; +import io.druid.query.filter.Filter; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQuerySpec; +import io.druid.segment.ColumnSelectorBitmapIndexSelector; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.filter.Filters; +import it.uniroma3.mat.extendedset.intset.ConciseSet; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +/** +*/ +public class SearchQueryRunner implements QueryRunner> +{ + private static final EmittingLogger log = new EmittingLogger(SearchQueryRunner.class); + + private final Segment segment; + + public SearchQueryRunner(Segment segment) + { + this.segment = segment; + } + + @Override + public Sequence> run(final Query> input) + { + if (!(input instanceof SearchQuery)) { + throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); + } + + final SearchQuery query = (SearchQuery) input; + final Filter filter = Filters.convertDimensionFilters(query.getDimensionsFilter()); + final List dimensions = query.getDimensions(); + final SearchQuerySpec searchQuerySpec = query.getQuery(); + final int limit = query.getLimit(); + + final QueryableIndex index = segment.asQueryableIndex(); + if (index != null) { + final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); + + Iterable dimsToSearch; + if (dimensions == null || dimensions.isEmpty()) { + dimsToSearch = index.getAvailableDimensions(); + } else { + dimsToSearch = dimensions; + } + + + final ImmutableConciseSet baseFilter; + if (filter == null) { + // Accept all, and work around https://github.com/metamx/extendedset/issues/1 + if (index.getNumRows() == 1) { + ConciseSet set = new ConciseSet(); + set.add(0); + baseFilter = ImmutableConciseSet.newImmutableFromMutable(set); + } + else { + baseFilter = ImmutableConciseSet.complement(new ImmutableConciseSet(), index.getNumRows()); + } + } + else { + baseFilter = filter.goConcise(new ColumnSelectorBitmapIndexSelector(index)); + } + + for (String dimension : dimsToSearch) { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + continue; + } + + final BitmapIndex bitmapIndex = column.getBitmapIndex(); + if (bitmapIndex != null) { + for (int i = 0; i < bitmapIndex.getCardinality(); ++i) { + String dimVal = Strings.nullToEmpty(bitmapIndex.getValue(i)); + if (searchQuerySpec.accept(dimVal) && + ImmutableConciseSet.intersection(baseFilter, bitmapIndex.getConciseSet(i)).size() > 0) { + retVal.add(new SearchHit(dimension, dimVal)); + if (retVal.size() >= limit) { + return makeReturnResult(limit, retVal); + } + } + } + } + } + + return makeReturnResult(limit, retVal); + } + + final StorageAdapter adapter = segment.asStorageAdapter(); + if (adapter != null) { + Iterable dimsToSearch; + if (dimensions == null || dimensions.isEmpty()) { + dimsToSearch = adapter.getAvailableDimensions(); + } else { + dimsToSearch = dimensions; + } + + final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); + + final Iterable cursors = adapter.makeCursors(filter, segment.getDataInterval(), QueryGranularity.ALL); + for (Cursor cursor : cursors) { + Map dimSelectors = Maps.newHashMap(); + for (String dim : dimsToSearch) { + dimSelectors.put(dim, cursor.makeDimensionSelector(dim)); + } + + while (!cursor.isDone()) { + for (Map.Entry entry : dimSelectors.entrySet()) { + final DimensionSelector selector = entry.getValue(); + final IndexedInts vals = selector.getRow(); + for (int i = 0; i < vals.size(); ++i) { + final String dimVal = selector.lookupName(vals.get(i)); + if (searchQuerySpec.accept(dimVal)) { + retVal.add(new SearchHit(entry.getKey(), dimVal)); + if (retVal.size() >= limit) { + return makeReturnResult(limit, retVal); + } + } + } + } + + cursor.advance(); + } + } + + return makeReturnResult(limit, retVal); + } + + log.makeAlert("WTF!? Unable to process search query on segment.") + .addData("segment", segment.getIdentifier()) + .addData("query", query); + return Sequences.empty(); + } + + private Sequence> makeReturnResult(int limit, TreeSet retVal) + { + return Sequences.simple( + ImmutableList.of( + new Result( + segment.getDataInterval().getStart(), + new SearchResultValue( + Lists.newArrayList(new FunctionalIterable(retVal).limit(limit)) + ) + ) + ) + ); + } +} diff --git a/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java new file mode 100644 index 00000000000..0c0fdab980d --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/SearchQueryRunnerFactory.java @@ -0,0 +1,68 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.search; + +import com.google.inject.Inject; +import io.druid.query.ChainedExecutionQueryRunner; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.search.search.SearchQuery; +import io.druid.segment.Segment; + +import java.util.concurrent.ExecutorService; + +/** + */ +public class SearchQueryRunnerFactory implements QueryRunnerFactory, SearchQuery> +{ + private final SearchQueryQueryToolChest toolChest; + + @Inject + public SearchQueryRunnerFactory( + SearchQueryQueryToolChest toolChest + ) + { + this.toolChest = toolChest; + } + + @Override + public QueryRunner> createRunner(final Segment segment) + { + return new SearchQueryRunner(segment); + } + + @Override + public QueryRunner> mergeRunners( + ExecutorService queryExecutor, Iterable>> queryRunners + ) + { + return new ChainedExecutionQueryRunner>( + queryExecutor, toolChest.getOrdering(), queryRunners + ); + } + + @Override + public QueryToolChest, SearchQuery> getToolchest() + { + return toolChest; + } +} diff --git a/client/src/main/java/com/metamx/druid/result/SearchResultValue.java b/processing/src/main/java/io/druid/query/search/SearchResultValue.java similarity index 93% rename from client/src/main/java/com/metamx/druid/result/SearchResultValue.java rename to processing/src/main/java/io/druid/query/search/SearchResultValue.java index 1403d111945..1ffbf510c96 100644 --- a/client/src/main/java/com/metamx/druid/result/SearchResultValue.java +++ b/processing/src/main/java/io/druid/query/search/SearchResultValue.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.result; +package io.druid.query.search; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; -import com.metamx.druid.query.search.SearchHit; +import io.druid.query.search.search.SearchHit; import java.util.Iterator; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java b/processing/src/main/java/io/druid/query/search/search/FragmentSearchQuerySpec.java similarity index 84% rename from client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java rename to processing/src/main/java/io/druid/query/search/search/FragmentSearchQuerySpec.java index c4a7b707712..6f5092d6d85 100644 --- a/client/src/main/java/com/metamx/druid/query/search/FragmentSearchQuerySpec.java +++ b/processing/src/main/java/io/druid/query/search/search/FragmentSearchQuerySpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -34,12 +34,10 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec private static final byte CACHE_TYPE_ID = 0x2; private final List values; - private final SearchSortSpec sortSpec; @JsonCreator public FragmentSearchQuerySpec( - @JsonProperty("values") List values, - @JsonProperty("sort") SearchSortSpec sortSpec + @JsonProperty("values") List values ) { this.values = Lists.transform( @@ -53,7 +51,6 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec } } ); - this.sortSpec = (sortSpec == null) ? new LexicographicSearchSortSpec() : sortSpec; } @JsonProperty @@ -62,13 +59,6 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec return values; } - @JsonProperty("sort") - @Override - public SearchSortSpec getSearchSortSpec() - { - return sortSpec; - } - @Override public boolean accept(String dimVal) { @@ -107,7 +97,6 @@ public class FragmentSearchQuerySpec implements SearchQuerySpec { return "FragmentSearchQuerySpec{" + "values=" + values + - ", sortSpec=" + sortSpec + "}"; } } diff --git a/client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java b/processing/src/main/java/io/druid/query/search/search/InsensitiveContainsSearchQuerySpec.java similarity index 79% rename from client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java rename to processing/src/main/java/io/druid/query/search/search/InsensitiveContainsSearchQuerySpec.java index 80c609cae76..acc23576fea 100644 --- a/client/src/main/java/com/metamx/druid/query/search/InsensitiveContainsSearchQuerySpec.java +++ b/processing/src/main/java/io/druid/query/search/search/InsensitiveContainsSearchQuerySpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -31,16 +31,13 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec private static final byte CACHE_TYPE_ID = 0x1; private final String value; - private final SearchSortSpec sortSpec; @JsonCreator public InsensitiveContainsSearchQuerySpec( - @JsonProperty("value") String value, - @JsonProperty("sort") SearchSortSpec sortSpec + @JsonProperty("value") String value ) { this.value = value.toLowerCase(); - this.sortSpec = (sortSpec == null) ? new LexicographicSearchSortSpec() : sortSpec; } @JsonProperty @@ -49,13 +46,6 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec return value; } - @JsonProperty("sort") - @Override - public SearchSortSpec getSearchSortSpec() - { - return sortSpec; - } - @Override public boolean accept(String dimVal) { @@ -81,7 +71,6 @@ public class InsensitiveContainsSearchQuerySpec implements SearchQuerySpec { return "InsensitiveContainsSearchQuerySpec{" + "value=" + value + - ", sortSpec=" + sortSpec + "}"; } } diff --git a/client/src/main/java/com/metamx/druid/query/search/LexicographicSearchSortSpec.java b/processing/src/main/java/io/druid/query/search/search/LexicographicSearchSortSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/search/LexicographicSearchSortSpec.java rename to processing/src/main/java/io/druid/query/search/search/LexicographicSearchSortSpec.java index 99ac489cadd..588aca7493a 100644 --- a/client/src/main/java/com/metamx/druid/query/search/LexicographicSearchSortSpec.java +++ b/processing/src/main/java/io/druid/query/search/search/LexicographicSearchSortSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchHit.java b/processing/src/main/java/io/druid/query/search/search/SearchHit.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/search/SearchHit.java rename to processing/src/main/java/io/druid/query/search/search/SearchHit.java index 1b2acad6f16..d9deba2b639 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchHit.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchHit.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchQuery.java b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java similarity index 90% rename from client/src/main/java/com/metamx/druid/query/search/SearchQuery.java rename to processing/src/main/java/io/druid/query/search/search/SearchQuery.java index 9a0d57cb82d..0ca276b53a6 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchQuery.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.metamx.druid.BaseQuery; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; +import io.druid.granularity.QueryGranularity; +import io.druid.query.BaseQuery; +import io.druid.query.Query; +import io.druid.query.Result; +import io.druid.query.filter.DimFilter; +import io.druid.query.search.SearchResultValue; +import io.druid.query.spec.QuerySegmentSpec; import javax.annotation.Nullable; import java.util.List; @@ -62,7 +62,7 @@ public class SearchQuery extends BaseQuery> { super(dataSource, querySegmentSpec, context); this.dimFilter = dimFilter; - this.sortSpec = sortSpec; + this.sortSpec = sortSpec == null ? new LexicographicSearchSortSpec() : sortSpec; this.granularity = granularity == null ? QueryGranularity.ALL : granularity; this.limit = (limit == 0) ? 1000 : limit; this.dimensions = (dimensions == null) ? null : Lists.transform( @@ -159,7 +159,7 @@ public class SearchQuery extends BaseQuery> @JsonProperty("sort") public SearchSortSpec getSort() { - return sortSpec == null ? querySpec.getSearchSortSpec() : sortSpec; + return sortSpec; } public SearchQuery withLimit(int newLimit) diff --git a/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java new file mode 100644 index 00000000000..611fecc2c11 --- /dev/null +++ b/processing/src/main/java/io/druid/query/search/search/SearchQueryConfig.java @@ -0,0 +1,38 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.search.search; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +/** + */ +public class SearchQueryConfig +{ + @JsonProperty + @Min(1) + private int maxSearchLimit = 1000; + + public int getMaxSearchLimit() + { + return maxSearchLimit; + } +} diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java b/processing/src/main/java/io/druid/query/search/search/SearchQuerySpec.java similarity index 72% rename from client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java rename to processing/src/main/java/io/druid/query/search/search/SearchQuerySpec.java index 68eb86dfb73..1a2b67cd4b5 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchQuerySpec.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchQuerySpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -31,18 +31,6 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; }) public interface SearchQuerySpec { - /** - * Deprecated! - * - * This has been moved to the SearchQuery and is only still here for backwards compatibility purposes. Search - * queries should be adjusted to use the sort parameter on the SearchQuery object itself rather than on this - * object. This method will eventually go away. - * - * @return - */ - @Deprecated - public SearchSortSpec getSearchSortSpec(); - public boolean accept(String dimVal); public byte[] getCacheKey(); diff --git a/client/src/main/java/com/metamx/druid/query/search/SearchSortSpec.java b/processing/src/main/java/io/druid/query/search/search/SearchSortSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/search/SearchSortSpec.java rename to processing/src/main/java/io/druid/query/search/search/SearchSortSpec.java index 7510cb582e1..e2e721a4acf 100644 --- a/client/src/main/java/com/metamx/druid/query/search/SearchSortSpec.java +++ b/processing/src/main/java/io/druid/query/search/search/SearchSortSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/client/src/main/java/com/metamx/druid/query/search/StrlenSearchSortSpec.java b/processing/src/main/java/io/druid/query/search/search/StrlenSearchSortSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/search/StrlenSearchSortSpec.java rename to processing/src/main/java/io/druid/query/search/search/StrlenSearchSortSpec.java index e269c241f2d..517463ab9fe 100644 --- a/client/src/main/java/com/metamx/druid/query/search/StrlenSearchSortSpec.java +++ b/processing/src/main/java/io/druid/query/search/search/StrlenSearchSortSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search.search; import java.util.Comparator; diff --git a/client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java b/processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java similarity index 98% rename from client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java rename to processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java index c67094a285f..3585ce55db4 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/LegacySegmentSpec.java +++ b/processing/src/main/java/io/druid/query/spec/LegacySegmentSpec.java @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; import com.fasterxml.jackson.annotation.JsonCreator; import com.google.common.base.Function; diff --git a/client/src/main/java/com/metamx/druid/query/segment/MultipleIntervalSegmentSpec.java b/processing/src/main/java/io/druid/query/spec/MultipleIntervalSegmentSpec.java similarity index 88% rename from client/src/main/java/com/metamx/druid/query/segment/MultipleIntervalSegmentSpec.java rename to processing/src/main/java/io/druid/query/spec/MultipleIntervalSegmentSpec.java index 51b77c7efc6..07d4436416c 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/MultipleIntervalSegmentSpec.java +++ b/processing/src/main/java/io/druid/query/spec/MultipleIntervalSegmentSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.Query; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.utils.JodaUtils; - +import io.druid.common.utils.JodaUtils; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; import org.joda.time.Interval; import java.util.Collections; diff --git a/client/src/main/java/com/metamx/druid/query/segment/MultipleSpecificSegmentSpec.java b/processing/src/main/java/io/druid/query/spec/MultipleSpecificSegmentSpec.java similarity index 89% rename from client/src/main/java/com/metamx/druid/query/segment/MultipleSpecificSegmentSpec.java rename to processing/src/main/java/io/druid/query/spec/MultipleSpecificSegmentSpec.java index 281f3222676..59dda1e0d67 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/MultipleSpecificSegmentSpec.java +++ b/processing/src/main/java/io/druid/query/spec/MultipleSpecificSegmentSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Function; import com.google.common.collect.Iterables; -import com.metamx.druid.Query; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.utils.JodaUtils; - +import io.druid.common.utils.JodaUtils; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/segment/QuerySegmentSpec.java b/processing/src/main/java/io/druid/query/spec/QuerySegmentSpec.java similarity index 88% rename from client/src/main/java/com/metamx/druid/query/segment/QuerySegmentSpec.java rename to processing/src/main/java/io/druid/query/spec/QuerySegmentSpec.java index 0498deb16f1..2a5014d76f2 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/QuerySegmentSpec.java +++ b/processing/src/main/java/io/druid/query/spec/QuerySegmentSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.Query; -import com.metamx.druid.query.QueryRunner; - +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; import org.joda.time.Interval; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/segment/QuerySegmentSpecs.java b/processing/src/main/java/io/druid/query/spec/QuerySegmentSpecs.java similarity index 93% rename from client/src/main/java/com/metamx/druid/query/segment/QuerySegmentSpecs.java rename to processing/src/main/java/io/druid/query/spec/QuerySegmentSpecs.java index 9516db4c508..bfb3fe77a3c 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/QuerySegmentSpecs.java +++ b/processing/src/main/java/io/druid/query/spec/QuerySegmentSpecs.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; import org.joda.time.Interval; diff --git a/client/src/main/java/com/metamx/druid/query/segment/SpecificSegmentQueryRunner.java b/processing/src/main/java/io/druid/query/spec/SpecificSegmentQueryRunner.java similarity index 96% rename from client/src/main/java/com/metamx/druid/query/segment/SpecificSegmentQueryRunner.java rename to processing/src/main/java/io/druid/query/spec/SpecificSegmentQueryRunner.java index f5289185646..e9bcd73139d 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/SpecificSegmentQueryRunner.java +++ b/processing/src/main/java/io/druid/query/spec/SpecificSegmentQueryRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; import com.google.common.base.Throwables; import com.metamx.common.guava.Accumulator; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Yielder; import com.metamx.common.guava.YieldingAccumulator; -import com.metamx.druid.Query; -import com.metamx.druid.query.QueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; import java.io.IOException; import java.util.concurrent.Callable; diff --git a/client/src/main/java/com/metamx/druid/query/segment/SpecificSegmentSpec.java b/processing/src/main/java/io/druid/query/spec/SpecificSegmentSpec.java similarity index 85% rename from client/src/main/java/com/metamx/druid/query/segment/SpecificSegmentSpec.java rename to processing/src/main/java/io/druid/query/spec/SpecificSegmentSpec.java index c85ef86aa2a..be48d55e651 100644 --- a/client/src/main/java/com/metamx/druid/query/segment/SpecificSegmentSpec.java +++ b/processing/src/main/java/io/druid/query/spec/SpecificSegmentSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.segment; +package io.druid.query.spec; -import com.metamx.druid.Query; -import com.metamx.druid.query.QueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; import java.util.Arrays; diff --git a/client/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java similarity index 92% rename from client/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQuery.java rename to processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java index de846108e40..957390acf15 100644 --- a/client/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,18 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeboundary; +package io.druid.query.timeboundary; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.druid.BaseQuery; -import com.metamx.druid.Query; -import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeBoundaryResultValue; - +import io.druid.query.BaseQuery; +import io.druid.query.Query; +import io.druid.query.Result; +import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.query.spec.QuerySegmentSpec; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/client/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryQueryToolChest.java similarity index 91% rename from client/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryQueryToolChest.java rename to processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryQueryToolChest.java index a5409b60887..ba2ab0feb9e 100644 --- a/client/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryQueryToolChest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeboundary; +package io.druid.query.timeboundary; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; @@ -29,17 +29,16 @@ import com.google.common.collect.Ordering; import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.LogicalSegment; -import com.metamx.druid.Query; -import com.metamx.druid.collect.OrderedMergeSequence; -import com.metamx.druid.query.BySegmentSkippingQueryRunner; -import com.metamx.druid.query.CacheStrategy; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeBoundaryResultValue; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.collections.OrderedMergeSequence; +import io.druid.query.BySegmentSkippingQueryRunner; +import io.druid.query.CacheStrategy; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.timeline.LogicalSegment; import org.joda.time.DateTime; import javax.annotation.Nullable; diff --git a/server/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java similarity index 86% rename from server/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java rename to processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java index 6c40e6774d9..cd732de18e4 100644 --- a/server/src/main/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeboundary; +package io.druid.query.timeboundary; import com.metamx.common.ISE; import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.Sequence; -import com.metamx.druid.Query; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.Segment; -import com.metamx.druid.query.ChainedExecutionQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeBoundaryResultValue; +import io.druid.query.ChainedExecutionQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; import java.util.Iterator; import java.util.concurrent.ExecutorService; diff --git a/client/src/main/java/com/metamx/druid/result/TimeBoundaryResultValue.java b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryResultValue.java similarity index 94% rename from client/src/main/java/com/metamx/druid/result/TimeBoundaryResultValue.java rename to processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryResultValue.java index 455119dcd28..1210e95af2c 100644 --- a/client/src/main/java/com/metamx/druid/result/TimeBoundaryResultValue.java +++ b/processing/src/main/java/io/druid/query/timeboundary/TimeBoundaryResultValue.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.result; +package io.druid.query.timeboundary; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; import com.metamx.common.IAE; -import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; - import org.joda.time.DateTime; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/TimeseriesBinaryFn.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesBinaryFn.java similarity index 88% rename from client/src/main/java/com/metamx/druid/TimeseriesBinaryFn.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesBinaryFn.java index 0824366f259..f8530ce334f 100644 --- a/client/src/main/java/com/metamx/druid/TimeseriesBinaryFn.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesBinaryFn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query.timeseries; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.AllGranularity; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.granularity.AllGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; import java.util.LinkedHashMap; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java similarity index 87% rename from client/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQuery.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java index 10fd403ef83..ab5b649a896 100644 --- a/client/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,22 +17,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeseries; +package io.druid.query.timeseries; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.collect.ImmutableList; -import com.metamx.druid.BaseQuery; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.query.Queries; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.granularity.QueryGranularity; +import io.druid.query.BaseQuery; +import io.druid.query.Queries; +import io.druid.query.Query; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.filter.DimFilter; +import io.druid.query.spec.QuerySegmentSpec; import java.util.List; import java.util.Map; diff --git a/server/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryEngine.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java similarity index 84% rename from server/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryEngine.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java index c95b7acf5df..b57105aa654 100644 --- a/server/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryEngine.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryEngine.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,21 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeseries; +package io.druid.query.timeseries; import com.google.common.base.Function; import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.Sequence; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.TimeseriesResultBuilder; -import com.metamx.druid.aggregation.Aggregator; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.index.brita.Filters; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.query.QueryRunnerHelper; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.query.QueryRunnerHelper; +import io.druid.query.Result; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.segment.Cursor; +import io.druid.segment.StorageAdapter; +import io.druid.segment.filter.Filters; import java.util.Iterator; import java.util.List; @@ -83,7 +81,7 @@ public class TimeseriesQueryEngine Result retVal = bob.build(); // cleanup - for(Aggregator agg : aggregators) { + for (Aggregator agg : aggregators) { agg.close(); } diff --git a/client/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java similarity index 89% rename from client/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryQueryToolChest.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java index c731bf1b166..71f5755449d 100644 --- a/client/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryQueryToolChest.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryQueryToolChest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeseries; +package io.druid.query.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; @@ -28,25 +28,22 @@ import com.google.common.collect.Ordering; import com.metamx.common.guava.MergeSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.ResultGranularTimestampComparator; -import com.metamx.druid.TimeseriesBinaryFn; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.collect.OrderedMergeSequence; -import com.metamx.druid.query.CacheStrategy; -import com.metamx.druid.query.IntervalChunkingQueryRunner; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryCacheHelper; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.ResultMergeQueryRunner; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; import com.metamx.emitter.service.ServiceMetricEvent; - +import io.druid.collections.OrderedMergeSequence; +import io.druid.granularity.QueryGranularity; +import io.druid.query.CacheStrategy; +import io.druid.query.IntervalChunkingQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryCacheHelper; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.ResultGranularTimestampComparator; +import io.druid.query.ResultMergeQueryRunner; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.filter.DimFilter; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Minutes; @@ -65,12 +62,11 @@ public class TimeseriesQueryQueryToolChest extends QueryToolChest> TYPE_REFERENCE = new TypeReference>() - { - }; - private static final TypeReference OBJECT_TYPE_REFERENCE = new TypeReference() - { - }; + private static final TypeReference OBJECT_TYPE_REFERENCE = + new TypeReference(){}; + + private static final TypeReference> TYPE_REFERENCE = + new TypeReference>() {}; @Override public QueryRunner> mergeResults(QueryRunner> queryRunner) diff --git a/server/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerFactory.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java similarity index 83% rename from server/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerFactory.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java index 9e60d1d49fa..8cd4f2aa1c0 100644 --- a/server/src/main/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerFactory.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesQueryRunnerFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,18 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeseries; +package io.druid.query.timeseries; import com.metamx.common.ISE; import com.metamx.common.guava.Sequence; -import com.metamx.druid.Query; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.Segment; -import com.metamx.druid.query.ChainedExecutionQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.query.ChainedExecutionQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; import java.util.concurrent.ExecutorService; diff --git a/server/src/main/java/com/metamx/druid/TimeseriesResultBuilder.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesResultBuilder.java similarity index 85% rename from server/src/main/java/com/metamx/druid/TimeseriesResultBuilder.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesResultBuilder.java index 076f40413bd..3df014deeec 100644 --- a/server/src/main/java/com/metamx/druid/TimeseriesResultBuilder.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesResultBuilder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query.timeseries; -import com.metamx.druid.aggregation.Aggregator; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.query.Result; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.PostAggregator; import org.joda.time.DateTime; import java.util.HashMap; diff --git a/client/src/main/java/com/metamx/druid/result/TimeseriesResultValue.java b/processing/src/main/java/io/druid/query/timeseries/TimeseriesResultValue.java similarity index 89% rename from client/src/main/java/com/metamx/druid/result/TimeseriesResultValue.java rename to processing/src/main/java/io/druid/query/timeseries/TimeseriesResultValue.java index add4bfe6b62..511494c04de 100644 --- a/client/src/main/java/com/metamx/druid/result/TimeseriesResultValue.java +++ b/processing/src/main/java/io/druid/query/timeseries/TimeseriesResultValue.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.result; +package io.druid.query.timeseries; import com.fasterxml.jackson.annotation.JsonCreator; +import io.druid.query.MetricValueExtractor; import java.util.Map; diff --git a/server/src/main/java/com/metamx/druid/Capabilities.java b/processing/src/main/java/io/druid/segment/Capabilities.java similarity index 89% rename from server/src/main/java/com/metamx/druid/Capabilities.java rename to processing/src/main/java/io/druid/segment/Capabilities.java index 7b70a8fbc1d..ca2764d1c9a 100644 --- a/server/src/main/java/com/metamx/druid/Capabilities.java +++ b/processing/src/main/java/io/druid/segment/Capabilities.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.segment; /** */ @@ -54,9 +54,9 @@ public class Capabilities return this; } - public Capabilities build() + public io.druid.segment.Capabilities build() { - return new Capabilities( + return new io.druid.segment.Capabilities( dimensionValuesSorted ); } diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelector.java similarity index 88% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ColumnSelector.java rename to processing/src/main/java/io/druid/segment/ColumnSelector.java index 0ed66850dcb..0710effb9d2 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment;import io.druid.segment.column.Column; /** */ diff --git a/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java new file mode 100644 index 00000000000..fc124621cb8 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorBitmapIndexSelector.java @@ -0,0 +1,142 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment; + +import com.google.common.io.Closeables; +import com.metamx.collections.spatial.ImmutableRTree; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.segment.column.Column; +import io.druid.segment.column.DictionaryEncodedColumn; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedIterable; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; + +import java.util.Iterator; + +/** +*/ +public class ColumnSelectorBitmapIndexSelector implements BitmapIndexSelector +{ + private final ColumnSelector index; + + public ColumnSelectorBitmapIndexSelector( + final ColumnSelector index + ) + { + this.index = index; + } + + @Override + public Indexed getDimensionValues(String dimension) + { + final Column columnDesc = index.getColumn(dimension.toLowerCase()); + if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) { + return null; + } + final DictionaryEncodedColumn column = columnDesc.getDictionaryEncoding(); + return new Indexed() + { + @Override + public Class getClazz() + { + return String.class; + } + + @Override + public int size() + { + return column.getCardinality(); + } + + @Override + public String get(int index) + { + return column.lookupName(index); + } + + @Override + public int indexOf(String value) + { + return column.lookupId(value); + } + + @Override + public Iterator iterator() + { + return IndexedIterable.create(this).iterator(); + } + }; + } + + @Override + public int getNumRows() + { + GenericColumn column = null; + try { + column = index.getTimeColumn().getGenericColumn(); + return column.length(); + } + finally { + Closeables.closeQuietly(column); + } + } + + @Override + public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + return new ImmutableConciseSet(); + } + if (!column.getCapabilities().hasBitmapIndexes()) { + return new ImmutableConciseSet(); + } + + return column.getBitmapIndex().getConciseSet(value); + } + + @Override + public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null) { + return new ImmutableConciseSet(); + } + if (!column.getCapabilities().hasBitmapIndexes()) { + return new ImmutableConciseSet(); + } + // This is a workaround given the current state of indexing, I feel shame + final int index1 = column.getBitmapIndex().hasNulls() ? idx + 1 : idx; + + return column.getBitmapIndex().getConciseSet(index1); + } + + @Override + public ImmutableRTree getSpatialIndex(String dimension) + { + final Column column = index.getColumn(dimension.toLowerCase()); + if (column == null || !column.getCapabilities().hasSpatialIndexes()) { + return new ImmutableRTree(); + } + + return column.getSpatialIndex().getRTree(); + } +} diff --git a/common/src/main/java/com/metamx/druid/processing/ColumnSelectorFactory.java b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java similarity index 80% rename from common/src/main/java/com/metamx/druid/processing/ColumnSelectorFactory.java rename to processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java index 250ab22ab11..df00a592b33 100644 --- a/common/src/main/java/com/metamx/druid/processing/ColumnSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/ColumnSelectorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.processing; +package io.druid.segment; /** * Factory class for MetricSelectors */ public interface ColumnSelectorFactory { - public FloatMetricSelector makeFloatMetricSelector(String metricName); - public ComplexMetricSelector makeComplexMetricSelector(String metricName); + public FloatColumnSelector makeFloatColumnSelector(String columnName); public ObjectColumnSelector makeObjectColumnSelector(String columnName); } diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedPools.java b/processing/src/main/java/io/druid/segment/CompressedPools.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedPools.java rename to processing/src/main/java/io/druid/segment/CompressedPools.java index a3061b67030..cccf216367e 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedPools.java +++ b/processing/src/main/java/io/druid/segment/CompressedPools.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.base.Supplier; import com.metamx.common.logger.Logger; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidPool; import com.ning.compress.lzf.ChunkEncoder; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidPool; import java.nio.ByteBuffer; import java.nio.ByteOrder; diff --git a/server/src/main/java/com/metamx/druid/index/v1/ConciseOffset.java b/processing/src/main/java/io/druid/segment/ConciseOffset.java similarity index 93% rename from server/src/main/java/com/metamx/druid/index/v1/ConciseOffset.java rename to processing/src/main/java/io/druid/segment/ConciseOffset.java index b08449e687c..2fd9120f22a 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/ConciseOffset.java +++ b/processing/src/main/java/io/druid/segment/ConciseOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; -import com.metamx.druid.index.v1.processing.Offset; +import io.druid.segment.data.Offset; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import it.uniroma3.mat.extendedset.intset.IntSet; diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/Cursor.java b/processing/src/main/java/io/druid/segment/Cursor.java similarity index 84% rename from server/src/main/java/com/metamx/druid/index/v1/processing/Cursor.java rename to processing/src/main/java/io/druid/segment/Cursor.java index c304d66b0d6..2ccf03827ff 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/Cursor.java +++ b/processing/src/main/java/io/druid/segment/Cursor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; - -import com.metamx.druid.processing.ColumnSelectorFactory; -import org.joda.time.DateTime; +package io.druid.segment;import org.joda.time.DateTime; /** */ diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/CursorFactory.java b/processing/src/main/java/io/druid/segment/CursorFactory.java similarity index 84% rename from server/src/main/java/com/metamx/druid/index/v1/processing/CursorFactory.java rename to processing/src/main/java/io/druid/segment/CursorFactory.java index b18fec4b322..05a72bad543 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/CursorFactory.java +++ b/processing/src/main/java/io/druid/segment/CursorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; - -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.index.brita.Filter; +package io.druid.segment;import io.druid.granularity.QueryGranularity; +import io.druid.query.filter.Filter; import org.joda.time.Interval; /** diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/DimensionSelector.java b/processing/src/main/java/io/druid/segment/DimensionSelector.java similarity index 93% rename from server/src/main/java/com/metamx/druid/index/v1/processing/DimensionSelector.java rename to processing/src/main/java/io/druid/segment/DimensionSelector.java index 8ced8f388dd..3922d345049 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/DimensionSelector.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; - -import com.metamx.druid.kv.IndexedInts; +package io.druid.segment;import io.druid.segment.data.IndexedInts; /** */ diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/DimensionSelectorFactory.java b/processing/src/main/java/io/druid/segment/DimensionSelectorFactory.java similarity index 90% rename from server/src/main/java/com/metamx/druid/index/v1/processing/DimensionSelectorFactory.java rename to processing/src/main/java/io/druid/segment/DimensionSelectorFactory.java index 9af5988a3ec..b4a1f8ca2fb 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/DimensionSelectorFactory.java +++ b/processing/src/main/java/io/druid/segment/DimensionSelectorFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; - -/** +package io.druid.segment;/** */ public interface DimensionSelectorFactory { diff --git a/common/src/main/java/com/metamx/druid/processing/FloatMetricSelector.java b/processing/src/main/java/io/druid/segment/FloatColumnSelector.java similarity index 85% rename from common/src/main/java/com/metamx/druid/processing/FloatMetricSelector.java rename to processing/src/main/java/io/druid/segment/FloatColumnSelector.java index d9a79603f33..2ff2ebca9d4 100644 --- a/common/src/main/java/com/metamx/druid/processing/FloatMetricSelector.java +++ b/processing/src/main/java/io/druid/segment/FloatColumnSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.processing; +package io.druid.segment; /** * An object that gets a metric value. Metric values are always floats and there is an assumption that the - * FloatMetricSelector has a handle onto some other stateful object (e.g. an Offset) which is changing between calls + * FloatColumnSelector has a handle onto some other stateful object (e.g. an Offset) which is changing between calls * to get() (though, that doesn't have to be the case if you always want the same value...). */ -public interface FloatMetricSelector +public interface FloatColumnSelector { public float get(); } diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/FloatMetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java similarity index 91% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/FloatMetricColumnSerializer.java rename to processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java index 20ec5a4d30d..4caa6b95fe2 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/FloatMetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/FloatMetricColumnSerializer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.io.Files; -import com.metamx.druid.kv.IOPeon; +import io.druid.segment.data.CompressedFloatsSupplierSerializer; +import io.druid.segment.data.IOPeon; import java.io.File; import java.io.IOException; diff --git a/server/src/main/java/com/metamx/druid/index/IncrementalIndexSegment.java b/processing/src/main/java/io/druid/segment/IncrementalIndexSegment.java similarity index 86% rename from server/src/main/java/com/metamx/druid/index/IncrementalIndexSegment.java rename to processing/src/main/java/io/druid/segment/IncrementalIndexSegment.java index 6e2d312aa75..9e04e2af836 100644 --- a/server/src/main/java/com/metamx/druid/index/IncrementalIndexSegment.java +++ b/processing/src/main/java/io/druid/segment/IncrementalIndexSegment.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.v1.IncrementalIndex; -import com.metamx.druid.index.v1.IncrementalIndexStorageAdapter; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexStorageAdapter; import org.joda.time.Interval; import java.io.IOException; diff --git a/server/src/main/java/com/metamx/druid/index/v1/IndexGranularity.java b/processing/src/main/java/io/druid/segment/IndexGranularity.java similarity index 99% rename from server/src/main/java/com/metamx/druid/index/v1/IndexGranularity.java rename to processing/src/main/java/io/druid/segment/IndexGranularity.java index 67ac3810534..29b3564b4fc 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IndexGranularity.java +++ b/processing/src/main/java/io/druid/segment/IndexGranularity.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.fasterxml.jackson.annotation.JsonCreator; -import com.metamx.druid.QueryGranularity; +import io.druid.granularity.QueryGranularity; import org.joda.time.DateTime; import org.joda.time.Days; import org.joda.time.Hours; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java b/processing/src/main/java/io/druid/segment/IndexIO.java similarity index 88% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java rename to processing/src/main/java/io/druid/segment/IndexIO.java index 07008e0a42b..990893c4b13 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/IndexIO.java +++ b/processing/src/main/java/io/druid/segment/IndexIO.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; @@ -39,32 +39,32 @@ import com.metamx.common.io.smoosh.Smoosh; import com.metamx.common.io.smoosh.SmooshedFileMapper; import com.metamx.common.io.smoosh.SmooshedWriter; import com.metamx.common.logger.Logger; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.SimpleQueryableIndex; -import com.metamx.druid.index.column.Column; -import com.metamx.druid.index.column.ColumnBuilder; -import com.metamx.druid.index.column.ColumnDescriptor; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.index.serde.BitmapIndexColumnPartSupplier; -import com.metamx.druid.index.serde.ComplexColumnPartSerde; -import com.metamx.druid.index.serde.ComplexColumnPartSupplier; -import com.metamx.druid.index.serde.DictionaryEncodedColumnPartSerde; -import com.metamx.druid.index.serde.DictionaryEncodedColumnSupplier; -import com.metamx.druid.index.serde.FloatGenericColumnPartSerde; -import com.metamx.druid.index.serde.FloatGenericColumnSupplier; -import com.metamx.druid.index.serde.LongGenericColumnPartSerde; -import com.metamx.druid.index.serde.LongGenericColumnSupplier; -import com.metamx.druid.index.serde.SpatialIndexColumnPartSupplier; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.kv.ArrayIndexed; -import com.metamx.druid.kv.ByteBufferSerializer; -import com.metamx.druid.kv.ConciseCompressedIndexedInts; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedIterable; -import com.metamx.druid.kv.IndexedRTree; -import com.metamx.druid.kv.VSizeIndexed; -import com.metamx.druid.kv.VSizeIndexedInts; -import com.metamx.druid.utils.SerializerUtils; +import com.metamx.emitter.EmittingLogger; +import io.druid.common.utils.SerializerUtils; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.column.ColumnDescriptor; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.ArrayIndexed; +import io.druid.segment.data.ByteBufferSerializer; +import io.druid.segment.data.CompressedLongsIndexedSupplier; +import io.druid.segment.data.ConciseCompressedIndexedInts; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.IndexedIterable; +import io.druid.segment.data.IndexedRTree; +import io.druid.segment.data.VSizeIndexed; +import io.druid.segment.data.VSizeIndexedInts; +import io.druid.segment.serde.BitmapIndexColumnPartSupplier; +import io.druid.segment.serde.ComplexColumnPartSerde; +import io.druid.segment.serde.ComplexColumnPartSupplier; +import io.druid.segment.serde.DictionaryEncodedColumnPartSerde; +import io.druid.segment.serde.DictionaryEncodedColumnSupplier; +import io.druid.segment.serde.FloatGenericColumnPartSerde; +import io.druid.segment.serde.FloatGenericColumnSupplier; +import io.druid.segment.serde.LongGenericColumnPartSerde; +import io.druid.segment.serde.LongGenericColumnSupplier; +import io.druid.segment.serde.SpatialIndexColumnPartSupplier; import it.uniroma3.mat.extendedset.intset.ConciseSet; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.joda.time.Interval; @@ -105,7 +105,7 @@ public class IndexIO .put(9, new V9IndexLoader()) .build(); - private static final Logger log = new Logger(IndexIO.class); + private static final EmittingLogger log = new EmittingLogger(IndexIO.class); private static final SerializerUtils serializerUtils = new SerializerUtils(); public static final ByteOrder BYTE_ORDER = ByteOrder.nativeOrder(); @@ -115,18 +115,6 @@ public class IndexIO private static volatile IndexIOHandler handler = null; public static final int CURRENT_VERSION_ID = V9_VERSION; - public static Index readIndex(File inDir) throws IOException - { - init(); - return handler.readIndex(inDir); - } - - public static boolean canBeMapped(final File inDir) throws IOException - { - init(); - return handler.canBeMapped(inDir); - } - @Deprecated public static MMappedIndex mapDir(final File inDir) throws IOException { @@ -148,11 +136,6 @@ public class IndexIO } } - public static void storeLatest(Index index, File file) throws IOException - { - handler.storeLatest(index, file); - } - public static boolean hasHandler() { return (IndexIO.handler != null); @@ -210,16 +193,10 @@ public class IndexIO case 1: case 2: case 3: - final String mappableDirName = "mappable"; - if (toConvert.getName().equals(mappableDirName)) { - throw new ISE("Infinite recursion at play! OMFG quit it, please, it hurts!"); - } - - File mappable = new File(toConvert, mappableDirName); - final Index index = readIndex(toConvert); - storeLatest(index, mappable); - - return convertSegment(mappable, converted); + log.makeAlert("Attempt to load segment of version <= 3.") + .addData("version", version) + .emit(); + return false; case 4: case 5: case 6: @@ -241,51 +218,13 @@ public class IndexIO public static interface IndexIOHandler { - /** - * This only exists for some legacy compatibility reasons, Metamarkets is working on getting rid of it in - * future versions - * - * @param inDir - * - * @return - */ - public Index readIndex(File inDir) throws IOException; - - /** - * This should really always return true, but it exists for legacy compatibility reasons, Metamarkets - * is working on getting rid of it in future versions - * - * @return - */ - public boolean canBeMapped(File inDir) throws IOException; - public MMappedIndex mapDir(File inDir) throws IOException; - - /** - * This only exists for some legacy compatibility reasons, Metamarkets is working on getting rid of it in - * future versions. Normal persisting of indexes is done via IndexMerger. - * - * @param file - */ - public void storeLatest(Index index, File file) throws IOException; } public static class DefaultIndexIOHandler implements IndexIOHandler { private static final Logger log = new Logger(DefaultIndexIOHandler.class); - @Override - public Index readIndex(File inDir) - { - throw new UnsupportedOperationException("Shouldn't ever happen in a cluster that is not owned by MMX."); - } - - @Override - public boolean canBeMapped(File inDir) - { - return true; - } - @Override public MMappedIndex mapDir(File inDir) throws IOException { @@ -384,12 +323,6 @@ public class IndexIO return retVal; } - @Override - public void storeLatest(Index index, File file) - { - throw new UnsupportedOperationException("Shouldn't ever happen in a cluster that is not owned by MMX."); - } - public static void convertV8toV9(File v8Dir, File v9Dir) throws IOException { log.info("Converting v8[%s] to v9[%s]", v8Dir, v9Dir); diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java b/processing/src/main/java/io/druid/segment/IndexMerger.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java rename to processing/src/main/java/io/druid/segment/IndexMerger.java index e662bf0224a..672cec724b5 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/IndexMerger.java +++ b/processing/src/main/java/io/druid/segment/IndexMerger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.base.Splitter; -import com.google.common.base.Stopwatch; import com.google.common.collect.Iterables; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; @@ -44,27 +43,30 @@ import com.metamx.common.guava.MergeIterable; import com.metamx.common.guava.nary.BinaryFn; import com.metamx.common.io.smoosh.Smoosh; import com.metamx.common.logger.Logger; -import com.metamx.druid.CombiningIterable; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.ToLowerCaseAggregatorFactory; -import com.metamx.druid.guava.FileOutputSupplier; -import com.metamx.druid.guava.GuavaUtils; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.index.v1.serde.ComplexMetrics; -import com.metamx.druid.kv.ByteBufferWriter; -import com.metamx.druid.kv.ConciseCompressedIndexedInts; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.IOPeon; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.kv.IndexedIterable; -import com.metamx.druid.kv.IndexedRTree; -import com.metamx.druid.kv.TmpFileIOPeon; -import com.metamx.druid.kv.VSizeIndexedWriter; -import com.metamx.druid.utils.JodaUtils; -import com.metamx.druid.utils.SerializerUtils; +import io.druid.collections.CombiningIterable; +import io.druid.common.guava.FileOutputSupplier; +import io.druid.common.guava.GuavaUtils; +import io.druid.common.utils.JodaUtils; +import io.druid.common.utils.SerializerUtils; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.ToLowerCaseAggregatorFactory; +import io.druid.segment.data.ByteBufferWriter; +import io.druid.segment.data.CompressedLongsSupplierSerializer; +import io.druid.segment.data.ConciseCompressedIndexedInts; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.GenericIndexedWriter; +import io.druid.segment.data.IOPeon; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedIterable; +import io.druid.segment.data.IndexedRTree; +import io.druid.segment.data.TmpFileIOPeon; +import io.druid.segment.data.VSizeIndexedWriter; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexAdapter; +import io.druid.segment.serde.ComplexMetricColumnSerializer; +import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.ComplexMetrics; import it.uniroma3.mat.extendedset.intset.ConciseSet; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.apache.commons.io.FileUtils; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/IndexableAdapter.java b/processing/src/main/java/io/druid/segment/IndexableAdapter.java similarity index 88% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/IndexableAdapter.java rename to processing/src/main/java/io/druid/segment/IndexableAdapter.java index 5c65a969957..a9c6f07f7a9 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/IndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/IndexableAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedInts; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; import org.joda.time.Interval; /** diff --git a/server/src/main/java/com/metamx/druid/index/v1/IndexedIntsOffset.java b/processing/src/main/java/io/druid/segment/IndexedIntsOffset.java similarity index 89% rename from server/src/main/java/com/metamx/druid/index/v1/IndexedIntsOffset.java rename to processing/src/main/java/io/druid/segment/IndexedIntsOffset.java index b258313c972..1a4b9e8868d 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IndexedIntsOffset.java +++ b/processing/src/main/java/io/druid/segment/IndexedIntsOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; -import com.metamx.druid.index.v1.processing.Offset; -import com.metamx.druid.kv.IndexedInts; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.Offset; /** */ diff --git a/processing/src/main/java/io/druid/segment/MMappedIndex.java b/processing/src/main/java/io/druid/segment/MMappedIndex.java new file mode 100644 index 00000000000..5f422ac1a84 --- /dev/null +++ b/processing/src/main/java/io/druid/segment/MMappedIndex.java @@ -0,0 +1,178 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment; + +import com.google.common.collect.Maps; +import com.metamx.collections.spatial.ImmutableRTree; +import com.metamx.common.io.smoosh.SmooshedFileMapper; +import com.metamx.common.logger.Logger; +import io.druid.segment.data.CompressedLongsIndexedSupplier; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.IndexedLongs; +import io.druid.segment.data.VSizeIndexed; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import org.joda.time.Interval; + +import java.io.IOException; +import java.util.Map; + +/** + */ +public class MMappedIndex +{ + private static final Logger log = new Logger(MMappedIndex.class); + private static final ImmutableConciseSet emptySet = new ImmutableConciseSet(); + + final GenericIndexed availableDimensions; + final GenericIndexed availableMetrics; + final Interval dataInterval; + final CompressedLongsIndexedSupplier timestamps; + final Map metrics; + final Map> dimValueLookups; + final Map dimColumns; + final Map> invertedIndexes; + final Map spatialIndexes; + final SmooshedFileMapper fileMapper; + + private final Map metricIndexes = Maps.newHashMap(); + + public MMappedIndex( + GenericIndexed availableDimensions, + GenericIndexed availableMetrics, + Interval dataInterval, + CompressedLongsIndexedSupplier timestamps, + Map metrics, + Map> dimValueLookups, + Map dimColumns, + Map> invertedIndexes, + Map spatialIndexes, + SmooshedFileMapper fileMapper + ) + { + this.availableDimensions = availableDimensions; + this.availableMetrics = availableMetrics; + this.dataInterval = dataInterval; + this.timestamps = timestamps; + this.metrics = metrics; + this.dimValueLookups = dimValueLookups; + this.dimColumns = dimColumns; + this.invertedIndexes = invertedIndexes; + this.spatialIndexes = spatialIndexes; + this.fileMapper = fileMapper; + + for (int i = 0; i < availableMetrics.size(); i++) { + metricIndexes.put(availableMetrics.get(i), i); + } + } + + public CompressedLongsIndexedSupplier getTimestamps() + { + return timestamps; + } + + public GenericIndexed getAvailableDimensions() + { + return availableDimensions; + } + + public GenericIndexed getAvailableMetrics() + { + return availableMetrics; + } + + public Map getMetrics() + { + return metrics; + } + + public Integer getMetricIndex(String metricName) + { + return metricIndexes.get(metricName); + } + + public Interval getDataInterval() + { + return dataInterval; + } + + public IndexedLongs getReadOnlyTimestamps() + { + return timestamps.get(); + } + + public MetricHolder getMetricHolder(String metric) + { + final MetricHolder retVal = metrics.get(metric); + + if (retVal == null) { + return null; + } + + return retVal; + } + + public GenericIndexed getDimValueLookup(String dimension) + { + return dimValueLookups.get(dimension); + } + + public VSizeIndexed getDimColumn(String dimension) + { + return dimColumns.get(dimension); + } + + public Map> getInvertedIndexes() + { + return invertedIndexes; + } + + public Map getSpatialIndexes() + { + return spatialIndexes; + } + + public ImmutableConciseSet getInvertedIndex(String dimension, String value) + { + final GenericIndexed lookup = dimValueLookups.get(dimension); + if (lookup == null) { + return emptySet; + } + + int indexOf = lookup.indexOf(value); + if (indexOf < 0) { + return emptySet; + } + + ImmutableConciseSet retVal = invertedIndexes.get(dimension).get(indexOf); + return (retVal == null) ? emptySet : retVal; + } + + public SmooshedFileMapper getFileMapper() + { + return fileMapper; + } + + public void close() throws IOException + { + if (fileMapper != null) { + fileMapper.close(); + } + } +} diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndexAdapter.java b/processing/src/main/java/io/druid/segment/MMappedIndexAdapter.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndexAdapter.java rename to processing/src/main/java/io/druid/segment/MMappedIndexAdapter.java index 8ea0dc21572..0db39fb9261 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/MMappedIndexAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.collect.Maps; import com.google.common.io.Closeables; -import com.metamx.druid.kv.ConciseCompressedIndexedInts; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedFloats; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.segment.data.ConciseCompressedIndexedInts; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedLongs; import org.joda.time.Interval; import java.util.Iterator; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/MetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/MetricColumnSerializer.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/MetricColumnSerializer.java rename to processing/src/main/java/io/druid/segment/MetricColumnSerializer.java index 2452f820b25..e55b8e38bc6 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/MetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/MetricColumnSerializer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import java.io.IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/MetricHolder.java b/processing/src/main/java/io/druid/segment/MetricHolder.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/MetricHolder.java rename to processing/src/main/java/io/druid/segment/MetricHolder.java index dc7002af24b..648f2fbe70c 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/MetricHolder.java +++ b/processing/src/main/java/io/druid/segment/MetricHolder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; @@ -25,14 +25,16 @@ import com.google.common.io.InputSupplier; import com.google.common.io.OutputSupplier; import com.metamx.common.IAE; import com.metamx.common.ISE; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.index.v1.serde.ComplexMetrics; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedFloats; -import com.metamx.druid.kv.ObjectStrategy; -import com.metamx.druid.utils.SerializerUtils; +import io.druid.common.utils.SerializerUtils; +import io.druid.segment.data.CompressedFloatsIndexedSupplier; +import io.druid.segment.data.CompressedFloatsSupplierSerializer; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.GenericIndexedWriter; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.ObjectStrategy; +import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.ComplexMetrics; import java.io.IOException; import java.io.InputStream; diff --git a/common/src/main/java/com/metamx/druid/processing/ObjectColumnSelector.java b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java similarity index 91% rename from common/src/main/java/com/metamx/druid/processing/ObjectColumnSelector.java rename to processing/src/main/java/io/druid/segment/ObjectColumnSelector.java index b91a3e0b4d1..215c86493e5 100644 --- a/common/src/main/java/com/metamx/druid/processing/ObjectColumnSelector.java +++ b/processing/src/main/java/io/druid/segment/ObjectColumnSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2013 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.processing; +package io.druid.segment; public interface ObjectColumnSelector { diff --git a/indexing-common/src/main/java/com/metamx/druid/index/QueryableIndex.java b/processing/src/main/java/io/druid/segment/QueryableIndex.java similarity index 84% rename from indexing-common/src/main/java/com/metamx/druid/index/QueryableIndex.java rename to processing/src/main/java/io/druid/segment/QueryableIndex.java index 8e429d45879..7dc1dcbe5d4 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/QueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndex.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; - -import com.metamx.druid.index.column.ColumnSelector; -import com.metamx.druid.kv.Indexed; +package io.druid.segment;import io.druid.segment.data.Indexed; import org.joda.time.Interval; -import java.io.Closeable; import java.io.IOException; /** @@ -37,7 +33,7 @@ public interface QueryableIndex extends ColumnSelector /** * The close method shouldn't actually be here as this is nasty. We will adjust it in the future. - * @throws IOException + * @throws java.io.IOException */ @Deprecated public void close() throws IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/QueryableIndexIndexableAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/QueryableIndexIndexableAdapter.java rename to processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java index 4f5633543b7..efe09b51f55 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/QueryableIndexIndexableAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexIndexableAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -25,20 +25,19 @@ import com.google.common.collect.Sets; import com.google.common.io.Closeables; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.column.BitmapIndex; -import com.metamx.druid.index.column.Column; -import com.metamx.druid.index.column.ComplexColumn; -import com.metamx.druid.index.column.DictionaryEncodedColumn; -import com.metamx.druid.index.column.GenericColumn; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.kv.ArrayBasedIndexedInts; -import com.metamx.druid.kv.ConciseCompressedIndexedInts; -import com.metamx.druid.kv.EmptyIndexedInts; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.kv.IndexedIterable; -import com.metamx.druid.kv.ListIndexed; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.column.Column; +import io.druid.segment.column.ComplexColumn; +import io.druid.segment.column.DictionaryEncodedColumn; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.ArrayBasedIndexedInts; +import io.druid.segment.data.ConciseCompressedIndexedInts; +import io.druid.segment.data.EmptyIndexedInts; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedIterable; +import io.druid.segment.data.ListIndexed; import org.joda.time.Interval; import java.io.Closeable; diff --git a/server/src/main/java/com/metamx/druid/index/QueryableIndexSegment.java b/processing/src/main/java/io/druid/segment/QueryableIndexSegment.java similarity index 89% rename from server/src/main/java/com/metamx/druid/index/QueryableIndexSegment.java rename to processing/src/main/java/io/druid/segment/QueryableIndexSegment.java index 118dd9ab285..c2a7ee18aa6 100644 --- a/server/src/main/java/com/metamx/druid/index/QueryableIndexSegment.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexSegment.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.v1.QueryableIndexStorageAdapter; import org.joda.time.Interval; import java.io.IOException; diff --git a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java similarity index 76% rename from server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java rename to processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java index 9c95eb27ed4..443c9cb353f 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/QueryableIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/QueryableIndexStorageAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,41 +17,28 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.base.Function; import com.google.common.base.Functions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.io.Closeables; -import com.metamx.collections.spatial.ImmutableRTree; import com.metamx.common.collect.MoreIterators; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.guava.FunctionalIterator; -import com.metamx.druid.BaseStorageAdapter; -import com.metamx.druid.Capabilities; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.brita.BitmapIndexSelector; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.column.Column; -import com.metamx.druid.index.column.ColumnCapabilities; -import com.metamx.druid.index.column.ColumnSelector; -import com.metamx.druid.index.column.ComplexColumn; -import com.metamx.druid.index.column.DictionaryEncodedColumn; -import com.metamx.druid.index.column.GenericColumn; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.index.v1.processing.DimensionSelector; -import com.metamx.druid.index.v1.processing.Offset; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.kv.IndexedIterable; -import com.metamx.druid.kv.SingleIndexedInts; -import com.metamx.druid.processing.ComplexMetricSelector; -import com.metamx.druid.processing.FloatMetricSelector; -import com.metamx.druid.processing.ObjectColumnSelector; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import io.druid.granularity.QueryGranularity; +import io.druid.query.filter.Filter; +import io.druid.segment.column.Column; +import io.druid.segment.column.ColumnCapabilities; +import io.druid.segment.column.ComplexColumn; +import io.druid.segment.column.DictionaryEncodedColumn; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.Offset; +import io.druid.segment.data.SingleIndexedInts; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -61,7 +48,7 @@ import java.util.Map; /** */ -public class QueryableIndexStorageAdapter extends BaseStorageAdapter +public class QueryableIndexStorageAdapter implements StorageAdapter { private final QueryableIndex index; @@ -84,6 +71,12 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return index.getDataInterval(); } + @Override + public Indexed getAvailableDimensions() + { + return index.getAvailableDimensions(); + } + @Override public int getDimensionCardinality(String dimension) { @@ -155,7 +148,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter if (filter == null) { iterable = new NoFilterCursorIterable(index, actualInterval, gran); } else { - Offset offset = new ConciseOffset(filter.goConcise(new MMappedBitmapIndexSelector(index))); + Offset offset = new ConciseOffset(filter.goConcise(new ColumnSelectorBitmapIndexSelector(index))); iterable = new CursorIterable(index, actualInterval, gran, offset); } @@ -163,102 +156,6 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return FunctionalIterable.create(iterable).keep(Functions.identity()); } - @Override - public Indexed getAvailableDimensions() - { - return index.getAvailableDimensions(); - } - - @Override - public Indexed getDimValueLookup(String dimension) - { - final Column column = index.getColumn(dimension.toLowerCase()); - - if (column == null || !column.getCapabilities().isDictionaryEncoded()) { - return null; - } - - final DictionaryEncodedColumn dictionary = column.getDictionaryEncoding(); - return new Indexed() - { - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return dictionary.getCardinality(); - } - - @Override - public String get(int index) - { - return dictionary.lookupName(index); - } - - @Override - public int indexOf(String value) - { - return dictionary.lookupId(value); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - }; - } - - @Override - public ImmutableConciseSet getInvertedIndex(String dimension, String dimVal) - { - final Column column = index.getColumn(dimension.toLowerCase()); - if (column == null) { - return new ImmutableConciseSet(); - } - if (!column.getCapabilities().hasBitmapIndexes()) { - return new ImmutableConciseSet(); - } - - return column.getBitmapIndex().getConciseSet(dimVal); - } - - @Override - public ImmutableConciseSet getInvertedIndex(String dimension, int idx) - { - final Column column = index.getColumn(dimension.toLowerCase()); - if (column == null) { - return new ImmutableConciseSet(); - } - if (!column.getCapabilities().hasBitmapIndexes()) { - return new ImmutableConciseSet(); - } - // This is a workaround given the current state of indexing, I feel shame - final int index = column.getBitmapIndex().hasNulls() ? idx + 1 : idx; - - return column.getBitmapIndex().getConciseSet(index); - } - - public ImmutableRTree getRTreeSpatialIndex(String dimension) - { - final Column column = index.getColumn(dimension.toLowerCase()); - if (column == null || !column.getCapabilities().hasSpatialIndexes()) { - return new ImmutableRTree(); - } - - return column.getSpatialIndex().getRTree(); - } - - @Override - public Offset getFilterOffset(Filter filter) - { - return new ConciseOffset(filter.goConcise(new MMappedBitmapIndexSelector(index))); - } - private static class CursorIterable implements Iterable { private final ColumnSelector index; @@ -412,9 +309,9 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter } @Override - public FloatMetricSelector makeFloatMetricSelector(String metric) + public FloatColumnSelector makeFloatColumnSelector(String columnName) { - final String metricName = metric.toLowerCase(); + final String metricName = columnName.toLowerCase(); GenericColumn cachedMetricVals = genericColumnCache.get(metricName); if (cachedMetricVals == null) { @@ -426,7 +323,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter } if (cachedMetricVals == null) { - return new FloatMetricSelector() + return new FloatColumnSelector() { @Override public float get() @@ -437,7 +334,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter } final GenericColumn metricVals = cachedMetricVals; - return new FloatMetricSelector() + return new FloatColumnSelector() { @Override public float get() @@ -447,41 +344,6 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter }; } - @Override - public ComplexMetricSelector makeComplexMetricSelector(String metric) - { - final String metricName = metric.toLowerCase(); - ComplexColumn cachedMetricVals = complexColumnCache.get(metricName); - - if (cachedMetricVals == null) { - Column holder = index.getColumn(metricName); - if (holder != null && holder.getCapabilities().getType() == ValueType.COMPLEX) { - cachedMetricVals = holder.getComplexColumn(); - complexColumnCache.put(metricName, cachedMetricVals); - } - } - - if (cachedMetricVals == null) { - return null; - } - - final ComplexColumn metricVals = cachedMetricVals; - return new ComplexMetricSelector() - { - @Override - public Class classOfObject() - { - return metricVals.getClazz(); - } - - @Override - public Object get() - { - return metricVals.getRowValue(cursorOffset.getOffset()); - } - }; - } - @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { @@ -831,9 +693,9 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter } @Override - public FloatMetricSelector makeFloatMetricSelector(String metric) + public FloatColumnSelector makeFloatColumnSelector(String columnName) { - final String metricName = metric.toLowerCase(); + final String metricName = columnName.toLowerCase(); GenericColumn cachedMetricVals = genericColumnCache.get(metricName); if (cachedMetricVals == null) { @@ -845,7 +707,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter } if (cachedMetricVals == null) { - return new FloatMetricSelector() + return new FloatColumnSelector() { @Override public float get() @@ -856,7 +718,7 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter } final GenericColumn metricVals = cachedMetricVals; - return new FloatMetricSelector() + return new FloatColumnSelector() { @Override public float get() @@ -866,41 +728,6 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter }; } - @Override - public ComplexMetricSelector makeComplexMetricSelector(String metric) - { - final String metricName = metric.toLowerCase(); - ComplexColumn cachedMetricVals = complexColumnCache.get(metricName); - - if (cachedMetricVals == null) { - Column holder = index.getColumn(metricName); - if (holder != null && holder.getCapabilities().getType() == ValueType.COMPLEX) { - cachedMetricVals = holder.getComplexColumn(); - complexColumnCache.put(metricName, cachedMetricVals); - } - } - - if (cachedMetricVals == null) { - return null; - } - - final ComplexColumn metricVals = cachedMetricVals; - return new ComplexMetricSelector() - { - @Override - public Class classOfObject() - { - return metricVals.getClazz(); - } - - @Override - public Object get() - { - return metricVals.getRowValue(currRow); - } - }; - } - @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { @@ -1081,87 +908,4 @@ public class QueryableIndexStorageAdapter extends BaseStorageAdapter return 0; } } - - private class MMappedBitmapIndexSelector implements BitmapIndexSelector - { - private final ColumnSelector index; - - public MMappedBitmapIndexSelector(final ColumnSelector index) - { - this.index = index; - } - - @Override - public Indexed getDimensionValues(String dimension) - { - final Column columnDesc = index.getColumn(dimension.toLowerCase()); - if (columnDesc == null || !columnDesc.getCapabilities().isDictionaryEncoded()) { - return null; - } - final DictionaryEncodedColumn column = columnDesc.getDictionaryEncoding(); - return new Indexed() - { - @Override - public Class getClazz() - { - return String.class; - } - - @Override - public int size() - { - return column.getCardinality(); - } - - @Override - public String get(int index) - { - return column.lookupName(index); - } - - @Override - public int indexOf(String value) - { - return column.lookupId(value); - } - - @Override - public Iterator iterator() - { - return IndexedIterable.create(this).iterator(); - } - }; - } - - @Override - public int getNumRows() - { - GenericColumn column = null; - try { - column = index.getTimeColumn().getGenericColumn(); - return column.length(); - } - finally { - Closeables.closeQuietly(column); - } - } - - @Override - public ImmutableConciseSet getConciseInvertedIndex(String dimension, String value) - { - return getInvertedIndex(dimension, value); - } - - @Override - public ImmutableConciseSet getConciseInvertedIndex(String dimension, int idx) - { - return getInvertedIndex(dimension, idx); - } - - @Override - public ImmutableRTree getSpatialIndex(String dimension) - { - return getRTreeSpatialIndex(dimension); - } - } } diff --git a/server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java b/processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java similarity index 97% rename from server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java rename to processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java index 967537a1dbb..3ed246e4edf 100644 --- a/server/src/main/java/com/metamx/druid/index/ReferenceCountingSegment.java +++ b/processing/src/main/java/io/druid/segment/ReferenceCountingSegment.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; -import com.metamx.druid.StorageAdapter; import com.metamx.emitter.EmittingLogger; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/index/ReferenceCountingSequence.java b/processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java similarity index 93% rename from server/src/main/java/com/metamx/druid/index/ReferenceCountingSequence.java rename to processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java index 58aeb068f90..2f51e4ddc10 100644 --- a/server/src/main/java/com/metamx/druid/index/ReferenceCountingSequence.java +++ b/processing/src/main/java/io/druid/segment/ReferenceCountingSequence.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; import com.metamx.common.guava.ResourceClosingYielder; import com.metamx.common.guava.Sequence; @@ -25,8 +25,6 @@ import com.metamx.common.guava.Yielder; import com.metamx.common.guava.YieldingAccumulator; import com.metamx.common.guava.YieldingSequenceBase; -import java.io.Closeable; - /** */ public class ReferenceCountingSequence extends YieldingSequenceBase diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/Rowboat.java b/processing/src/main/java/io/druid/segment/Rowboat.java similarity index 97% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/Rowboat.java rename to processing/src/main/java/io/druid/segment/Rowboat.java index b4615a4ef93..f6367ffcd61 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/Rowboat.java +++ b/processing/src/main/java/io/druid/segment/Rowboat.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.collect.Maps; import com.google.common.collect.Sets; diff --git a/server/src/main/java/com/metamx/druid/index/v1/RowboatFilteringIndexAdapter.java b/processing/src/main/java/io/druid/segment/RowboatFilteringIndexAdapter.java similarity index 93% rename from server/src/main/java/com/metamx/druid/index/v1/RowboatFilteringIndexAdapter.java rename to processing/src/main/java/io/druid/segment/RowboatFilteringIndexAdapter.java index 95061ae39a8..7cfa6c9c217 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/RowboatFilteringIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/RowboatFilteringIndexAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.base.Predicate; import com.google.common.collect.Iterables; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedInts; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; import org.joda.time.Interval; /** diff --git a/server/src/main/java/com/metamx/druid/index/Segment.java b/processing/src/main/java/io/druid/segment/Segment.java similarity index 89% rename from server/src/main/java/com/metamx/druid/index/Segment.java rename to processing/src/main/java/io/druid/segment/Segment.java index 9b9e6c94cd1..78c34541ebc 100644 --- a/server/src/main/java/com/metamx/druid/index/Segment.java +++ b/processing/src/main/java/io/druid/segment/Segment.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; -import com.metamx.druid.StorageAdapter; import org.joda.time.Interval; import java.io.Closeable; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/SimpleQueryableIndex.java b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/index/SimpleQueryableIndex.java rename to processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java index f9898d3a607..2e3bc85958f 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/SimpleQueryableIndex.java +++ b/processing/src/main/java/io/druid/segment/SimpleQueryableIndex.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; import com.metamx.common.io.smoosh.SmooshedFileMapper; -import com.metamx.druid.index.column.Column; -import com.metamx.druid.kv.Indexed; +import io.druid.segment.column.Column; +import io.druid.segment.data.Indexed; import org.joda.time.Interval; import java.io.IOException; diff --git a/server/src/main/java/com/metamx/druid/StorageAdapter.java b/processing/src/main/java/io/druid/segment/StorageAdapter.java similarity index 81% rename from server/src/main/java/com/metamx/druid/StorageAdapter.java rename to processing/src/main/java/io/druid/segment/StorageAdapter.java index 285d17b5b05..ef6ed7c6bd6 100644 --- a/server/src/main/java/com/metamx/druid/StorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/StorageAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; - -import com.metamx.druid.index.v1.Searchable; -import com.metamx.druid.index.v1.processing.CursorFactory; +package io.druid.segment;import io.druid.segment.data.Indexed; import org.joda.time.DateTime; import org.joda.time.Interval; /** */ -public interface StorageAdapter extends CursorFactory, Searchable +public interface StorageAdapter extends CursorFactory { public String getSegmentIdentifier(); public Interval getInterval(); + public Indexed getAvailableDimensions(); public int getDimensionCardinality(String dimension); public DateTime getMinTime(); public DateTime getMaxTime(); diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java b/processing/src/main/java/io/druid/segment/column/AbstractColumn.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java rename to processing/src/main/java/io/druid/segment/column/AbstractColumn.java index c44c59a5e47..b43f5119c3e 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/AbstractColumn.java +++ b/processing/src/main/java/io/druid/segment/column/AbstractColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java b/processing/src/main/java/io/druid/segment/column/BitmapIndex.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java rename to processing/src/main/java/io/druid/segment/column/BitmapIndex.java index ae9ae8a17ed..573b38982ce 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/BitmapIndex.java +++ b/processing/src/main/java/io/druid/segment/column/BitmapIndex.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/Column.java b/processing/src/main/java/io/druid/segment/column/Column.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/index/column/Column.java rename to processing/src/main/java/io/druid/segment/column/Column.java index 8de06439af7..14de694e149 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/Column.java +++ b/processing/src/main/java/io/druid/segment/column/Column.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java similarity index 97% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java rename to processing/src/main/java/io/druid/segment/column/ColumnBuilder.java index 180099f108f..ba8c329b3ac 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnBuilder.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnBuilder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java rename to processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java index 3fbd439ffca..fbb3e87fd31 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilities.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilities.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java rename to processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java index 1e1554c030d..f7b09d698c1 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnCapabilitiesImpl.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnCapabilitiesImpl.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java rename to processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java index 72918aee9fa..e352d1d161e 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ColumnDescriptor.java +++ b/processing/src/main/java/io/druid/segment/column/ColumnDescriptor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.metamx.common.IAE; -import com.metamx.druid.index.serde.ColumnPartSerde; +import io.druid.segment.serde.ColumnPartSerde; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ComplexColumn.java b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java similarity index 91% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ComplexColumn.java rename to processing/src/main/java/io/druid/segment/column/ComplexColumn.java index f7cfb706e44..c4dcad927e4 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/ComplexColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; import java.io.Closeable; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ComplexColumnImpl.java b/processing/src/main/java/io/druid/segment/column/ComplexColumnImpl.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/column/ComplexColumnImpl.java rename to processing/src/main/java/io/druid/segment/column/ComplexColumnImpl.java index 46f665c57c6..98f80288151 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ComplexColumnImpl.java +++ b/processing/src/main/java/io/druid/segment/column/ComplexColumnImpl.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.Indexed; +import io.druid.segment.data.Indexed; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/DictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java similarity index 89% rename from indexing-common/src/main/java/com/metamx/druid/index/column/DictionaryEncodedColumn.java rename to processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java index 62057cb8e10..4925df0f62d 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/DictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/DictionaryEncodedColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.IndexedInts; +import io.druid.segment.data.IndexedInts; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/FloatColumn.java b/processing/src/main/java/io/druid/segment/column/FloatColumn.java similarity index 90% rename from indexing-common/src/main/java/com/metamx/druid/index/column/FloatColumn.java rename to processing/src/main/java/io/druid/segment/column/FloatColumn.java index 44ffd7e970a..ee72160274f 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/FloatColumn.java +++ b/processing/src/main/java/io/druid/segment/column/FloatColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.index.v1.CompressedFloatsIndexedSupplier; +import io.druid.segment.data.CompressedFloatsIndexedSupplier; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/GenericColumn.java b/processing/src/main/java/io/druid/segment/column/GenericColumn.java similarity index 85% rename from indexing-common/src/main/java/com/metamx/druid/index/column/GenericColumn.java rename to processing/src/main/java/io/druid/segment/column/GenericColumn.java index 530eb2fc232..8afa8624599 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/GenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/GenericColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedFloats; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedLongs; import java.io.Closeable; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/IndexedComplexColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedComplexColumn.java similarity index 91% rename from indexing-common/src/main/java/com/metamx/druid/index/column/IndexedComplexColumn.java rename to processing/src/main/java/io/druid/segment/column/IndexedComplexColumn.java index bafb6977dd0..a48f2f2659c 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/IndexedComplexColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedComplexColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.Indexed; +import io.druid.segment.data.Indexed; import java.io.IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/IndexedFloatsGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java similarity index 90% rename from indexing-common/src/main/java/com/metamx/druid/index/column/IndexedFloatsGenericColumn.java rename to processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java index 5df228435c1..8aa260e4e53 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/IndexedFloatsGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedFloatsGenericColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedFloats; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedLongs; import java.io.IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/IndexedLongsGenericColumn.java b/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java similarity index 90% rename from indexing-common/src/main/java/com/metamx/druid/index/column/IndexedLongsGenericColumn.java rename to processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java index 211dab35a19..368e7b11ebe 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/IndexedLongsGenericColumn.java +++ b/processing/src/main/java/io/druid/segment/column/IndexedLongsGenericColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedFloats; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedFloats; +import io.druid.segment.data.IndexedLongs; import java.io.IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/LongColumn.java b/processing/src/main/java/io/druid/segment/column/LongColumn.java similarity index 90% rename from indexing-common/src/main/java/com/metamx/druid/index/column/LongColumn.java rename to processing/src/main/java/io/druid/segment/column/LongColumn.java index 1ec297ea61c..7923c0c303c 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/LongColumn.java +++ b/processing/src/main/java/io/druid/segment/column/LongColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.index.v1.CompressedLongsIndexedSupplier; +import io.druid.segment.data.CompressedLongsIndexedSupplier; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/RunLengthColumn.java b/processing/src/main/java/io/druid/segment/column/RunLengthColumn.java similarity index 91% rename from indexing-common/src/main/java/com/metamx/druid/index/column/RunLengthColumn.java rename to processing/src/main/java/io/druid/segment/column/RunLengthColumn.java index 5242e994008..e71e41e59b4 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/RunLengthColumn.java +++ b/processing/src/main/java/io/druid/segment/column/RunLengthColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java similarity index 97% rename from indexing-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java rename to processing/src/main/java/io/druid/segment/column/SimpleColumn.java index 5fd9f1c381e..597069a3b93 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/SimpleColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; import com.google.common.base.Supplier; import com.google.common.io.Closeables; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/SimpleDictionaryEncodedColumn.java b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java similarity index 88% rename from indexing-common/src/main/java/com/metamx/druid/index/column/SimpleDictionaryEncodedColumn.java rename to processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java index fbc3877c5fe..5e3bb3d0b9f 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/SimpleDictionaryEncodedColumn.java +++ b/processing/src/main/java/io/druid/segment/column/SimpleDictionaryEncodedColumn.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; +package io.druid.segment.column; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.kv.VSizeIndexed; -import com.metamx.druid.kv.VSizeIndexedInts; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.VSizeIndexed; +import io.druid.segment.data.VSizeIndexedInts; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/SpatialIndex.java b/processing/src/main/java/io/druid/segment/column/SpatialIndex.java similarity index 91% rename from indexing-common/src/main/java/com/metamx/druid/index/column/SpatialIndex.java rename to processing/src/main/java/io/druid/segment/column/SpatialIndex.java index 27502e13f35..115a4a341c2 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/SpatialIndex.java +++ b/processing/src/main/java/io/druid/segment/column/SpatialIndex.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -16,7 +16,8 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.column; + +package io.druid.segment.column; import com.metamx.collections.spatial.ImmutableRTree; diff --git a/processing/src/main/java/io/druid/segment/column/ValueType.java b/processing/src/main/java/io/druid/segment/column/ValueType.java new file mode 100644 index 00000000000..a59d6a7e20b --- /dev/null +++ b/processing/src/main/java/io/druid/segment/column/ValueType.java @@ -0,0 +1,30 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.column; + +/** +*/ +public enum ValueType +{ + FLOAT, + LONG, + STRING, + COMPLEX +} diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ArrayBasedIndexedInts.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/kv/ArrayBasedIndexedInts.java rename to processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java index 9e38b4a84db..6616ec859ed 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ArrayBasedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedIndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.util.Iterator; diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/ArrayBasedOffset.java b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java similarity index 94% rename from server/src/main/java/com/metamx/druid/index/v1/processing/ArrayBasedOffset.java rename to processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java index 42ac85ed869..d1f1fd42829 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/ArrayBasedOffset.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayBasedOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ArrayIndexed.java b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/kv/ArrayIndexed.java rename to processing/src/main/java/io/druid/segment/data/ArrayIndexed.java index 753db6cb5a3..eda5a70e824 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ArrayIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ArrayIndexed.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.util.Arrays; import java.util.Iterator; diff --git a/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java new file mode 100644 index 00000000000..0145c339f1d --- /dev/null +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferSerializer.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.data; + +import com.google.common.primitives.Ints; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +/** + */ +public class ByteBufferSerializer +{ + public static T read(ByteBuffer buffer, ObjectStrategy strategy) + { + int size = buffer.getInt(); + ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); + bufferToUse.limit(bufferToUse.position() + size); + buffer.position(bufferToUse.limit()); + + return strategy.fromByteBuffer(bufferToUse, size); + } + + public static void writeToChannel(T obj, ObjectStrategy strategy, WritableByteChannel channel) + throws IOException + { + byte[] toWrite = strategy.toBytes(obj); + channel.write(ByteBuffer.allocate(Ints.BYTES).putInt(0, toWrite.length)); + channel.write(ByteBuffer.wrap(toWrite)); + } +} diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferWriter.java b/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java similarity index 74% rename from indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferWriter.java rename to processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java index 0b384b3a852..91897402f80 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferWriter.java +++ b/processing/src/main/java/io/druid/segment/data/ByteBufferWriter.java @@ -1,4 +1,23 @@ -package com.metamx.druid.kv; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.data; import com.google.common.base.Function; import com.google.common.base.Preconditions; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatBufferObjectStrategy.java rename to processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java index 6aa2355dc15..7a9906cd364 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatBufferObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedFloatBufferObjectStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.collect.Ordering; import com.google.common.primitives.Floats; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java rename to processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java index c93d09d4725..c969d479068 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedFloatsIndexedSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; @@ -25,10 +25,8 @@ import com.google.common.io.Closeables; import com.google.common.primitives.Floats; import com.google.common.primitives.Ints; import com.metamx.common.IAE; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidResourceHolder; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedFloats; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidResourceHolder; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/CompressedFloatsSupplierSerializer.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsSupplierSerializer.java rename to processing/src/main/java/io/druid/segment/data/CompressedFloatsSupplierSerializer.java index 441ea2d0519..d0bf42ee451 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedFloatsSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedFloatsSupplierSerializer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; import com.google.common.io.OutputSupplier; import com.google.common.primitives.Ints; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidResourceHolder; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.IOPeon; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidResourceHolder; import java.io.IOException; import java.io.OutputStream; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongBufferObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedLongBufferObjectStrategy.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongBufferObjectStrategy.java rename to processing/src/main/java/io/druid/segment/data/CompressedLongBufferObjectStrategy.java index 0fed227d5a3..823c86a0516 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongBufferObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedLongBufferObjectStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.collect.Ordering; import com.google.common.primitives.Longs; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java rename to processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java index ee752e4a20f..c9d433b7740 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplier.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedLongsIndexedSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; @@ -25,10 +25,8 @@ import com.google.common.io.Closeables; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import com.metamx.common.IAE; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidResourceHolder; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidResourceHolder; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsSupplierSerializer.java b/processing/src/main/java/io/druid/segment/data/CompressedLongsSupplierSerializer.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsSupplierSerializer.java rename to processing/src/main/java/io/druid/segment/data/CompressedLongsSupplierSerializer.java index 3358e209db0..558c81afa47 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedLongsSupplierSerializer.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedLongsSupplierSerializer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; import com.google.common.io.OutputSupplier; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidResourceHolder; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.IOPeon; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidResourceHolder; import java.io.IOException; import java.io.OutputStream; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedObjectStrategy.java rename to processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java index 7733ec7fb2f..2c42ca1327f 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/CompressedObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/CompressedObjectStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.base.Throwables; import com.google.common.io.Closeables; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.kv.ObjectStrategy; import com.ning.compress.lzf.ChunkEncoder; import com.ning.compress.lzf.LZFChunk; import com.ning.compress.lzf.LZFDecoder; +import io.druid.collections.ResourceHolder; +import io.druid.segment.CompressedPools; import java.io.IOException; import java.nio.Buffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ConciseCompressedIndexedInts.java b/processing/src/main/java/io/druid/segment/data/ConciseCompressedIndexedInts.java similarity index 97% rename from indexing-common/src/main/java/com/metamx/druid/kv/ConciseCompressedIndexedInts.java rename to processing/src/main/java/io/druid/segment/data/ConciseCompressedIndexedInts.java index 2fdaeda30c0..e81e9389a32 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ConciseCompressedIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/ConciseCompressedIndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.collect.Ordering; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/EmptyIndexedInts.java b/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/kv/EmptyIndexedInts.java rename to processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java index cd83f0b481d..fa0b09cc8e5 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/EmptyIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/EmptyIndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.collect.ImmutableList; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java similarity index 98% rename from indexing-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java rename to processing/src/main/java/io/druid/segment/data/GenericIndexed.java index 99e003e844c..1ae8ec8ea01 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/GenericIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexed.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.base.Charsets; import com.google.common.collect.Ordering; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/GenericIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java similarity index 98% rename from indexing-common/src/main/java/com/metamx/druid/kv/GenericIndexedWriter.java rename to processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java index dd970ee403e..bd6258b2a52 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/GenericIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/GenericIndexedWriter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.base.Function; import com.google.common.base.Preconditions; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IOPeon.java b/processing/src/main/java/io/druid/segment/data/IOPeon.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/kv/IOPeon.java rename to processing/src/main/java/io/druid/segment/data/IOPeon.java index c57be9d1bb9..baac06ca3f8 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IOPeon.java +++ b/processing/src/main/java/io/druid/segment/data/IOPeon.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.io.IOException; import java.io.InputStream; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/InMemoryCompressedFloats.java b/processing/src/main/java/io/druid/segment/data/InMemoryCompressedFloats.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/InMemoryCompressedFloats.java rename to processing/src/main/java/io/druid/segment/data/InMemoryCompressedFloats.java index a46b990a2ba..4e6aaac55fb 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/InMemoryCompressedFloats.java +++ b/processing/src/main/java/io/druid/segment/data/InMemoryCompressedFloats.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.Closeables; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidResourceHolder; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedFloats; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidResourceHolder; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/InMemoryCompressedLongs.java b/processing/src/main/java/io/druid/segment/data/InMemoryCompressedLongs.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/InMemoryCompressedLongs.java rename to processing/src/main/java/io/druid/segment/data/InMemoryCompressedLongs.java index 9a408b46a4c..67a05b17508 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/InMemoryCompressedLongs.java +++ b/processing/src/main/java/io/druid/segment/data/InMemoryCompressedLongs.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.Closeables; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.collect.StupidResourceHolder; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidResourceHolder; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/Indexed.java b/processing/src/main/java/io/druid/segment/data/Indexed.java similarity index 80% rename from indexing-common/src/main/java/com/metamx/druid/kv/Indexed.java rename to processing/src/main/java/io/druid/segment/data/Indexed.java index 44ef9dc103a..f8def0481b7 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/Indexed.java +++ b/processing/src/main/java/io/druid/segment/data/Indexed.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; /** - * An Integer-indexed random-access collection. - * Typically wraps an {@link Indexed}. - * * @param */ -public interface Indexed extends Iterable { +public interface Indexed extends Iterable +{ Class getClazz(); int size(); T get(int index); diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedFloats.java b/processing/src/main/java/io/druid/segment/data/IndexedFloats.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedFloats.java rename to processing/src/main/java/io/druid/segment/data/IndexedFloats.java index 5bfffda48cf..cc01524f86f 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedFloats.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedFloats.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.io.Closeable; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedInts.java b/processing/src/main/java/io/druid/segment/data/IndexedInts.java similarity index 87% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedInts.java rename to processing/src/main/java/io/druid/segment/data/IndexedInts.java index a868f33838b..de0f213fb4b 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; /** * Get a int an index (array or list lookup abstraction without boxing). - * Typically wraps an {@link com.metamx.druid.kv.Indexed}. */ public interface IndexedInts extends Iterable { diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedIntsIterator.java b/processing/src/main/java/io/druid/segment/data/IndexedIntsIterator.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedIntsIterator.java rename to processing/src/main/java/io/druid/segment/data/IndexedIntsIterator.java index 02d5c47f663..6b5a40b2818 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedIntsIterator.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedIntsIterator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; + import java.util.Iterator; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedIterable.java b/processing/src/main/java/io/druid/segment/data/IndexedIterable.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedIterable.java rename to processing/src/main/java/io/druid/segment/data/IndexedIterable.java index 6f5efd7fbb2..f8d331e0deb 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedIterable.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedIterable.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.util.Iterator; import java.util.NoSuchElementException; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedList.java b/processing/src/main/java/io/druid/segment/data/IndexedList.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedList.java rename to processing/src/main/java/io/druid/segment/data/IndexedList.java index c65556cc50d..8839231231a 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedList.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedList.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.util.AbstractList; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedLongs.java b/processing/src/main/java/io/druid/segment/data/IndexedLongs.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedLongs.java rename to processing/src/main/java/io/druid/segment/data/IndexedLongs.java index fbdebb63693..eabd9be5a82 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedLongs.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedLongs.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.io.Closeable; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedRTree.java b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java similarity index 68% rename from indexing-common/src/main/java/com/metamx/druid/kv/IndexedRTree.java rename to processing/src/main/java/io/druid/segment/data/IndexedRTree.java index 8c8e3c46d50..b5cb07388fd 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IndexedRTree.java +++ b/processing/src/main/java/io/druid/segment/data/IndexedRTree.java @@ -1,4 +1,23 @@ -package com.metamx.druid.kv; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.data; import com.google.common.collect.Ordering; import com.metamx.collections.spatial.ImmutableRTree; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/Indexedids.java b/processing/src/main/java/io/druid/segment/data/Indexedids.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/kv/Indexedids.java rename to processing/src/main/java/io/druid/segment/data/Indexedids.java index 0b3f3a2c000..9ced3b72c4e 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/Indexedids.java +++ b/processing/src/main/java/io/druid/segment/data/Indexedids.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/IntBufferIndexedInts.java b/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/kv/IntBufferIndexedInts.java rename to processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java index a6c7e42b049..bb121602e85 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/IntBufferIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/IntBufferIndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.collect.Ordering; import com.google.common.primitives.Ints; -import com.metamx.druid.IntList; +import io.druid.collections.IntList; import java.nio.ByteBuffer; import java.nio.IntBuffer; diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/IntersectingOffset.java b/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java similarity index 95% rename from server/src/main/java/com/metamx/druid/index/v1/processing/IntersectingOffset.java rename to processing/src/main/java/io/druid/segment/data/IntersectingOffset.java index 25cacff2093..0f5a7f7b885 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/IntersectingOffset.java +++ b/processing/src/main/java/io/druid/segment/data/IntersectingOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ListIndexed.java b/processing/src/main/java/io/druid/segment/data/ListIndexed.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/kv/ListIndexed.java rename to processing/src/main/java/io/druid/segment/data/ListIndexed.java index df939fe88f8..b58a9ed516c 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ListIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/ListIndexed.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.util.Iterator; import java.util.List; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ObjectStrategy.java b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/kv/ObjectStrategy.java rename to processing/src/main/java/io/druid/segment/data/ObjectStrategy.java index 57c7a9426a7..7e2753523ba 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ObjectStrategy.java +++ b/processing/src/main/java/io/druid/segment/data/ObjectStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import java.nio.ByteBuffer; import java.util.Comparator; diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/Offset.java b/processing/src/main/java/io/druid/segment/data/Offset.java similarity index 92% rename from server/src/main/java/com/metamx/druid/index/v1/processing/Offset.java rename to processing/src/main/java/io/druid/segment/data/Offset.java index 17d0a2ebace..db178f04f30 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/Offset.java +++ b/processing/src/main/java/io/druid/segment/data/Offset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; /** * The "mutable" version of a ReadableOffset. Introduces "increment()" and "withinBounds()" methods, which are diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/ReadableOffset.java b/processing/src/main/java/io/druid/segment/data/ReadableOffset.java similarity index 87% rename from server/src/main/java/com/metamx/druid/index/v1/processing/ReadableOffset.java rename to processing/src/main/java/io/druid/segment/data/ReadableOffset.java index aea02d16fb6..e77d74f3ba6 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/ReadableOffset.java +++ b/processing/src/main/java/io/druid/segment/data/ReadableOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; /** * A ReadableOffset is an object that provides an integer offset, ostensibly as an index into an array. * * See the companion class Offset, for more context on how this could be useful. A ReadableOffset should be - * given to classes (e.g. FloatMetricSelector objects) by something which keeps a reference to the base Offset object + * given to classes (e.g. FloatColumnSelector objects) by something which keeps a reference to the base Offset object * and increments it. */ public interface ReadableOffset diff --git a/server/src/main/java/com/metamx/druid/loading/SegmentLoaderConfig.java b/processing/src/main/java/io/druid/segment/data/SingleIndexedInts.java similarity index 60% rename from server/src/main/java/com/metamx/druid/loading/SegmentLoaderConfig.java rename to processing/src/main/java/io/druid/segment/data/SingleIndexedInts.java index 8a0e32484e1..796f7ef655d 100644 --- a/server/src/main/java/com/metamx/druid/loading/SegmentLoaderConfig.java +++ b/processing/src/main/java/io/druid/segment/data/SingleIndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,26 +17,37 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.data; -import org.skife.config.Config; +import com.google.common.collect.Iterators; + +import java.util.Iterator; /** - */ -public abstract class SegmentLoaderConfig +*/ +public class SingleIndexedInts implements IndexedInts { - @Config({"druid.paths.indexCache", "druid.segmentCache.path"}) - public abstract String getCacheDirectory(); + private final int value; - @Config("druid.server.maxSize") - public long getServerMaxSize() - { - return Long.MAX_VALUE; + public SingleIndexedInts(int value) { + this.value = value; } - @Config("druid.segmentCache.deleteOnRemove") - public boolean deleteOnRemove() + @Override + public int size() { - return true; + return 1; + } + + @Override + public int get(int index) + { + return value; + } + + @Override + public Iterator iterator() + { + return Iterators.singletonIterator(value); } } diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/StartLimitedOffset.java b/processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java similarity index 94% rename from server/src/main/java/com/metamx/druid/index/v1/processing/StartLimitedOffset.java rename to processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java index 0bb84122334..dd4d34585c5 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/StartLimitedOffset.java +++ b/processing/src/main/java/io/druid/segment/data/StartLimitedOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/TmpFileIOPeon.java b/processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/kv/TmpFileIOPeon.java rename to processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java index 9af7e0b0337..a60ed66838c 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/TmpFileIOPeon.java +++ b/processing/src/main/java/io/druid/segment/data/TmpFileIOPeon.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.collect.Maps; diff --git a/server/src/main/java/com/metamx/druid/index/v1/processing/UnioningOffset.java b/processing/src/main/java/io/druid/segment/data/UnioningOffset.java similarity index 97% rename from server/src/main/java/com/metamx/druid/index/v1/processing/UnioningOffset.java rename to processing/src/main/java/io/druid/segment/data/UnioningOffset.java index 263aa665ff9..a12256519c6 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/processing/UnioningOffset.java +++ b/processing/src/main/java/io/druid/segment/data/UnioningOffset.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexed.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java similarity index 95% rename from indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexed.java rename to processing/src/main/java/io/druid/segment/data/VSizeIndexed.java index 9ef63f39ac0..8c1f481b2d1 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexed.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexed.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.primitives.Ints; import com.metamx.common.IAE; @@ -27,7 +27,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; -import java.util.Arrays; import java.util.Iterator; /** @@ -36,11 +35,6 @@ public class VSizeIndexed implements Indexed { private static final byte version = 0x1; - public static VSizeIndexed fromArray(VSizeIndexedInts[] objects) - { - return fromIterable(Arrays.asList(objects)); - } - public static VSizeIndexed fromIterable(Iterable objectsIterable) { Iterator objects = objectsIterable.iterator(); diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexedInts.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java similarity index 98% rename from indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexedInts.java rename to processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java index 616065d163c..a2c106ff309 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexedInts.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedInts.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.primitives.Ints; import com.metamx.common.IAE; diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexedWriter.java b/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java similarity index 98% rename from indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexedWriter.java rename to processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java index c50ea15ad53..87fcc127155 100644 --- a/indexing-common/src/main/java/com/metamx/druid/kv/VSizeIndexedWriter.java +++ b/processing/src/main/java/io/druid/segment/data/VSizeIndexedWriter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.base.Function; import com.google.common.base.Preconditions; diff --git a/server/src/main/java/com/metamx/druid/index/brita/AndFilter.java b/processing/src/main/java/io/druid/segment/filter/AndFilter.java similarity index 89% rename from server/src/main/java/com/metamx/druid/index/brita/AndFilter.java rename to processing/src/main/java/io/druid/segment/filter/AndFilter.java index 2de8823d9ca..70b374459db 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/AndFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/AndFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.collect.Lists; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/index/brita/BooleanValueMatcher.java b/processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java similarity index 88% rename from server/src/main/java/com/metamx/druid/index/brita/BooleanValueMatcher.java rename to processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java index 0f7d7adc71d..c60a72befff 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/BooleanValueMatcher.java +++ b/processing/src/main/java/io/druid/segment/filter/BooleanValueMatcher.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; + +import io.druid.query.filter.ValueMatcher; /** */ diff --git a/server/src/main/java/com/metamx/druid/index/brita/DimensionPredicateFilter.java b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java similarity index 88% rename from server/src/main/java/com/metamx/druid/index/brita/DimensionPredicateFilter.java rename to processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java index d88dd06d3dc..b3f06b949a3 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/DimensionPredicateFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/DimensionPredicateFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,16 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.base.Function; import com.google.common.base.Predicate; import com.metamx.common.guava.FunctionalIterable; -import com.metamx.druid.kv.Indexed; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; +import io.druid.segment.data.Indexed; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import javax.annotation.Nullable; diff --git a/server/src/main/java/com/metamx/druid/index/brita/ExtractionFilter.java b/processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java similarity index 85% rename from server/src/main/java/com/metamx/druid/index/brita/ExtractionFilter.java rename to processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java index 883e19226e8..62a92ca193a 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/ExtractionFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/ExtractionFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.collect.Lists; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.DimExtractionFn; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; +import io.druid.segment.data.Indexed; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/index/brita/Filters.java b/processing/src/main/java/io/druid/segment/filter/Filters.java similarity index 84% rename from server/src/main/java/com/metamx/druid/index/brita/Filters.java rename to processing/src/main/java/io/druid/segment/filter/Filters.java index c27940b06fd..21c4722d1fd 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/Filters.java +++ b/processing/src/main/java/io/druid/segment/filter/Filters.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.base.Function; import com.google.common.collect.Lists; -import com.metamx.druid.query.filter.AndDimFilter; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.query.filter.ExtractionDimFilter; -import com.metamx.druid.query.filter.JavaScriptDimFilter; -import com.metamx.druid.query.filter.NotDimFilter; -import com.metamx.druid.query.filter.OrDimFilter; -import com.metamx.druid.query.filter.RegexDimFilter; -import com.metamx.druid.query.filter.SearchQueryDimFilter; -import com.metamx.druid.query.filter.SelectorDimFilter; -import com.metamx.druid.query.filter.SpatialDimFilter; +import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.ExtractionDimFilter; +import io.druid.query.filter.Filter; +import io.druid.query.filter.JavaScriptDimFilter; +import io.druid.query.filter.NotDimFilter; +import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.RegexDimFilter; +import io.druid.query.filter.SearchQueryDimFilter; +import io.druid.query.filter.SelectorDimFilter; +import io.druid.query.filter.SpatialDimFilter; import javax.annotation.Nullable; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java similarity index 78% rename from server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java rename to processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java index 7c465d35a79..743a07421cb 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/JavaScriptFilter.java @@ -1,8 +1,31 @@ -package com.metamx.druid.index.brita; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.filter; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.metamx.common.guava.FunctionalIterable; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; diff --git a/server/src/main/java/com/metamx/druid/index/brita/NotFilter.java b/processing/src/main/java/io/druid/segment/filter/NotFilter.java similarity index 85% rename from server/src/main/java/com/metamx/druid/index/brita/NotFilter.java rename to processing/src/main/java/io/druid/segment/filter/NotFilter.java index d4d335add0c..d517613f978 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/NotFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/NotFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; /** diff --git a/server/src/main/java/com/metamx/druid/index/brita/OrFilter.java b/processing/src/main/java/io/druid/segment/filter/OrFilter.java similarity index 89% rename from server/src/main/java/com/metamx/druid/index/brita/OrFilter.java rename to processing/src/main/java/io/druid/segment/filter/OrFilter.java index c9312b0b6f1..8476066a933 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/OrFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/OrFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.collect.Lists; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/index/brita/RegexFilter.java b/processing/src/main/java/io/druid/segment/filter/RegexFilter.java similarity index 91% rename from server/src/main/java/com/metamx/druid/index/brita/RegexFilter.java rename to processing/src/main/java/io/druid/segment/filter/RegexFilter.java index b8880f4289e..05b5e70fad6 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/RegexFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/RegexFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.base.Predicate; -import javax.annotation.Nullable; import java.util.regex.Pattern; /** diff --git a/server/src/main/java/com/metamx/druid/index/brita/SearchQueryFilter.java b/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java similarity index 91% rename from server/src/main/java/com/metamx/druid/index/brita/SearchQueryFilter.java rename to processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java index a94a518dc61..0f80683187e 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/SearchQueryFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SearchQueryFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Predicate; -import com.metamx.druid.query.search.SearchQuerySpec; +import io.druid.query.search.search.SearchQuerySpec; import javax.annotation.Nullable; diff --git a/server/src/main/java/com/metamx/druid/index/brita/SelectorFilter.java b/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java similarity index 83% rename from server/src/main/java/com/metamx/druid/index/brita/SelectorFilter.java rename to processing/src/main/java/io/druid/segment/filter/SelectorFilter.java index a43c7fa5afc..f57d040a1c0 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/SelectorFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SelectorFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; /** diff --git a/server/src/main/java/com/metamx/druid/index/brita/SpatialFilter.java b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java similarity index 84% rename from server/src/main/java/com/metamx/druid/index/brita/SpatialFilter.java rename to processing/src/main/java/io/druid/segment/filter/SpatialFilter.java index 4d7e4e36f6a..538b133be57 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/SpatialFilter.java +++ b/processing/src/main/java/io/druid/segment/filter/SpatialFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -16,13 +16,14 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.metamx.collections.spatial.search.Bound; +import io.druid.query.filter.BitmapIndexSelector; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; -import it.uniroma3.mat.extendedset.intset.IntSet; - -import java.util.Iterator; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java similarity index 89% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java rename to processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java index 4141ce9d3a1..77d99e44cbc 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndex.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndex.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.incremental; import com.google.common.base.Function; import com.google.common.base.Joiner; @@ -33,20 +33,19 @@ import com.google.common.primitives.Longs; import com.metamx.common.IAE; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.Aggregator; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.index.v1.serde.ComplexMetricExtractor; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.index.v1.serde.ComplexMetrics; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.processing.ColumnSelectorFactory; -import com.metamx.druid.processing.ComplexMetricSelector; -import com.metamx.druid.processing.FloatMetricSelector; -import com.metamx.druid.processing.ObjectColumnSelector; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.segment.ColumnSelectorFactory; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.serde.ComplexMetricExtractor; +import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.ComplexMetrics; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -197,10 +196,10 @@ public class IncrementalIndex implements Iterable new ColumnSelectorFactory() { @Override - public FloatMetricSelector makeFloatMetricSelector(String metric) + public FloatColumnSelector makeFloatColumnSelector(String columnName) { - final String metricName = metric.toLowerCase(); - return new FloatMetricSelector() + final String metricName = columnName.toLowerCase(); + return new FloatColumnSelector() { @Override public float get() @@ -210,36 +209,6 @@ public class IncrementalIndex implements Iterable }; } - @Override - public ComplexMetricSelector makeComplexMetricSelector(final String metric) - { - final String typeName = agg.getTypeName(); - - final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(typeName); - - if (serde == null) { - throw new ISE("Don't know how to handle type[%s]", typeName); - } - - final ComplexMetricExtractor extractor = serde.getExtractor(); - final String metricName = metric.toLowerCase(); - - return new ComplexMetricSelector() - { - @Override - public Class classOfObject() - { - return extractor.extractedClass(); - } - - @Override - public Object get() - { - return extractor.extractValue(in, metricName); - } - }; - } - @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndexAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndexAdapter.java rename to processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java index 6831f17fdda..4ebea50e24d 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndexAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexAdapter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.incremental; import com.google.common.base.Function; import com.google.common.collect.Maps; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.logger.Logger; -import com.metamx.druid.aggregation.Aggregator; -import com.metamx.druid.kv.EmptyIndexedInts; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.kv.IndexedIterable; -import com.metamx.druid.kv.ListIndexed; +import io.druid.query.aggregation.Aggregator; +import io.druid.segment.IndexableAdapter; +import io.druid.segment.Rowboat; +import io.druid.segment.data.EmptyIndexedInts; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.IndexedIterable; +import io.druid.segment.data.ListIndexed; import it.uniroma3.mat.extendedset.intset.ConciseSet; import it.uniroma3.mat.extendedset.intset.IntSet; import org.joda.time.Interval; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndexSchema.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java similarity index 74% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndexSchema.java rename to processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java index 5bb0b63103b..5bd0250c90f 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/IncrementalIndexSchema.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexSchema.java @@ -1,10 +1,29 @@ -package com.metamx.druid.index.v1; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.incremental; import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; import java.util.Collections; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java similarity index 74% rename from server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java rename to processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java index 62eba78ff61..fd43d81d698 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapter.java +++ b/processing/src/main/java/io/druid/segment/incremental/IncrementalIndexStorageAdapter.java @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.incremental; import com.google.common.base.Function; import com.google.common.base.Predicate; @@ -25,30 +25,25 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.metamx.collections.spatial.search.Bound; -import com.metamx.common.IAE; -import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.guava.FunctionalIterator; -import com.metamx.druid.Capabilities; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.aggregation.Aggregator; -import com.metamx.druid.index.brita.BooleanValueMatcher; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.brita.ValueMatcher; -import com.metamx.druid.index.brita.ValueMatcherFactory; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.index.v1.processing.DimensionSelector; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.index.v1.serde.ComplexMetrics; -import com.metamx.druid.kv.IndexedInts; -import com.metamx.druid.processing.ComplexMetricSelector; -import com.metamx.druid.processing.FloatMetricSelector; -import com.metamx.druid.processing.ObjectColumnSelector; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQuerySpec; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.Aggregator; +import io.druid.query.filter.Filter; +import io.druid.query.filter.ValueMatcher; +import io.druid.query.filter.ValueMatcherFactory; +import io.druid.segment.Capabilities; +import io.druid.segment.Cursor; +import io.druid.segment.DimensionSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.ObjectColumnSelector; +import io.druid.segment.StorageAdapter; +import io.druid.segment.data.Indexed; +import io.druid.segment.data.IndexedInts; +import io.druid.segment.data.ListIndexed; +import io.druid.segment.filter.BooleanValueMatcher; +import io.druid.segment.serde.ComplexMetricSerde; +import io.druid.segment.serde.ComplexMetrics; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -57,7 +52,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.TreeSet; import java.util.concurrent.ConcurrentNavigableMap; /** @@ -87,6 +81,12 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter return index.getInterval(); } + @Override + public Indexed getAvailableDimensions() + { + return new ListIndexed(index.getDimensions(), String.class); + } + @Override public int getDimensionCardinality(String dimension) { @@ -310,12 +310,12 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter } @Override - public FloatMetricSelector makeFloatMetricSelector(String metric) + public FloatColumnSelector makeFloatColumnSelector(String columnName) { - final String metricName = metric.toLowerCase(); + final String metricName = columnName.toLowerCase(); final Integer metricIndexInt = index.getMetricIndex(metricName); if (metricIndexInt == null) { - return new FloatMetricSelector() + return new FloatColumnSelector() { @Override public float get() @@ -327,7 +327,7 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter final int metricIndex = metricIndexInt; - return new FloatMetricSelector() + return new FloatColumnSelector() { @Override public float get() @@ -337,35 +337,6 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter }; } - @Override - public ComplexMetricSelector makeComplexMetricSelector(String metric) - { - final String metricName = metric.toLowerCase(); - final Integer metricIndexInt = index.getMetricIndex(metricName); - if (metricIndexInt == null) { - return null; - } - - final int metricIndex = metricIndexInt; - - final ComplexMetricSerde serde = ComplexMetrics.getSerdeForType(index.getMetricType(metricName)); - - return new ComplexMetricSelector() - { - @Override - public Class classOfObject() - { - return serde.getObjectStrategy().getClazz(); - } - - @Override - public Object get() - { - return currEntry.getValue()[metricIndex].get(); - } - }; - } - @Override public ObjectColumnSelector makeObjectColumnSelector(String column) { @@ -432,92 +403,6 @@ public class IncrementalIndexStorageAdapter implements StorageAdapter }; } - @Override - public Iterable searchDimensions(final SearchQuery query, final Filter filter) - { - final List dimensions = query.getDimensions(); - final int[] dimensionIndexes; - final String[] dimensionNames; - final List dimensionOrder = index.getDimensions(); - if (dimensions == null || dimensions.isEmpty()) { - dimensionIndexes = new int[dimensionOrder.size()]; - dimensionNames = new String[dimensionIndexes.length]; - - Iterator dimensionOrderIter = dimensionOrder.iterator(); - for (int i = 0; i < dimensionIndexes.length; ++i) { - dimensionNames[i] = dimensionOrderIter.next(); - dimensionIndexes[i] = index.getDimensionIndex(dimensionNames[i]); - } - } else { - int[] tmpDimensionIndexes = new int[dimensions.size()]; - String[] tmpDimensionNames = new String[dimensions.size()]; - int i = 0; - for (String dimension : dimensions) { - Integer dimIndex = index.getDimensionIndex(dimension.toLowerCase()); - if (dimIndex != null) { - tmpDimensionNames[i] = dimension; - tmpDimensionIndexes[i] = dimIndex; - ++i; - } - } - - if (i != tmpDimensionIndexes.length) { - dimensionIndexes = new int[i]; - dimensionNames = new String[i]; - System.arraycopy(tmpDimensionIndexes, 0, dimensionIndexes, 0, i); - System.arraycopy(tmpDimensionNames, 0, dimensionNames, 0, i); - } else { - dimensionIndexes = tmpDimensionIndexes; - dimensionNames = tmpDimensionNames; - } - } - - final List queryIntervals = query.getIntervals(); - if (queryIntervals.size() != 1) { - throw new IAE("Can only handle one interval, got query[%s]", query); - } - - final Interval queryInterval = queryIntervals.get(0); - final long intervalStart = queryInterval.getStartMillis(); - final long intervalEnd = queryInterval.getEndMillis(); - - final EntryHolder holder = new EntryHolder(); - final ValueMatcher theMatcher = makeFilterMatcher(filter, holder); - final SearchQuerySpec searchQuerySpec = query.getQuery(); - final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); - - ConcurrentNavigableMap facts = index.getSubMap( - new IncrementalIndex.TimeAndDims(intervalStart, new String[][]{}), - new IncrementalIndex.TimeAndDims(intervalEnd, new String[][]{}) - ); - - for (Map.Entry entry : facts.entrySet()) { - holder.set(entry); - final IncrementalIndex.TimeAndDims key = holder.getKey(); - final long timestamp = key.getTimestamp(); - - if (timestamp >= intervalStart && timestamp < intervalEnd && theMatcher.matches()) { - final String[][] dims = key.getDims(); - - for (int i = 0; i < dimensionIndexes.length; ++i) { - if (dimensionIndexes[i] < dims.length) { - final String[] dimVals = dims[dimensionIndexes[i]]; - if (dimVals != null) { - for (int j = 0; j < dimVals.length; ++j) { - if (searchQuerySpec.accept(dimVals[j])) { - retVal.add(new SearchHit(dimensionNames[i], dimVals[j])); - } - } - } - } - } - } - } - - - return new FunctionalIterable(retVal).limit(query.getLimit()); - } - private ValueMatcher makeFilterMatcher(final Filter filter, final EntryHolder holder) { return filter == null diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionRowFormatter.java b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowFormatter.java similarity index 96% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionRowFormatter.java rename to processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowFormatter.java index 864a0087831..d5d8f4662de 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionRowFormatter.java +++ b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionRowFormatter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.incremental; import com.google.common.base.Function; import com.google.common.base.Joiner; @@ -29,10 +29,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Floats; import com.metamx.common.ISE; -import com.metamx.druid.input.InputRow; -import org.joda.time.DateTime; +import io.druid.data.input.InputRow; -import javax.annotation.Nullable; import java.util.Arrays; import java.util.List; import java.util.Map; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionSchema.java b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionSchema.java similarity index 51% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionSchema.java rename to processing/src/main/java/io/druid/segment/incremental/SpatialDimensionSchema.java index 311626e5bc0..6ef1e788c6c 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/SpatialDimensionSchema.java +++ b/processing/src/main/java/io/druid/segment/incremental/SpatialDimensionSchema.java @@ -1,4 +1,23 @@ -package com.metamx.druid.index.v1; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.incremental; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java rename to processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java index 831fc887eb8..d36e6a684e1 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/BitmapIndexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/BitmapIndexColumnPartSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.druid.index.column.BitmapIndex; -import com.metamx.druid.kv.GenericIndexed; +import io.druid.segment.column.BitmapIndex; +import io.druid.segment.data.GenericIndexed; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; /** diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java index 556fab5ea4d..c22685ddc65 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/ColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ColumnPartSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.index.column.ColumnBuilder; +import io.druid.segment.column.ColumnBuilder; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java similarity index 87% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java index 03e13cfe371..080978e77ab 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.index.column.ColumnBuilder; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.index.v1.serde.ComplexMetrics; -import com.metamx.druid.kv.GenericIndexed; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.data.GenericIndexed; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java similarity index 84% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSupplier.java rename to processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java index 4a4cfce4f77..7251e557ab4 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/ComplexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexColumnPartSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.druid.index.column.ComplexColumn; -import com.metamx.druid.index.column.IndexedComplexColumn; -import com.metamx.druid.kv.GenericIndexed; +import io.druid.segment.column.ComplexColumn; +import io.druid.segment.column.IndexedComplexColumn; +import io.druid.segment.data.GenericIndexed; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/ComplexMetricColumnSerializer.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java similarity index 87% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/ComplexMetricColumnSerializer.java rename to processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java index 55032f54da0..dd5e5939bea 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/ComplexMetricColumnSerializer.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricColumnSerializer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.serde; import com.google.common.io.Files; -import com.metamx.druid.index.v1.serde.ComplexMetricSerde; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.IOPeon; +import io.druid.segment.IndexIO; +import io.druid.segment.MetricColumnSerializer; +import io.druid.segment.MetricHolder; +import io.druid.segment.data.GenericIndexedWriter; +import io.druid.segment.data.IOPeon; import java.io.File; import java.io.IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetricExtractor.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java similarity index 87% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetricExtractor.java rename to processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java index b2c5545db73..c571d7627d6 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetricExtractor.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricExtractor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.serde; +package io.druid.segment.serde; -import com.metamx.druid.input.InputRow; +import io.druid.data.input.InputRow; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetricSerde.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java similarity index 92% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetricSerde.java rename to processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java index eb39ece5a70..2ac3b309f62 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetricSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetricSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.serde; +package io.druid.segment.serde; import com.google.common.base.Function; -import com.metamx.druid.index.column.ColumnBuilder; -import com.metamx.druid.index.serde.ColumnPartSerde; -import com.metamx.druid.kv.ObjectStrategy; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.data.ObjectStrategy; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetrics.java b/processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java similarity index 93% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetrics.java rename to processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java index 33093f8f825..ae7cee1679a 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/ComplexMetrics.java +++ b/processing/src/main/java/io/druid/segment/serde/ComplexMetrics.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.serde; +package io.druid.segment.serde; import com.google.common.collect.Maps; import com.metamx.common.ISE; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java similarity index 91% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 82fe3cd5cea..391bc62ffdb 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,21 +17,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.primitives.Ints; import com.metamx.collections.spatial.ImmutableRTree; import com.metamx.common.IAE; -import com.metamx.druid.index.column.ColumnBuilder; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.kv.ByteBufferSerializer; -import com.metamx.druid.kv.ConciseCompressedIndexedInts; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.IndexedRTree; -import com.metamx.druid.kv.VSizeIndexed; -import com.metamx.druid.kv.VSizeIndexedInts; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.ByteBufferSerializer; +import io.druid.segment.data.ConciseCompressedIndexedInts; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.IndexedRTree; +import io.druid.segment.data.VSizeIndexed; +import io.druid.segment.data.VSizeIndexedInts; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import java.io.IOException; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java similarity index 81% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnSupplier.java rename to processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java index 40285cf550d..d2a5ee82a85 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/DictionaryEncodedColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/DictionaryEncodedColumnSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.druid.index.column.DictionaryEncodedColumn; -import com.metamx.druid.index.column.SimpleDictionaryEncodedColumn; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.VSizeIndexed; -import com.metamx.druid.kv.VSizeIndexedInts; +import io.druid.segment.column.DictionaryEncodedColumn; +import io.druid.segment.column.SimpleDictionaryEncodedColumn; +import io.druid.segment.data.GenericIndexed; +import io.druid.segment.data.VSizeIndexed; +import io.druid.segment.data.VSizeIndexedInts; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java similarity index 90% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java index 13f86775f30..0a0951af793 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnPartSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.index.column.ColumnBuilder; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.index.v1.CompressedFloatsIndexedSupplier; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.CompressedFloatsIndexedSupplier; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java similarity index 82% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnSupplier.java rename to processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java index c13a5c11942..41d7c17fcdb 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/FloatGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/FloatGenericColumnSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.druid.index.column.GenericColumn; -import com.metamx.druid.index.column.IndexedFloatsGenericColumn; -import com.metamx.druid.index.v1.CompressedFloatsIndexedSupplier; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.IndexedFloatsGenericColumn; +import io.druid.segment.data.CompressedFloatsIndexedSupplier; import java.nio.ByteOrder; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java similarity index 90% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java rename to processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java index e37f08dff81..92e25b17e79 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnPartSerde.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnPartSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.index.column.ColumnBuilder; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.index.v1.CompressedLongsIndexedSupplier; +import io.druid.segment.column.ColumnBuilder; +import io.druid.segment.column.ValueType; +import io.druid.segment.data.CompressedLongsIndexedSupplier; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnSupplier.java b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java similarity index 81% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnSupplier.java rename to processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java index 0f080d74564..edc8483efb3 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/LongGenericColumnSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/LongGenericColumnSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.google.common.base.Supplier; -import com.metamx.druid.index.column.GenericColumn; -import com.metamx.druid.index.column.IndexedLongsGenericColumn; -import com.metamx.druid.index.v1.CompressedLongsIndexedSupplier; +import io.druid.segment.column.GenericColumn; +import io.druid.segment.column.IndexedLongsGenericColumn; +import io.druid.segment.data.CompressedLongsIndexedSupplier; /** */ diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/Registererer.java b/processing/src/main/java/io/druid/segment/serde/Registererer.java similarity index 94% rename from indexing-common/src/main/java/com/metamx/druid/index/v1/serde/Registererer.java rename to processing/src/main/java/io/druid/segment/serde/Registererer.java index 8a18b7033cb..6c1fdcceb91 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/serde/Registererer.java +++ b/processing/src/main/java/io/druid/segment/serde/Registererer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.serde; +package io.druid.segment.serde; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/client/src/main/java/com/metamx/druid/RegisteringNode.java b/processing/src/main/java/io/druid/segment/serde/Registererers.java similarity index 71% rename from client/src/main/java/com/metamx/druid/RegisteringNode.java rename to processing/src/main/java/io/druid/segment/serde/Registererers.java index 0d09de2f0a3..e56e42605f2 100644 --- a/client/src/main/java/com/metamx/druid/RegisteringNode.java +++ b/processing/src/main/java/io/druid/segment/serde/Registererers.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.segment.serde; import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.druid.index.v1.serde.Registererer; - -import java.util.Arrays; -import java.util.List; /** */ -public class RegisteringNode +public class Registererers { public static void registerHandlers(Iterable registererers, Iterable mappers) { @@ -43,17 +39,4 @@ public class RegisteringNode } private static boolean doneRegister = false; - - private final List mappers; - - public RegisteringNode(List mappers) - { - this.mappers = mappers; - } - - public RegisteringNode registerHandlers(Registererer... registererers) - { - registerHandlers(Arrays.asList(registererers), mappers); - return this; - } } diff --git a/indexing-common/src/main/java/com/metamx/druid/index/serde/SpatialIndexColumnPartSupplier.java b/processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java similarity index 88% rename from indexing-common/src/main/java/com/metamx/druid/index/serde/SpatialIndexColumnPartSupplier.java rename to processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java index 246da53ab7b..48014bc8bb6 100644 --- a/indexing-common/src/main/java/com/metamx/druid/index/serde/SpatialIndexColumnPartSupplier.java +++ b/processing/src/main/java/io/druid/segment/serde/SpatialIndexColumnPartSupplier.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -16,12 +16,11 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.serde; +package io.druid.segment.serde; import com.google.common.base.Supplier; import com.metamx.collections.spatial.ImmutableRTree; -import com.metamx.druid.index.column.SpatialIndex; -import com.metamx.druid.kv.GenericIndexed; +import io.druid.segment.column.SpatialIndex; /** */ diff --git a/client/src/test/java/com/metamx/druid/CombiningIterableTest.java b/processing/src/test/java/io/druid/collections/CombiningIterableTest.java similarity index 89% rename from client/src/test/java/com/metamx/druid/CombiningIterableTest.java rename to processing/src/test/java/io/druid/collections/CombiningIterableTest.java index 94650467656..e7c5e94d5af 100644 --- a/client/src/test/java/com/metamx/druid/CombiningIterableTest.java +++ b/processing/src/test/java/io/druid/collections/CombiningIterableTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.collections; import com.metamx.common.guava.nary.BinaryFn; -import com.metamx.druid.result.Result; +import io.druid.query.Result; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; @@ -69,7 +69,10 @@ public class CombiningIterableTest return arg1; } - return new Result(arg1.getTimestamp(), ((Long) arg1.getValue()).longValue() + ((Long) arg2.getValue()).longValue()); + return new Result( + arg1.getTimestamp(), + ((Long) arg1.getValue()).longValue() + ((Long) arg2.getValue()).longValue() + ); } } ); diff --git a/indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java b/processing/src/test/java/io/druid/data/input/InputRowParserSerdeTest.java similarity index 71% rename from indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java rename to processing/src/test/java/io/druid/data/input/InputRowParserSerdeTest.java index 418661ce9d6..a57962a6f37 100644 --- a/indexing-common/src/test/java/com/metamx/druid/indexer/data/InputRowParserSerdeTest.java +++ b/processing/src/test/java/io/druid/data/input/InputRowParserSerdeTest.java @@ -1,12 +1,30 @@ -package com.metamx.druid.indexer.data; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.data.input; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.index.v1.SpatialDimensionSchema; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.incremental.SpatialDimensionSchema; import junit.framework.Assert; import org.joda.time.DateTime; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/indexer/data/ProtoBufInputRowParserTest.java b/processing/src/test/java/io/druid/data/input/ProtoBufInputRowParserTest.java similarity index 69% rename from indexing-common/src/test/java/com/metamx/druid/indexer/data/ProtoBufInputRowParserTest.java rename to processing/src/test/java/io/druid/data/input/ProtoBufInputRowParserTest.java index 237f89edf14..aa98a3bccb9 100644 --- a/indexing-common/src/test/java/com/metamx/druid/indexer/data/ProtoBufInputRowParserTest.java +++ b/processing/src/test/java/io/druid/data/input/ProtoBufInputRowParserTest.java @@ -1,18 +1,33 @@ -package com.metamx.druid.indexer.data; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -import static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ONE; -import static org.junit.Assert.assertEquals; +package io.druid.data.input; + +import org.joda.time.DateTime; +import org.junit.Test; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; -import com.google.protobuf.ByteString; -import org.joda.time.DateTime; -import org.junit.Test; - -import com.metamx.druid.input.InputRow; +import static org.junit.Assert.assertEquals; public class ProtoBufInputRowParserTest { @@ -44,7 +59,7 @@ public class ProtoBufInputRowParserTest { DateTime dateTime = new DateTime(2012, 07, 12, 9, 30); ProtoTestEventWrapper.ProtoTestEvent event = ProtoTestEventWrapper.ProtoTestEvent.newBuilder() .setDescription("description") - .setEventType(CATEGORY_ONE) + .setEventType(ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ONE) .setId(4711L) .setIsValid(true) .setSomeOtherId(4712) @@ -66,7 +81,7 @@ public class ProtoBufInputRowParserTest { assertDimensionEquals(row, "isValid", "true"); assertDimensionEquals(row, "someOtherId", "4712"); assertDimensionEquals(row, "description", "description"); - assertDimensionEquals(row, "eventType", CATEGORY_ONE.name()); + assertDimensionEquals(row, "eventType", ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ONE.name()); assertEquals(47.11F, row.getFloatMetric("someFloatColumn"), 0.0); diff --git a/indexing-common/src/test/java/com/metamx/druid/indexer/data/ProtoTestEventWrapper.java b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java similarity index 86% rename from indexing-common/src/test/java/com/metamx/druid/indexer/data/ProtoTestEventWrapper.java rename to processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java index 9c86246398a..965859fecfc 100644 --- a/indexing-common/src/test/java/com/metamx/druid/indexer/data/ProtoTestEventWrapper.java +++ b/processing/src/test/java/io/druid/data/input/ProtoTestEventWrapper.java @@ -1,7 +1,28 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + // Generated by the protocol buffer compiler. DO NOT EDIT! // source: ProtoTest.proto -package com.metamx.druid.indexer.data; +package io.druid.data.input; + +import com.google.protobuf.AbstractMessage; public final class ProtoTestEventWrapper { private ProtoTestEventWrapper() {} @@ -13,7 +34,7 @@ public final class ProtoTestEventWrapper { // required .prototest.ProtoTestEvent.EventCategory eventType = 1; boolean hasEventType(); - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory getEventType(); + ProtoTestEventWrapper.ProtoTestEvent.EventCategory getEventType(); // required uint64 id = 2; boolean hasId(); @@ -67,12 +88,12 @@ public final class ProtoTestEventWrapper { public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor; + return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor; } protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_fieldAccessorTable; + return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_fieldAccessorTable; } public enum EventCategory @@ -120,7 +141,7 @@ public final class ProtoTestEventWrapper { } public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.getDescriptor().getEnumTypes().get(0); + return ProtoTestEventWrapper.ProtoTestEvent.getDescriptor().getEnumTypes().get(0); } private static final EventCategory[] VALUES = { @@ -150,11 +171,11 @@ public final class ProtoTestEventWrapper { private int bitField0_; // required .prototest.ProtoTestEvent.EventCategory eventType = 1; public static final int EVENTTYPE_FIELD_NUMBER = 1; - private com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory eventType_; + private ProtoTestEventWrapper.ProtoTestEvent.EventCategory eventType_; public boolean hasEventType() { return ((bitField0_ & 0x00000001) == 0x00000001); } - public com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory getEventType() { + public ProtoTestEventWrapper.ProtoTestEvent.EventCategory getEventType() { return eventType_; } @@ -283,7 +304,7 @@ public final class ProtoTestEventWrapper { } private void initFields() { - eventType_ = com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; + eventType_ = ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; id_ = 0L; timestamp_ = ""; someOtherId_ = 0; @@ -401,41 +422,41 @@ public final class ProtoTestEventWrapper { return super.writeReplace(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom( com.google.protobuf.ByteString data) throws com.google.protobuf.InvalidProtocolBufferException { return newBuilder().mergeFrom(data).buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom( com.google.protobuf.ByteString data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException { return newBuilder().mergeFrom(data, extensionRegistry) .buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom(byte[] data) + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException { return newBuilder().mergeFrom(data).buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom( byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws com.google.protobuf.InvalidProtocolBufferException { return newBuilder().mergeFrom(data, extensionRegistry) .buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom(java.io.InputStream input) + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom(java.io.InputStream input) throws java.io.IOException { return newBuilder().mergeFrom(input).buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom( java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { return newBuilder().mergeFrom(input, extensionRegistry) .buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseDelimitedFrom(java.io.InputStream input) + public static ProtoTestEventWrapper.ProtoTestEvent parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException { Builder builder = newBuilder(); if (builder.mergeDelimitedFrom(input)) { @@ -444,7 +465,7 @@ public final class ProtoTestEventWrapper { return null; } } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseDelimitedFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseDelimitedFrom( java.io.InputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { @@ -455,12 +476,12 @@ public final class ProtoTestEventWrapper { return null; } } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom( com.google.protobuf.CodedInputStream input) throws java.io.IOException { return newBuilder().mergeFrom(input).buildParsed(); } - public static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent parseFrom( + public static ProtoTestEventWrapper.ProtoTestEvent parseFrom( com.google.protobuf.CodedInputStream input, com.google.protobuf.ExtensionRegistryLite extensionRegistry) throws java.io.IOException { @@ -470,7 +491,7 @@ public final class ProtoTestEventWrapper { public static Builder newBuilder() { return Builder.create(); } public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder(com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent prototype) { + public static Builder newBuilder(ProtoTestEventWrapper.ProtoTestEvent prototype) { return newBuilder().mergeFrom(prototype); } public Builder toBuilder() { return newBuilder(this); } @@ -483,18 +504,18 @@ public final class ProtoTestEventWrapper { } public static final class Builder extends com.google.protobuf.GeneratedMessage.Builder - implements com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEventOrBuilder { + implements ProtoTestEventWrapper.ProtoTestEventOrBuilder { public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor; + return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_descriptor; } protected com.google.protobuf.GeneratedMessage.FieldAccessorTable internalGetFieldAccessorTable() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_fieldAccessorTable; + return ProtoTestEventWrapper.internal_static_prototest_ProtoTestEvent_fieldAccessorTable; } - // Construct using com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.newBuilder() + // Construct using io.druid.data.input.ProtoTestEventWrapper.ProtoTestEvent.newBuilder() private Builder() { maybeForceBuilderInitialization(); } @@ -513,7 +534,7 @@ public final class ProtoTestEventWrapper { public Builder clear() { super.clear(); - eventType_ = com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; + eventType_ = ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; bitField0_ = (bitField0_ & ~0x00000001); id_ = 0L; bitField0_ = (bitField0_ & ~0x00000002); @@ -540,33 +561,34 @@ public final class ProtoTestEventWrapper { public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.getDescriptor(); + return ProtoTestEventWrapper.ProtoTestEvent.getDescriptor(); } - public com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent getDefaultInstanceForType() { - return com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.getDefaultInstance(); + public ProtoTestEventWrapper.ProtoTestEvent getDefaultInstanceForType() { + return ProtoTestEventWrapper.ProtoTestEvent.getDefaultInstance(); } - public com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent build() { - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent result = buildPartial(); + public ProtoTestEventWrapper.ProtoTestEvent build() { + ProtoTestEventWrapper.ProtoTestEvent result = buildPartial(); if (!result.isInitialized()) { - throw newUninitializedMessageException(result); + throw AbstractMessage.Builder.newUninitializedMessageException(result); } return result; } - private com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent buildParsed() + private ProtoTestEventWrapper.ProtoTestEvent buildParsed() throws com.google.protobuf.InvalidProtocolBufferException { - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent result = buildPartial(); + ProtoTestEventWrapper.ProtoTestEvent result = buildPartial(); if (!result.isInitialized()) { - throw newUninitializedMessageException( - result).asInvalidProtocolBufferException(); + throw AbstractMessage.Builder.newUninitializedMessageException( + result + ).asInvalidProtocolBufferException(); } return result; } - public com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent buildPartial() { - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent result = new com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent(this); + public ProtoTestEventWrapper.ProtoTestEvent buildPartial() { + ProtoTestEventWrapper.ProtoTestEvent result = new ProtoTestEventWrapper.ProtoTestEvent(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0; if (((from_bitField0_ & 0x00000001) == 0x00000001)) { @@ -611,16 +633,16 @@ public final class ProtoTestEventWrapper { } public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent) { - return mergeFrom((com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent)other); + if (other instanceof ProtoTestEventWrapper.ProtoTestEvent) { + return mergeFrom((ProtoTestEventWrapper.ProtoTestEvent)other); } else { super.mergeFrom(other); return this; } } - public Builder mergeFrom(com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent other) { - if (other == com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.getDefaultInstance()) return this; + public Builder mergeFrom(ProtoTestEventWrapper.ProtoTestEvent other) { + if (other == ProtoTestEventWrapper.ProtoTestEvent.getDefaultInstance()) return this; if (other.hasEventType()) { setEventType(other.getEventType()); } @@ -693,7 +715,7 @@ public final class ProtoTestEventWrapper { } case 8: { int rawValue = input.readEnum(); - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory value = com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.valueOf(rawValue); + ProtoTestEventWrapper.ProtoTestEvent.EventCategory value = ProtoTestEventWrapper.ProtoTestEvent.EventCategory.valueOf(rawValue); if (value == null) { unknownFields.mergeVarintField(1, rawValue); } else { @@ -749,14 +771,14 @@ public final class ProtoTestEventWrapper { private int bitField0_; // required .prototest.ProtoTestEvent.EventCategory eventType = 1; - private com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory eventType_ = com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; + private ProtoTestEventWrapper.ProtoTestEvent.EventCategory eventType_ = ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; public boolean hasEventType() { return ((bitField0_ & 0x00000001) == 0x00000001); } - public com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory getEventType() { + public ProtoTestEventWrapper.ProtoTestEvent.EventCategory getEventType() { return eventType_; } - public Builder setEventType(com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory value) { + public Builder setEventType(ProtoTestEventWrapper.ProtoTestEvent.EventCategory value) { if (value == null) { throw new NullPointerException(); } @@ -767,7 +789,7 @@ public final class ProtoTestEventWrapper { } public Builder clearEventType() { bitField0_ = (bitField0_ & ~0x00000001); - eventType_ = com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; + eventType_ = ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ZERO; onChanged(); return this; } @@ -1017,8 +1039,8 @@ public final class ProtoTestEventWrapper { com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_prototest_ProtoTestEvent_descriptor, new java.lang.String[] { "EventType", "Id", "Timestamp", "SomeOtherId", "IsValid", "Description", "SomeFloatColumn", "SomeIntColumn", "SomeLongColumn", }, - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.class, - com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.Builder.class); + ProtoTestEventWrapper.ProtoTestEvent.class, + ProtoTestEventWrapper.ProtoTestEvent.Builder.class); return null; } }; diff --git a/common/src/test/java/com/metamx/druid/QueryGranularityTest.java b/processing/src/test/java/io/druid/granularity/QueryGranularityTest.java similarity index 99% rename from common/src/test/java/com/metamx/druid/QueryGranularityTest.java rename to processing/src/test/java/io/druid/granularity/QueryGranularityTest.java index 84c1aed555e..9d0eb0287f0 100644 --- a/common/src/test/java/com/metamx/druid/QueryGranularityTest.java +++ b/processing/src/test/java/io/druid/granularity/QueryGranularityTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.granularity; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.druid.jackson.DefaultObjectMapper; - +import io.druid.jackson.DefaultObjectMapper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Days; diff --git a/common/src/test/java/com/metamx/druid/jackson/DefaultObjectMapperTest.java b/processing/src/test/java/io/druid/jackson/DefaultObjectMapperTest.java similarity index 93% rename from common/src/test/java/com/metamx/druid/jackson/DefaultObjectMapperTest.java rename to processing/src/test/java/io/druid/jackson/DefaultObjectMapperTest.java index aff4ad208c6..2a87ae87853 100644 --- a/common/src/test/java/com/metamx/druid/jackson/DefaultObjectMapperTest.java +++ b/processing/src/test/java/io/druid/jackson/DefaultObjectMapperTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.jackson; +package io.druid.jackson; import com.fasterxml.jackson.databind.ObjectMapper; import junit.framework.Assert; diff --git a/server/src/test/java/com/metamx/druid/query/NoopQueryRunnerFactoryConglomerate.java b/processing/src/test/java/io/druid/query/NoopQueryRunnerFactoryConglomerate.java similarity index 90% rename from server/src/test/java/com/metamx/druid/query/NoopQueryRunnerFactoryConglomerate.java rename to processing/src/test/java/io/druid/query/NoopQueryRunnerFactoryConglomerate.java index 6960b921ad7..e3854b8ca4f 100644 --- a/server/src/test/java/com/metamx/druid/query/NoopQueryRunnerFactoryConglomerate.java +++ b/processing/src/test/java/io/druid/query/NoopQueryRunnerFactoryConglomerate.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; - -import com.metamx.druid.Query; +package io.druid.query; /** */ diff --git a/client/src/test/java/com/metamx/druid/query/PrioritizedExecutorServiceTest.java b/processing/src/test/java/io/druid/query/PrioritizedExecutorServiceTest.java similarity index 97% rename from client/src/test/java/com/metamx/druid/query/PrioritizedExecutorServiceTest.java rename to processing/src/test/java/io/druid/query/PrioritizedExecutorServiceTest.java index b6ce2a9ee1c..0209fafa5ec 100644 --- a/client/src/test/java/com/metamx/druid/query/PrioritizedExecutorServiceTest.java +++ b/processing/src/test/java/io/druid/query/PrioritizedExecutorServiceTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.collect.ImmutableList; import com.metamx.common.concurrent.ExecutorServiceConfig; diff --git a/client/src/test/java/com/metamx/druid/query/QueriesTest.java b/processing/src/test/java/io/druid/query/QueriesTest.java similarity index 92% rename from client/src/test/java/com/metamx/druid/query/QueriesTest.java rename to processing/src/test/java/io/druid/query/QueriesTest.java index 02acaa71834..eaac2446d76 100644 --- a/client/src/test/java/com/metamx/druid/query/QueriesTest.java +++ b/processing/src/test/java/io/druid/query/QueriesTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.aggregation.DoubleSumAggregatorFactory; -import com.metamx.druid.aggregation.post.ArithmeticPostAggregator; -import com.metamx.druid.aggregation.post.ConstantPostAggregator; -import com.metamx.druid.aggregation.post.FieldAccessPostAggregator; -import com.metamx.druid.aggregation.post.PostAggregator; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.aggregation.post.ArithmeticPostAggregator; +import io.druid.query.aggregation.post.ConstantPostAggregator; +import io.druid.query.aggregation.post.FieldAccessPostAggregator; import junit.framework.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/query/QueryRunnerTestHelper.java b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java similarity index 85% rename from server/src/test/java/com/metamx/druid/query/QueryRunnerTestHelper.java rename to processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java index 18c647c1c70..58c1e97034c 100644 --- a/server/src/test/java/com/metamx/druid/query/QueryRunnerTestHelper.java +++ b/processing/src/test/java/io/druid/query/QueryRunnerTestHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,26 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query; import com.google.common.collect.Lists; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.aggregation.DoubleSumAggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.aggregation.post.ArithmeticPostAggregator; -import com.metamx.druid.aggregation.post.ConstantPostAggregator; -import com.metamx.druid.aggregation.post.FieldAccessPostAggregator; -import com.metamx.druid.index.IncrementalIndexSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.QueryableIndexSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.v1.IncrementalIndex; -import com.metamx.druid.index.v1.TestIndex; -import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentSpec; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.aggregation.post.ArithmeticPostAggregator; +import io.druid.query.aggregation.post.ConstantPostAggregator; +import io.druid.query.aggregation.post.FieldAccessPostAggregator; +import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.query.spec.QuerySegmentSpec; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.Segment; +import io.druid.segment.TestIndex; +import io.druid.segment.incremental.IncrementalIndex; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/client/src/test/java/com/metamx/druid/ResultGranularTimestampComparatorTest.java b/processing/src/test/java/io/druid/query/ResultGranularTimestampComparatorTest.java similarity index 95% rename from client/src/test/java/com/metamx/druid/ResultGranularTimestampComparatorTest.java rename to processing/src/test/java/io/druid/query/ResultGranularTimestampComparatorTest.java index 13f99f601ac..02665b7491d 100644 --- a/client/src/test/java/com/metamx/druid/ResultGranularTimestampComparatorTest.java +++ b/processing/src/test/java/io/druid/query/ResultGranularTimestampComparatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query; -import com.metamx.druid.result.Result; +import io.druid.granularity.QueryGranularity; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; diff --git a/common/src/test/java/com/metamx/druid/aggregation/CountAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/CountAggregatorTest.java similarity index 95% rename from common/src/test/java/com/metamx/druid/aggregation/CountAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/CountAggregatorTest.java index 8a780146504..2ca8584c939 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/CountAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/CountAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import org.junit.Assert; import org.junit.Test; diff --git a/common/src/test/java/com/metamx/druid/aggregation/DoubleSumAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java similarity index 89% rename from common/src/test/java/com/metamx/druid/aggregation/DoubleSumAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java index d1a87ac9c2d..ff30afbec26 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/DoubleSumAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/DoubleSumAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import org.junit.Assert; import org.junit.Test; @@ -28,7 +28,7 @@ import java.util.Comparator; */ public class DoubleSumAggregatorTest { - private void aggregate(TestFloatMetricSelector selector, DoubleSumAggregator agg) + private void aggregate(TestFloatColumnSelector selector, DoubleSumAggregator agg) { agg.aggregate(); selector.increment(); @@ -38,7 +38,7 @@ public class DoubleSumAggregatorTest public void testAggregate() { final float[] values = {0.15f, 0.27f}; - final TestFloatMetricSelector selector = new TestFloatMetricSelector(values); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); DoubleSumAggregator agg = new DoubleSumAggregator("billy", selector); Assert.assertEquals("billy", agg.getName()); @@ -62,7 +62,7 @@ public class DoubleSumAggregatorTest @Test public void testComparator() { - final TestFloatMetricSelector selector = new TestFloatMetricSelector(new float[]{0.15f, 0.27f}); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(new float[]{0.15f, 0.27f}); DoubleSumAggregator agg = new DoubleSumAggregator("billy", selector); Assert.assertEquals("billy", agg.getName()); diff --git a/common/src/test/java/com/metamx/druid/aggregation/HistogramAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/HistogramAggregatorTest.java similarity index 95% rename from common/src/test/java/com/metamx/druid/aggregation/HistogramAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/HistogramAggregatorTest.java index d72ca031efc..598b0f68ce8 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/HistogramAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/HistogramAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.collect.Lists; import org.junit.Assert; @@ -28,7 +28,7 @@ import java.util.ArrayList; public class HistogramAggregatorTest { - private void aggregate(TestFloatMetricSelector selector, Aggregator agg) + private void aggregate(TestFloatColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); @@ -39,7 +39,7 @@ public class HistogramAggregatorTest final float[] values = {0.55f, 0.27f, -0.3f, -.1f, -0.8f, -.7f, -.5f, 0.25f, 0.1f, 2f, -3f}; final float[] breaks = {-1f, -0.5f, 0.0f, 0.5f, 1f}; - final TestFloatMetricSelector selector = new TestFloatMetricSelector(values); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); HistogramAggregator agg = new HistogramAggregator("billy", selector, breaks); @@ -76,7 +76,7 @@ public class HistogramAggregatorTest Assert.assertArrayEquals(new long[]{1,3,2,3,1,1}, ((Histogram)agg.get()).bins); } - private void aggregateBuffer(TestFloatMetricSelector selector, BufferAggregator agg, ByteBuffer buf, int position) + private void aggregateBuffer(TestFloatColumnSelector selector, BufferAggregator agg, ByteBuffer buf, int position) { agg.aggregate(buf, position); selector.increment(); @@ -87,7 +87,7 @@ public class HistogramAggregatorTest final float[] values = {0.55f, 0.27f, -0.3f, -.1f, -0.8f, -.7f, -.5f, 0.25f, 0.1f, 2f, -3f}; final float[] breaks = {-1f, -0.5f, 0.0f, 0.5f, 1f}; - final TestFloatMetricSelector selector = new TestFloatMetricSelector(values); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); ArrayList b = Lists.newArrayList(); for (int i = 0; i < breaks.length; ++i) { diff --git a/common/src/test/java/com/metamx/druid/histogram/HistogramTest.java b/processing/src/test/java/io/druid/query/aggregation/HistogramTest.java similarity index 93% rename from common/src/test/java/com/metamx/druid/histogram/HistogramTest.java rename to processing/src/test/java/io/druid/query/aggregation/HistogramTest.java index f221b0e26df..5f944167450 100644 --- a/common/src/test/java/com/metamx/druid/histogram/HistogramTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/HistogramTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.histogram; +package io.druid.query.aggregation; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; -import com.metamx.druid.aggregation.Histogram; -import com.metamx.druid.jackson.DefaultObjectMapper; - +import io.druid.jackson.DefaultObjectMapper; import org.junit.Assert; import org.junit.Test; diff --git a/common/src/test/java/com/metamx/druid/aggregation/JavaScriptAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java similarity index 85% rename from common/src/test/java/com/metamx/druid/aggregation/JavaScriptAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java index 98e4576a731..2435211dfe9 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/JavaScriptAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/JavaScriptAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.druid.processing.ObjectColumnSelector; +import io.druid.segment.ObjectColumnSelector; import org.junit.Assert; import org.junit.Test; @@ -44,15 +44,15 @@ public class JavaScriptAggregatorTest scriptDoubleSum.put("fnCombine", "function combine(a,b) { return a + b }"); } - private static void aggregate(TestFloatMetricSelector selector1, TestFloatMetricSelector selector2, Aggregator agg) + private static void aggregate(TestFloatColumnSelector selector1, TestFloatColumnSelector selector2, Aggregator agg) { agg.aggregate(); selector1.increment(); selector2.increment(); } - private void aggregateBuffer(TestFloatMetricSelector selector1, - TestFloatMetricSelector selector2, + private void aggregateBuffer(TestFloatColumnSelector selector1, + TestFloatColumnSelector selector2, BufferAggregator agg, ByteBuffer buf, int position) { @@ -61,7 +61,7 @@ public class JavaScriptAggregatorTest selector2.increment(); } - private static void aggregate(TestFloatMetricSelector selector, Aggregator agg) + private static void aggregate(TestFloatColumnSelector selector, Aggregator agg) { agg.aggregate(); selector.increment(); @@ -70,8 +70,8 @@ public class JavaScriptAggregatorTest @Test public void testAggregate() { - final TestFloatMetricSelector selector1 = new TestFloatMetricSelector(new float[]{42.12f, 9f}); - final TestFloatMetricSelector selector2 = new TestFloatMetricSelector(new float[]{2f, 3f}); + final TestFloatColumnSelector selector1 = new TestFloatColumnSelector(new float[]{42.12f, 9f}); + final TestFloatColumnSelector selector2 = new TestFloatColumnSelector(new float[]{2f, 3f}); Map script = sumLogATimesBPlusTen; @@ -108,8 +108,8 @@ public class JavaScriptAggregatorTest @Test public void testBufferAggregate() { - final TestFloatMetricSelector selector1 = new TestFloatMetricSelector(new float[]{42.12f, 9f}); - final TestFloatMetricSelector selector2 = new TestFloatMetricSelector(new float[]{2f, 3f}); + final TestFloatColumnSelector selector1 = new TestFloatColumnSelector(new float[]{42.12f, 9f}); + final TestFloatColumnSelector selector2 = new TestFloatColumnSelector(new float[]{2f, 3f}); Map script = sumLogATimesBPlusTen; JavaScriptBufferAggregator agg = new JavaScriptBufferAggregator( @@ -142,7 +142,7 @@ public class JavaScriptAggregatorTest } public static void main(String... args) throws Exception { - final LoopingFloatMetricSelector selector = new LoopingFloatMetricSelector(new float[]{42.12f, 9f}); + final LoopingFloatColumnSelector selector = new LoopingFloatColumnSelector(new float[]{42.12f, 9f}); /* memory usage test List aggs = Lists.newLinkedList(); @@ -150,7 +150,7 @@ public class JavaScriptAggregatorTest for(int i = 0; i < 100000; ++i) { JavaScriptAggregator a = new JavaScriptAggregator( "billy", - Lists.asList(selector, new FloatMetricSelector[]{}), + Lists.asList(selector, new FloatColumnSelector[]{}), JavaScriptAggregatorFactory.compileScript(scriptDoubleSum) ); //aggs.add(a); @@ -206,12 +206,12 @@ public class JavaScriptAggregatorTest System.out.println(String.format("JavaScript is %2.1fx slower", (double)t1 / t2)); } - static class LoopingFloatMetricSelector extends TestFloatMetricSelector + static class LoopingFloatColumnSelector extends TestFloatColumnSelector { private final float[] floats; private long index = 0; - public LoopingFloatMetricSelector(float[] floats) + public LoopingFloatColumnSelector(float[] floats) { super(floats); this.floats = floats; diff --git a/common/src/test/java/com/metamx/druid/aggregation/LongSumAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java similarity index 88% rename from common/src/test/java/com/metamx/druid/aggregation/LongSumAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java index 8db1a4a4755..294eef07b24 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/LongSumAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/LongSumAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import org.junit.Assert; import org.junit.Test; @@ -28,7 +28,7 @@ import java.util.Comparator; */ public class LongSumAggregatorTest { - private void aggregate(TestFloatMetricSelector selector, LongSumAggregator agg) + private void aggregate(TestFloatColumnSelector selector, LongSumAggregator agg) { agg.aggregate(); selector.increment(); @@ -37,7 +37,7 @@ public class LongSumAggregatorTest @Test public void testAggregate() { - final TestFloatMetricSelector selector = new TestFloatMetricSelector(new float[]{24.15f, 20f}); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(new float[]{24.15f, 20f}); LongSumAggregator agg = new LongSumAggregator("billy", selector); Assert.assertEquals("billy", agg.getName()); @@ -58,7 +58,7 @@ public class LongSumAggregatorTest @Test public void testComparator() { - final TestFloatMetricSelector selector = new TestFloatMetricSelector(new float[]{18293f}); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(new float[]{18293f}); LongSumAggregator agg = new LongSumAggregator("billy", selector); Assert.assertEquals("billy", agg.getName()); diff --git a/common/src/test/java/com/metamx/druid/aggregation/MaxAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/MaxAggregatorTest.java similarity index 86% rename from common/src/test/java/com/metamx/druid/aggregation/MaxAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/MaxAggregatorTest.java index ed2a55efbb2..c53fe74f6d6 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/MaxAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/MaxAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import org.junit.Assert; import org.junit.Test; @@ -26,7 +26,7 @@ import org.junit.Test; */ public class MaxAggregatorTest { - private void aggregate(TestFloatMetricSelector selector, MaxAggregator agg) + private void aggregate(TestFloatColumnSelector selector, MaxAggregator agg) { agg.aggregate(); selector.increment(); @@ -36,7 +36,7 @@ public class MaxAggregatorTest public void testAggregate() throws Exception { final float[] values = {0.15f, 0.27f, 0.0f, 0.93f}; - final TestFloatMetricSelector selector = new TestFloatMetricSelector(values); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); MaxAggregator agg = new MaxAggregator("billy", selector); Assert.assertEquals("billy", agg.getName()); diff --git a/common/src/test/java/com/metamx/druid/aggregation/MetricSelectorUtils.java b/processing/src/test/java/io/druid/query/aggregation/MetricSelectorUtils.java similarity index 75% rename from common/src/test/java/com/metamx/druid/aggregation/MetricSelectorUtils.java rename to processing/src/test/java/io/druid/query/aggregation/MetricSelectorUtils.java index da49af820e2..1f7648379f1 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/MetricSelectorUtils.java +++ b/processing/src/test/java/io/druid/query/aggregation/MetricSelectorUtils.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2013 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.ComplexMetricSelector; -import com.metamx.druid.processing.FloatMetricSelector; -import com.metamx.druid.processing.ObjectColumnSelector; +import io.druid.segment.FloatColumnSelector; +import io.druid.segment.ObjectColumnSelector; public class MetricSelectorUtils { - public static ObjectColumnSelector wrap(final FloatMetricSelector selector) + public static ObjectColumnSelector wrap(final FloatColumnSelector selector) { return new ObjectColumnSelector() { diff --git a/common/src/test/java/com/metamx/druid/aggregation/MinAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/MinAggregatorTest.java similarity index 86% rename from common/src/test/java/com/metamx/druid/aggregation/MinAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/MinAggregatorTest.java index 9544595ec82..aea142ca8b3 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/MinAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/MinAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; import org.junit.Assert; import org.junit.Test; @@ -26,7 +26,7 @@ import org.junit.Test; */ public class MinAggregatorTest { - private void aggregate(TestFloatMetricSelector selector, MinAggregator agg) + private void aggregate(TestFloatColumnSelector selector, MinAggregator agg) { agg.aggregate(); selector.increment(); @@ -36,7 +36,7 @@ public class MinAggregatorTest public void testAggregate() throws Exception { final float[] values = {0.15f, 0.27f, 0.0f, 0.93f}; - final TestFloatMetricSelector selector = new TestFloatMetricSelector(values); + final TestFloatColumnSelector selector = new TestFloatColumnSelector(values); MinAggregator agg = new MinAggregator("billy", selector); Assert.assertEquals("billy", agg.getName()); diff --git a/common/src/test/java/com/metamx/druid/aggregation/TestFloatMetricSelector.java b/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java similarity index 79% rename from common/src/test/java/com/metamx/druid/aggregation/TestFloatMetricSelector.java rename to processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java index 616b3c5ecca..8aea1f41503 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/TestFloatMetricSelector.java +++ b/processing/src/test/java/io/druid/query/aggregation/TestFloatColumnSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation; +package io.druid.query.aggregation; -import com.metamx.druid.processing.FloatMetricSelector; +import io.druid.segment.FloatColumnSelector; /** */ -public class TestFloatMetricSelector implements FloatMetricSelector +public class TestFloatColumnSelector implements FloatColumnSelector { private final float[] floats; private int index = 0; - public TestFloatMetricSelector(float[] floats) + public TestFloatColumnSelector(float[] floats) { this.floats = floats; } diff --git a/common/src/test/java/com/metamx/druid/aggregation/post/ArithmeticPostAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java similarity index 94% rename from common/src/test/java/com/metamx/druid/aggregation/post/ArithmeticPostAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java index c8365de19b6..6c0f49e96bd 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/post/ArithmeticPostAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/post/ArithmeticPostAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; +package io.druid.query.aggregation.post; import com.google.common.collect.Lists; -import com.metamx.druid.aggregation.CountAggregator; +import io.druid.query.aggregation.CountAggregator; +import io.druid.query.aggregation.PostAggregator; import org.junit.Assert; import org.junit.Test; diff --git a/common/src/test/java/com/metamx/druid/aggregation/post/ConstantPostAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/post/ConstantPostAggregatorTest.java similarity index 95% rename from common/src/test/java/com/metamx/druid/aggregation/post/ConstantPostAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/post/ConstantPostAggregatorTest.java index 0de6e2c62e7..08d86452230 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/post/ConstantPostAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/post/ConstantPostAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; +package io.druid.query.aggregation.post; import org.junit.Assert; import org.junit.Test; diff --git a/common/src/test/java/com/metamx/druid/aggregation/post/FieldAccessPostAggregatorTest.java b/processing/src/test/java/io/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java similarity index 91% rename from common/src/test/java/com/metamx/druid/aggregation/post/FieldAccessPostAggregatorTest.java rename to processing/src/test/java/io/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java index e413504b1ad..5e20d488559 100644 --- a/common/src/test/java/com/metamx/druid/aggregation/post/FieldAccessPostAggregatorTest.java +++ b/processing/src/test/java/io/druid/query/aggregation/post/FieldAccessPostAggregatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.aggregation.post; +package io.druid.query.aggregation.post; -import com.metamx.druid.aggregation.CountAggregator; +import io.druid.query.aggregation.CountAggregator; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/query/extraction/PartialDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/extraction/PartialDimExtractionFnTest.java similarity index 89% rename from client/src/test/java/com/metamx/druid/query/extraction/PartialDimExtractionFnTest.java rename to processing/src/test/java/io/druid/query/extraction/extraction/PartialDimExtractionFnTest.java index 061f8122f2e..8c9cb6b093a 100644 --- a/client/src/test/java/com/metamx/druid/query/extraction/PartialDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/extraction/PartialDimExtractionFnTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction.extraction; import com.google.common.collect.Sets; +import io.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.PartialDimExtractionFn; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/query/extraction/RegexDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/extraction/RegexDimExtractionFnTest.java similarity index 93% rename from client/src/test/java/com/metamx/druid/query/extraction/RegexDimExtractionFnTest.java rename to processing/src/test/java/io/druid/query/extraction/extraction/RegexDimExtractionFnTest.java index 2ec3f3db239..43f90a1ac01 100644 --- a/client/src/test/java/com/metamx/druid/query/extraction/RegexDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/extraction/RegexDimExtractionFnTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction.extraction; import com.google.common.collect.Sets; +import io.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.RegexDimExtractionFn; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/extraction/SearchQuerySpecDimExtractionFnTest.java similarity index 83% rename from client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java rename to processing/src/test/java/io/druid/query/extraction/extraction/SearchQuerySpecDimExtractionFnTest.java index 808c3603b6b..30104f523a9 100644 --- a/client/src/test/java/com/metamx/druid/query/extraction/SearchQuerySpecDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/extraction/SearchQuerySpecDimExtractionFnTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction.extraction; import com.google.common.collect.Sets; -import com.metamx.druid.query.search.FragmentSearchQuerySpec; -import com.metamx.druid.query.search.LexicographicSearchSortSpec; -import com.metamx.druid.query.search.SearchQuerySpec; +import io.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.SearchQuerySpecDimExtractionFn; +import io.druid.query.search.search.FragmentSearchQuerySpec; +import io.druid.query.search.search.SearchQuerySpec; import org.junit.Assert; import org.junit.Test; @@ -49,7 +50,7 @@ public class SearchQuerySpecDimExtractionFnTest public void testExtraction() { SearchQuerySpec spec = new FragmentSearchQuerySpec( - Arrays.asList("to", "yo"), new LexicographicSearchSortSpec() + Arrays.asList("to", "yo") ); DimExtractionFn dimExtractionFn = new SearchQuerySpecDimExtractionFn(spec); List expected = Arrays.asList("Kyoto", "Tokyo", "Toyokawa", "Yorktown"); diff --git a/client/src/test/java/com/metamx/druid/query/extraction/TimeDimExtractionFnTest.java b/processing/src/test/java/io/druid/query/extraction/extraction/TimeDimExtractionFnTest.java similarity index 91% rename from client/src/test/java/com/metamx/druid/query/extraction/TimeDimExtractionFnTest.java rename to processing/src/test/java/io/druid/query/extraction/extraction/TimeDimExtractionFnTest.java index 7f5b6eabe1c..1ac839a7e4c 100644 --- a/client/src/test/java/com/metamx/druid/query/extraction/TimeDimExtractionFnTest.java +++ b/processing/src/test/java/io/druid/query/extraction/extraction/TimeDimExtractionFnTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.extraction; +package io.druid.query.extraction.extraction; import com.google.common.collect.Sets; +import io.druid.query.extraction.DimExtractionFn; +import io.druid.query.extraction.TimeDimExtractionFn; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java similarity index 92% rename from server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java rename to processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java index 3e3b6006da5..e1556e94da7 100644 --- a/server/src/test/java/com/metamx/druid/query/group/GroupByQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByQueryRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group; +package io.druid.query.groupby; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; @@ -29,32 +30,33 @@ import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.PeriodGranularity; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.TestHelper; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.DoubleSumAggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.input.MapBasedRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerTestHelper; -import com.metamx.druid.query.dimension.DefaultDimensionSpec; -import com.metamx.druid.query.dimension.DimensionSpec; -import com.metamx.druid.query.filter.RegexDimFilter; -import com.metamx.druid.query.group.having.EqualToHavingSpec; -import com.metamx.druid.query.group.having.GreaterThanHavingSpec; -import com.metamx.druid.query.group.having.OrHavingSpec; -import com.metamx.druid.query.group.orderby.DefaultLimitSpec; -import com.metamx.druid.query.group.orderby.LimitSpec; -import com.metamx.druid.query.group.orderby.OrderByColumnSpec; -import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; +import io.druid.collections.StupidPool; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.granularity.PeriodGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.RegexDimFilter; +import io.druid.query.groupby.having.EqualToHavingSpec; +import io.druid.query.groupby.having.GreaterThanHavingSpec; +import io.druid.query.groupby.having.OrHavingSpec; +import io.druid.query.groupby.orderby.DefaultLimitSpec; +import io.druid.query.groupby.orderby.LimitSpec; +import io.druid.query.groupby.orderby.OrderByColumnSpec; +import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.TestHelper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; import org.joda.time.Period; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -73,20 +75,24 @@ public class GroupByQueryRunnerTest { private final QueryRunner runner; private GroupByQueryRunnerFactory factory; + private Supplier configSupplier; + + @Before + public void setUp() throws Exception + { + configSupplier = Suppliers.ofInstance(new GroupByQueryConfig()); + } @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { + GroupByQueryConfig config = new GroupByQueryConfig(); + config.setMaxIntermediateRows(10000); + + final Supplier configSupplier = Suppliers.ofInstance(config); final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( new GroupByQueryEngine( - new GroupByQueryEngineConfig() - { - @Override - public int getMaxIntermediateRows() - { - return 10000; - } - }, + configSupplier, new StupidPool( new Supplier() { @@ -98,9 +104,8 @@ public class GroupByQueryRunnerTest } ) ), - new GroupByQueryRunnerFactoryConfig() - { - } + configSupplier, + new GroupByQueryQueryToolChest(configSupplier) ); return Lists.newArrayList( @@ -250,7 +255,7 @@ public class GroupByQueryRunnerTest final GroupByQuery fullQuery = builder.build(); final GroupByQuery allGranQuery = builder.copy().setGranularity(QueryGranularity.ALL).build(); - QueryRunner mergedRunner = new GroupByQueryQueryToolChest().mergeResults( + QueryRunner mergedRunner = factory.getToolchest().mergeResults( new QueryRunner() { @Override @@ -337,7 +342,7 @@ public class GroupByQueryRunnerTest createExpectedRow("2011-04-01", "alias", "travel", "rows", 2L, "idx", 243L) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, limit), mergeRunner.run(fullQuery), String.format("limit: %d", limit) @@ -442,7 +447,7 @@ public class GroupByQueryRunnerTest final GroupByQuery fullQuery = builder.build(); - QueryRunner mergedRunner = new GroupByQueryQueryToolChest().mergeResults( + QueryRunner mergedRunner = factory.getToolchest().mergeResults( new QueryRunner() { @Override @@ -495,7 +500,7 @@ public class GroupByQueryRunnerTest createExpectedRow("2011-04-01", "alias", "mezzanine", "rows", 6L, "idx", 4420L) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query), "no-limit"); TestHelper.assertExpectedObjects( @@ -535,7 +540,7 @@ public class GroupByQueryRunnerTest createExpectedRow("2011-04-01", "alias", "automotive", "rows", 2L, "idx", 269L) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query), "no-limit"); TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, 5), mergeRunner.run(builder.limit(5).build()), "limited" @@ -574,7 +579,7 @@ public class GroupByQueryRunnerTest createExpectedRow("2011-04-01", "alias", "technology", "rows", 2L, "idx", 178.24917602539062D) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = factory.getToolchest().mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query), "no-limit"); TestHelper.assertExpectedObjects( Iterables.limit(expectedResults, 5), mergeRunner.run(builder.limit(5).build()), "limited" @@ -613,7 +618,7 @@ public class GroupByQueryRunnerTest final GroupByQuery fullQuery = builder.build(); - QueryRunner mergedRunner = new GroupByQueryQueryToolChest().mergeResults( + QueryRunner mergedRunner = factory.getToolchest().mergeResults( new QueryRunner() { @Override @@ -656,7 +661,7 @@ public class GroupByQueryRunnerTest createExpectedRow("2011-04-01", "quality", "automotive", "rows", 2L) ); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = new GroupByQueryQueryToolChest(configSupplier).mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query), "no-limit"); } @@ -691,12 +696,12 @@ public class GroupByQueryRunnerTest ); TestHelper.assertExpectedObjects(expectedResults, runner.run(query), "normal"); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = new GroupByQueryQueryToolChest(configSupplier).mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query), "no-limit"); } @Test - public void testGroupByWithNonexistantDimension() throws Exception + public void testGroupByWithNonexistentDimension() throws Exception { GroupByQuery.Builder builder = GroupByQuery .builder() @@ -726,7 +731,7 @@ public class GroupByQueryRunnerTest ); TestHelper.assertExpectedObjects(expectedResults, runner.run(query), "normal"); - QueryRunner mergeRunner = new GroupByQueryQueryToolChest().mergeResults(runner); + QueryRunner mergeRunner = new GroupByQueryQueryToolChest(configSupplier).mergeResults(runner); TestHelper.assertExpectedObjects(expectedResults, mergeRunner.run(query), "no-limit"); } diff --git a/server/src/test/java/com/metamx/druid/query/group/GroupByTimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java similarity index 86% rename from server/src/test/java/com/metamx/druid/query/group/GroupByTimeseriesQueryRunnerTest.java rename to processing/src/test/java/io/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java index ca6dab39c98..562527556a9 100644 --- a/server/src/test/java/com/metamx/druid/query/group/GroupByTimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/groupby/GroupByTimeseriesQueryRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,23 +17,24 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.group; +package io.druid.query.groupby; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Query; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.input.MapBasedRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerTestHelper; -import com.metamx.druid.query.timeseries.TimeseriesQuery; -import com.metamx.druid.query.timeseries.TimeseriesQueryRunnerTest; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.collections.StupidPool; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryRunnerTest; +import io.druid.query.timeseries.TimeseriesResultValue; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -51,17 +52,13 @@ public class GroupByTimeseriesQueryRunnerTest extends TimeseriesQueryRunnerTest @Parameterized.Parameters public static Collection constructorFeeder() throws IOException { + GroupByQueryConfig config = new GroupByQueryConfig(); + config.setMaxIntermediateRows(10000); + final Supplier configSupplier = Suppliers.ofInstance(config); final GroupByQueryRunnerFactory factory = new GroupByQueryRunnerFactory( new GroupByQueryEngine( - new GroupByQueryEngineConfig() - { - @Override - public int getMaxIntermediateRows() - { - return 10000; - } - }, + configSupplier, new StupidPool( new Supplier() { @@ -73,9 +70,8 @@ public class GroupByTimeseriesQueryRunnerTest extends TimeseriesQueryRunnerTest } ) ), - new GroupByQueryRunnerFactoryConfig() - { - } + configSupplier, + new GroupByQueryQueryToolChest(configSupplier) ); final Collection objects = QueryRunnerTestHelper.makeQueryRunners(factory); diff --git a/client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java b/processing/src/test/java/io/druid/query/groupby/having/HavingSpecTest.java similarity index 86% rename from client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java rename to processing/src/test/java/io/druid/query/groupby/having/HavingSpecTest.java index 28c91878e57..460ace869b7 100644 --- a/client/src/test/java/com/metamx/druid/query/group/having/HavingSpecTest.java +++ b/processing/src/test/java/io/druid/query/groupby/having/HavingSpecTest.java @@ -1,12 +1,31 @@ -package com.metamx.druid.query.group.having; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.having; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.Row; +import io.druid.jackson.DefaultObjectMapper; import org.junit.Test; import java.util.ArrayList; diff --git a/client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java b/processing/src/test/java/io/druid/query/groupby/orderby/TopNSorterTest.java similarity index 73% rename from client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java rename to processing/src/test/java/io/druid/query/groupby/orderby/TopNSorterTest.java index 872155e5049..7e2eeb051bf 100644 --- a/client/src/test/java/com/metamx/druid/query/group/orderby/TopNSorterTest.java +++ b/processing/src/test/java/io/druid/query/groupby/orderby/TopNSorterTest.java @@ -1,4 +1,23 @@ -package com.metamx.druid.query.group.orderby; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.groupby.orderby; import com.google.common.base.Splitter; import com.google.common.collect.Lists; diff --git a/server/src/test/java/com/metamx/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java similarity index 83% rename from server/src/test/java/com/metamx/druid/query/metadata/SegmentAnalyzerTest.java rename to processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java index 6805bb6c23b..55ed5887480 100644 --- a/server/src/test/java/com/metamx/druid/query/metadata/SegmentAnalyzerTest.java +++ b/processing/src/test/java/io/druid/query/metadata/SegmentAnalyzerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,22 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.metadata; +package io.druid.query.metadata; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequences; -import com.metamx.druid.index.IncrementalIndexSegment; -import com.metamx.druid.index.QueryableIndexSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.column.ValueType; -import com.metamx.druid.index.v1.TestIndex; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerTestHelper; -import com.metamx.druid.query.segment.QuerySegmentSpecs; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.metadata.metadata.ColumnAnalysis; +import io.druid.query.metadata.metadata.SegmentAnalysis; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.query.spec.QuerySegmentSpecs; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.Segment; +import io.druid.segment.TestIndex; +import io.druid.segment.column.ValueType; import junit.framework.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/query/search/LexicographicSearchSortSpecTest.java b/processing/src/test/java/io/druid/query/search/LexicographicSearchSortSpecTest.java similarity index 84% rename from client/src/test/java/com/metamx/druid/query/search/LexicographicSearchSortSpecTest.java rename to processing/src/test/java/io/druid/query/search/LexicographicSearchSortSpecTest.java index 46c8a7537ff..0ec86b46343 100644 --- a/client/src/test/java/com/metamx/druid/query/search/LexicographicSearchSortSpecTest.java +++ b/processing/src/test/java/io/druid/query/search/LexicographicSearchSortSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search; +import io.druid.query.search.search.LexicographicSearchSortSpec; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.SearchSortSpec; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/SearchBinaryFnTest.java b/processing/src/test/java/io/druid/query/search/SearchBinaryFnTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/SearchBinaryFnTest.java rename to processing/src/test/java/io/druid/query/search/SearchBinaryFnTest.java index 4b6b4181d26..9f277bb267b 100644 --- a/client/src/test/java/com/metamx/druid/SearchBinaryFnTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchBinaryFnTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query.search; import com.google.common.collect.ImmutableList; -import com.metamx.druid.query.search.LexicographicSearchSortSpec; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.StrlenSearchSortSpec; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Result; +import io.druid.query.search.search.LexicographicSearchSortSpec; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.StrlenSearchSortSpec; import junit.framework.Assert; import org.joda.time.DateTime; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java similarity index 93% rename from server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java rename to processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java index 0802a99bb9a..2e5623b4365 100644 --- a/server/src/test/java/com/metamx/druid/query/search/SearchQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/search/SearchQueryRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,22 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Druids; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerTestHelper; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; +import io.druid.query.Druids; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; +import io.druid.query.filter.DimFilter; +import io.druid.query.search.search.FragmentSearchQuerySpec; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQueryConfig; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; @@ -43,7 +46,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.TreeMap; /** */ @@ -54,7 +56,7 @@ public class SearchQueryRunnerTest public static Collection constructorFeeder() throws IOException { return QueryRunnerTestHelper.makeQueryRunners( - new SearchQueryRunnerFactory() + new SearchQueryRunnerFactory(new SearchQueryQueryToolChest(new SearchQueryConfig())) ); } @@ -95,7 +97,7 @@ public class SearchQueryRunnerTest .dataSource(QueryRunnerTestHelper.dataSource) .granularity(QueryRunnerTestHelper.allGran) .intervals(QueryRunnerTestHelper.fullOnInterval) - .query(new FragmentSearchQuerySpec(Arrays.asList("auto", "ve"), null)) + .query(new FragmentSearchQuerySpec(Arrays.asList("auto", "ve"))) .build(); Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); @@ -105,7 +107,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimension1() + public void testSearchWithDimensionQuality() { Map> expectedResults = new HashMap>(); expectedResults.put( @@ -129,7 +131,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimension2() + public void testSearchWithDimensionProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); @@ -147,7 +149,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimensions1() + public void testSearchWithDimensionsQualityAndProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); expectedResults.putAll( @@ -183,7 +185,7 @@ public class SearchQueryRunnerTest } @Test - public void testSearchWithDimensions2() + public void testSearchWithDimensionsPlacementAndProvider() { Map> expectedResults = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); expectedResults.put(QueryRunnerTestHelper.providerDimension, new HashSet(Arrays.asList("total_market"))); @@ -391,7 +393,7 @@ public class SearchQueryRunnerTest for (Map.Entry> entry : expectedResults.entrySet()) { Assert.assertTrue( String.format( - "Dimension %s should have had everything removed, still has[%s]", entry.getKey(), entry.getValue() + "Dimension[%s] should have had everything removed, still has[%s]", entry.getKey(), entry.getValue() ), entry.getValue().isEmpty() ); diff --git a/client/src/test/java/com/metamx/druid/query/search/StrlenSearchSortSpecTest.java b/processing/src/test/java/io/druid/query/search/StrlenSearchSortSpecTest.java similarity index 84% rename from client/src/test/java/com/metamx/druid/query/search/StrlenSearchSortSpecTest.java rename to processing/src/test/java/io/druid/query/search/StrlenSearchSortSpecTest.java index 36eaba1330d..dcdf9074b8a 100644 --- a/client/src/test/java/com/metamx/druid/query/search/StrlenSearchSortSpecTest.java +++ b/processing/src/test/java/io/druid/query/search/StrlenSearchSortSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.search; +package io.druid.query.search; +import io.druid.query.search.search.SearchHit; +import io.druid.query.search.search.SearchSortSpec; +import io.druid.query.search.search.StrlenSearchSortSpec; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/query/QuerySegmentSpecTest.java b/processing/src/test/java/io/druid/query/spec/QuerySegmentSpecTest.java similarity index 90% rename from client/src/test/java/com/metamx/druid/query/QuerySegmentSpecTest.java rename to processing/src/test/java/io/druid/query/spec/QuerySegmentSpecTest.java index 654ea2e6b79..0c43bd1176f 100644 --- a/client/src/test/java/com/metamx/druid/query/QuerySegmentSpecTest.java +++ b/processing/src/test/java/io/druid/query/spec/QuerySegmentSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query; +package io.druid.query.spec; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.segment.LegacySegmentSpec; -import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; -import com.metamx.druid.query.segment.MultipleSpecificSegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.query.segment.SegmentDescriptor; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryQueryToolChestTest.java b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryQueryToolChestTest.java similarity index 95% rename from client/src/test/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryQueryToolChestTest.java rename to processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryQueryToolChestTest.java index 03492e32716..616d8d05938 100644 --- a/client/src/test/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryQueryToolChestTest.java +++ b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryQueryToolChestTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeboundary; +package io.druid.query.timeboundary; -import com.metamx.druid.LogicalSegment; +import io.druid.timeline.LogicalSegment; import junit.framework.Assert; import org.joda.time.Interval; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java similarity index 87% rename from server/src/test/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java rename to processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java index 999c4202073..9b92826ac34 100644 --- a/server/src/test/java/com/metamx/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeboundary/TimeBoundaryQueryRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeboundary; +package io.druid.query.timeboundary; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Druids; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerTestHelper; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeBoundaryResultValue; +import io.druid.query.Druids; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Test; diff --git a/client/src/test/java/com/metamx/druid/TimeseriesBinaryFnTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java similarity index 91% rename from client/src/test/java/com/metamx/druid/TimeseriesBinaryFnTest.java rename to processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java index f278153a643..03211178d25 100644 --- a/client/src/test/java/com/metamx/druid/TimeseriesBinaryFnTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesBinaryFnTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.query.timeseries; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.aggregation.post.ArithmeticPostAggregator; -import com.metamx.druid.aggregation.post.ConstantPostAggregator; -import com.metamx.druid.aggregation.post.FieldAccessPostAggregator; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.aggregation.post.ArithmeticPostAggregator; +import io.druid.query.aggregation.post.ConstantPostAggregator; +import io.druid.query.aggregation.post.FieldAccessPostAggregator; import junit.framework.Assert; import org.joda.time.DateTime; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java similarity index 69% rename from server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java rename to processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java index 91865f47c7d..1be909b1c68 100644 --- a/server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerBonusTest.java @@ -1,23 +1,41 @@ -package com.metamx.druid.query.timeseries; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.query.timeseries; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Druids; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.index.IncrementalIndexSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.v1.IncrementalIndex; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.data.input.MapBasedInputRow; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Druids; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.Segment; +import io.druid.segment.incremental.IncrementalIndex; import junit.framework.Assert; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerTest.java b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java similarity index 98% rename from server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerTest.java rename to processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java index c1eb9a249eb..e825c306b02 100644 --- a/server/src/test/java/com/metamx/druid/query/timeseries/TimeseriesQueryRunnerTest.java +++ b/processing/src/test/java/io/druid/query/timeseries/TimeseriesQueryRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,28 +17,27 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.query.timeseries; +package io.druid.query.timeseries; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequences; -import com.metamx.druid.Druids; -import com.metamx.druid.PeriodGranularity; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.TestHelper; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.aggregation.MaxAggregatorFactory; -import com.metamx.druid.aggregation.MinAggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerTestHelper; -import com.metamx.druid.query.filter.AndDimFilter; -import com.metamx.druid.query.filter.DimFilter; -import com.metamx.druid.query.filter.RegexDimFilter; -import com.metamx.druid.query.segment.MultipleIntervalSegmentSpec; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.granularity.PeriodGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Druids; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerTestHelper; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.aggregation.MaxAggregatorFactory; +import io.druid.query.aggregation.MinAggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.RegexDimFilter; +import io.druid.query.spec.MultipleIntervalSegmentSpec; +import io.druid.segment.TestHelper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Interval; @@ -52,7 +51,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; -import java.util.Map; /** */ diff --git a/server/src/test/java/com/metamx/druid/index/v1/ConciseOffsetTest.java b/processing/src/test/java/io/druid/segment/ConciseOffsetTest.java similarity index 93% rename from server/src/test/java/com/metamx/druid/index/v1/ConciseOffsetTest.java rename to processing/src/test/java/io/druid/segment/ConciseOffsetTest.java index f32ecae3a47..4e2a2085c82 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/ConciseOffsetTest.java +++ b/processing/src/test/java/io/druid/segment/ConciseOffsetTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; -import com.metamx.druid.index.v1.processing.Offset; +import io.druid.segment.data.Offset; import it.uniroma3.mat.extendedset.intset.ConciseSet; import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; import org.junit.Assert; diff --git a/server/src/test/java/com/metamx/druid/index/v1/EmptyIndexTest.java b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java similarity index 88% rename from server/src/test/java/com/metamx/druid/index/v1/EmptyIndexTest.java rename to processing/src/test/java/io/druid/segment/EmptyIndexTest.java index 55c4b7be6e5..d4e835566a9 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/EmptyIndexTest.java +++ b/processing/src/test/java/io/druid/segment/EmptyIndexTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.index.QueryableIndex; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexAdapter; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/index/v1/IndexMergerTest.java b/processing/src/test/java/io/druid/segment/IndexMergerTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/index/v1/IndexMergerTest.java rename to processing/src/test/java/io/druid/segment/IndexMergerTest.java index 8af66f30eb4..a21eb92c718 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/IndexMergerTest.java +++ b/processing/src/test/java/io/druid/segment/IndexMergerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.io.Files; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.input.MapBasedInputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.data.IncrementalIndexTest; +import io.druid.segment.incremental.IncrementalIndex; import junit.framework.Assert; import org.apache.commons.io.FileUtils; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/index/ReferenceCountingSegmentTest.java b/processing/src/test/java/io/druid/segment/ReferenceCountingSegmentTest.java similarity index 96% rename from server/src/test/java/com/metamx/druid/index/ReferenceCountingSegmentTest.java rename to processing/src/test/java/io/druid/segment/ReferenceCountingSegmentTest.java index ef986166cb1..a9e1b629829 100644 --- a/server/src/test/java/com/metamx/druid/index/ReferenceCountingSegmentTest.java +++ b/processing/src/test/java/io/druid/segment/ReferenceCountingSegmentTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index; +package io.druid.segment; import com.google.common.base.Throwables; -import com.metamx.druid.StorageAdapter; import junit.framework.Assert; import org.joda.time.DateTime; import org.joda.time.Days; diff --git a/server/src/test/java/com/metamx/druid/TestHelper.java b/processing/src/test/java/io/druid/segment/TestHelper.java similarity index 97% rename from server/src/test/java/com/metamx/druid/TestHelper.java rename to processing/src/test/java/io/druid/segment/TestHelper.java index 8a15a73f724..a1fab7efecb 100644 --- a/server/src/test/java/com/metamx/druid/TestHelper.java +++ b/processing/src/test/java/io/druid/segment/TestHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.segment; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.result.Result; +import io.druid.query.Result; import org.junit.Assert; import java.util.Iterator; diff --git a/server/src/test/java/com/metamx/druid/index/v1/TestIndex.java b/processing/src/test/java/io/druid/segment/TestIndex.java similarity index 90% rename from server/src/test/java/com/metamx/druid/index/v1/TestIndex.java rename to processing/src/test/java/io/druid/segment/TestIndex.java index df5761681af..8b13c6ce9e8 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/TestIndex.java +++ b/processing/src/test/java/io/druid/segment/TestIndex.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,34 +17,29 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment; import com.google.common.base.Charsets; -import com.google.common.base.Function; import com.google.common.base.Throwables; -import com.google.common.collect.Maps; import com.google.common.io.CharStreams; import com.google.common.io.InputSupplier; import com.google.common.io.LineProcessor; import com.metamx.common.logger.Logger; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.DoubleSumAggregatorFactory; -import com.metamx.druid.client.RangeIterable; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.indexer.data.DelimitedDataSpec; -import com.metamx.druid.indexer.data.StringInputRowParser; -import com.metamx.druid.indexer.data.TimestampSpec; +import io.druid.data.input.DelimitedDataSpec; +import io.druid.data.input.StringInputRowParser; +import io.druid.data.input.TimestampSpec; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.segment.incremental.IncrementalIndex; import org.joda.time.DateTime; import org.joda.time.Interval; -import javax.annotation.Nullable; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.util.Arrays; -import java.util.Map; import java.util.concurrent.atomic.AtomicLong; /** diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedFloatsIndexedSupplierTest.java similarity index 98% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplierTest.java rename to processing/src/test/java/io/druid/segment/data/CompressedFloatsIndexedSupplierTest.java index 125aa7618f8..d9cf4c5b6ab 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedFloatsIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedFloatsIndexedSupplierTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.io.Closeables; import com.google.common.primitives.Floats; -import com.metamx.druid.kv.IndexedFloats; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedFloatsSupplierSerializerTest.java b/processing/src/test/java/io/druid/segment/data/CompressedFloatsSupplierSerializerTest.java similarity index 91% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedFloatsSupplierSerializerTest.java rename to processing/src/test/java/io/druid/segment/data/CompressedFloatsSupplierSerializerTest.java index 8ddf982fc7c..7b670110269 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedFloatsSupplierSerializerTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedFloatsSupplierSerializerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.io.OutputSupplier; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.IndexedFloats; +import io.druid.collections.ResourceHolder; import org.junit.Assert; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplierTest.java b/processing/src/test/java/io/druid/segment/data/CompressedLongsIndexedSupplierTest.java similarity index 97% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplierTest.java rename to processing/src/test/java/io/druid/segment/data/CompressedLongsIndexedSupplierTest.java index 0f06cff2b64..873d3b3d423 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedLongsIndexedSupplierTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedLongsIndexedSupplierTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.io.Closeables; import com.google.common.primitives.Longs; -import com.metamx.druid.kv.IndexedLongs; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -31,7 +30,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.FloatBuffer; import java.nio.LongBuffer; import java.nio.channels.Channels; import java.util.concurrent.CountDownLatch; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedLongsSupplierSerializerTest.java b/processing/src/test/java/io/druid/segment/data/CompressedLongsSupplierSerializerTest.java similarity index 91% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedLongsSupplierSerializerTest.java rename to processing/src/test/java/io/druid/segment/data/CompressedLongsSupplierSerializerTest.java index c387db96618..029297e73e8 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/CompressedLongsSupplierSerializerTest.java +++ b/processing/src/test/java/io/druid/segment/data/CompressedLongsSupplierSerializerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.io.OutputSupplier; -import com.metamx.druid.collect.ResourceHolder; -import com.metamx.druid.kv.GenericIndexedWriter; -import com.metamx.druid.kv.IndexedLongs; +import io.druid.collections.ResourceHolder; import org.junit.Assert; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/kv/GenericIndexedTest.java b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java similarity index 98% rename from indexing-common/src/test/java/com/metamx/druid/kv/GenericIndexedTest.java rename to processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java index fad903abc96..61d64eb1d9a 100644 --- a/indexing-common/src/test/java/com/metamx/druid/kv/GenericIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/GenericIndexedTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.collect.Maps; import org.junit.Assert; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/IOPeonForTesting.java b/processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java similarity index 94% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/IOPeonForTesting.java rename to processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java index 6229d222e83..132c13a23b4 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/IOPeonForTesting.java +++ b/processing/src/test/java/io/druid/segment/data/IOPeonForTesting.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.collect.Maps; -import com.metamx.druid.kv.IOPeon; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/InMemoryCompressedFloatsTest.java b/processing/src/test/java/io/druid/segment/data/InMemoryCompressedFloatsTest.java similarity index 95% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/InMemoryCompressedFloatsTest.java rename to processing/src/test/java/io/druid/segment/data/InMemoryCompressedFloatsTest.java index bb52785b910..18c2b8d75d3 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/InMemoryCompressedFloatsTest.java +++ b/processing/src/test/java/io/druid/segment/data/InMemoryCompressedFloatsTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; -import com.metamx.druid.kv.IndexedFloats; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/InMemoryCompressedLongsTest.java b/processing/src/test/java/io/druid/segment/data/InMemoryCompressedLongsTest.java similarity index 95% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/InMemoryCompressedLongsTest.java rename to processing/src/test/java/io/druid/segment/data/InMemoryCompressedLongsTest.java index 00c01ac52be..1c8199ccc29 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/InMemoryCompressedLongsTest.java +++ b/processing/src/test/java/io/druid/segment/data/InMemoryCompressedLongsTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; -import com.metamx.druid.kv.IndexedLongs; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/index/v1/IncrementalIndexTest.java b/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java similarity index 88% rename from indexing-common/src/test/java/com/metamx/druid/index/v1/IncrementalIndexTest.java rename to processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java index 17f089ab3ee..5a7b448a4a8 100644 --- a/indexing-common/src/test/java/com/metamx/druid/index/v1/IncrementalIndexTest.java +++ b/processing/src/test/java/io/druid/segment/data/IncrementalIndexTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.data; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.input.Row; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.Row; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.segment.incremental.IncrementalIndex; import junit.framework.Assert; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/kv/IndexedIntsTest.java b/processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java similarity index 95% rename from indexing-common/src/test/java/com/metamx/druid/kv/IndexedIntsTest.java rename to processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java index d7341cb5c28..629cf72b722 100644 --- a/indexing-common/src/test/java/com/metamx/druid/kv/IndexedIntsTest.java +++ b/processing/src/test/java/io/druid/segment/data/IndexedIntsTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/index/v1/processing/IntersectingOffsetTest.java b/processing/src/test/java/io/druid/segment/data/IntersectingOffsetTest.java similarity index 96% rename from server/src/test/java/com/metamx/druid/index/v1/processing/IntersectingOffsetTest.java rename to processing/src/test/java/io/druid/segment/data/IntersectingOffsetTest.java index 3d5d5100b84..53d8c3bbe79 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/processing/IntersectingOffsetTest.java +++ b/processing/src/test/java/io/druid/segment/data/IntersectingOffsetTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; import com.google.common.collect.Lists; import org.junit.Assert; diff --git a/server/src/test/java/com/metamx/druid/index/v1/processing/UnioningOffsetTest.java b/processing/src/test/java/io/druid/segment/data/UnioningOffsetTest.java similarity index 97% rename from server/src/test/java/com/metamx/druid/index/v1/processing/UnioningOffsetTest.java rename to processing/src/test/java/io/druid/segment/data/UnioningOffsetTest.java index f470e5ae37a..6581aceff9b 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/processing/UnioningOffsetTest.java +++ b/processing/src/test/java/io/druid/segment/data/UnioningOffsetTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1.processing; +package io.druid.segment.data; import com.google.common.collect.Lists; import org.junit.Assert; diff --git a/indexing-common/src/test/java/com/metamx/druid/kv/VSizeIndexedIntsTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java similarity index 96% rename from indexing-common/src/test/java/com/metamx/druid/kv/VSizeIndexedIntsTest.java rename to processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java index d953958116b..543cd956b42 100644 --- a/indexing-common/src/test/java/com/metamx/druid/kv/VSizeIndexedIntsTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedIntsTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import org.junit.Assert; import org.junit.Test; diff --git a/indexing-common/src/test/java/com/metamx/druid/kv/VSizeIndexedTest.java b/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java similarity index 95% rename from indexing-common/src/test/java/com/metamx/druid/kv/VSizeIndexedTest.java rename to processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java index 631ec03869f..4d2f1a95710 100644 --- a/indexing-common/src/test/java/com/metamx/druid/kv/VSizeIndexedTest.java +++ b/processing/src/test/java/io/druid/segment/data/VSizeIndexedTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.kv; +package io.druid.segment.data; import com.google.common.base.Function; import com.google.common.collect.Iterables; @@ -27,7 +27,6 @@ import org.junit.Test; import java.io.ByteArrayOutputStream; import java.nio.ByteBuffer; import java.nio.channels.Channels; -import java.nio.channels.WritableByteChannel; import java.util.Arrays; import java.util.List; diff --git a/server/src/test/java/com/metamx/druid/index/brita/SpatialFilterBonusTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java similarity index 93% rename from server/src/test/java/com/metamx/druid/index/brita/SpatialFilterBonusTest.java rename to processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java index f4ef6249b39..1f8e4afe8e2 100644 --- a/server/src/test/java/com/metamx/druid/index/brita/SpatialFilterBonusTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterBonusTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,36 +17,36 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.collections.spatial.search.RadiusBound; import com.metamx.collections.spatial.search.RectangularBound; -import com.metamx.druid.Druids; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.TestHelper; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.index.IncrementalIndexSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.QueryableIndexSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.v1.IncrementalIndex; -import com.metamx.druid.index.v1.IncrementalIndexSchema; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.index.v1.IndexMerger; -import com.metamx.druid.index.v1.SpatialDimensionSchema; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.filter.SpatialDimFilter; -import com.metamx.druid.query.timeseries.TimeseriesQuery; -import com.metamx.druid.query.timeseries.TimeseriesQueryRunnerFactory; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.data.input.MapBasedInputRow; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Druids; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.QueryRunner; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.filter.SpatialDimFilter; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesResultValue; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.IndexIO; +import io.druid.segment.IndexMerger; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.Segment; +import io.druid.segment.TestHelper; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexSchema; +import io.druid.segment.incremental.SpatialDimensionSchema; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/index/brita/SpatialFilterTest.java b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java similarity index 93% rename from server/src/test/java/com/metamx/druid/index/brita/SpatialFilterTest.java rename to processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java index 5261c19d861..d319baad108 100644 --- a/server/src/test/java/com/metamx/druid/index/brita/SpatialFilterTest.java +++ b/processing/src/test/java/io/druid/segment/filter/SpatialFilterTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,36 +17,36 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.brita; +package io.druid.segment.filter; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.collections.spatial.search.RadiusBound; import com.metamx.collections.spatial.search.RectangularBound; -import com.metamx.druid.Druids; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.TestHelper; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.index.IncrementalIndexSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.QueryableIndexSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.v1.IncrementalIndex; -import com.metamx.druid.index.v1.IncrementalIndexSchema; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.index.v1.IndexMerger; -import com.metamx.druid.index.v1.SpatialDimensionSchema; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.filter.SpatialDimFilter; -import com.metamx.druid.query.timeseries.TimeseriesQuery; -import com.metamx.druid.query.timeseries.TimeseriesQueryRunnerFactory; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; +import io.druid.data.input.MapBasedInputRow; +import io.druid.granularity.QueryGranularity; +import io.druid.query.Druids; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.QueryRunner; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.filter.SpatialDimFilter; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesResultValue; +import io.druid.segment.IncrementalIndexSegment; +import io.druid.segment.IndexIO; +import io.druid.segment.IndexMerger; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.Segment; +import io.druid.segment.TestHelper; +import io.druid.segment.incremental.IncrementalIndex; +import io.druid.segment.incremental.IncrementalIndexSchema; +import io.druid.segment.incremental.SpatialDimensionSchema; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapterTest.java b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java similarity index 81% rename from server/src/test/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapterTest.java rename to processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java index 0072c27d73e..f4f089c92a4 100644 --- a/server/src/test/java/com/metamx/druid/index/v1/IncrementalIndexStorageAdapterTest.java +++ b/processing/src/test/java/io/druid/segment/incremental/IncrementalIndexStorageAdapterTest.java @@ -17,24 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.segment.incremental; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.CountAggregatorFactory; -import com.metamx.druid.aggregation.LongSumAggregatorFactory; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.input.MapBasedRow; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.group.GroupByQuery; -import com.metamx.druid.query.group.GroupByQueryEngine; -import com.metamx.druid.query.group.GroupByQueryEngineConfig; +import io.druid.collections.StupidPool; +import io.druid.data.input.MapBasedInputRow; +import io.druid.data.input.MapBasedRow; +import io.druid.data.input.Row; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.LongSumAggregatorFactory; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryEngine; import junit.framework.Assert; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -70,15 +71,16 @@ public class IncrementalIndexStorageAdapterTest ); GroupByQueryEngine engine = new GroupByQueryEngine( - new GroupByQueryEngineConfig() + Suppliers.ofInstance(new GroupByQueryConfig() { + @Override public int getMaxIntermediateRows() { return 5; } - }, - new StupidPool( + }), + new StupidPool( new Supplier() { @Override diff --git a/indexing-common/src/test/resources/ProtoTest.proto b/processing/src/test/resources/ProtoTest.proto similarity index 90% rename from indexing-common/src/test/resources/ProtoTest.proto rename to processing/src/test/resources/ProtoTest.proto index c7822f9ff1b..956db5259ac 100644 --- a/indexing-common/src/test/resources/ProtoTest.proto +++ b/processing/src/test/resources/ProtoTest.proto @@ -1,6 +1,6 @@ package prototest; -option java_package = "com.metamx.druid.indexer.data"; +option java_package = "io.druid.indexer.data"; option java_outer_classname = "ProtoTestEventWrapper"; diff --git a/indexing-common/src/test/resources/druid.sample.tsv b/processing/src/test/resources/druid.sample.tsv similarity index 100% rename from indexing-common/src/test/resources/druid.sample.tsv rename to processing/src/test/resources/druid.sample.tsv diff --git a/indexing-common/src/test/resources/druid.sample.tsv.bottom b/processing/src/test/resources/druid.sample.tsv.bottom similarity index 100% rename from indexing-common/src/test/resources/druid.sample.tsv.bottom rename to processing/src/test/resources/druid.sample.tsv.bottom diff --git a/indexing-common/src/test/resources/druid.sample.tsv.top b/processing/src/test/resources/druid.sample.tsv.top similarity index 100% rename from indexing-common/src/test/resources/druid.sample.tsv.top rename to processing/src/test/resources/druid.sample.tsv.top diff --git a/indexing-common/src/test/resources/prototest.desc b/processing/src/test/resources/prototest.desc similarity index 100% rename from indexing-common/src/test/resources/prototest.desc rename to processing/src/test/resources/prototest.desc diff --git a/doc/publications/vldb/.gitignore b/publications/vldb/.gitignore similarity index 100% rename from doc/publications/vldb/.gitignore rename to publications/vldb/.gitignore diff --git a/doc/publications/vldb/Makefile b/publications/vldb/Makefile similarity index 100% rename from doc/publications/vldb/Makefile rename to publications/vldb/Makefile diff --git a/doc/publications/vldb/druid.bib b/publications/vldb/druid.bib similarity index 100% rename from doc/publications/vldb/druid.bib rename to publications/vldb/druid.bib diff --git a/doc/publications/vldb/druid.tex b/publications/vldb/druid.tex similarity index 100% rename from doc/publications/vldb/druid.tex rename to publications/vldb/druid.tex diff --git a/doc/publications/vldb/figures/Druid_cluster.png b/publications/vldb/figures/Druid_cluster.png similarity index 100% rename from doc/publications/vldb/figures/Druid_cluster.png rename to publications/vldb/figures/Druid_cluster.png diff --git a/doc/publications/vldb/figures/cluster_scan_rate.pdf b/publications/vldb/figures/cluster_scan_rate.pdf similarity index 100% rename from doc/publications/vldb/figures/cluster_scan_rate.pdf rename to publications/vldb/figures/cluster_scan_rate.pdf diff --git a/doc/publications/vldb/figures/concise_plot.pdf b/publications/vldb/figures/concise_plot.pdf similarity index 100% rename from doc/publications/vldb/figures/concise_plot.pdf rename to publications/vldb/figures/concise_plot.pdf diff --git a/doc/publications/vldb/figures/core_scan_rate.pdf b/publications/vldb/figures/core_scan_rate.pdf similarity index 100% rename from doc/publications/vldb/figures/core_scan_rate.pdf rename to publications/vldb/figures/core_scan_rate.pdf diff --git a/doc/publications/vldb/figures/data_reading.png b/publications/vldb/figures/data_reading.png similarity index 100% rename from doc/publications/vldb/figures/data_reading.png rename to publications/vldb/figures/data_reading.png diff --git a/doc/publications/vldb/figures/druid_message_bus.png b/publications/vldb/figures/druid_message_bus.png similarity index 100% rename from doc/publications/vldb/figures/druid_message_bus.png rename to publications/vldb/figures/druid_message_bus.png diff --git a/doc/publications/vldb/figures/druid_realtime_flow.png b/publications/vldb/figures/druid_realtime_flow.png similarity index 100% rename from doc/publications/vldb/figures/druid_realtime_flow.png rename to publications/vldb/figures/druid_realtime_flow.png diff --git a/doc/publications/vldb/figures/druid_segment.png b/publications/vldb/figures/druid_segment.png similarity index 100% rename from doc/publications/vldb/figures/druid_segment.png rename to publications/vldb/figures/druid_segment.png diff --git a/doc/publications/vldb/figures/zookeeper.png b/publications/vldb/figures/zookeeper.png similarity index 100% rename from doc/publications/vldb/figures/zookeeper.png rename to publications/vldb/figures/zookeeper.png diff --git a/doc/publications/vldb/src/druid_plot.R b/publications/vldb/src/druid_plot.R similarity index 100% rename from doc/publications/vldb/src/druid_plot.R rename to publications/vldb/src/druid_plot.R diff --git a/doc/publications/vldb/src/druid_tables.R b/publications/vldb/src/druid_tables.R similarity index 100% rename from doc/publications/vldb/src/druid_tables.R rename to publications/vldb/src/druid_tables.R diff --git a/doc/publications/vldb/vldb.cls b/publications/vldb/vldb.cls similarity index 100% rename from doc/publications/vldb/vldb.cls rename to publications/vldb/vldb.cls diff --git a/realtime/pom.xml b/realtime/pom.xml index 6f0e0cf12d9..9d8e461c631 100644 --- a/realtime/pom.xml +++ b/realtime/pom.xml @@ -1,7 +1,7 @@ - + 4.0.0 com.metamx.druid druid-server @@ -28,18 +29,13 @@ com.metamx druid - 0.5.59-SNAPSHOT + 0.6.0-SNAPSHOT com.metamx.druid - druid-client - ${project.parent.version} - - - com.metamx.druid - druid-indexing-common + druid-processing ${project.parent.version} @@ -72,6 +68,10 @@ commons-io commons-io + + com.amazonaws + aws-java-sdk + com.ning compress-lzf @@ -100,10 +100,6 @@ com.google.inject guice - - com.google.inject.extensions - guice-servlet - com.fasterxml.jackson.core jackson-core @@ -145,8 +141,8 @@ jets3t - org.mortbay.jetty - jetty + org.eclipse.jetty + jetty-server joda-time @@ -162,7 +158,7 @@ javax.servlet - servlet-api + javax.servlet-api org.slf4j @@ -172,6 +168,15 @@ org.apache.hadoop hadoop-core + + io.tesla.aether + tesla-aether + + + org.eclipse.aether + aether-api + + @@ -199,7 +204,30 @@ astyanax 1.0.1 - + + org.antlr + antlr4-runtime + + + com.google.code.simple-spring-memcached + spymemcached + + + net.jpountz.lz4 + lz4 + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.eclipse.jetty + jetty-servlets + @@ -214,14 +242,7 @@ com.metamx.druid - druid-indexing-common - ${project.parent.version} - test-jar - test - - - com.metamx.druid - druid-client + druid-processing ${project.parent.version} test-jar test @@ -231,6 +252,11 @@ curator-test test + + com.google.caliper + caliper + test + @@ -247,6 +273,17 @@ + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + + diff --git a/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 similarity index 84% rename from client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 rename to server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 index 9dd297dc8c0..1cafead058d 100644 --- a/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 +++ b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 @@ -1,18 +1,50 @@ grammar DruidSQL; @header { -import com.metamx.druid.aggregation.post.*; -import com.metamx.druid.aggregation.*; -import com.metamx.druid.query.filter.*; -import com.metamx.druid.query.dimension.*; -import com.metamx.druid.*; - -import com.google.common.base.*; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import org.joda.time.*; +import io.druid.granularity.PeriodGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.MaxAggregatorFactory; +import io.druid.query.aggregation.MinAggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.aggregation.post.ArithmeticPostAggregator; +import io.druid.query.aggregation.post.ConstantPostAggregator; +import io.druid.query.aggregation.post.FieldAccessPostAggregator; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.NotDimFilter; +import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.RegexDimFilter; +import io.druid.query.filter.SelectorDimFilter; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNSimulator; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.joda.time.DateTime; +import org.joda.time.Period; -import java.text.*; -import java.util.*; +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; } @parser::members { diff --git a/server/src/main/java/com/metamx/TsvToJson.java b/server/src/main/java/com/metamx/TsvToJson.java deleted file mode 100644 index d0778bf1476..00000000000 --- a/server/src/main/java/com/metamx/TsvToJson.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.collect.Maps; -import com.metamx.common.IAE; - -import org.joda.time.DateTime; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.util.Map; - -/** - */ -public class TsvToJson -{ - public static void main(String[] args) throws IOException - { - ObjectMapper mapper = new ObjectMapper(); - - String[] fields = args[0].split(","); - File inFile = new File(args[1]); - File outFile = new File(args[2]); - - FieldHandler[] handlers = new FieldHandler[fields.length]; - for (int i = 0; i < fields.length; i++) { - String field = fields[i]; - String[] fieldParts = field.split(":"); - String fieldName = fieldParts[0]; - if (fieldParts.length < 2 || "string".equalsIgnoreCase(fieldParts[1])) { - handlers[i] = new StringField(fieldName); - } - else if ("number".equalsIgnoreCase(fieldParts[1])) { - handlers[i] = new NumberField(fieldName); - } - else if ("ISO8601".equals(fieldParts[1])) { - handlers[i] = new IsoToNumberField(fieldName); - } - else { - throw new IAE("Unknown type[%s]", fieldParts[1]); - } - } - - BufferedReader in = null; - BufferedWriter out = null; - try { - in = new BufferedReader(new InputStreamReader(new FileInputStream(inFile), Charsets.UTF_8)); - out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFile), Charsets.UTF_8)); - String line = null; - int count = 0; - long currTime = System.currentTimeMillis(); - long startTime = currTime; - while ((line = in.readLine()) != null) { - if (count % 1000000 == 0) { - long nowTime = System.currentTimeMillis(); - System.out.printf("Processed [%,d] lines in %,d millis. Incremental time %,d millis.%n", count, nowTime - startTime, nowTime - currTime); - currTime = nowTime; - } - ++count; - String[] splits = line.split("\t"); - - if (splits.length == 30) { - continue; - } - - if (splits.length != handlers.length) { - throw new IAE("splits.length[%d] != handlers.length[%d]; line[%s]", splits.length, handlers.length, line); - } - - Map jsonMap = Maps.newLinkedHashMap(); - for (int i = 0; i < handlers.length; ++i) { - jsonMap.put(handlers[i].getFieldName(), handlers[i].process(splits[i])); - } - - final String str = mapper.writeValueAsString(jsonMap); - out.write(str); - out.write("\n"); - } - System.out.printf("Completed %,d lines in %,d millis.%n", count, System.currentTimeMillis() - startTime); - out.flush(); - } finally { - if (out != null) { - out.close(); - } - if (in != null) { - in.close(); - } - } - } - - public static interface FieldHandler - { - public String getFieldName(); - public Object process(String value); - } - - public static class StringField implements FieldHandler - { - private final String fieldName; - - public StringField( - String fieldName - ) - { - this.fieldName = fieldName; - } - - @Override - public String getFieldName() - { - return fieldName; - } - - @Override - public Object process(String value) - { - return value; - } - } - - public static class NumberField implements FieldHandler - { - private final String fieldName; - - public NumberField( - String fieldName - ) - { - this.fieldName = fieldName; - } - - @Override - public String getFieldName() - { - return fieldName; - } - - @Override - public Object process(String value) - { - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - return Double.parseDouble(value); - } - } - } - - public static class IsoToNumberField implements FieldHandler - { - - private final String fieldName; - - public IsoToNumberField( - String fieldName - ) - { - this.fieldName = fieldName; - } - - @Override - public String getFieldName() - { - return fieldName; - } - - @Override - public Object process(String value) - { - return new DateTime(value).getMillis(); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/BaseServerNode.java b/server/src/main/java/com/metamx/druid/BaseServerNode.java deleted file mode 100644 index 17eee84dc6e..00000000000 --- a/server/src/main/java/com/metamx/druid/BaseServerNode.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.metamx.common.concurrent.ExecutorServiceConfig; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.initialization.ServerInit; -import com.metamx.druid.query.DefaultQueryRunnerFactoryConglomerate; -import com.metamx.druid.query.MetricsEmittingExecutorService; -import com.metamx.druid.query.PrioritizedExecutorService; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; - -import com.metamx.emitter.service.ServiceMetricEvent; -import org.skife.config.ConfigurationObjectFactory; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutorService; - -/** - */ -public abstract class BaseServerNode extends QueryableNode -{ - private final Map, QueryRunnerFactory> additionalFactories = Maps.newLinkedHashMap(); - private DruidProcessingConfig processingConfig = null; - private QueryRunnerFactoryConglomerate conglomerate = null; - private StupidPool computeScratchPool = null; - private ExecutorService queryExecutorService = null; - - public BaseServerNode( - String nodeType, - Logger log, - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super(nodeType, log, props, lifecycle, jsonMapper, smileMapper, configFactory); - } - - public QueryRunnerFactoryConglomerate getConglomerate() - { - initializeQueryRunnerFactoryConglomerate(); - return conglomerate; - } - - public StupidPool getComputeScratchPool() - { - initializeComputeScratchPool(); - return computeScratchPool; - } - - public DruidProcessingConfig getProcessingConfig() - { - initializeProcessingConfig(); - return processingConfig; - } - - public ExecutorService getQueryExecutorService() - { - initializeQueryExecutorService(); - return queryExecutorService; - } - - @SuppressWarnings("unchecked") - public T setConglomerate(QueryRunnerFactoryConglomerate conglomerate) - { - checkFieldNotSetAndSet("conglomerate", conglomerate); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setComputeScratchPool(StupidPool computeScratchPool) - { - checkFieldNotSetAndSet("computeScratchPool", computeScratchPool); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setProcessingConfig(DruidProcessingConfig processingConfig) - { - checkFieldNotSetAndSet("processingConfig", processingConfig); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setQueryExecutorService(ExecutorService queryExecutorService) - { - checkFieldNotSetAndSet("queryExecutorService", queryExecutorService); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T registerQueryRunnerFactory(Class queryClazz, QueryRunnerFactory factory) - { - Preconditions.checkState( - conglomerate == null, - "Registering a QueryRunnerFactory only works when a separate conglomerate is not specified." - ); - Preconditions.checkState( - !additionalFactories.containsKey(queryClazz), "Registered factory for class[%s] multiple times", queryClazz - ); - additionalFactories.put(queryClazz, factory); - return (T) this; - } - - private void initializeComputeScratchPool() - { - if (computeScratchPool == null) { - setComputeScratchPool(ServerInit.makeComputeScratchPool(getProcessingConfig())); - } - } - - private void initializeQueryRunnerFactoryConglomerate() - { - if (conglomerate == null) { - final Map, QueryRunnerFactory> factories = ServerInit.initDefaultQueryTypes( - getConfigFactory(), getComputeScratchPool() - ); - - for (Map.Entry, QueryRunnerFactory> entry : additionalFactories.entrySet()) { - factories.put(entry.getKey(), entry.getValue()); - } - - setConglomerate(new DefaultQueryRunnerFactoryConglomerate(factories)); - } - } - - private void initializeProcessingConfig() - { - if (processingConfig == null) { - setProcessingConfig( - getConfigFactory().buildWithReplacements( - DruidProcessingConfig.class, ImmutableMap.of("base_path", "druid.processing") - ) - ); - } - } - - private void initializeQueryExecutorService() - { - if (queryExecutorService == null) { - final PrioritizedExecutorService innerExecutorService = PrioritizedExecutorService.create( - getLifecycle(), - getConfigFactory().buildWithReplacements( - ExecutorServiceConfig.class, ImmutableMap.of("base_path", "druid.processing") - ) - ); - - setQueryExecutorService( - new MetricsEmittingExecutorService( - innerExecutorService, - getEmitter(), - new ServiceMetricEvent.Builder() - ) - ); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java b/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java deleted file mode 100644 index 2aa7e4f59bf..00000000000 --- a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.google.common.collect.Sets; -import com.metamx.common.guava.FunctionalIterable; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.v1.ConciseOffset; -import com.metamx.druid.index.v1.processing.IntersectingOffset; -import com.metamx.druid.index.v1.processing.Offset; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQuerySpec; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; - -import java.util.List; -import java.util.TreeSet; - -/** - */ -public abstract class BaseStorageAdapter implements StorageAdapter -{ - public abstract Indexed getAvailableDimensions(); - - public abstract Indexed getDimValueLookup(String dimension); - - public abstract ImmutableConciseSet getInvertedIndex(String dimension, String dimVal); - - public abstract ImmutableConciseSet getInvertedIndex(String dimension, int idx); - - public abstract Offset getFilterOffset(Filter filter); - - @Override - public Iterable searchDimensions(final SearchQuery query, final Filter filter) - { - final List dimensions = query.getDimensions(); - final SearchQuerySpec searchQuerySpec = query.getQuery(); - - final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); - - Iterable dimsToSearch; - if (dimensions == null || dimensions.isEmpty()) { - dimsToSearch = getAvailableDimensions(); - } else { - dimsToSearch = dimensions; - } - - Offset filterOffset = (filter == null) ? null : getFilterOffset(filter); - - for (String dimension : dimsToSearch) { - Iterable dims = getDimValueLookup(dimension); - if (dims != null) { - for (String dimVal : dims) { - dimVal = dimVal == null ? "" : dimVal; - if (searchQuerySpec.accept(dimVal)) { - if (filterOffset != null) { - Offset lhs = new ConciseOffset(getInvertedIndex(dimension, dimVal)); - Offset rhs = filterOffset.clone(); - - if (new IntersectingOffset(lhs, rhs).withinBounds()) { - retVal.add(new SearchHit(dimension, dimVal)); - } - } else { - retVal.add(new SearchHit(dimension, dimVal)); - } - } - } - } - } - - return new FunctionalIterable(retVal).limit(query.getLimit()); - } -} - diff --git a/server/src/main/java/com/metamx/druid/http/ComputeMain.java b/server/src/main/java/com/metamx/druid/http/ComputeMain.java deleted file mode 100644 index 828d1568214..00000000000 --- a/server/src/main/java/com/metamx/druid/http/ComputeMain.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.log.LogLevelAdjuster; - -/** - */ -public class ComputeMain -{ - private static final Logger log = new Logger(ComputeMain.class); - - public static void main(String[] args) throws Exception - { - LogLevelAdjuster.register(); - - Lifecycle lifecycle = new Lifecycle(); - - lifecycle.addManagedInstance( - ComputeNode.builder().build() - ); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.info(t, "Throwable caught at startup, committing seppuku"); - System.exit(2); - } - - lifecycle.join(); - } -} diff --git a/server/src/main/java/com/metamx/druid/http/ComputeNode.java b/server/src/main/java/com/metamx/druid/http/ComputeNode.java deleted file mode 100644 index e677db381e3..00000000000 --- a/server/src/main/java/com/metamx/druid/http/ComputeNode.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; -import com.metamx.common.ISE; -import com.metamx.common.concurrent.ExecutorServiceConfig; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.BaseServerNode; -import com.metamx.druid.coordination.ServerManager; -import com.metamx.druid.coordination.ZkCoordinator; -import com.metamx.druid.coordination.ZkCoordinatorConfig; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServerInit; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.metrics.ServerMonitor; -import com.metamx.druid.query.MetricsEmittingExecutorService; -import com.metamx.druid.query.PrioritizedExecutorService; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.Monitor; -import org.jets3t.service.S3ServiceException; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.ServletHolder; -import org.skife.config.ConfigurationObjectFactory; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; - -/** - */ -public class ComputeNode extends BaseServerNode -{ - private static final Logger log = new Logger(ComputeNode.class); - - public static Builder builder() - { - return new Builder(); - } - - private SegmentLoader segmentLoader; - - public ComputeNode( - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super("historical", log, props, lifecycle, jsonMapper, smileMapper, configFactory); - } - - public ComputeNode setSegmentLoader(SegmentLoader segmentLoader) - { - Preconditions.checkState(this.segmentLoader == null, "Cannot set segmentLoader once it has already been set."); - this.segmentLoader = segmentLoader; - return this; - } - - public SegmentLoader getSegmentLoader() - { - initializeSegmentLoader(); - return segmentLoader; - } - - protected void doInit() throws Exception - { - final Lifecycle lifecycle = getLifecycle(); - final ServiceEmitter emitter = getEmitter(); - final List monitors = getMonitors(); - final QueryRunnerFactoryConglomerate conglomerate = getConglomerate(); - - final ServerManager serverManager = new ServerManager( - getSegmentLoader(), - conglomerate, - emitter, - getQueryExecutorService() - ); - - final ZkCoordinator coordinator = new ZkCoordinator( - getJsonMapper(), - getConfigFactory().build(ZkCoordinatorConfig.class), - getZkPaths(), - getDruidServerMetadata(), - getAnnouncer(), - getCuratorFramework(), - serverManager - ); - lifecycle.addManagedInstance(coordinator); - - monitors.add(new ServerMonitor(getDruidServerMetadata(), serverManager)); - startMonitoring(monitors); - - final Context root = new Context(getServer(), "/", Context.SESSIONS); - root.addServlet(new ServletHolder(new StatusServlet()), "/status"); - root.addServlet( - new ServletHolder( - new QueryServlet(getJsonMapper(), getSmileMapper(), serverManager, emitter, getRequestLogger()) - ), - "/druid/v2/*" - ); - } - - private void initializeSegmentLoader() - { - if (segmentLoader == null) { - try { - setSegmentLoader( - ServerInit.makeDefaultQueryableLoader(getConfigFactory(), getProps()) - ); - } - catch (S3ServiceException e) { - throw Throwables.propagate(e); - } - } - } - - public static class Builder - { - private ObjectMapper jsonMapper = null; - private ObjectMapper smileMapper = null; - private Lifecycle lifecycle = null; - private Properties props = null; - private ConfigurationObjectFactory configFactory = null; - - public Builder withMappers(ObjectMapper jsonMapper, ObjectMapper smileMapper) - { - this.jsonMapper = jsonMapper; - this.smileMapper = smileMapper; - return this; - } - - public Builder withProps(Properties props) - { - this.props = props; - return this; - } - - public Builder withConfigFactory(ConfigurationObjectFactory configFactory) - { - this.configFactory = configFactory; - return this; - } - - public ComputeNode build() - { - if (jsonMapper == null && smileMapper == null) { - jsonMapper = new DefaultObjectMapper(); - smileMapper = new DefaultObjectMapper(new SmileFactory()); - smileMapper.getJsonFactory().setCodec(smileMapper); - } else if (jsonMapper == null || smileMapper == null) { - throw new ISE( - "Only jsonMapper[%s] or smileMapper[%s] was set, must set neither or both.", - jsonMapper, - smileMapper - ); - } - - if (lifecycle == null) { - lifecycle = new Lifecycle(); - } - - if (props == null) { - props = Initialization.loadProperties(); - } - - if (configFactory == null) { - configFactory = Config.createFactory(props); - } - - return new ComputeNode(props, lifecycle, jsonMapper, smileMapper, configFactory); - } - } -} \ No newline at end of file diff --git a/server/src/main/java/com/metamx/druid/http/MasterMain.java b/server/src/main/java/com/metamx/druid/http/MasterMain.java deleted file mode 100644 index 53382fe81bb..00000000000 --- a/server/src/main/java/com/metamx/druid/http/MasterMain.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.servlet.GuiceFilter; -import com.metamx.common.IAE; -import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.common.concurrent.ScheduledExecutors; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.BatchServerInventoryView; -import com.metamx.druid.client.ServerInventoryView; -import com.metamx.druid.client.ServerInventoryViewConfig; -import com.metamx.druid.client.SingleServerInventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.config.ConfigManager; -import com.metamx.druid.config.ConfigManagerConfig; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.curator.discovery.ServiceAnnouncer; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseRuleManagerConfig; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.db.DatabaseSegmentManagerConfig; -import com.metamx.druid.db.DbConnector; -import com.metamx.druid.db.DbConnectorConfig; -import com.metamx.druid.initialization.CuratorConfig; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServerConfig; -import com.metamx.druid.initialization.ServiceDiscoveryConfig; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.log.LogLevelAdjuster; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterConfig; -import com.metamx.druid.master.LoadQueueTaskMaster; -import com.metamx.druid.utils.PropUtils; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.Emitters; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; -import com.metamx.http.client.response.ToStringResponseHandler; -import com.metamx.metrics.JvmMonitor; -import com.metamx.metrics.Monitor; -import com.metamx.metrics.MonitorScheduler; -import com.metamx.metrics.MonitorSchedulerConfig; -import com.metamx.metrics.SysMonitor; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.apache.curator.x.discovery.ServiceProvider; -import org.joda.time.Duration; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.DefaultServlet; -import org.mortbay.jetty.servlet.FilterHolder; -import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.servlet.GzipFilter; -import org.skife.config.ConfigurationObjectFactory; -import org.skife.jdbi.v2.DBI; - -import java.net.URL; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -/** - */ -public class MasterMain -{ - private static final Logger log = new Logger(MasterMain.class); - - public static void main(String[] args) throws Exception - { - LogLevelAdjuster.register(); - - final ObjectMapper jsonMapper = new DefaultObjectMapper(); - final Properties props = Initialization.loadProperties(); - final ConfigurationObjectFactory configFactory = Config.createFactory(props); - final Lifecycle lifecycle = new Lifecycle(); - - final HttpClientConfig.Builder httpClientConfigBuilder = HttpClientConfig.builder().withNumConnections(1); - - final String emitterTimeout = props.getProperty("druid.emitter.timeOut"); - if (emitterTimeout != null) { - httpClientConfigBuilder.withReadTimeout(new Duration(emitterTimeout)); - } - final HttpClient httpClient = HttpClientInit.createClient(httpClientConfigBuilder.build(), lifecycle); - - final ServiceEmitter emitter = new ServiceEmitter( - PropUtils.getProperty(props, "druid.service"), - PropUtils.getProperty(props, "druid.host"), - Emitters.create(props, httpClient, jsonMapper, lifecycle) - ); - EmittingLogger.registerEmitter(emitter); - - final ScheduledExecutorFactory scheduledExecutorFactory = ScheduledExecutors.createFactory(lifecycle); - - final ServiceDiscoveryConfig serviceDiscoveryConfig = configFactory.build(ServiceDiscoveryConfig.class); - CuratorFramework serviceDiscoveryCuratorFramework = Initialization.makeCuratorFramework( - serviceDiscoveryConfig, - lifecycle - ); - final CuratorConfig curatorConfig = configFactory.build(CuratorConfig.class); - CuratorFramework curatorFramework = Initialization.makeCuratorFramework( - curatorConfig, - lifecycle - ); - - final ZkPathsConfig zkPaths = configFactory.build(ZkPathsConfig.class); - - final ExecutorService exec = Executors.newFixedThreadPool( - 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ServerInventoryView-%s").build() - ); - - final ServerInventoryViewConfig serverInventoryViewConfig = configFactory.build(ServerInventoryViewConfig.class); - final String announcerType = serverInventoryViewConfig.getAnnouncerType(); - - final ServerInventoryView serverInventoryView; - if ("legacy".equalsIgnoreCase(announcerType)) { - serverInventoryView = new SingleServerInventoryView( - serverInventoryViewConfig, - zkPaths, - curatorFramework, - exec, - jsonMapper - ); - } else if ("batch".equalsIgnoreCase(announcerType)) { - serverInventoryView = new BatchServerInventoryView( - serverInventoryViewConfig, - zkPaths, - curatorFramework, - exec, - jsonMapper - ); - } else { - throw new IAE("Unknown type %s", announcerType); - } - - lifecycle.addManagedInstance(serverInventoryView); - - final DbConnectorConfig dbConnectorConfig = configFactory.build(DbConnectorConfig.class); - final DatabaseRuleManagerConfig databaseRuleManagerConfig = configFactory.build(DatabaseRuleManagerConfig.class); - final DBI dbi = new DbConnector(dbConnectorConfig).getDBI(); - DbConnector.createSegmentTable(dbi, PropUtils.getProperty(props, "druid.database.segmentTable")); - DbConnector.createRuleTable(dbi, PropUtils.getProperty(props, "druid.database.ruleTable")); - DatabaseRuleManager.createDefaultRule( - dbi, databaseRuleManagerConfig.getRuleTable(), databaseRuleManagerConfig.getDefaultDatasource(), jsonMapper - ); - - final DatabaseSegmentManager databaseSegmentManager = new DatabaseSegmentManager( - jsonMapper, - scheduledExecutorFactory.create(1, "DatabaseSegmentManager-Exec--%d"), - configFactory.build(DatabaseSegmentManagerConfig.class), - dbi - ); - final DatabaseRuleManager databaseRuleManager = new DatabaseRuleManager( - jsonMapper, - scheduledExecutorFactory.create(1, "DatabaseRuleManager-Exec--%d"), - databaseRuleManagerConfig, - dbi - ); - - final ScheduledExecutorService globalScheduledExec = scheduledExecutorFactory.create(1, "Global--%d"); - final List monitors = Lists.newArrayList(); - monitors.add(new JvmMonitor()); - if (Boolean.parseBoolean(props.getProperty("druid.monitoring.monitorSystem", "false"))) { - monitors.add(new SysMonitor()); - } - - final MonitorScheduler healthMonitor = new MonitorScheduler( - configFactory.build(MonitorSchedulerConfig.class), - globalScheduledExec, - emitter, - monitors - ); - lifecycle.addManagedInstance(healthMonitor); - - final DruidMasterConfig druidMasterConfig = configFactory.build(DruidMasterConfig.class); - - final ServiceDiscovery serviceDiscovery = Initialization.makeServiceDiscoveryClient( - serviceDiscoveryCuratorFramework, - serviceDiscoveryConfig, - lifecycle - ); - final ServiceAnnouncer serviceAnnouncer = Initialization.makeServiceAnnouncer( - serviceDiscoveryConfig, serviceDiscovery - ); - Initialization.announceDefaultService(serviceDiscoveryConfig, serviceAnnouncer, lifecycle); - - ServiceProvider serviceProvider = null; - if (druidMasterConfig.getMergerServiceName() != null) { - serviceProvider = Initialization.makeServiceProvider( - druidMasterConfig.getMergerServiceName(), - serviceDiscovery, - lifecycle - ); - } - IndexingServiceClient indexingServiceClient = new IndexingServiceClient(httpClient, jsonMapper, serviceProvider); - - final ConfigManagerConfig configManagerConfig = configFactory.build(ConfigManagerConfig.class); - DbConnector.createConfigTable(dbi, configManagerConfig.getConfigTable()); - JacksonConfigManager configManager = new JacksonConfigManager( - new ConfigManager(dbi, configManagerConfig), jsonMapper - ); - - final LoadQueueTaskMaster taskMaster = new LoadQueueTaskMaster( - curatorFramework, - jsonMapper, - scheduledExecutorFactory.create(1, "Master-PeonExec--%d"), - druidMasterConfig - ); - - final DruidMaster master = new DruidMaster( - druidMasterConfig, - zkPaths, - configManager, - databaseSegmentManager, - serverInventoryView, - databaseRuleManager, - curatorFramework, - emitter, - scheduledExecutorFactory, - indexingServiceClient, - taskMaster - ); - lifecycle.addManagedInstance(master); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.error(t, "Error when starting up. Failing."); - System.exit(1); - } - - Runtime.getRuntime().addShutdownHook( - new Thread( - new Runnable() - { - @Override - public void run() - { - log.info("Running shutdown hook"); - lifecycle.stop(); - } - } - ) - ); - - final Injector injector = Guice.createInjector( - new MasterServletModule( - serverInventoryView, - databaseSegmentManager, - databaseRuleManager, - master, - jsonMapper, - indexingServiceClient, - configManager - ) - ); - - final Server server = Initialization.makeJettyServer(configFactory.build(ServerConfig.class)); - - final RedirectInfo redirectInfo = new RedirectInfo() - { - @Override - public boolean doLocal() - { - return master.isClusterMaster(); - } - - @Override - public URL getRedirectURL(String queryString, String requestURI) - { - try { - final String currentMaster = master.getCurrentMaster(); - if (currentMaster == null) { - return null; - } - - String location = String.format("http://%s%s", currentMaster, requestURI); - - if (queryString != null) { - location = String.format("%s?%s", location, queryString); - } - - return new URL(location); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - }; - - final Context staticContext = new Context(server, "/static", Context.SESSIONS); - staticContext.addServlet(new ServletHolder(new RedirectServlet(redirectInfo)), "/*"); - - staticContext.setResourceBase(ComputeMain.class.getClassLoader().getResource("static").toExternalForm()); - - final Context root = new Context(server, "/", Context.SESSIONS); - root.addServlet(new ServletHolder(new StatusServlet()), "/status"); - root.addServlet(new ServletHolder(new DefaultServlet()), "/*"); - root.addEventListener(new GuiceServletConfig(injector)); - root.addFilter(GzipFilter.class, "/*", 0); - root.addFilter( - new FilterHolder( - new RedirectFilter( - new ToStringResponseHandler(Charsets.UTF_8), - redirectInfo - ) - ), "/*", 0 - ); - root.addFilter(GuiceFilter.class, "/info/*", 0); - root.addFilter(GuiceFilter.class, "/master/*", 0); - - server.start(); - server.join(); - } -} diff --git a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java b/server/src/main/java/com/metamx/druid/http/MasterServletModule.java deleted file mode 100644 index 51d57492914..00000000000 --- a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; -import com.google.inject.Provides; -import com.google.inject.util.Providers; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.master.DruidMaster; -import com.sun.jersey.guice.JerseyServletModule; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; - -import javax.inject.Singleton; - -/** - */ -public class MasterServletModule extends JerseyServletModule -{ - private final InventoryView serverInventoryView; - private final DatabaseSegmentManager segmentInventoryManager; - private final DatabaseRuleManager databaseRuleManager; - private final DruidMaster master; - private final ObjectMapper jsonMapper; - private final IndexingServiceClient indexingServiceClient; - private final JacksonConfigManager configManager; - - - public MasterServletModule( - InventoryView serverInventoryView, - DatabaseSegmentManager segmentInventoryManager, - DatabaseRuleManager databaseRuleManager, - DruidMaster master, - ObjectMapper jsonMapper, - IndexingServiceClient indexingServiceClient, - JacksonConfigManager configManager - ) - { - this.serverInventoryView = serverInventoryView; - this.segmentInventoryManager = segmentInventoryManager; - this.databaseRuleManager = databaseRuleManager; - this.master = master; - this.jsonMapper = jsonMapper; - this.indexingServiceClient = indexingServiceClient; - this.configManager = configManager; - } - - @Override - protected void configureServlets() - { - bind(MasterSegmentSettingsResource.class); - bind(InfoResource.class); - bind(MasterResource.class); - bind(InventoryView.class).toInstance(serverInventoryView); - bind(DatabaseSegmentManager.class).toInstance(segmentInventoryManager); - bind(DatabaseRuleManager.class).toInstance(databaseRuleManager); - bind(DruidMaster.class).toInstance(master); - bind(JacksonConfigManager.class).toInstance(configManager); - if (indexingServiceClient == null) { - bind(IndexingServiceClient.class).toProvider(Providers.of(null)); - } - else { - bind(IndexingServiceClient.class).toInstance(indexingServiceClient); - } - - serve("/*").with(GuiceContainer.class); - } - - @Provides - @Singleton - public JacksonJsonProvider getJacksonJsonProvider() - { - final JacksonJsonProvider provider = new JacksonJsonProvider(); - provider.setMapper(jsonMapper); - return provider; - } -} diff --git a/server/src/main/java/com/metamx/druid/http/ServerMain.java b/server/src/main/java/com/metamx/druid/http/ServerMain.java deleted file mode 100644 index 11dcb879584..00000000000 --- a/server/src/main/java/com/metamx/druid/http/ServerMain.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -/** - */ -@Deprecated -public class ServerMain -{ - public static void main(String[] args) throws Exception - { - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("K thx bye."); - ComputeMain.main(args); - } -} diff --git a/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java deleted file mode 100644 index 1e47528d801..00000000000 --- a/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.index.v1; - -import com.metamx.druid.Capabilities; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import org.joda.time.DateTime; -import org.joda.time.Interval; - -/** - */ -public class SegmentIdAttachedStorageAdapter implements StorageAdapter -{ - private final String segmentId; - private final StorageAdapter delegate; - - public SegmentIdAttachedStorageAdapter( - String segmentId, - StorageAdapter delegate - ) - { - this.segmentId = segmentId; - this.delegate = delegate; - } - - @Override - public String getSegmentIdentifier() - { - return segmentId; - } - - @Override - public Interval getInterval() - { - return delegate.getInterval(); - } - - @Override - public Iterable searchDimensions(SearchQuery query, Filter filter) - { - return delegate.searchDimensions(query, filter); - } - - @Override - public Iterable makeCursors(Filter filter, Interval interval, QueryGranularity gran) - { - return delegate.makeCursors(filter, interval, gran); - } - - @Override - public Capabilities getCapabilities() - { - return delegate.getCapabilities(); - } - - @Override - public DateTime getMaxTime() - { - return delegate.getMaxTime(); - } - - @Override - public DateTime getMinTime() - { - return delegate.getMinTime(); - } - - @Override - public int getDimensionCardinality(String dimension) - { - return delegate.getDimensionCardinality(dimension); - } - - public StorageAdapter getDelegate() - { - return delegate; - } -} diff --git a/server/src/main/java/com/metamx/druid/initialization/ServerInit.java b/server/src/main/java/com/metamx/druid/initialization/ServerInit.java deleted file mode 100644 index 0ac5e67a4de..00000000000 --- a/server/src/main/java/com/metamx/druid/initialization/ServerInit.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.initialization; - -import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.conf.Configuration; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.security.AWSCredentials; -import org.skife.config.ConfigurationObjectFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.metamx.common.ISE; -import com.metamx.common.logger.Logger; -import com.metamx.druid.DruidProcessingConfig; -import com.metamx.druid.Query; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.loading.DataSegmentPusher; -import com.metamx.druid.loading.DelegatingSegmentLoader; -import com.metamx.druid.loading.HdfsDataSegmentPuller; -import com.metamx.druid.loading.HdfsDataSegmentPusher; -import com.metamx.druid.loading.HdfsDataSegmentPusherConfig; -import com.metamx.druid.loading.LocalDataSegmentPuller; -import com.metamx.druid.loading.LocalDataSegmentPusher; -import com.metamx.druid.loading.LocalDataSegmentPusherConfig; -import com.metamx.druid.loading.MMappedQueryableIndexFactory; -import com.metamx.druid.loading.QueryableIndexFactory; -import com.metamx.druid.loading.S3DataSegmentPuller; -import com.metamx.druid.loading.S3DataSegmentPusher; -import com.metamx.druid.loading.S3DataSegmentPusherConfig; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.loading.SegmentLoaderConfig; -import com.metamx.druid.loading.SingleSegmentLoader; -import com.metamx.druid.loading.cassandra.CassandraDataSegmentConfig; -import com.metamx.druid.loading.cassandra.CassandraDataSegmentPuller; -import com.metamx.druid.loading.cassandra.CassandraDataSegmentPusher; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.group.GroupByQuery; -import com.metamx.druid.query.group.GroupByQueryEngine; -import com.metamx.druid.query.group.GroupByQueryEngineConfig; -import com.metamx.druid.query.group.GroupByQueryRunnerFactory; -import com.metamx.druid.query.group.GroupByQueryRunnerFactoryConfig; -import com.metamx.druid.query.metadata.SegmentMetadataQuery; -import com.metamx.druid.query.metadata.SegmentMetadataQueryRunnerFactory; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQueryRunnerFactory; -import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; -import com.metamx.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; -import com.metamx.druid.query.timeseries.TimeseriesQuery; -import com.metamx.druid.query.timeseries.TimeseriesQueryRunnerFactory; -import com.metamx.druid.utils.PropUtils; - -/** - */ -public class ServerInit -{ - private static Logger log = new Logger(ServerInit.class); - - public static SegmentLoader makeDefaultQueryableLoader( - final ConfigurationObjectFactory configFactory, - final Properties props - ) throws S3ServiceException - { - SegmentLoaderConfig config = configFactory.build(SegmentLoaderConfig.class); - DelegatingSegmentLoader delegateLoader = new DelegatingSegmentLoader(); - final QueryableIndexFactory factory = new MMappedQueryableIndexFactory(); - - final RestS3Service s3Client = new RestS3Service( - new AWSCredentials( - props.getProperty("com.metamx.aws.accessKey", ""), - props.getProperty("com.metamx.aws.secretKey", "") - ) - ); - final S3DataSegmentPuller segmentGetter = new S3DataSegmentPuller(s3Client); - final SingleSegmentLoader s3segmentLoader = new SingleSegmentLoader(segmentGetter, factory, config); - - delegateLoader.setLoaderTypes( - ImmutableMap.builder() - .put("local", new SingleSegmentLoader(new LocalDataSegmentPuller(), factory, config)) - .put("hdfs", new SingleSegmentLoader(new HdfsDataSegmentPuller(new Configuration()), factory, config)) - .put("s3", s3segmentLoader) - .put("s3_zip", s3segmentLoader) - .put("c*",new SingleSegmentLoader(new CassandraDataSegmentPuller(configFactory.build(CassandraDataSegmentConfig.class)), factory, config)) - .build() - ); - return delegateLoader; - } - - public static StupidPool makeComputeScratchPool(DruidProcessingConfig config) - { - try { - Class vmClass = Class.forName("sun.misc.VM"); - Object maxDirectMemoryObj = vmClass.getMethod("maxDirectMemory").invoke(null); - - if (maxDirectMemoryObj == null || !(maxDirectMemoryObj instanceof Number)) { - log.info("Cannot determine maxDirectMemory from[%s]", maxDirectMemoryObj); - } else { - long maxDirectMemory = ((Number) maxDirectMemoryObj).longValue(); - - final long memoryNeeded = (long) config.intermediateComputeSizeBytes() * (config.getNumThreads() + 1); - if (maxDirectMemory < memoryNeeded) { - throw new ISE( - "Not enough direct memory. Please adjust -XX:MaxDirectMemorySize or druid.computation.buffer.size: " - + "maxDirectMemory[%,d], memoryNeeded[%,d], druid.computation.buffer.size[%,d], numThreads[%,d]", - maxDirectMemory, memoryNeeded, config.intermediateComputeSizeBytes(), config.getNumThreads() - ); - } - } - } - catch (ClassNotFoundException e) { - log.info("No VM class, cannot do memory check."); - } - catch (NoSuchMethodException e) { - log.info("VM.maxDirectMemory doesn't exist, cannot do memory check."); - } - catch (InvocationTargetException e) { - log.warn(e, "static method shouldn't throw this"); - } - catch (IllegalAccessException e) { - log.warn(e, "public method, shouldn't throw this"); - } - - return new ComputeScratchPool(config.intermediateComputeSizeBytes()); - } - - public static Map, QueryRunnerFactory> initDefaultQueryTypes( - ConfigurationObjectFactory configFactory, - StupidPool computationBufferPool - ) - { - Map, QueryRunnerFactory> queryRunners = Maps.newLinkedHashMap(); - queryRunners.put(TimeseriesQuery.class, new TimeseriesQueryRunnerFactory()); - queryRunners.put( - GroupByQuery.class, - new GroupByQueryRunnerFactory( - new GroupByQueryEngine( - configFactory.build(GroupByQueryEngineConfig.class), - computationBufferPool - ), - configFactory.build(GroupByQueryRunnerFactoryConfig.class) - ) - ); - queryRunners.put(SearchQuery.class, new SearchQueryRunnerFactory()); - queryRunners.put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory()); - queryRunners.put(SegmentMetadataQuery.class, new SegmentMetadataQueryRunnerFactory()); - return queryRunners; - } - - public static DataSegmentPusher getSegmentPusher( - final Properties props, - final ConfigurationObjectFactory configFactory, - final ObjectMapper jsonMapper - ) - { - if (Boolean.parseBoolean(props.getProperty("druid.pusher.local", "false"))) { - return new LocalDataSegmentPusher(configFactory.build(LocalDataSegmentPusherConfig.class), jsonMapper); - } - else if (Boolean.parseBoolean(props.getProperty("druid.pusher.cassandra", "false"))) { - final CassandraDataSegmentConfig config = configFactory.build(CassandraDataSegmentConfig.class); - - return new CassandraDataSegmentPusher(config, jsonMapper); - } - else if (Boolean.parseBoolean(props.getProperty("druid.pusher.hdfs", "false"))) { - final HdfsDataSegmentPusherConfig config = configFactory.build(HdfsDataSegmentPusherConfig.class); - - return new HdfsDataSegmentPusher(config, new Configuration(), jsonMapper); - } - else { - - final RestS3Service s3Client; - try { - s3Client = new RestS3Service( - new AWSCredentials( - PropUtils.getProperty(props, "com.metamx.aws.accessKey"), - PropUtils.getProperty(props, "com.metamx.aws.secretKey") - ) - ); - } - catch (S3ServiceException e) { - throw Throwables.propagate(e); - } - - return new S3DataSegmentPusher(s3Client, configFactory.build(S3DataSegmentPusherConfig.class), jsonMapper); - } - } - - private static class ComputeScratchPool extends StupidPool - { - private static final Logger log = new Logger(ComputeScratchPool.class); - - public ComputeScratchPool(final int computationBufferSize) - { - super( - new Supplier() - { - final AtomicLong count = new AtomicLong(0); - - @Override - public ByteBuffer get() - { - log.info( - "Allocating new computeScratchPool[%,d] of size[%,d]", count.getAndIncrement(), computationBufferSize - ); - return ByteBuffer.allocateDirect(computationBufferSize); - } - } - ); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/loading/DataSegmentPuller.java b/server/src/main/java/com/metamx/druid/loading/DataSegmentPuller.java deleted file mode 100644 index 306d7f449af..00000000000 --- a/server/src/main/java/com/metamx/druid/loading/DataSegmentPuller.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.loading; - -import com.metamx.druid.client.DataSegment; - -import java.io.File; - -/** - */ -public interface DataSegmentPuller -{ - /** - * Pull down segment files for the given DataSegment and put them in the given directory. - * - * @param segment The segment to pull down files for - * @param dir The directory to store the files in - * @throws SegmentLoadingException if there are any errors - */ - public void getSegmentFiles(DataSegment segment, File dir) throws SegmentLoadingException; - - /** - * Returns the last modified time of the given segment. - * - * Note, this is not actually used at this point and doesn't need to actually be implemented. It's just still here - * to not break compatibility. - * - * @param segment The segment to check the last modified time for - * @return the last modified time in millis from the epoch - * @throws SegmentLoadingException if there are any errors - */ - @Deprecated - public long getLastModified(DataSegment segment) throws SegmentLoadingException; -} diff --git a/server/src/main/java/com/metamx/druid/loading/DelegatingSegmentLoader.java b/server/src/main/java/com/metamx/druid/loading/DelegatingSegmentLoader.java deleted file mode 100644 index f113051d506..00000000000 --- a/server/src/main/java/com/metamx/druid/loading/DelegatingSegmentLoader.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.loading; - -import com.metamx.common.MapUtils; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.Segment; - -import javax.inject.Inject; -import java.util.Map; - -/** - */ -public class DelegatingSegmentLoader implements SegmentLoader -{ - private static final Logger log = new Logger(DelegatingSegmentLoader.class); - - private volatile Map loaderTypes; - - @Inject - public void setLoaderTypes( - Map loaderTypes - ) - { - this.loaderTypes = loaderTypes; - } - - @Override - public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException - { - return getLoader(segment.getLoadSpec()).isSegmentLoaded(segment); - } - - @Override - public Segment getSegment(DataSegment segment) throws SegmentLoadingException - { - return getLoader(segment.getLoadSpec()).getSegment(segment); - } - - @Override - public void cleanup(DataSegment segment) throws SegmentLoadingException - { - getLoader(segment.getLoadSpec()).cleanup(segment); - } - - private SegmentLoader getLoader(Map loadSpec) throws SegmentLoadingException - { - String type = MapUtils.getString(loadSpec, "type"); - SegmentLoader loader = loaderTypes.get(type); - - if (loader == null) { - throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, loaderTypes.keySet()); - } - return loader; - } -} diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusherConfig.java b/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusherConfig.java deleted file mode 100644 index 3fbbe2d311f..00000000000 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusherConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.loading; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class S3DataSegmentPusherConfig -{ - @Config("druid.pusher.s3.bucket") - public abstract String getBucket(); - - @Config("druid.pusher.s3.baseKey") - @Default("") - public abstract String getBaseKey(); - - @Config("druid.pusher.s3.disableAcl") - @Default("false") - public abstract boolean getDisableAcl(); -} diff --git a/server/src/main/java/com/metamx/druid/master/BalancerStrategy.java b/server/src/main/java/com/metamx/druid/master/BalancerStrategy.java deleted file mode 100644 index 84944114b49..00000000000 --- a/server/src/main/java/com/metamx/druid/master/BalancerStrategy.java +++ /dev/null @@ -1,35 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -package com.metamx.druid.master; - -import com.metamx.druid.client.DataSegment; - -import java.util.List; - -public interface BalancerStrategy -{ - public ServerHolder findNewSegmentHomeBalancer(final DataSegment proposalSegment, final List serverHolders); - - public ServerHolder findNewSegmentHomeReplicator(final DataSegment proposalSegment, final List serverHolders); - - public BalancerSegmentHolder pickSegmentToMove(final List serverHolders); - - public void emitStats(String tier, MasterStats stats, List serverHolderList); -} diff --git a/server/src/main/java/com/metamx/druid/master/BalancerStrategyFactory.java b/server/src/main/java/com/metamx/druid/master/BalancerStrategyFactory.java deleted file mode 100644 index c215c1b39a9..00000000000 --- a/server/src/main/java/com/metamx/druid/master/BalancerStrategyFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; - -import org.joda.time.DateTime; - -public interface BalancerStrategyFactory -{ - public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp); -} diff --git a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategyFactory.java b/server/src/main/java/com/metamx/druid/master/CostBalancerStrategyFactory.java deleted file mode 100644 index 76e315b1625..00000000000 --- a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategyFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; - -import org.joda.time.DateTime; - -public class CostBalancerStrategyFactory implements BalancerStrategyFactory -{ - - @Override - public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) - { - return new CostBalancerStrategy(referenceTimestamp); - } -} diff --git a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategyFactory.java b/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategyFactory.java deleted file mode 100644 index 1994c0a5fea..00000000000 --- a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategyFactory.java +++ /dev/null @@ -1,30 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; - -import org.joda.time.DateTime; - -public class RandomBalancerStrategyFactory implements BalancerStrategyFactory -{ - @Override - public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) - { - return new RandomBalancerStrategy(); - } -} diff --git a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactoryConfig.java b/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactoryConfig.java deleted file mode 100644 index 52e3227c1ff..00000000000 --- a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactoryConfig.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.metamx.druid.query.group; - -import org.skife.config.Config; - -/** - */ -public abstract class GroupByQueryRunnerFactoryConfig -{ - @Config("druid.query.groupBy.singleThreaded") - public boolean isSingleThreaded() - { - return false; - } -} diff --git a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java b/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java deleted file mode 100644 index 516742ba59b..00000000000 --- a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.query.search; - -import com.google.common.collect.Iterators; -import com.metamx.common.ISE; -import com.metamx.common.guava.BaseSequence; -import com.metamx.common.guava.Sequence; -import com.metamx.druid.Query; -import com.metamx.druid.SearchResultBuilder; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.brita.Filters; -import com.metamx.druid.query.ChainedExecutionQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; - -import java.util.Iterator; -import java.util.concurrent.ExecutorService; - -/** - */ -public class SearchQueryRunnerFactory implements QueryRunnerFactory, SearchQuery> -{ - private static final SearchQueryQueryToolChest toolChest = new SearchQueryQueryToolChest(); - - @Override - public QueryRunner> createRunner(final Segment segment) - { - return new SearchQueryRunner(segment); - } - - @Override - public QueryRunner> mergeRunners( - ExecutorService queryExecutor, Iterable>> queryRunners - ) - { - return new ChainedExecutionQueryRunner>( - queryExecutor, toolChest.getOrdering(), queryRunners - ); - } - - @Override - public QueryToolChest, SearchQuery> getToolchest() - { - return toolChest; - } - - private static class SearchQueryRunner implements QueryRunner> - { - private final StorageAdapter adapter; - - public SearchQueryRunner(Segment segment) - { - this.adapter = segment.asStorageAdapter(); - } - - @Override - public Sequence> run(final Query> input) - { - if (!(input instanceof SearchQuery)) { - throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); - } - - final SearchQuery query = (SearchQuery) input; - - return new BaseSequence, Iterator>>( - new BaseSequence.IteratorMaker, Iterator>>() - { - @Override - public Iterator> make() - { - return Iterators.singletonIterator( - new SearchResultBuilder( - adapter.getInterval().getStart(), - adapter.searchDimensions( - query, - Filters.convertDimensionFilters(query.getDimensionsFilter()) - ) - ).build() - ); - } - - @Override - public void cleanup(Iterator> toClean) - { - - } - } - ); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/utils/CLI.java b/server/src/main/java/com/metamx/druid/utils/CLI.java deleted file mode 100644 index 3c2b5ae1497..00000000000 --- a/server/src/main/java/com/metamx/druid/utils/CLI.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.utils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; - -/** - */ -public class CLI -{ - private final Options opts; - - public CLI() - { - this.opts = new Options(); - } - - public CLI addOptionGroup(OptionGroup group) - { - opts.addOptionGroup(group); - return this; - } - - public CLI addOption(String opt, boolean hasArg, String description) - { - opts.addOption(opt, hasArg, description); - return this; - } - - public CLI addOption(String opt, String longOpt, boolean hasArg, String description) - { - opts.addOption(opt, longOpt, hasArg, description); - return this; - } - - public CLI addOption(Option opt) - { - opts.addOption(opt); - return this; - } - - public CLI addRequiredOption(String opt, String longOpt, boolean hasArg, String description) - { - opts.addOption(new RequiredOption(opt, longOpt, hasArg, description)); - return this; - } - - public CommandLine parse(String[] args) - { - if (args.length == 0) { - new HelpFormatter().printHelp("", opts); - return null; - } - - CommandLine cli = null; - try { - cli = new PosixParser().parse(opts, args, false); - } - catch (ParseException e) { - System.out.println(e.getMessage()); - new HelpFormatter().printHelp("", opts); - return null; - } - - if (cli.hasOption("help")) { - new HelpFormatter().printHelp("", opts); - return null; - } - - return cli; - } -} diff --git a/server/src/main/java/com/metamx/druid/utils/DruidSetup.java b/server/src/main/java/com/metamx/druid/utils/DruidSetup.java deleted file mode 100644 index ef5185ac703..00000000000 --- a/server/src/main/java/com/metamx/druid/utils/DruidSetup.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.utils; - -import com.google.common.base.Charsets; -import com.google.common.base.Throwables; -import com.google.common.collect.Sets; -import com.google.common.io.Closeables; -import com.metamx.common.config.Config; -import com.metamx.druid.initialization.ZkPathsConfig; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryOneTime; -import org.skife.config.ConfigurationObjectFactory; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintStream; -import java.util.Properties; -import java.util.Set; - -/** - * Set up the shared Druid ensemble space. - * This affects the Zookeeper which holds common properties, and znode paths for coordination, - * and also performs metadata table creation in the database (MySQL). - * By storing ensemble-wide properties in zookeeper, cluster administration is simplified. - * Each service instance can also have local property overrides in the file runtime.properties - * located in the classpath. - *

- * The design rules are noted here with rationale - *

- *

- * 

- * Design Rule Notes:
- * (a) Properties set on the commandline of services take precedence over  runtime.properties which
- *       takes precedence over properties stored in zookeeper.
- *
- *       Rationale:  organizing principle.
- *
- * (a) Services load properties on startup only.
- *
- *       Rationale: stepwise changes are safer and easier to manage.
- *
- * (b) Only DruidSetup creates properties and znode paths (zpaths) on zookeeper and no other tool or service
- *       will make ensemble-wide settings automatically.
- *
- *       Rationale: one place for this logic, under manual control, and avoid accidental
- *       namespace/partition creation.
- *
- * (c) DruidSetup creates reasonable zpaths but supports overrides to enable tactical
- *   version transitions (just in case).  If zpaths are overridden, then they must all be
- *   overridden together since they are not independent.
- *
- *       Rationale:  convention beats configuration most of the time; sometimes configuration is needed
- *       negotiate unusual cases.
- *
- * (d) Properties settings stored on zookeeper are not cumulative; previous properties are removed before
- *   new ones are stored.
- *       Rationale:  Keep the operations at the granularity of a file of properties, avoid
- *       dependence on order of setup operations, enable dumping of current settings.
- *
- * - * @author pbaclace - */ -public class DruidSetup -{ - private final static String MODIFIED_PROP = "__MODIFIED"; - private final static Set IGNORED_PROPS = Sets.newHashSet(MODIFIED_PROP); - - public static void main(final String[] args) - { - CuratorFramework curator = null; - - try { - if (args.length < 2 || args.length > 3) { - printUsage(); - System.exit(1); - } - String cmd = args[0]; - if ("dump".equals(cmd) && args.length == 3) { - final String zkConnect = args[1]; - curator = connectToZK(zkConnect); - curator.start(); - String zpathBase = args[2]; - dumpFromZk(curator, zkConnect, zpathBase, System.out); - } else if ("put".equals(cmd) && args.length == 3) { - final String zkConnect = args[1]; - curator = connectToZK(zkConnect); - curator.start(); - final String pfile = args[2]; - putToZk(curator, pfile); - } else { - printUsage(); - System.exit(1); - } - } - finally { - Closeables.closeQuietly(curator); - } - } - - /** - * Load properties from local file, validate and tweak. - *

- * This can only be used for setup, not service run time because of some assembly here. - * - * @param pfile path to runtime.properties file to be read. - */ - private static Properties loadProperties(String pfile) - { - InputStream is = null; - try { - is = new FileInputStream(pfile); - } - catch (FileNotFoundException e) { - System.err.println("File not found: " + pfile); - System.err.println("No changes made."); - System.exit(4); - } - - try { - Properties props = new Properties(); - props.load(new InputStreamReader(is, Charsets.UTF_8)); - return props; - } - catch (IOException e) { - throw reportErrorAndExit(pfile, e); - } - finally { - Closeables.closeQuietly(is); - } - } - - /** - * @param curator zookeeper client. - * @param zPathBase znode base path. - * @param zkConnect ZK coordinates in the form host1:port1[,host2:port2[, ...]] - * @param out - */ - private static void dumpFromZk(CuratorFramework curator, String zkConnect, final String zPathBase, PrintStream out) - { - ZkPathsConfig config = new ZkPathsConfig() - { - @Override - public String getZkBasePath() - { - return zPathBase; - } - }; - - try { - if (curator.checkExists().forPath(config.getPropertiesPath()) != null) { - byte[] data = curator.getData().forPath(config.getPropertiesPath()); - Properties currProps = new Properties(); - currProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - - if (! currProps.isEmpty()) { - out.printf("# Begin Properties Listing for zpath[%s]%n", config.getPropertiesPath()); - try { - currProps.store(new OutputStreamWriter(out, Charsets.UTF_8), "Druid"); - } - catch (IOException ignored) { - } - out.printf("# End Properties for zkConnect[%s] zpath[%s]%n", zkConnect, config.getPropertiesPath()); - } - else { - out.printf("# Properties at zpath[%s] empty.%n", config.getPropertiesPath()); - } - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - private static void putToZk(CuratorFramework curator, String pfile) - { - final Properties props = loadProperties(pfile); - ConfigurationObjectFactory configFactory = Config.createFactory(props); - final ZkPathsConfig zkPaths = configFactory.build(ZkPathsConfig.class); - - createZNodes(curator, zkPaths, System.out); - updatePropertiesZK(curator, zkPaths, props, System.out); - } - - /** - * @param curator zookeeper client. - * @param zkPaths znode base path. - * @param props the properties to store. - * @param out the PrintStream for human readable update summary (usually System.out). - */ - private static void updatePropertiesZK(CuratorFramework curator, ZkPathsConfig zkPaths, Properties props, PrintStream out) - { - Properties currProps = new Properties(); - try { - if (curator.checkExists().forPath(zkPaths.getPropertiesPath()) != null) { - final byte[] data = curator.getData().forPath(zkPaths.getPropertiesPath()); - currProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - } - boolean propsDiffer = false; - if (currProps.isEmpty()) { - out.println("No properties currently stored in zk"); - propsDiffer = true; - } else { // determine whether anything is different - int countNew = 0; - int countDiffer = 0; - int countRemoved = 0; - int countNoChange = 0; - StringBuilder changes = new StringBuilder(1024); - for (String pname : props.stringPropertyNames()) { - if (IGNORED_PROPS.contains(pname)) { - continue; // ignore meta props, if any - } - final String pvalue = props.getProperty(pname); - final String pvalueCurr = currProps.getProperty(pname); - if (pvalueCurr == null) { - countNew++; - } else { - if (pvalueCurr.equals(pvalue)) { - countNoChange++; - } else { - countDiffer++; - changes.append(String.format("CHANGED[%s]: PREV=%s --- NOW=%s%n", pname, pvalueCurr, pvalue)); - } - } - } - for (String pname : currProps.stringPropertyNames()) { - if (IGNORED_PROPS.contains(pname)) { - continue; // ignore meta props, if any - } - if (props.getProperty(pname) == null) { - countRemoved++; - changes.append(String.format("REMOVED: %s=%s%n", pname, currProps.getProperty(pname))); - } - } - if (countNew + countRemoved + countDiffer > 0) { - out.printf( - "Properties differ: %,d new, %,d changed, %,d removed, %,d unchanged, previously updated %s%n", - countNew, countDiffer, countRemoved, countNoChange, currProps.getProperty(MODIFIED_PROP) - ); - out.println(changes); - propsDiffer = true; - } else { - out.printf("Current properties identical to file given, %,d total properties set.%n", countNoChange); - } - } - if (propsDiffer) { - ByteArrayOutputStream propsBytes = new ByteArrayOutputStream(); - props.store(new OutputStreamWriter(propsBytes, Charsets.UTF_8), "Common Druid properties"); - - if (currProps.isEmpty()) { - curator.setData().forPath(zkPaths.getPropertiesPath(), propsBytes.toByteArray()); - } - else { - curator.create().forPath(zkPaths.getPropertiesPath(), propsBytes.toByteArray()); - } - out.printf("Properties updated, %,d total properties set.%n", props.size()); - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - /** - * @param curator zookeeper client. - * @param zkPaths znode base path. - * @param out the PrintStream for human readable update summary. - */ - private static void createZNodes(CuratorFramework curator, ZkPathsConfig zkPaths, PrintStream out) - { - createPath(curator, zkPaths.getAnnouncementsPath(), out); - createPath(curator, zkPaths.getMasterPath(), out); - createPath(curator, zkPaths.getLoadQueuePath(), out); - createPath(curator, zkPaths.getServedSegmentsPath(), out); - createPath(curator, zkPaths.getLiveSegmentsPath(), out); - createPath(curator, zkPaths.getPropertiesPath(), out); - } - - private static void createPath(CuratorFramework curator, String thePath, PrintStream out) - { - try { - if (curator.checkExists().forPath(thePath) != null) { - out.printf("Path[%s] exists already%n", thePath); - } else { - out.printf("Creating ZK path[%s]%n", thePath); - curator.create().creatingParentsIfNeeded().forPath(thePath); - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - private static RuntimeException reportErrorAndExit(String pfile, IOException ioe) - { - System.err.println("Could not read file: " + pfile); - System.err.println(" because of: " + ioe); - System.err.println("No changes made."); - System.exit(4); - - return new RuntimeException(); - } - - private static CuratorFramework connectToZK(String zkConnect) - { - return CuratorFrameworkFactory.builder() - .connectString(zkConnect) - .retryPolicy(new RetryOneTime(5000)) - .build(); - } - - /** - * Print usage to stdout. - */ - private static void printUsage() - { - System.out.println( - "Usage: CMD [args]\n" - + " Where CMD is a particular command:\n" - + " CMD choices:\n" - + " dump zkConnect baseZkPath # dump info from zk at given coordinates\n" - + " put zkConnect propfile # store paths and propfile into zk at given coordinates\n" - + " args:\n" - + " zkConnect: ZK coordinates in the form host1:port1[,host2:port2[, ...]]\n" - + " baseZkPath: like /druid or /mydruid etc. to uniquely identify a Druid ensemble\n" - + " and should be equal to property druid.zk.paths.base\n" - + " propfile: Java properties file with common properties for all services in ensemble\n" - + " Notes:\n" - + " dump command makes no modifications and shows zk properties at baseZkPath.\n" - + " put command can safely be invoked more than once, will not disturb existing queues,\n" - + " and properties are not cumulative.\n" - + " A zookeeper can service more than one Druid ensemble if baseZkPath is distinct.\n" - + " Druid services only load properties during process startup.\n" - + " Properties defined on a service command line take precedence over the runtime.properties\n" - + " file which takes precedence over properties stored in zookeeper.\n" - + "" - ); - } -} diff --git a/server/src/main/java/com/metamx/druid/utils/RequiredOption.java b/server/src/main/java/com/metamx/druid/utils/RequiredOption.java deleted file mode 100644 index 5ec23aac081..00000000000 --- a/server/src/main/java/com/metamx/druid/utils/RequiredOption.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.utils; - -import org.apache.commons.cli.Option; - -/** - */ -public class RequiredOption extends Option -{ - public RequiredOption(String opt, String description) - throws IllegalArgumentException - { - super(opt, description); - setRequired(true); - } - - public RequiredOption(String opt, boolean hasArg, String description) - throws IllegalArgumentException - { - super(opt, hasArg, description); - setRequired(true); - } - - public RequiredOption(String opt, String longOpt, boolean hasArg, String description) - throws IllegalArgumentException - { - super(opt, longOpt, hasArg, description); - setRequired(true); - } -} diff --git a/client/src/main/java/com/metamx/druid/client/BatchServerInventoryView.java b/server/src/main/java/io/druid/client/BatchServerInventoryView.java similarity index 80% rename from client/src/main/java/com/metamx/druid/client/BatchServerInventoryView.java rename to server/src/main/java/io/druid/client/BatchServerInventoryView.java index 58497e293db..3a1bf636aa2 100644 --- a/client/src/main/java/com/metamx/druid/client/BatchServerInventoryView.java +++ b/server/src/main/java/io/druid/client/BatchServerInventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,61 +17,46 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.MapMaker; import com.google.common.collect.Sets; +import com.google.inject.Inject; import com.metamx.common.ISE; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; import com.metamx.emitter.EmittingLogger; +import io.druid.guice.ManageLifecycle; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import java.util.Set; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; /** */ +@ManageLifecycle public class BatchServerInventoryView extends ServerInventoryView> { private static final EmittingLogger log = new EmittingLogger(BatchServerInventoryView.class); final ConcurrentMap> zNodes = new MapMaker().makeMap(); + @Inject public BatchServerInventoryView( - final ServerInventoryViewConfig config, final ZkPathsConfig zkPaths, final CuratorFramework curator, - final ExecutorService exec, final ObjectMapper jsonMapper ) { super( - config, log, - new InventoryManagerConfig() - { - @Override - public String getContainerPath() - { - return zkPaths.getAnnouncementsPath(); - } - - @Override - public String getInventoryPath() - { - return zkPaths.getLiveSegmentsPath(); - } - }, + zkPaths.getAnnouncementsPath(), + zkPaths.getLiveSegmentsPath(), curator, - exec, jsonMapper, - new TypeReference>() - { - } + new TypeReference>(){} ); } diff --git a/server/src/main/java/io/druid/client/BatchServerInventoryViewProvider.java b/server/src/main/java/io/druid/client/BatchServerInventoryViewProvider.java new file mode 100644 index 00000000000..609a47754aa --- /dev/null +++ b/server/src/main/java/io/druid/client/BatchServerInventoryViewProvider.java @@ -0,0 +1,50 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.server.initialization.ZkPathsConfig; +import org.apache.curator.framework.CuratorFramework; + +import javax.validation.constraints.NotNull; + +/** + */ +public class BatchServerInventoryViewProvider implements ServerInventoryViewProvider +{ + @JacksonInject + @NotNull + private ZkPathsConfig zkPaths = null; + + @JacksonInject + @NotNull + private CuratorFramework curator = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public ServerInventoryView get() + { + return new BatchServerInventoryView(zkPaths, curator, jsonMapper); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/BrokerServerView.java b/server/src/main/java/io/druid/client/BrokerServerView.java similarity index 92% rename from client/src/main/java/com/metamx/druid/client/BrokerServerView.java rename to server/src/main/java/io/druid/client/BrokerServerView.java index a61b6b0c00f..a60cfb18fea 100644 --- a/client/src/main/java/com/metamx/druid/client/BrokerServerView.java +++ b/server/src/main/java/io/druid/client/BrokerServerView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,23 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.selector.QueryableDruidServer; -import com.metamx.druid.client.selector.ServerSelector; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChestWarehouse; import com.metamx.http.client.HttpClient; +import io.druid.client.selector.QueryableDruidServer; +import io.druid.client.selector.ServerSelector; +import io.druid.concurrent.Execs; +import io.druid.guice.annotations.Client; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.timeline.DataSegment; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; import java.util.Iterator; import java.util.Map; @@ -54,12 +58,12 @@ public class BrokerServerView implements TimelineServerView private final HttpClient httpClient; private final ServerView baseView; + @Inject public BrokerServerView( QueryToolChestWarehouse warehose, ObjectMapper smileMapper, - HttpClient httpClient, - ServerView baseView, - ExecutorService exec + @Client HttpClient httpClient, + ServerView baseView ) { this.warehose = warehose; @@ -71,6 +75,7 @@ public class BrokerServerView implements TimelineServerView this.selectors = Maps.newHashMap(); this.timelines = Maps.newHashMap(); + ExecutorService exec = Execs.singleThreaded("BrokerServerView-%s"); baseView.registerSegmentCallback( exec, new ServerView.SegmentCallback() diff --git a/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java similarity index 93% rename from client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java rename to server/src/main/java/io/druid/client/CachingClusteredClient.java index 72519cbeb8c..7560494170a 100644 --- a/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -32,32 +32,31 @@ import com.google.common.collect.Maps; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.common.ISE; +import com.google.inject.Inject; import com.metamx.common.Pair; import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.LazySequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.common.logger.Logger; -import com.metamx.druid.Query; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.client.cache.Cache; -import com.metamx.druid.client.selector.QueryableDruidServer; -import com.metamx.druid.client.selector.ServerSelector; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.query.CacheStrategy; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.Queries; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.segment.MultipleSpecificSegmentSpec; -import com.metamx.druid.query.segment.SegmentDescriptor; -import com.metamx.druid.result.BySegmentResultValueClass; -import com.metamx.druid.result.Result; import com.metamx.emitter.EmittingLogger; +import io.druid.client.cache.Cache; +import io.druid.client.selector.QueryableDruidServer; +import io.druid.client.selector.ServerSelector; +import io.druid.query.BySegmentResultValueClass; +import io.druid.query.CacheStrategy; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.Result; +import io.druid.query.SegmentDescriptor; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.spec.MultipleSpecificSegmentSpec; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -82,6 +81,7 @@ public class CachingClusteredClient implements QueryRunner private final Cache cache; private final ObjectMapper objectMapper; + @Inject public CachingClusteredClient( QueryToolChestWarehouse warehouse, TimelineServerView serverView, diff --git a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java similarity index 94% rename from client/src/main/java/com/metamx/druid/client/DirectDruidClient.java rename to server/src/main/java/io/druid/client/DirectDruidClient.java index 92a4e5b9426..80844ccbeda 100644 --- a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; @@ -35,18 +35,18 @@ import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.logger.Logger; -import com.metamx.druid.Query; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.result.BySegmentResultValueClass; -import com.metamx.druid.result.Result; import com.metamx.http.client.HttpClient; import com.metamx.http.client.io.AppendableByteArrayInputStream; import com.metamx.http.client.response.ClientResponse; import com.metamx.http.client.response.InputStreamResponseHandler; +import io.druid.query.BySegmentResultValueClass; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; import org.jboss.netty.handler.codec.http.HttpChunk; import org.jboss.netty.handler.codec.http.HttpHeaders; import org.jboss.netty.handler.codec.http.HttpResponse; @@ -199,14 +199,16 @@ public class DirectDruidClient implements QueryRunner if (!isBySegment) { retVal = Sequences.map( retVal, - toolChest.makeMetricManipulatorFn(query, new MetricManipulationFn() + toolChest.makeMetricManipulatorFn( + query, new MetricManipulationFn() { @Override public Object manipulate(AggregatorFactory factory, Object object) { return factory.deserialize(object); } - }) + } + ) ); } diff --git a/client/src/main/java/com/metamx/druid/client/DruidDataSource.java b/server/src/main/java/io/druid/client/DruidDataSource.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/DruidDataSource.java rename to server/src/main/java/io/druid/client/DruidDataSource.java index ebe566052f8..45137eb7da1 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidDataSource.java +++ b/server/src/main/java/io/druid/client/DruidDataSource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Maps; +import io.druid.timeline.DataSegment; import java.util.Collections; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/client/DruidServer.java b/server/src/main/java/io/druid/client/DruidServer.java similarity index 95% rename from client/src/main/java/com/metamx/druid/client/DruidServer.java rename to server/src/main/java/io/druid/client/DruidServer.java index a3acc3c00d8..53acc501513 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidServer.java +++ b/server/src/main/java/io/druid/client/DruidServer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; import com.metamx.common.logger.Logger; -import com.metamx.druid.coordination.DruidServerMetadata; +import io.druid.server.DruidNode; +import io.druid.server.coordination.DruidServerMetadata; +import io.druid.timeline.DataSegment; import java.util.Collections; import java.util.Map; @@ -47,13 +49,14 @@ public class DruidServer implements Comparable private volatile long currSize; public DruidServer( + DruidNode node, DruidServerConfig config, String type ) { this( - config.getServerName(), - config.getHost(), + node.getHost(), + node.getHost(), config.getMaxSize(), type, config.getTier() diff --git a/server/src/main/java/com/metamx/druid/loading/SegmentLoadingException.java b/server/src/main/java/io/druid/client/DruidServerConfig.java similarity index 65% rename from server/src/main/java/com/metamx/druid/loading/SegmentLoadingException.java rename to server/src/main/java/io/druid/client/DruidServerConfig.java index d52fd6e3a82..14cfaee290a 100644 --- a/server/src/main/java/com/metamx/druid/loading/SegmentLoadingException.java +++ b/server/src/main/java/io/druid/client/DruidServerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,26 +17,30 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.client; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; /** */ -public class SegmentLoadingException extends Exception +public class DruidServerConfig { - public SegmentLoadingException( - String formatString, - Object... objs - ) + @JsonProperty + @Min(0) + private long maxSize = -1; + + @JsonProperty + private String tier = "_default_tier"; + + public long getMaxSize() { - super(String.format(formatString, objs)); + return maxSize; } - public SegmentLoadingException( - Throwable cause, - String formatString, - Object... objs - ) + public String getTier() { - super(String.format(formatString, objs), cause); + return tier; } } diff --git a/server/src/main/java/com/metamx/druid/loading/DataSegmentKiller.java b/server/src/main/java/io/druid/client/InventoryView.java similarity index 77% rename from server/src/main/java/com/metamx/druid/loading/DataSegmentKiller.java rename to server/src/main/java/io/druid/client/InventoryView.java index 85483eaa505..d955ea21c42 100644 --- a/server/src/main/java/com/metamx/druid/loading/DataSegmentKiller.java +++ b/server/src/main/java/io/druid/client/InventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; - -import com.metamx.druid.client.DataSegment; +package io.druid.client; /** */ -public interface DataSegmentKiller +public interface InventoryView { - public void kill(DataSegment segments) throws SegmentLoadingException; + public DruidServer getInventoryValue(String string); + public Iterable getInventory(); } diff --git a/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java b/server/src/main/java/io/druid/client/ServerInventoryView.java similarity index 85% rename from client/src/main/java/com/metamx/druid/client/ServerInventoryView.java rename to server/src/main/java/io/druid/client/ServerInventoryView.java index bd5cd70c668..f9187828ac3 100644 --- a/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java +++ b/server/src/main/java/io/druid/client/ServerInventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -27,51 +27,59 @@ import com.google.common.base.Throwables; import com.google.common.collect.MapMaker; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.inventory.CuratorInventoryManager; -import com.metamx.druid.curator.inventory.CuratorInventoryManagerStrategy; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; import com.metamx.emitter.EmittingLogger; +import io.druid.concurrent.Execs; +import io.druid.curator.inventory.CuratorInventoryManager; +import io.druid.curator.inventory.CuratorInventoryManagerStrategy; +import io.druid.curator.inventory.InventoryManagerConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; /** */ public abstract class ServerInventoryView implements ServerView, InventoryView { - private final ServerInventoryViewConfig config; - private final Logger log; + + private final EmittingLogger log; private final CuratorInventoryManager inventoryManager; private final AtomicBoolean started = new AtomicBoolean(false); private final ConcurrentMap serverCallbacks = new MapMaker().makeMap(); private final ConcurrentMap segmentCallbacks = new MapMaker().makeMap(); - private final Map removedSegments = new MapMaker().makeMap(); - public ServerInventoryView( - final ServerInventoryViewConfig config, - final Logger log, - final InventoryManagerConfig inventoryManagerConfig, + final EmittingLogger log, + final String announcementsPath, + final String inventoryPath, final CuratorFramework curator, - final ExecutorService exec, final ObjectMapper jsonMapper, final TypeReference typeReference ) { - this.config = config; this.log = log; - this.inventoryManager = new CuratorInventoryManager( + this.inventoryManager = new CuratorInventoryManager<>( curator, - inventoryManagerConfig, - exec, + new InventoryManagerConfig() + { + @Override + public String getContainerPath() + { + return announcementsPath; + } + + @Override + public String getInventoryPath() + { + return inventoryPath; + } + }, + Execs.singleThreaded("ServerInventoryView-%s"), new CuratorInventoryManagerStrategy() { @Override @@ -164,26 +172,6 @@ public abstract class ServerInventoryView implements ServerView, ); } - public int lookupSegmentLifetime(DataSegment segment) - { - Integer lifetime = removedSegments.get(segment.getIdentifier()); - return (lifetime == null) ? 0 : lifetime; - } - - public void decrementRemovedSegmentsLifetime() - { - for (Iterator> mapIter = removedSegments.entrySet().iterator(); mapIter.hasNext(); ) { - Map.Entry segment = mapIter.next(); - int lifetime = segment.getValue() - 1; - - if (lifetime < 0) { - mapIter.remove(); - } else { - segment.setValue(lifetime); - } - } - } - @LifecycleStart public void start() throws Exception { @@ -335,8 +323,6 @@ public abstract class ServerInventoryView implements ServerView, } } ); - - removedSegments.put(inventoryKey, config.getRemovedSegmentLifetime()); } protected abstract DruidServer addInnerInventory( diff --git a/server/src/main/java/io/druid/client/ServerInventoryViewProvider.java b/server/src/main/java/io/druid/client/ServerInventoryViewProvider.java new file mode 100644 index 00000000000..fa48fba4661 --- /dev/null +++ b/server/src/main/java/io/druid/client/ServerInventoryViewProvider.java @@ -0,0 +1,35 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = SingleServerInventoryProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "legacy", value = SingleServerInventoryProvider.class), + @JsonSubTypes.Type(name = "batch", value = BatchServerInventoryViewProvider.class) +}) +public interface ServerInventoryViewProvider extends Provider +{ +} diff --git a/client/src/main/java/com/metamx/druid/client/ServerView.java b/server/src/main/java/io/druid/client/ServerView.java similarity index 97% rename from client/src/main/java/com/metamx/druid/client/ServerView.java rename to server/src/main/java/io/druid/client/ServerView.java index 61afd6d1222..cbd740269b8 100644 --- a/client/src/main/java/com/metamx/druid/client/ServerView.java +++ b/server/src/main/java/io/druid/client/ServerView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; + +import io.druid.timeline.DataSegment; import java.util.concurrent.Executor; diff --git a/server/src/main/java/io/druid/client/SingleServerInventoryProvider.java b/server/src/main/java/io/druid/client/SingleServerInventoryProvider.java new file mode 100644 index 00000000000..7ebce938791 --- /dev/null +++ b/server/src/main/java/io/druid/client/SingleServerInventoryProvider.java @@ -0,0 +1,50 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.server.initialization.ZkPathsConfig; +import org.apache.curator.framework.CuratorFramework; + +import javax.validation.constraints.NotNull; + +/** + */ +public class SingleServerInventoryProvider implements ServerInventoryViewProvider +{ + @JacksonInject + @NotNull + private ZkPathsConfig zkPaths = null; + + @JacksonInject + @NotNull + private CuratorFramework curator = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public ServerInventoryView get() + { + return new SingleServerInventoryView(zkPaths, curator, jsonMapper); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/SingleServerInventoryView.java b/server/src/main/java/io/druid/client/SingleServerInventoryView.java similarity index 74% rename from client/src/main/java/com/metamx/druid/client/SingleServerInventoryView.java rename to server/src/main/java/io/druid/client/SingleServerInventoryView.java index 781c4ed0c77..801b78c5d2a 100644 --- a/client/src/main/java/com/metamx/druid/client/SingleServerInventoryView.java +++ b/server/src/main/java/io/druid/client/SingleServerInventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,50 +17,36 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; +import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; +import io.druid.guice.ManageLifecycle; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; -import java.util.concurrent.ExecutorService; - /** */ +@ManageLifecycle public class SingleServerInventoryView extends ServerInventoryView { private static final EmittingLogger log = new EmittingLogger(SingleServerInventoryView.class); + @Inject public SingleServerInventoryView( - final ServerInventoryViewConfig config, final ZkPathsConfig zkPaths, final CuratorFramework curator, - final ExecutorService exec, final ObjectMapper jsonMapper ) { super( - config, log, - new InventoryManagerConfig() - { - @Override - public String getContainerPath() - { - return zkPaths.getAnnouncementsPath(); - } - - @Override - public String getInventoryPath() - { - return zkPaths.getServedSegmentsPath(); - } - }, + zkPaths.getAnnouncementsPath(), + zkPaths.getServedSegmentsPath(), curator, - exec, jsonMapper, new TypeReference(){} ); diff --git a/client/src/main/java/com/metamx/druid/client/TimelineServerView.java b/server/src/main/java/io/druid/client/TimelineServerView.java similarity index 81% rename from client/src/main/java/com/metamx/druid/client/TimelineServerView.java rename to server/src/main/java/io/druid/client/TimelineServerView.java index 33747f18af1..7082c599c75 100644 --- a/client/src/main/java/com/metamx/druid/client/TimelineServerView.java +++ b/server/src/main/java/io/druid/client/TimelineServerView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.selector.ServerSelector; -import com.metamx.druid.query.QueryRunner; +import io.druid.client.selector.ServerSelector; +import io.druid.query.QueryRunner; +import io.druid.timeline.VersionedIntervalTimeline; /** */ diff --git a/client/src/main/java/com/metamx/druid/client/cache/ByteCountingLRUMap.java b/server/src/main/java/io/druid/client/cache/ByteCountingLRUMap.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/ByteCountingLRUMap.java rename to server/src/main/java/io/druid/client/cache/ByteCountingLRUMap.java index 59a60f060e5..b351dfc6837 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/ByteCountingLRUMap.java +++ b/server/src/main/java/io/druid/client/cache/ByteCountingLRUMap.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.metamx.common.logger.Logger; diff --git a/client/src/main/java/com/metamx/druid/client/cache/Cache.java b/server/src/main/java/io/druid/client/cache/Cache.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/Cache.java rename to server/src/main/java/io/druid/client/cache/Cache.java index 6e9463deb56..6ed875edb0b 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/Cache.java +++ b/server/src/main/java/io/druid/client/cache/Cache.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheMonitor.java b/server/src/main/java/io/druid/client/cache/CacheMonitor.java similarity index 95% rename from client/src/main/java/com/metamx/druid/client/cache/CacheMonitor.java rename to server/src/main/java/io/druid/client/cache/CacheMonitor.java index e89c1113e8d..baa4920e60f 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheMonitor.java +++ b/server/src/main/java/io/druid/client/cache/CacheMonitor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; +import com.google.inject.Inject; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.metrics.AbstractMonitor; @@ -31,6 +32,7 @@ public class CacheMonitor extends AbstractMonitor private volatile CacheStats prevCacheStats = null; + @Inject public CacheMonitor( Cache cache ) diff --git a/server/src/main/java/io/druid/client/cache/CacheProvider.java b/server/src/main/java/io/druid/client/cache/CacheProvider.java new file mode 100644 index 00000000000..87597dba1c7 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/CacheProvider.java @@ -0,0 +1,33 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LocalCacheProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "local", value = LocalCacheProvider.class), + @JsonSubTypes.Type(name = "memcached", value = MemcachedCacheProvider.class) +}) +public interface CacheProvider extends Provider +{ +} diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheSerde.java b/server/src/main/java/io/druid/client/cache/CacheSerde.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/cache/CacheSerde.java rename to server/src/main/java/io/druid/client/cache/CacheSerde.java index e7e60fcc096..16dbce6ad6d 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheSerde.java +++ b/server/src/main/java/io/druid/client/cache/CacheSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; /** */ diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheStats.java b/server/src/main/java/io/druid/client/cache/CacheStats.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/CacheStats.java rename to server/src/main/java/io/druid/client/cache/CacheStats.java index 33f0a145082..ea917649154 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheStats.java +++ b/server/src/main/java/io/druid/client/cache/CacheStats.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; /** */ diff --git a/client/src/main/java/com/metamx/druid/client/cache/LZ4Transcoder.java b/server/src/main/java/io/druid/client/cache/LZ4Transcoder.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/LZ4Transcoder.java rename to server/src/main/java/io/druid/client/cache/LZ4Transcoder.java index 4728430b4e7..7427c2b3db5 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/LZ4Transcoder.java +++ b/server/src/main/java/io/druid/client/cache/LZ4Transcoder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2013 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.primitives.Ints; import net.jpountz.lz4.LZ4Compressor; diff --git a/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java new file mode 100644 index 00000000000..4f78457b5b9 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +/** + */ +public class LocalCacheProvider implements CacheProvider +{ + @JsonProperty + @Min(0) + private long sizeInBytes = 0; + + @JsonProperty + @Min(0) + private int initialSize = 500000; + + @JsonProperty + @Min(0) + private int logEvictionCount = 0; + + + @Override + public Cache get() + { + return new MapCache(new ByteCountingLRUMap(initialSize, logEvictionCount, sizeInBytes)); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/cache/MapCache.java b/server/src/main/java/io/druid/client/cache/MapCache.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/cache/MapCache.java rename to server/src/main/java/io/druid/client/cache/MapCache.java index bf549ec31b9..dda2929bc89 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/MapCache.java +++ b/server/src/main/java/io/druid/client/cache/MapCache.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -33,6 +33,11 @@ import java.util.concurrent.atomic.AtomicLong; */ public class MapCache implements Cache { + public static Cache create(long sizeInBytes) + { + return new MapCache(new ByteCountingLRUMap(sizeInBytes)); + } + private final Map baseMap; private final ByteCountingLRUMap byteCountingLRUMap; @@ -44,17 +49,6 @@ public class MapCache implements Cache private final AtomicLong hitCount = new AtomicLong(0); private final AtomicLong missCount = new AtomicLong(0); - public static com.metamx.druid.client.cache.Cache create(final MapCacheConfig config) - { - return new MapCache( - new ByteCountingLRUMap( - config.getInitialSize(), - config.getLogEvictionCount(), - config.getSizeInBytes() - ) - ); - } - MapCache( ByteCountingLRUMap byteCountingLRUMap ) diff --git a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java b/server/src/main/java/io/druid/client/cache/MemcachedCache.java similarity index 98% rename from client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java rename to server/src/main/java/io/druid/client/cache/MemcachedCache.java index fb6fa72ce46..155c75e3f86 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java +++ b/server/src/main/java/io/druid/client/cache/MemcachedCache.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -32,7 +32,6 @@ import net.spy.memcached.FailureMode; import net.spy.memcached.MemcachedClient; import net.spy.memcached.MemcachedClientIF; import net.spy.memcached.internal.BulkFuture; -import net.spy.memcached.transcoders.SerializingTranscoder; import org.apache.commons.codec.digest.DigestUtils; import javax.annotation.Nullable; diff --git a/server/src/main/java/io/druid/client/cache/MemcachedCacheConfig.java b/server/src/main/java/io/druid/client/cache/MemcachedCacheConfig.java new file mode 100644 index 00000000000..2cc06cf3637 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/MemcachedCacheConfig.java @@ -0,0 +1,68 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.NotNull; + +public class MemcachedCacheConfig +{ + @JsonProperty + private int expiration = 2592000; // What is this number? + + @JsonProperty + private int timeout = 500; + + @JsonProperty + @NotNull + private String hosts; + + @JsonProperty + private int maxObjectSize = 50 * 1024 * 1024; + + @JsonProperty + private String memcachedPrefix = "druid"; + + public int getExpiration() + { + return expiration; + } + + public int getTimeout() + { + return timeout; + } + + public String getHosts() + { + return hosts; + } + + public int getMaxObjectSize() + { + return maxObjectSize; + } + + public String getMemcachedPrefix() + { + return memcachedPrefix; + } +} diff --git a/server/src/main/java/io/druid/client/cache/MemcachedCacheProvider.java b/server/src/main/java/io/druid/client/cache/MemcachedCacheProvider.java new file mode 100644 index 00000000000..afdab26dd09 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/MemcachedCacheProvider.java @@ -0,0 +1,29 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +public class MemcachedCacheProvider extends MemcachedCacheConfig implements CacheProvider +{ + @Override + public Cache get() + { + return MemcachedCache.create(this); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientAppendQuery.java b/server/src/main/java/io/druid/client/indexing/ClientAppendQuery.java similarity index 92% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientAppendQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientAppendQuery.java index 6e9a2eb4163..1f83252a681 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientAppendQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientAppendQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientConversionQuery.java b/server/src/main/java/io/druid/client/indexing/ClientConversionQuery.java similarity index 51% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientConversionQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientConversionQuery.java index c0b96bb80b4..631d0aca86f 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientConversionQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientConversionQuery.java @@ -1,7 +1,26 @@ -package com.metamx.druid.client.indexing; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; /** diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientKillQuery.java b/server/src/main/java/io/druid/client/indexing/ClientKillQuery.java similarity index 94% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientKillQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientKillQuery.java index 3ae8dffb225..6f280cef7fa 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientKillQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientKillQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientMergeQuery.java b/server/src/main/java/io/druid/client/indexing/ClientMergeQuery.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientMergeQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientMergeQuery.java index e363618a107..3aa6efefe19 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientMergeQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientMergeQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.client.DataSegment; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.timeline.DataSegment; import java.util.List; diff --git a/server/src/main/java/io/druid/client/indexing/IndexingService.java b/server/src/main/java/io/druid/client/indexing/IndexingService.java new file mode 100644 index 00000000000..e79bab7fbee --- /dev/null +++ b/server/src/main/java/io/druid/client/indexing/IndexingService.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + */ +@BindingAnnotation +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface IndexingService +{ +} diff --git a/client/src/main/java/com/metamx/druid/client/indexing/IndexingServiceClient.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java similarity index 85% rename from client/src/main/java/com/metamx/druid/client/indexing/IndexingServiceClient.java rename to server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java index ea17b6b6838..628ed978c42 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/IndexingServiceClient.java +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; +import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.ISE; -import com.metamx.druid.client.DataSegment; import com.metamx.http.client.HttpClient; import com.metamx.http.client.response.InputStreamResponseHandler; -import org.apache.curator.x.discovery.ServiceInstance; -import org.apache.curator.x.discovery.ServiceProvider; +import io.druid.client.selector.DiscoverySelector; +import io.druid.client.selector.Server; +import io.druid.guice.annotations.Global; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import java.io.InputStream; @@ -41,12 +43,13 @@ public class IndexingServiceClient private final HttpClient client; private final ObjectMapper jsonMapper; - private final ServiceProvider serviceProvider; + private final DiscoverySelector serviceProvider; + @Inject public IndexingServiceClient( - HttpClient client, + @Global HttpClient client, ObjectMapper jsonMapper, - ServiceProvider serviceProvider + @IndexingService DiscoverySelector serviceProvider ) { this.client = client; @@ -103,12 +106,12 @@ public class IndexingServiceClient private String baseUrl() { try { - final ServiceInstance instance = serviceProvider.getInstance(); + final Server instance = serviceProvider.pick(); if (instance == null) { throw new ISE("Cannot find instance of indexingService"); } - return String.format("http://%s:%s/druid/indexer/v1", instance.getAddress(), instance.getPort()); + return String.format("http://%s:%s/druid/indexer/v1", instance.getHost(), instance.getPort()); } catch (Exception e) { throw Throwables.propagate(e); diff --git a/server/src/main/java/io/druid/client/indexing/IndexingServiceSelector.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelector.java new file mode 100644 index 00000000000..7c5eaaf8d72 --- /dev/null +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelector.java @@ -0,0 +1,94 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; + +import com.google.inject.Inject; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.logger.Logger; +import io.druid.client.selector.DiscoverySelector; +import io.druid.client.selector.Server; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceProvider; + +import javax.annotation.Nullable; +import java.io.IOException; + +/** +*/ +public class IndexingServiceSelector implements DiscoverySelector +{ + private static final Logger log = new Logger(IndexingServiceSelector.class); + + private final ServiceProvider serviceProvider; + + @Inject + public IndexingServiceSelector( + @Nullable @IndexingService ServiceProvider serviceProvider + ) { + this.serviceProvider = serviceProvider; + } + + @Override + public Server pick() + { + final ServiceInstance instance; + try { + instance = serviceProvider.getInstance(); + } + catch (Exception e) { + log.info(e, ""); + return null; + } + + return new Server() + { + @Override + public String getHost() + { + return instance.getAddress(); + } + + @Override + public int getPort() + { + return instance.getPort(); + } + + @Override + public String getScheme() + { + return "http"; + } + }; + } + + @LifecycleStart + public void start() throws Exception + { + serviceProvider.start(); + } + + @LifecycleStop + public void stop() throws IOException + { + serviceProvider.close(); + } +} diff --git a/server/src/main/java/io/druid/client/indexing/IndexingServiceSelectorConfig.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelectorConfig.java new file mode 100644 index 00000000000..976225f6665 --- /dev/null +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelectorConfig.java @@ -0,0 +1,35 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class IndexingServiceSelectorConfig +{ + @JsonProperty + private String serviceName = null; + + public String getServiceName() + { + return serviceName; + } +} diff --git a/server/src/main/java/io/druid/client/selector/DiscoverySelector.java b/server/src/main/java/io/druid/client/selector/DiscoverySelector.java new file mode 100644 index 00000000000..d878d7ed921 --- /dev/null +++ b/server/src/main/java/io/druid/client/selector/DiscoverySelector.java @@ -0,0 +1,27 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.selector; + +/** + */ +public interface DiscoverySelector +{ + public T pick(); +} diff --git a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java b/server/src/main/java/io/druid/client/selector/QueryableDruidServer.java similarity index 86% rename from client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java rename to server/src/main/java/io/druid/client/selector/QueryableDruidServer.java index 2528facb8d8..8f4f7f48e82 100644 --- a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java +++ b/server/src/main/java/io/druid/client/selector/QueryableDruidServer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.selector; +package io.druid.client.selector; -import com.metamx.druid.client.DirectDruidClient; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DirectDruidClient; +import io.druid.client.DruidServer; /** */ diff --git a/server/src/main/java/io/druid/client/selector/Server.java b/server/src/main/java/io/druid/client/selector/Server.java new file mode 100644 index 00000000000..664d621136e --- /dev/null +++ b/server/src/main/java/io/druid/client/selector/Server.java @@ -0,0 +1,29 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.selector; + +/** + */ +public interface Server +{ + public String getScheme(); + public String getHost(); + public int getPort(); +} diff --git a/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java b/server/src/main/java/io/druid/client/selector/ServerSelector.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java rename to server/src/main/java/io/druid/client/selector/ServerSelector.java index 388761aaaa2..d10c561c72d 100644 --- a/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/io/druid/client/selector/ServerSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.selector; +package io.druid.client.selector; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.timeline.DataSegment; import java.util.Collections; import java.util.Comparator; @@ -30,7 +29,7 @@ import java.util.Set; /** */ -public class ServerSelector +public class ServerSelector implements DiscoverySelector { private static final Comparator comparator = new Comparator() { diff --git a/client/src/main/java/com/metamx/druid/initialization/CuratorConfig.java b/server/src/main/java/io/druid/curator/CuratorConfig.java similarity index 93% rename from client/src/main/java/com/metamx/druid/initialization/CuratorConfig.java rename to server/src/main/java/io/druid/curator/CuratorConfig.java index be46aea41f6..7ac0247b342 100644 --- a/client/src/main/java/com/metamx/druid/initialization/CuratorConfig.java +++ b/server/src/main/java/io/druid/curator/CuratorConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.initialization; +package io.druid.curator; import org.skife.config.Config; import org.skife.config.Default; diff --git a/server/src/main/java/io/druid/curator/CuratorModule.java b/server/src/main/java/io/druid/curator/CuratorModule.java new file mode 100644 index 00000000000..94fba357dd2 --- /dev/null +++ b/server/src/main/java/io/druid/curator/CuratorModule.java @@ -0,0 +1,79 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.common.logger.Logger; +import io.druid.guice.ConfigProvider; +import io.druid.guice.LazySingleton; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.BoundedExponentialBackoffRetry; + +import java.io.IOException; + +/** + */ +public class CuratorModule implements Module +{ + private static final Logger log = new Logger(CuratorModule.class); + + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, CuratorConfig.class); + } + + @Provides @LazySingleton + public CuratorFramework makeCurator(CuratorConfig config, Lifecycle lifecycle) throws IOException + { + final CuratorFramework framework = + CuratorFrameworkFactory.builder() + .connectString(config.getZkHosts()) + .sessionTimeoutMs(config.getZkSessionTimeoutMs()) + .retryPolicy(new BoundedExponentialBackoffRetry(1000, 45000, 30)) + .compressionProvider(new PotentiallyGzippedCompressionProvider(config.enableCompression())) + .build(); + + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + log.info("Starting Curator"); + framework.start(); + } + + @Override + public void stop() + { + log.info("Stopping Curator"); + framework.close(); + } + } + ); + + return framework; + } +} diff --git a/client/src/main/java/com/metamx/druid/curator/PotentiallyGzippedCompressionProvider.java b/server/src/main/java/io/druid/curator/PotentiallyGzippedCompressionProvider.java similarity index 93% rename from client/src/main/java/com/metamx/druid/curator/PotentiallyGzippedCompressionProvider.java rename to server/src/main/java/io/druid/curator/PotentiallyGzippedCompressionProvider.java index abddd9574fc..c2fda7139be 100644 --- a/client/src/main/java/com/metamx/druid/curator/PotentiallyGzippedCompressionProvider.java +++ b/server/src/main/java/io/druid/curator/PotentiallyGzippedCompressionProvider.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator; +package io.druid.curator; import org.apache.curator.framework.api.CompressionProvider; import org.apache.curator.framework.imps.GzipCompressionProvider; import java.io.IOException; -import java.util.zip.ZipException; /** */ diff --git a/client/src/main/java/com/metamx/druid/curator/ShutdownNowIgnoringExecutorService.java b/server/src/main/java/io/druid/curator/ShutdownNowIgnoringExecutorService.java similarity index 74% rename from client/src/main/java/com/metamx/druid/curator/ShutdownNowIgnoringExecutorService.java rename to server/src/main/java/io/druid/curator/ShutdownNowIgnoringExecutorService.java index b4a8128f6d9..7ac77fcc5ea 100644 --- a/client/src/main/java/com/metamx/druid/curator/ShutdownNowIgnoringExecutorService.java +++ b/server/src/main/java/io/druid/curator/ShutdownNowIgnoringExecutorService.java @@ -1,4 +1,23 @@ -package com.metamx.druid.curator; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator; import com.google.common.collect.ImmutableList; diff --git a/client/src/main/java/com/metamx/druid/curator/announcement/Announcer.java b/server/src/main/java/io/druid/curator/announcement/Announcer.java similarity index 97% rename from client/src/main/java/com/metamx/druid/curator/announcement/Announcer.java rename to server/src/main/java/io/druid/curator/announcement/Announcer.java index abb96b76f68..8b017c41a24 100644 --- a/client/src/main/java/com/metamx/druid/curator/announcement/Announcer.java +++ b/server/src/main/java/io/druid/curator/announcement/Announcer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.announcement; +package io.druid.curator.announcement; import com.google.common.base.Throwables; import com.google.common.collect.Lists; @@ -30,9 +30,9 @@ import com.metamx.common.Pair; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.ShutdownNowIgnoringExecutorService; -import com.metamx.druid.curator.cache.PathChildrenCacheFactory; -import com.metamx.druid.curator.cache.SimplePathChildrenCacheFactory; +import io.druid.curator.ShutdownNowIgnoringExecutorService; +import io.druid.curator.cache.PathChildrenCacheFactory; +import io.druid.curator.cache.SimplePathChildrenCacheFactory; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/client/src/main/java/com/metamx/druid/curator/cache/PathChildrenCacheFactory.java b/server/src/main/java/io/druid/curator/cache/PathChildrenCacheFactory.java similarity index 92% rename from client/src/main/java/com/metamx/druid/curator/cache/PathChildrenCacheFactory.java rename to server/src/main/java/io/druid/curator/cache/PathChildrenCacheFactory.java index 37335d8aa88..d722ba0ba5d 100644 --- a/client/src/main/java/com/metamx/druid/curator/cache/PathChildrenCacheFactory.java +++ b/server/src/main/java/io/druid/curator/cache/PathChildrenCacheFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.cache; +package io.druid.curator.cache; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/client/src/main/java/com/metamx/druid/curator/cache/SimplePathChildrenCacheFactory.java b/server/src/main/java/io/druid/curator/cache/SimplePathChildrenCacheFactory.java similarity index 96% rename from client/src/main/java/com/metamx/druid/curator/cache/SimplePathChildrenCacheFactory.java rename to server/src/main/java/io/druid/curator/cache/SimplePathChildrenCacheFactory.java index fb7a3044ee4..ef0d3c50b9b 100644 --- a/client/src/main/java/com/metamx/druid/curator/cache/SimplePathChildrenCacheFactory.java +++ b/server/src/main/java/io/druid/curator/cache/SimplePathChildrenCacheFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.cache; +package io.druid.curator.cache; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java new file mode 100644 index 00000000000..633b987a88a --- /dev/null +++ b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java @@ -0,0 +1,122 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.discovery; + +import com.google.common.base.Throwables; +import com.google.common.collect.Maps; +import com.google.inject.Inject; +import com.metamx.emitter.EmittingLogger; +import io.druid.server.DruidNode; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceInstance; + +import java.util.Map; + +/** + * Uses the Curator Service Discovery recipe to announce services. + */ +public class CuratorServiceAnnouncer implements ServiceAnnouncer +{ + private static final EmittingLogger log = new EmittingLogger(CuratorServiceAnnouncer.class); + + private final ServiceDiscovery discovery; + private final Map> instanceMap = Maps.newHashMap(); + private final Object monitor = new Object(); + + @Inject + public CuratorServiceAnnouncer( + ServiceDiscovery discovery + ) + { + this.discovery = discovery; + } + + @Override + public void announce(DruidNode service) + { + final String serviceName = getServiceName(service); + + final ServiceInstance instance; + synchronized (monitor) { + if (instanceMap.containsKey(serviceName)) { + log.warn("Ignoring request to announce service[%s]", service); + return; + } else { + try { + instance = ServiceInstance.builder() + .name(serviceName) + .address(service.getHost()) + .port(service.getPort()) + .build(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + instanceMap.put(serviceName, instance); + } + } + + try { + log.info("Announcing service[%s]", service); + discovery.registerService(instance); + } + catch (Exception e) { + log.warn("Failed to announce service[%s]", service); + synchronized (monitor) { + instanceMap.remove(serviceName); + } + } + } + + @Override + public void unannounce(DruidNode service) + { + final String serviceName = getServiceName(service); + final ServiceInstance instance; + + synchronized (monitor) { + instance = instanceMap.get(serviceName); + if (instance == null) { + log.warn("Ignoring request to unannounce service[%s]", service); + return; + } + } + + log.info("Unannouncing service[%s]", service); + try { + discovery.unregisterService(instance); + } + catch (Exception e) { + log.makeAlert(e, "Failed to unannounce service[%s], zombie znode perhaps in existence.", serviceName) + .addData("service", service) + .emit(); + } + finally { + synchronized (monitor) { + instanceMap.remove(serviceName); + } + } + } + + private String getServiceName(DruidNode service) { + return service.getServiceName().replaceAll("/", ":"); + } +} diff --git a/server/src/main/java/io/druid/curator/discovery/DiscoveryModule.java b/server/src/main/java/io/druid/curator/discovery/DiscoveryModule.java new file mode 100644 index 00000000000..20d52e8e77f --- /dev/null +++ b/server/src/main/java/io/druid/curator/discovery/DiscoveryModule.java @@ -0,0 +1,233 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.discovery; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.metamx.common.lifecycle.Lifecycle; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.server.DruidNode; +import io.druid.server.initialization.CuratorDiscoveryConfig; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceDiscoveryBuilder; + +import javax.annotation.Nullable; +import java.lang.annotation.Annotation; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * The DiscoveryModule allows for the registration of Keys of DruidNode objects, which it intends to be + * automatically announced at the end of the lifecycle start. + * + * In order for this to work a ServiceAnnouncer instance *must* be injected and instantiated first. + * This can often be achieved by registering ServiceAnnouncer.class with the LifecycleModule. + */ +public class DiscoveryModule implements Module +{ + private static final String NAME = "DiscoveryModule:internal"; + + public final List>> nodesToAnnounce = new CopyOnWriteArrayList>>(); + public boolean configured = false; + + /** + * Requests that the un-annotated DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @return this, for chaining. + */ + public DiscoveryModule registerDefault() + { + return registerKey(Key.get(new TypeLiteral>(){})); + } + + /** + * Requests that the annotated DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class, annotation)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @param annotation The annotation instance to use in finding the DruidNode instance, usually a Named annotation + * @return this, for chaining. + */ + public DiscoveryModule register(Annotation annotation) + { + return registerKey(Key.get(new TypeLiteral>(){}, annotation)); + } + + /** + * Requests that the annotated DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class, annotation)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @param annotation The annotation class to use in finding the DruidNode instance + * @return this, for chaining + */ + public DiscoveryModule register(Class annotation) + { + return registerKey(Key.get(new TypeLiteral>(){}, annotation)); + } + + /** + * Requests that the keyed DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class, annotation)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @param key The key to use in finding the DruidNode instance + * @return this, for chaining + */ + public DiscoveryModule registerKey(Key> key) + { + synchronized (nodesToAnnounce) { + Preconditions.checkState(!configured, "Cannot register key[%s] after configuration.", key); + } + nodesToAnnounce.add(key); + return this; + } + + @Override + public void configure(Binder binder) + { + synchronized (nodesToAnnounce) { + configured = true; + JsonConfigProvider.bind(binder, "druid.discovery.curator", CuratorDiscoveryConfig.class); + + binder.bind(CuratorServiceAnnouncer.class).in(LazySingleton.class); + + // We bind this eagerly so that it gets instantiated and registers stuff with Lifecycle as a side-effect + binder.bind(ServiceAnnouncer.class) + .to(Key.get(CuratorServiceAnnouncer.class, Names.named(NAME))) + .asEagerSingleton(); + } + } + + @Provides @LazySingleton @Named(NAME) + public CuratorServiceAnnouncer getServiceAnnouncer( + final CuratorServiceAnnouncer announcer, + final Injector injector, + final Lifecycle lifecycle + ) + { + lifecycle.addHandler( + new Lifecycle.Handler() + { + private volatile List> nodes = null; + + @Override + public void start() throws Exception + { + if (nodes == null) { + nodes = Lists.transform( + nodesToAnnounce, + new Function>, Supplier>() + { + @Nullable + @Override + public Supplier apply( + @Nullable Key> input + ) + { + return injector.getInstance(input); + } + } + ); + } + + for (Supplier node : nodes) { + announcer.announce(node.get()); + } + } + + @Override + public void stop() + { + if (nodes != null) { + for (Supplier node : nodes) { + announcer.unannounce(node.get()); + } + } + } + }, + Lifecycle.Stage.LAST + ); + + return announcer; + } + + @Provides @LazySingleton + public ServiceDiscovery getServiceDiscovery( + CuratorFramework curator, + Supplier config, + Lifecycle lifecycle + ) throws Exception + { + final ServiceDiscovery serviceDiscovery = + ServiceDiscoveryBuilder.builder(Void.class) + .basePath(config.get().getPath()) + .client(curator) + .build(); + + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + serviceDiscovery.start(); + } + + @Override + public void stop() + { + try { + serviceDiscovery.close(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + }, + Lifecycle.Stage.LAST + ); + + return serviceDiscovery; + } +} diff --git a/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java new file mode 100644 index 00000000000..672b900d5bf --- /dev/null +++ b/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java @@ -0,0 +1,40 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.discovery; + +import io.druid.server.DruidNode; + +/** + * Does nothing. + */ +public class NoopServiceAnnouncer implements ServiceAnnouncer +{ + @Override + public void announce(DruidNode node) + { + + } + + @Override + public void unannounce(DruidNode node) + { + + } +} diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManagerConfig.java b/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java similarity index 65% rename from server/src/main/java/com/metamx/druid/db/DatabaseSegmentManagerConfig.java rename to server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java index 25e90cd16bd..46da8f5b0ff 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManagerConfig.java +++ b/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.curator.discovery; -import org.joda.time.Duration; -import org.skife.config.Config; -import org.skife.config.Default; +import io.druid.server.DruidNode; /** + * Announces our ability to serve a particular function. Multiple users may announce the same service, in which + * case they are treated as interchangeable instances of that service. */ -public abstract class DatabaseSegmentManagerConfig +public interface ServiceAnnouncer { - @Config("druid.database.segmentTable") - public abstract String getSegmentTable(); + public void announce(DruidNode node); - @Config("druid.database.poll.duration") - @Default("PT1M") - public abstract Duration getPollDuration(); + public void unannounce(DruidNode node); } diff --git a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManager.java b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java similarity index 97% rename from client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManager.java rename to server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java index 7c35ad12618..dcff14024e3 100644 --- a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManager.java +++ b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.inventory; +package io.druid.curator.inventory; import com.google.common.base.Function; import com.google.common.collect.Iterables; @@ -26,9 +26,9 @@ import com.google.common.collect.MapMaker; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.ShutdownNowIgnoringExecutorService; -import com.metamx.druid.curator.cache.PathChildrenCacheFactory; -import com.metamx.druid.curator.cache.SimplePathChildrenCacheFactory; +import io.druid.curator.ShutdownNowIgnoringExecutorService; +import io.druid.curator.cache.PathChildrenCacheFactory; +import io.druid.curator.cache.SimplePathChildrenCacheFactory; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerStrategy.java b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManagerStrategy.java similarity index 94% rename from client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerStrategy.java rename to server/src/main/java/io/druid/curator/inventory/CuratorInventoryManagerStrategy.java index 8cab619e16f..785e35534df 100644 --- a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerStrategy.java +++ b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManagerStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.inventory; +package io.druid.curator.inventory; /** */ diff --git a/client/src/main/java/com/metamx/druid/curator/inventory/InventoryManagerConfig.java b/server/src/main/java/io/druid/curator/inventory/InventoryManagerConfig.java similarity index 95% rename from client/src/main/java/com/metamx/druid/curator/inventory/InventoryManagerConfig.java rename to server/src/main/java/io/druid/curator/inventory/InventoryManagerConfig.java index 418c402ff83..78158b713be 100644 --- a/client/src/main/java/com/metamx/druid/curator/inventory/InventoryManagerConfig.java +++ b/server/src/main/java/io/druid/curator/inventory/InventoryManagerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.inventory; +package io.druid.curator.inventory; /** */ diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManager.java b/server/src/main/java/io/druid/db/DatabaseRuleManager.java similarity index 81% rename from server/src/main/java/com/metamx/druid/db/DatabaseRuleManager.java rename to server/src/main/java/io/druid/db/DatabaseRuleManager.java index 44ea802082c..c6968362caa 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManager.java +++ b/server/src/main/java/io/druid/db/DatabaseRuleManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,28 +17,32 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.db; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.concurrent.ScheduledExecutors; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.master.rules.PeriodLoadRule; -import com.metamx.druid.master.rules.Rule; - +import io.druid.concurrent.Execs; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.annotations.Json; +import io.druid.server.master.rules.PeriodLoadRule; +import io.druid.server.master.rules.Rule; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Period; -import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.FoldController; import org.skife.jdbi.v2.Folder3; import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.tweak.HandleCallback; @@ -52,12 +56,13 @@ import java.util.concurrent.atomic.AtomicReference; /** */ +@ManageLifecycle public class DatabaseRuleManager { public static void createDefaultRule( - final DBI dbi, + final IDBI dbi, final String ruleTable, - final String defaultDatasource, + final String defaultTier, final ObjectMapper jsonMapper ) { @@ -72,7 +77,7 @@ public class DatabaseRuleManager String.format( "SELECT id from %s where datasource='%s';", ruleTable, - defaultDatasource + defaultTier ) ); @@ -94,8 +99,8 @@ public class DatabaseRuleManager ruleTable ) ) - .bind("id", String.format("%s_%s", defaultDatasource, version)) - .bind("dataSource", defaultDatasource) + .bind("id", String.format("%s_%s", defaultTier, version)) + .bind("dataSource", defaultTier) .bind("version", version) .bind("payload", jsonMapper.writeValueAsString(defaultRules)) .execute(); @@ -113,25 +118,28 @@ public class DatabaseRuleManager private static final Logger log = new Logger(DatabaseRuleManager.class); private final ObjectMapper jsonMapper; - private final ScheduledExecutorService exec; - private final DatabaseRuleManagerConfig config; - private final DBI dbi; + private final Supplier config; + private final Supplier dbTables; + private final IDBI dbi; private final AtomicReference>> rules; + private volatile ScheduledExecutorService exec; + private final Object lock = new Object(); private volatile boolean started = false; + @Inject public DatabaseRuleManager( - ObjectMapper jsonMapper, - ScheduledExecutorService exec, - DatabaseRuleManagerConfig config, - DBI dbi + @Json ObjectMapper jsonMapper, + Supplier config, + Supplier dbTables, + IDBI dbi ) { this.jsonMapper = jsonMapper; - this.exec = exec; this.config = config; + this.dbTables = dbTables; this.dbi = dbi; this.rules = new AtomicReference>>( @@ -147,10 +155,13 @@ public class DatabaseRuleManager return; } + this.exec = Execs.scheduledSingleThreaded("DatabaseRuleManager-Exec--%d"); + + createDefaultRule(dbi, getRulesTable(), config.get().getDefaultTier(), jsonMapper); ScheduledExecutors.scheduleWithFixedDelay( exec, new Duration(0), - config.getRulesPollDuration(), + config.get().getPollDuration().toStandardDuration(), new Runnable() { @Override @@ -176,6 +187,8 @@ public class DatabaseRuleManager rules.set(new ConcurrentHashMap>()); started = false; + exec.shutdownNow(); + exec = null; } } @@ -192,8 +205,11 @@ public class DatabaseRuleManager return handle.createQuery( // Return latest version rule by dataSource String.format( - "SELECT %1$s.dataSource, %1$s.payload FROM %1$s INNER JOIN(SELECT dataSource, max(version) as version FROM %1$s GROUP BY dataSource) ds ON %1$s.datasource = ds.datasource and %1$s.version = ds.version", - config.getRuleTable() + "SELECT r.dataSource, r.payload " + + "FROM %1$s r " + + "INNER JOIN(SELECT dataSource, max(version) as version, payload FROM %1$s GROUP BY dataSource) ds " + + "ON r.datasource = ds.datasource and r.version = ds.version", + getRulesTable() ) ).fold( Maps.>newHashMap(), @@ -255,8 +271,8 @@ public class DatabaseRuleManager if (theRules.get(dataSource) != null) { retVal.addAll(theRules.get(dataSource)); } - if (theRules.get(config.getDefaultDatasource()) != null) { - retVal.addAll(theRules.get(config.getDefaultDatasource())); + if (theRules.get(config.get().getDefaultTier()) != null) { + retVal.addAll(theRules.get(config.get().getDefaultTier())); } return retVal; } @@ -275,7 +291,7 @@ public class DatabaseRuleManager handle.createStatement( String.format( "INSERT INTO %s (id, dataSource, version, payload) VALUES (:id, :dataSource, :version, :payload)", - config.getRuleTable() + getRulesTable() ) ) .bind("id", String.format("%s_%s", dataSource, version)) @@ -303,4 +319,6 @@ public class DatabaseRuleManager return true; } + + private String getRulesTable() {return dbTables.get().getRulesTable();} } diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java b/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java new file mode 100644 index 00000000000..8b770a57955 --- /dev/null +++ b/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java @@ -0,0 +1,44 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.db; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +/** + */ +public class DatabaseRuleManagerConfig +{ + @JsonProperty + private String defaultTier = "_default"; + + @JsonProperty + private Period pollDuration = new Period("PT1M"); + + public String getDefaultTier() + { + return defaultTier; + } + + public Period getPollDuration() + { + return pollDuration; + } +} diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java b/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java new file mode 100644 index 00000000000..dfc25bea283 --- /dev/null +++ b/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java @@ -0,0 +1,85 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.db; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.metamx.common.lifecycle.Lifecycle; + +/** + */ +public class DatabaseRuleManagerProvider implements Provider +{ + private final ObjectMapper jsonMapper; + private final Supplier config; + private final Supplier dbTables; + private final DbConnector dbConnector; + private final Lifecycle lifecycle; + + @Inject + public DatabaseRuleManagerProvider( + ObjectMapper jsonMapper, + Supplier config, + Supplier dbTables, + DbConnector dbConnector, + Lifecycle lifecycle + ) + { + this.jsonMapper = jsonMapper; + this.config = config; + this.dbTables = dbTables; + this.dbConnector = dbConnector; + this.lifecycle = lifecycle; + } + + @Override + public DatabaseRuleManager get() + { + try { + lifecycle.addMaybeStartHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + dbConnector.createRulesTable(); + DatabaseRuleManager.createDefaultRule( + dbConnector.getDBI(), dbTables.get().getRulesTable(), config.get().getDefaultTier(), jsonMapper + ); + } + + @Override + public void stop() + { + + } + } + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + return new DatabaseRuleManager(jsonMapper, config, dbTables, dbConnector.getDBI()); + } +} diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManager.java b/server/src/main/java/io/druid/db/DatabaseSegmentManager.java similarity index 88% rename from server/src/main/java/com/metamx/druid/db/DatabaseSegmentManager.java rename to server/src/main/java/io/druid/db/DatabaseSegmentManager.java index 2096e0bd70a..203d082b806 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManager.java +++ b/server/src/main/java/io/druid/db/DatabaseSegmentManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,30 +17,34 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.db; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; +import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.concurrent.ScheduledExecutors; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.partition.PartitionChunk; +import io.druid.client.DruidDataSource; +import io.druid.concurrent.Execs; +import io.druid.guice.ManageLifecycle; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Interval; import org.skife.jdbi.v2.Batch; -import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.FoldController; import org.skife.jdbi.v2.Folder3; import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.tweak.HandleCallback; @@ -55,6 +59,7 @@ import java.util.concurrent.atomic.AtomicReference; /** */ +@ManageLifecycle public class DatabaseSegmentManager { private static final Logger log = new Logger(DatabaseSegmentManager.class); @@ -62,23 +67,26 @@ public class DatabaseSegmentManager private final Object lock = new Object(); private final ObjectMapper jsonMapper; - private final ScheduledExecutorService exec; - private final DatabaseSegmentManagerConfig config; + private final Supplier config; + private final Supplier dbTables; private final AtomicReference> dataSources; - private final DBI dbi; + private final IDBI dbi; + + private volatile ScheduledExecutorService exec; private volatile boolean started = false; + @Inject public DatabaseSegmentManager( ObjectMapper jsonMapper, - ScheduledExecutorService exec, - DatabaseSegmentManagerConfig config, - DBI dbi + Supplier config, + Supplier dbTables, + IDBI dbi ) { this.jsonMapper = jsonMapper; - this.exec = exec; this.config = config; + this.dbTables = dbTables; this.dataSources = new AtomicReference>( new ConcurrentHashMap() ); @@ -93,10 +101,13 @@ public class DatabaseSegmentManager return; } + this.exec = Execs.scheduledSingleThreaded("DatabaseSegmentManager-Exec--%d"); + + final Duration delay = config.get().getPollDuration().toStandardDuration(); ScheduledExecutors.scheduleWithFixedDelay( exec, - new Duration(0), - config.getPollDuration(), + delay, + delay, new Runnable() { @Override @@ -121,6 +132,8 @@ public class DatabaseSegmentManager started = false; dataSources.set(new ConcurrentHashMap()); + exec.shutdownNow(); + exec = null; } } @@ -134,7 +147,7 @@ public class DatabaseSegmentManager public VersionedIntervalTimeline withHandle(Handle handle) throws Exception { return handle.createQuery( - String.format("SELECT payload FROM %s WHERE dataSource = :dataSource", config.getSegmentTable()) + String.format("SELECT payload FROM %s WHERE dataSource = :dataSource", getSegmentsTable()) ) .bind("dataSource", ds) .fold( @@ -200,8 +213,8 @@ public class DatabaseSegmentManager for (DataSegment segment : segments) { batch.add( String.format( - "UPDATE %s SET used=true WHERE id = '%s'", - config.getSegmentTable(), + "UPDATE %s SET used=1 WHERE id = '%s'", + getSegmentsTable(), segment.getIdentifier() ) ); @@ -231,7 +244,7 @@ public class DatabaseSegmentManager public Void withHandle(Handle handle) throws Exception { handle.createStatement( - String.format("UPDATE %s SET used=true WHERE id = :id", config.getSegmentTable()) + String.format("UPDATE %s SET used=1 WHERE id = :id", getSegmentsTable()) ) .bind("id", segmentId) .execute(); @@ -265,7 +278,7 @@ public class DatabaseSegmentManager public Void withHandle(Handle handle) throws Exception { handle.createStatement( - String.format("UPDATE %s SET used=false WHERE dataSource = :dataSource", config.getSegmentTable()) + String.format("UPDATE %s SET used=0 WHERE dataSource = :dataSource", getSegmentsTable()) ) .bind("dataSource", ds) .execute(); @@ -295,7 +308,7 @@ public class DatabaseSegmentManager public Void withHandle(Handle handle) throws Exception { handle.createStatement( - String.format("UPDATE %s SET used=false WHERE id = :segmentID", config.getSegmentTable()) + String.format("UPDATE %s SET used=0 WHERE id = :segmentID", getSegmentsTable()) ).bind("segmentID", segmentID) .execute(); @@ -351,7 +364,7 @@ public class DatabaseSegmentManager public List withHandle(Handle handle) throws Exception { return handle.createQuery( - String.format("SELECT DISTINCT(datasource) FROM %s", config.getSegmentTable()) + String.format("SELECT DISTINCT(datasource) FROM %s", getSegmentsTable()) ) .fold( Lists.newArrayList(), @@ -395,7 +408,7 @@ public class DatabaseSegmentManager public List> withHandle(Handle handle) throws Exception { return handle.createQuery( - String.format("SELECT payload FROM %s WHERE used=true", config.getSegmentTable()) + String.format("SELECT payload FROM %s WHERE used=1", getSegmentsTable()) ).list(); } } @@ -448,4 +461,8 @@ public class DatabaseSegmentManager log.error(e, "Problem polling DB."); } } + + private String getSegmentsTable() { + return dbTables.get().getSegmentsTable(); + } } \ No newline at end of file diff --git a/server/src/main/java/com/metamx/druid/index/v1/Searchable.java b/server/src/main/java/io/druid/db/DatabaseSegmentManagerConfig.java similarity index 70% rename from server/src/main/java/com/metamx/druid/index/v1/Searchable.java rename to server/src/main/java/io/druid/db/DatabaseSegmentManagerConfig.java index a4f6ad70c78..a87d2db59e5 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/Searchable.java +++ b/server/src/main/java/io/druid/db/DatabaseSegmentManagerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.db; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; /** */ -public interface Searchable +public class DatabaseSegmentManagerConfig { - public Iterable searchDimensions(SearchQuery query, Filter filter); + @JsonProperty + private Period pollDuration = new Period("PT1M"); + + public Period getPollDuration() + { + return pollDuration; + } } diff --git a/server/src/main/java/io/druid/db/DatabaseSegmentManagerProvider.java b/server/src/main/java/io/druid/db/DatabaseSegmentManagerProvider.java new file mode 100644 index 00000000000..c7cda31f8d5 --- /dev/null +++ b/server/src/main/java/io/druid/db/DatabaseSegmentManagerProvider.java @@ -0,0 +1,81 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.db; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.metamx.common.lifecycle.Lifecycle; + +/** + */ +public class DatabaseSegmentManagerProvider implements Provider +{ + private final ObjectMapper jsonMapper; + private final Supplier config; + private final Supplier dbTables; + private final DbConnector dbConnector; + private final Lifecycle lifecycle; + + @Inject + public DatabaseSegmentManagerProvider( + ObjectMapper jsonMapper, + Supplier config, + Supplier dbTables, + DbConnector dbConnector, + Lifecycle lifecycle + ) + { + this.jsonMapper = jsonMapper; + this.config = config; + this.dbTables = dbTables; + this.dbConnector = dbConnector; + this.lifecycle = lifecycle; + } + + @Override + public DatabaseSegmentManager get() + { + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + dbConnector.createSegmentTable(); + } + + @Override + public void stop() + { + + } + } + ); + + return new DatabaseSegmentManager( + jsonMapper, + config, + dbTables, + dbConnector.getDBI() + ); + } +} diff --git a/server/src/main/java/io/druid/guice/AWSModule.java b/server/src/main/java/io/druid/guice/AWSModule.java new file mode 100644 index 00000000000..9a9b6d3b0f0 --- /dev/null +++ b/server/src/main/java/io/druid/guice/AWSModule.java @@ -0,0 +1,76 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2Client; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import io.druid.segment.loading.AWSCredentialsConfig; +import org.jets3t.service.S3ServiceException; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; + +/** + */ +public class AWSModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class); + } + + @Provides + @LazySingleton + public AWSCredentials getAWSCredentials(AWSCredentialsConfig config) + { + return new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); + } + + @Provides + @LazySingleton + public org.jets3t.service.security.AWSCredentials getJets3tAWSCredentials(AWSCredentialsConfig config) + { + return new org.jets3t.service.security.AWSCredentials(config.getAccessKey(), config.getSecretKey()); + } + + @Provides + @LazySingleton + public RestS3Service getRestS3Service(org.jets3t.service.security.AWSCredentials credentials) + { + try { + return new RestS3Service(credentials); + } + catch (S3ServiceException e) { + throw new ProvisionException("Unable to create a RestS3Service", e); + } + } + + @Provides + @LazySingleton + public AmazonEC2 getEc2Client(AWSCredentials credentials) + { + return new AmazonEC2Client(credentials); + } +} diff --git a/server/src/main/java/io/druid/guice/AnnouncerModule.java b/server/src/main/java/io/druid/guice/AnnouncerModule.java new file mode 100644 index 00000000000..379cc29392e --- /dev/null +++ b/server/src/main/java/io/druid/guice/AnnouncerModule.java @@ -0,0 +1,54 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import io.druid.concurrent.Execs; +import io.druid.curator.announcement.Announcer; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DataSegmentAnnouncer; +import io.druid.server.coordination.DataSegmentAnnouncerProvider; +import io.druid.server.coordination.SingleDataSegmentAnnouncer; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import org.apache.curator.framework.CuratorFramework; + +/** + */ +public class AnnouncerModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.announcer", BatchDataSegmentAnnouncerConfig.class); + JsonConfigProvider.bind(binder, "druid.announcer", DataSegmentAnnouncerProvider.class); + binder.bind(DataSegmentAnnouncer.class).toProvider(DataSegmentAnnouncerProvider.class); + binder.bind(BatchDataSegmentAnnouncer.class).in(ManageLifecycleLast.class); + binder.bind(SingleDataSegmentAnnouncer.class).in(ManageLifecycleLast.class); + } + + @Provides + @ManageLifecycle + public Announcer getAnnouncer(CuratorFramework curator) + { + return new Announcer(curator, Execs.singleThreaded("Announcer-%s")); + } +} diff --git a/server/src/main/java/io/druid/guice/BrokerModule.java b/server/src/main/java/io/druid/guice/BrokerModule.java new file mode 100644 index 00000000000..8ebc51f9832 --- /dev/null +++ b/server/src/main/java/io/druid/guice/BrokerModule.java @@ -0,0 +1,75 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; +import io.druid.client.BrokerServerView; +import io.druid.client.CachingClusteredClient; +import io.druid.client.TimelineServerView; +import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheProvider; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidPool; +import io.druid.guice.annotations.Global; +import io.druid.query.MapQueryToolChestWarehouse; +import io.druid.query.QueryToolChestWarehouse; + +import java.nio.ByteBuffer; + +/** + */ +public class BrokerModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class); + + binder.bind(CachingClusteredClient.class).in(LazySingleton.class); + binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); + + binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); + + // This is a workaround and needs to be made better in the near future. + binder.bind( + new TypeLiteral>() + { + } + ).annotatedWith(Global.class).toInstance(new NoopStupidPool(null)); + } + + private static class NoopStupidPool extends StupidPool + { + public NoopStupidPool(Supplier generator) + { + super(generator); + } + + @Override + public ResourceHolder take() + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/server/src/main/java/io/druid/guice/CassandraDataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/CassandraDataSegmentPusherProvider.java new file mode 100644 index 00000000000..7ce7e2219fa --- /dev/null +++ b/server/src/main/java/io/druid/guice/CassandraDataSegmentPusherProvider.java @@ -0,0 +1,47 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.cassandra.CassandraDataSegmentConfig; +import io.druid.segment.loading.cassandra.CassandraDataSegmentPusher; + +import javax.validation.constraints.NotNull; + +/** + */ +public class CassandraDataSegmentPusherProvider implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private CassandraDataSegmentConfig config = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new CassandraDataSegmentPusher(config, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/CoordinatorModule.java b/server/src/main/java/io/druid/guice/CoordinatorModule.java new file mode 100644 index 00000000000..df0e36a53c5 --- /dev/null +++ b/server/src/main/java/io/druid/guice/CoordinatorModule.java @@ -0,0 +1,80 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.concurrent.ScheduledExecutorFactory; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.db.DatabaseRuleManager; +import io.druid.db.DatabaseRuleManagerConfig; +import io.druid.db.DatabaseRuleManagerProvider; +import io.druid.db.DatabaseSegmentManager; +import io.druid.db.DatabaseSegmentManagerConfig; +import io.druid.db.DatabaseSegmentManagerProvider; +import io.druid.server.http.MasterRedirectInfo; +import io.druid.server.http.RedirectFilter; +import io.druid.server.http.RedirectInfo; +import io.druid.server.http.RedirectServlet; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterConfig; +import io.druid.server.master.LoadQueueTaskMaster; +import org.apache.curator.framework.CuratorFramework; + +/** + */ +public class CoordinatorModule implements Module +{ + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, DruidMasterConfig.class); + + JsonConfigProvider.bind(binder, "druid.manager.segment", DatabaseSegmentManagerConfig.class); + JsonConfigProvider.bind(binder, "druid.manager.rules", DatabaseRuleManagerConfig.class); + + binder.bind(RedirectServlet.class).in(LazySingleton.class); + binder.bind(RedirectFilter.class).in(LazySingleton.class); + + binder.bind(DatabaseSegmentManager.class) + .toProvider(DatabaseSegmentManagerProvider.class) + .in(ManageLifecycle.class); + + binder.bind(DatabaseRuleManager.class) + .toProvider(DatabaseRuleManagerProvider.class) + .in(ManageLifecycle.class); + + binder.bind(IndexingServiceClient.class).in(LazySingleton.class); + + binder.bind(RedirectInfo.class).to(MasterRedirectInfo.class).in(LazySingleton.class); + + binder.bind(DruidMaster.class); + } + + @Provides @LazySingleton + public LoadQueueTaskMaster getLoadQueueTaskMaster( + CuratorFramework curator, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, DruidMasterConfig config + ) + { + return new LoadQueueTaskMaster(curator, jsonMapper, factory.create(1, "Master-PeonExec--%d"), config); + } +} diff --git a/server/src/main/java/io/druid/guice/DataSegmentPullerModule.java b/server/src/main/java/io/druid/guice/DataSegmentPullerModule.java new file mode 100644 index 00000000000..3fd70ceecab --- /dev/null +++ b/server/src/main/java/io/druid/guice/DataSegmentPullerModule.java @@ -0,0 +1,81 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.segment.loading.HdfsDataSegmentPuller; +import io.druid.segment.loading.LocalDataSegmentPuller; +import io.druid.segment.loading.OmniSegmentLoader; +import io.druid.segment.loading.S3DataSegmentPuller; +import io.druid.segment.loading.SegmentLoader; +import io.druid.segment.loading.cassandra.CassandraDataSegmentConfig; +import io.druid.segment.loading.cassandra.CassandraDataSegmentPuller; +import org.apache.hadoop.conf.Configuration; + +/** + */ +public class DataSegmentPullerModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(SegmentLoader.class).to(OmniSegmentLoader.class).in(LazySingleton.class); + + bindDeepStorageLocal(binder); + bindDeepStorageS3(binder); + bindDeepStorageHdfs(binder); + bindDeepStorageCassandra(binder); + } + + private static void bindDeepStorageLocal(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("local") + .to(LocalDataSegmentPuller.class) + .in(LazySingleton.class); + } + + private static void bindDeepStorageS3(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("s3_zip") + .to(S3DataSegmentPuller.class) + .in(LazySingleton.class); + } + + private static void bindDeepStorageHdfs(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("hdfs") + .to(HdfsDataSegmentPuller.class) + .in(LazySingleton.class); + binder.bind(Configuration.class).toInstance(new Configuration()); + } + + private static void bindDeepStorageCassandra(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("c*") + .to(CassandraDataSegmentPuller.class) + .in(LazySingleton.class); + ConfigProvider.bind(binder, CassandraDataSegmentConfig.class); + } +} diff --git a/server/src/main/java/io/druid/guice/DataSegmentPusherModule.java b/server/src/main/java/io/druid/guice/DataSegmentPusherModule.java new file mode 100644 index 00000000000..cd65b35b7ee --- /dev/null +++ b/server/src/main/java/io/druid/guice/DataSegmentPusherModule.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.HdfsDataSegmentPusherConfig; +import io.druid.segment.loading.S3DataSegmentPusherConfig; +import io.druid.segment.loading.cassandra.CassandraDataSegmentConfig; +import org.apache.hadoop.conf.Configuration; + +/** + */ +public class DataSegmentPusherModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.pusher", DataSegmentPusherProvider.class); + + JsonConfigProvider.bind(binder, "druid.pusher.s3", S3DataSegmentPusherConfig.class); + binder.bind(Configuration.class).toInstance(new Configuration()); + + JsonConfigProvider.bind(binder, "druid.pusher.hdfs", HdfsDataSegmentPusherConfig.class); + + JsonConfigProvider.bind(binder, "druid.pusher.cassandra", CassandraDataSegmentConfig.class); + + binder.bind(DataSegmentPusher.class).toProvider(DataSegmentPusherProvider.class); + } +} diff --git a/server/src/main/java/io/druid/guice/DataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/DataSegmentPusherProvider.java new file mode 100644 index 00000000000..564f5002e94 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DataSegmentPusherProvider.java @@ -0,0 +1,37 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; +import io.druid.segment.loading.DataSegmentPusher; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LocalDataSegmentPusherProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "s3_zip", value = S3DataSegmentPusherProvider.class), + @JsonSubTypes.Type(name = "hdfs", value = HdfsDataSegmentPusherProvider.class), + @JsonSubTypes.Type(name = "c*", value = CassandraDataSegmentPusherProvider.class) +}) +public interface DataSegmentPusherProvider extends Provider +{ +} diff --git a/server/src/main/java/io/druid/guice/DruidBinders.java b/server/src/main/java/io/druid/guice/DruidBinders.java new file mode 100644 index 00000000000..5bc84227cd5 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DruidBinders.java @@ -0,0 +1,58 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; +import io.druid.query.Query; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.segment.loading.DataSegmentPuller; + +/** + */ +public class DruidBinders +{ + public static MapBinder, QueryRunnerFactory> queryRunnerFactoryBinder(Binder binder) + { + return MapBinder.newMapBinder( + binder, new TypeLiteral>() + { + }, TypeLiteral.get(QueryRunnerFactory.class) + ); + } + + public static MapBinder, QueryToolChest> queryToolChestBinder(Binder binder) + { + return MapBinder.newMapBinder( + binder, new TypeLiteral>() + { + }, new TypeLiteral() + { + } + ); + } + + public static MapBinder dataSegmentPullerBinder(Binder binder) + { + return MapBinder.newMapBinder(binder, String.class, DataSegmentPuller.class); + } +} diff --git a/server/src/main/java/io/druid/guice/DruidProcessingModule.java b/server/src/main/java/io/druid/guice/DruidProcessingModule.java new file mode 100644 index 00000000000..cf0157de9e0 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DruidProcessingModule.java @@ -0,0 +1,136 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import com.metamx.common.concurrent.ExecutorServiceConfig; +import com.metamx.common.logger.Logger; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.collections.StupidPool; +import io.druid.concurrent.Execs; +import io.druid.guice.annotations.Global; +import io.druid.guice.annotations.Processing; +import io.druid.query.MetricsEmittingExecutorService; +import io.druid.server.DruidProcessingConfig; + +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + */ +public class DruidProcessingModule implements Module +{ + private static final Logger log = new Logger(DruidProcessingModule.class); + + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, DruidProcessingConfig.class, ImmutableMap.of("base_path", "druid.processing")); + binder.bind(ExecutorServiceConfig.class).to(DruidProcessingConfig.class); + } + + @Provides + @Processing + @ManageLifecycle + public ExecutorService getProcessingExecutorService(ExecutorServiceConfig config, ServiceEmitter emitter) + { + return new MetricsEmittingExecutorService( + Executors.newFixedThreadPool(config.getNumThreads(), Execs.makeThreadFactory(config.getFormatString())), + emitter, + new ServiceMetricEvent.Builder() + ); + } + + @Provides + @LazySingleton + @Global + public StupidPool getIntermediateResultsPool(DruidProcessingConfig config) + { + try { + Class vmClass = Class.forName("sun.misc.VM"); + Object maxDirectMemoryObj = vmClass.getMethod("maxDirectMemory").invoke(null); + + if (maxDirectMemoryObj == null || !(maxDirectMemoryObj instanceof Number)) { + log.info("Cannot determine maxDirectMemory from[%s]", maxDirectMemoryObj); + } else { + long maxDirectMemory = ((Number) maxDirectMemoryObj).longValue(); + + final long memoryNeeded = (long) config.intermediateComputeSizeBytes() * (config.getNumThreads() + 1); + if (maxDirectMemory < memoryNeeded) { + throw new ProvisionException( + String.format( + "Not enough direct memory. Please adjust -XX:MaxDirectMemorySize or druid.computation.buffer.size: " + + "maxDirectMemory[%,d], memoryNeeded[%,d], druid.computation.buffer.size[%,d], numThreads[%,d]", + maxDirectMemory, memoryNeeded, config.intermediateComputeSizeBytes(), config.getNumThreads() + ) + ); + } + } + } + catch (ClassNotFoundException e) { + log.info("No VM class, cannot do memory check."); + } + catch (NoSuchMethodException e) { + log.info("VM.maxDirectMemory doesn't exist, cannot do memory check."); + } + catch (InvocationTargetException e) { + log.warn(e, "static method shouldn't throw this"); + } + catch (IllegalAccessException e) { + log.warn(e, "public method, shouldn't throw this"); + } + + return new IntermediateProcessingBufferPool(config.intermediateComputeSizeBytes()); + } + + private static class IntermediateProcessingBufferPool extends StupidPool + { + private static final Logger log = new Logger(IntermediateProcessingBufferPool.class); + + public IntermediateProcessingBufferPool(final int computationBufferSize) + { + super( + new Supplier() + { + final AtomicLong count = new AtomicLong(0); + + @Override + public ByteBuffer get() + { + log.info( + "Allocating new intermediate processing buffer[%,d] of size[%,d]", + count.getAndIncrement(), computationBufferSize + ); + return ByteBuffer.allocateDirect(computationBufferSize); + } + } + ); + } + } +} diff --git a/server/src/main/java/io/druid/guice/HdfsDataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/HdfsDataSegmentPusherProvider.java new file mode 100644 index 00000000000..c6cbf50d16a --- /dev/null +++ b/server/src/main/java/io/druid/guice/HdfsDataSegmentPusherProvider.java @@ -0,0 +1,52 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.HdfsDataSegmentPusher; +import io.druid.segment.loading.HdfsDataSegmentPusherConfig; +import org.apache.hadoop.conf.Configuration; + +import javax.validation.constraints.NotNull; + +/** + */ +public class HdfsDataSegmentPusherProvider implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private HdfsDataSegmentPusherConfig hdfsDataSegmentPusherConfig = null; + + @JacksonInject + @NotNull + private Configuration config = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new HdfsDataSegmentPusher(hdfsDataSegmentPusherConfig, config, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/HistoricalModule.java b/server/src/main/java/io/druid/guice/HistoricalModule.java new file mode 100644 index 00000000000..0e00526351b --- /dev/null +++ b/server/src/main/java/io/druid/guice/HistoricalModule.java @@ -0,0 +1,40 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.metamx.common.logger.Logger; +import io.druid.server.coordination.ServerManager; +import io.druid.server.coordination.ZkCoordinator; + +/** + */ +public class HistoricalModule implements Module +{ + private static final Logger log = new Logger(HistoricalModule.class); + + @Override + public void configure(Binder binder) + { + binder.bind(ServerManager.class).in(LazySingleton.class); + binder.bind(ZkCoordinator.class).in(ManageLifecycle.class); + } +} diff --git a/server/src/main/java/io/druid/guice/HttpClientModule.java b/server/src/main/java/io/druid/guice/HttpClientModule.java new file mode 100644 index 00000000000..8bf7c70b698 --- /dev/null +++ b/server/src/main/java/io/druid/guice/HttpClientModule.java @@ -0,0 +1,171 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Binding; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.HttpClientConfig; +import com.metamx.http.client.HttpClientInit; +import io.druid.guice.annotations.Global; +import org.joda.time.Duration; +import org.joda.time.Period; + +import javax.net.ssl.SSLContext; +import javax.validation.constraints.Min; +import java.lang.annotation.Annotation; + +/** + */ +public class HttpClientModule implements Module +{ + public static HttpClientModule global() + { + return new HttpClientModule("druid.global.http", Global.class); + } + + private final String propertyPrefix; + private Annotation annotation = null; + private Class annotationClazz = null; + + public HttpClientModule(String propertyPrefix) + { + this.propertyPrefix = propertyPrefix; + } + + public HttpClientModule(String propertyPrefix, Class annotation) + { + this.propertyPrefix = propertyPrefix; + this.annotationClazz = annotation; + } + + public HttpClientModule(String propertyPrefix, Annotation annotation) + { + this.propertyPrefix = propertyPrefix; + this.annotation = annotation; + } + + @Override + public void configure(Binder binder) + { + if (annotation != null) { + JsonConfigProvider.bind(binder, propertyPrefix, DruidHttpClientConfig.class, annotation); + binder.bind(HttpClient.class) + .annotatedWith(annotation) + .toProvider(new HttpClientProvider(annotation)) + .in(LazySingleton.class); + } + else if (annotationClazz != null) { + JsonConfigProvider.bind(binder, propertyPrefix, DruidHttpClientConfig.class, annotationClazz); + binder.bind(HttpClient.class) + .annotatedWith(annotationClazz) + .toProvider(new HttpClientProvider(annotationClazz)) + .in(LazySingleton.class); + } + else { + JsonConfigProvider.bind(binder, propertyPrefix, DruidHttpClientConfig.class); + binder.bind(HttpClient.class) + .toProvider(new HttpClientProvider()) + .in(LazySingleton.class); + } + } + + public static class DruidHttpClientConfig + { + @JsonProperty + @Min(0) + private int numConnections = 5; + + @JsonProperty + private Period readTimeout = null; + + public int getNumConnections() + { + return numConnections; + } + + public Duration getReadTimeout() + { + return readTimeout == null ? null : readTimeout.toStandardDuration(); + } + } + + public static class HttpClientProvider implements Provider + { + private final Key> configKey; + private final Key sslContextKey; + + private Provider> configProvider; + private Provider lifecycleProvider; + private Binding sslContextBinding; + + public HttpClientProvider() + { + configKey = Key.get(new TypeLiteral>(){}); + sslContextKey = Key.get(SSLContext.class); + } + + public HttpClientProvider(Annotation annotation) + { + configKey = Key.get(new TypeLiteral>(){}, annotation); + sslContextKey = Key.get(SSLContext.class, annotation); + } + + public HttpClientProvider(Class annotation) + { + configKey = Key.get(new TypeLiteral>(){}, annotation); + sslContextKey = Key.get(SSLContext.class, annotation); + } + + @Inject + public void configure(Injector injector) + { + configProvider = injector.getProvider(configKey); + sslContextBinding = injector.getExistingBinding(sslContextKey); + lifecycleProvider = injector.getProvider(Lifecycle.class); + } + + @Override + public HttpClient get() + { + final DruidHttpClientConfig config = configProvider.get().get(); + + final HttpClientConfig.Builder builder = HttpClientConfig + .builder() + .withNumConnections(config.getNumConnections()) + .withReadTimeout(config.getReadTimeout()); + + if (sslContextBinding != null) { + builder.withSslContext(sslContextBinding.getProvider().get()); + } + + return HttpClientInit.createClient(builder.build(), lifecycleProvider.get()); + } + } +} diff --git a/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java b/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java new file mode 100644 index 00000000000..60e0c15281b --- /dev/null +++ b/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java @@ -0,0 +1,83 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import io.druid.client.indexing.IndexingService; +import io.druid.client.indexing.IndexingServiceSelector; +import io.druid.client.indexing.IndexingServiceSelectorConfig; +import io.druid.client.selector.DiscoverySelector; +import io.druid.client.selector.Server; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceProvider; + +import java.io.IOException; + +/** + */ +public class IndexingServiceDiscoveryModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.selectors.indexing", IndexingServiceSelectorConfig.class); + binder.bind(new TypeLiteral>(){}) + .annotatedWith(IndexingService.class) + .to(IndexingServiceSelector.class); + + binder.bind(IndexingServiceSelector.class).in(ManageLifecycle.class); + } + + @Provides + @LazySingleton @IndexingService + public ServiceProvider getServiceProvider( + IndexingServiceSelectorConfig config, + ServiceDiscovery serviceDiscovery + ) + { + if (config.getServiceName() == null) { + return new ServiceProvider() + { + @Override + public void start() throws Exception + { + + } + + @Override + public ServiceInstance getInstance() throws Exception + { + return null; + } + + @Override + public void close() throws IOException + { + + } + }; + } + return serviceDiscovery.serviceProviderBuilder().serviceName(config.getServiceName()).build(); + } +} diff --git a/server/src/main/java/io/druid/guice/LocalDataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/LocalDataSegmentPusherProvider.java new file mode 100644 index 00000000000..3af8510dd1d --- /dev/null +++ b/server/src/main/java/io/druid/guice/LocalDataSegmentPusherProvider.java @@ -0,0 +1,43 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.LocalDataSegmentPusher; +import io.druid.segment.loading.LocalDataSegmentPusherConfig; + +import javax.validation.constraints.NotNull; + +/** + */ +public class LocalDataSegmentPusherProvider extends LocalDataSegmentPusherConfig implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new LocalDataSegmentPusher(this, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java new file mode 100644 index 00000000000..0f9970a6f30 --- /dev/null +++ b/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java @@ -0,0 +1,70 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.multibindings.MapBinder; +import io.druid.query.Query; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryEngine; +import io.druid.query.groupby.GroupByQueryRunnerFactory; +import io.druid.query.metadata.SegmentMetadataQueryRunnerFactory; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.query.search.SearchQueryRunnerFactory; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; + +import java.util.Map; + +/** + */ +public class QueryRunnerFactoryModule extends QueryToolChestModule +{ + private static final Map, Class> mappings = + ImmutableMap., Class>builder() + .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) + .put(SearchQuery.class, SearchQueryRunnerFactory.class) + .put(TimeBoundaryQuery.class, TimeBoundaryQueryRunnerFactory.class) + .put(SegmentMetadataQuery.class, SegmentMetadataQueryRunnerFactory.class) + .put(GroupByQuery.class, GroupByQueryRunnerFactory.class) + .build(); + + @Override + public void configure(Binder binder) + { + super.configure(binder); + + final MapBinder, QueryRunnerFactory> queryFactoryBinder = DruidBinders.queryRunnerFactoryBinder( + binder + ); + + for (Map.Entry, Class> entry : mappings.entrySet()) { + queryFactoryBinder.addBinding(entry.getKey()).to(entry.getValue()); + binder.bind(entry.getValue()).in(LazySingleton.class); + } + + binder.bind(GroupByQueryEngine.class).in(LazySingleton.class); + } +} diff --git a/server/src/main/java/io/druid/guice/QueryToolChestModule.java b/server/src/main/java/io/druid/guice/QueryToolChestModule.java new file mode 100644 index 00000000000..4da4eb2537f --- /dev/null +++ b/server/src/main/java/io/druid/guice/QueryToolChestModule.java @@ -0,0 +1,69 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.multibindings.MapBinder; +import io.druid.query.Query; +import io.druid.query.QueryToolChest; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryQueryToolChest; +import io.druid.query.metadata.SegmentMetadataQueryQueryToolChest; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.query.search.SearchQueryQueryToolChest; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQueryConfig; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.query.timeboundary.TimeBoundaryQueryQueryToolChest; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; + +import java.util.Map; + +/** + */ +public class QueryToolChestModule implements Module +{ + public final Map, Class> mappings = + ImmutableMap., Class>builder() + .put(TimeseriesQuery.class, TimeseriesQueryQueryToolChest.class) + .put(SearchQuery.class, SearchQueryQueryToolChest.class) + .put(TimeBoundaryQuery.class, TimeBoundaryQueryQueryToolChest.class) + .put(SegmentMetadataQuery.class, SegmentMetadataQueryQueryToolChest.class) + .put(GroupByQuery.class, GroupByQueryQueryToolChest.class) + .build(); + + @Override + public void configure(Binder binder) + { + MapBinder, QueryToolChest> toolChests = DruidBinders.queryToolChestBinder(binder); + + for (Map.Entry, Class> entry : mappings.entrySet()) { + toolChests.addBinding(entry.getKey()).to(entry.getValue()); + binder.bind(entry.getValue()).in(LazySingleton.class); + } + + JsonConfigProvider.bind(binder, "druid.query.groupBy", GroupByQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.query.search", SearchQueryConfig.class); + } +} diff --git a/server/src/main/java/io/druid/guice/QueryableModule.java b/server/src/main/java/io/druid/guice/QueryableModule.java new file mode 100644 index 00000000000..184f80a1b58 --- /dev/null +++ b/server/src/main/java/io/druid/guice/QueryableModule.java @@ -0,0 +1,66 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import io.druid.initialization.DruidModule; +import io.druid.query.QuerySegmentWalker; +import io.druid.server.QueryServlet; +import io.druid.server.log.EmittingRequestLoggerProvider; +import io.druid.server.log.FileRequestLoggerProvider; +import io.druid.server.log.RequestLogger; +import io.druid.server.log.RequestLoggerProvider; + +import java.util.Arrays; +import java.util.List; + +/** + */ +public class QueryableModule implements DruidModule +{ + private final Class walkerClass; + + public QueryableModule( + Class walkerClass + ) + { + this.walkerClass = walkerClass; + } + + @Override + public void configure(Binder binder) + { + binder.bind(QueryServlet.class).in(LazySingleton.class); + binder.bind(QuerySegmentWalker.class).to(walkerClass).in(LazySingleton.class); + binder.bind(RequestLogger.class).toProvider(RequestLoggerProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.request.logging", RequestLoggerProvider.class); + } + + @Override + public List getJacksonModules() + { + return Arrays.asList( + new SimpleModule("QueryableModule") + .registerSubtypes(EmittingRequestLoggerProvider.class, FileRequestLoggerProvider.class) + ); + } +} diff --git a/server/src/main/java/io/druid/guice/S3DataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/S3DataSegmentPusherProvider.java new file mode 100644 index 00000000000..2cb9765cbf7 --- /dev/null +++ b/server/src/main/java/io/druid/guice/S3DataSegmentPusherProvider.java @@ -0,0 +1,52 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.S3DataSegmentPusher; +import io.druid.segment.loading.S3DataSegmentPusherConfig; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; + +import javax.validation.constraints.NotNull; + +/** + */ +public class S3DataSegmentPusherProvider implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private RestS3Service restS3Service = null; + + @JacksonInject + @NotNull + private S3DataSegmentPusherConfig config = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new S3DataSegmentPusher(restS3Service, config, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/ServerModule.java b/server/src/main/java/io/druid/guice/ServerModule.java new file mode 100644 index 00000000000..3417ef3d82e --- /dev/null +++ b/server/src/main/java/io/druid/guice/ServerModule.java @@ -0,0 +1,70 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.metamx.common.concurrent.ScheduledExecutorFactory; +import com.metamx.common.concurrent.ScheduledExecutors; +import com.metamx.common.lifecycle.Lifecycle; +import io.druid.guice.annotations.Self; +import io.druid.initialization.DruidModule; +import io.druid.server.DruidNode; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.partition.LinearShardSpec; +import io.druid.timeline.partition.NumberedShardSpec; +import io.druid.timeline.partition.SingleDimensionShardSpec; + +import java.util.Arrays; +import java.util.List; + +/** + */ +public class ServerModule implements DruidModule +{ + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, ZkPathsConfig.class); + + JsonConfigProvider.bind(binder, "druid", DruidNode.class, Self.class); + } + + @Provides @LazySingleton + public ScheduledExecutorFactory getScheduledExecutorFactory(Lifecycle lifecycle) + { + return ScheduledExecutors.createFactory(lifecycle); + } + + @Override + public List getJacksonModules() + { + return Arrays.asList( + new SimpleModule() + .registerSubtypes( + new NamedType(SingleDimensionShardSpec.class, "single"), + new NamedType(LinearShardSpec.class, "linear"), + new NamedType(NumberedShardSpec.class, "numbered") + ) + ); + } +} diff --git a/realtime/src/main/java/com/metamx/druid/realtime/RealtimeMain.java b/server/src/main/java/io/druid/guice/ServerViewModule.java similarity index 51% rename from realtime/src/main/java/com/metamx/druid/realtime/RealtimeMain.java rename to server/src/main/java/io/druid/guice/ServerViewModule.java index c81ab548a2e..9ccee0984e0 100644 --- a/realtime/src/main/java/com/metamx/druid/realtime/RealtimeMain.java +++ b/server/src/main/java/io/druid/guice/ServerViewModule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,36 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.realtime; +package io.druid.guice; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.log.LogLevelAdjuster; +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.client.InventoryView; +import io.druid.client.ServerInventoryView; +import io.druid.client.ServerInventoryViewProvider; +import io.druid.client.ServerView; /** */ -public class RealtimeMain +public class ServerViewModule implements Module { - private static final Logger log = new Logger(RealtimeMain.class); - - public static void main(String[] args) throws Exception + @Override + public void configure(Binder binder) { - LogLevelAdjuster.register(); - - Lifecycle lifecycle = new Lifecycle(); - - lifecycle.addManagedInstance( - RealtimeNode.builder().build() - ); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.info(t, "Throwable caught at startup, committing seppuku"); - System.exit(2); - } - - lifecycle.join(); + JsonConfigProvider.bind(binder, "druid.announcer", ServerInventoryViewProvider.class); + binder.bind(InventoryView.class).to(ServerInventoryView.class); + binder.bind(ServerView.class).to(ServerInventoryView.class); + binder.bind(ServerInventoryView.class).toProvider(ServerInventoryViewProvider.class).in(ManageLifecycle.class); } -} \ No newline at end of file +} diff --git a/server/src/main/java/io/druid/guice/StorageNodeModule.java b/server/src/main/java/io/druid/guice/StorageNodeModule.java new file mode 100644 index 00000000000..2ee270cc330 --- /dev/null +++ b/server/src/main/java/io/druid/guice/StorageNodeModule.java @@ -0,0 +1,71 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import io.druid.client.DruidServerConfig; +import io.druid.guice.annotations.Self; +import io.druid.query.DefaultQueryRunnerFactoryConglomerate; +import io.druid.query.QueryRunnerFactoryConglomerate; +import io.druid.segment.loading.MMappedQueryableIndexFactory; +import io.druid.segment.loading.QueryableIndexFactory; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.server.DruidNode; +import io.druid.server.coordination.DruidServerMetadata; + +/** + */ +public class StorageNodeModule implements Module +{ + private final String nodeType; + + public StorageNodeModule(String nodeType) + { + this.nodeType = nodeType; + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.server", DruidServerConfig.class); + JsonConfigProvider.bind(binder, "druid.segmentCache", SegmentLoaderConfig.class); + + binder.bind(QueryableIndexFactory.class).to(MMappedQueryableIndexFactory.class).in(LazySingleton.class); + + binder.bind(QueryRunnerFactoryConglomerate.class) + .to(DefaultQueryRunnerFactoryConglomerate.class) + .in(LazySingleton.class); + } + + @Provides + @LazySingleton + public DruidServerMetadata getMetadata(@Self DruidNode node, DruidServerConfig config) + { + return new DruidServerMetadata( + node.getHost(), + node.getHost(), + config.getMaxSize(), + nodeType, + config.getTier() + ); + } +} diff --git a/server/src/main/java/io/druid/guice/annotations/Client.java b/server/src/main/java/io/druid/guice/annotations/Client.java new file mode 100644 index 00000000000..240a0a8bcd5 --- /dev/null +++ b/server/src/main/java/io/druid/guice/annotations/Client.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice.annotations; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + */ +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@BindingAnnotation +public @interface Client +{ +} diff --git a/server/src/main/java/io/druid/guice/annotations/Processing.java b/server/src/main/java/io/druid/guice/annotations/Processing.java new file mode 100644 index 00000000000..e386476c1c9 --- /dev/null +++ b/server/src/main/java/io/druid/guice/annotations/Processing.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice.annotations; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + */ +@BindingAnnotation +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Processing +{ +} diff --git a/server/src/main/java/io/druid/segment/loading/AWSCredentialsConfig.java b/server/src/main/java/io/druid/segment/loading/AWSCredentialsConfig.java new file mode 100644 index 00000000000..b9a032fd4ea --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/AWSCredentialsConfig.java @@ -0,0 +1,43 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class AWSCredentialsConfig +{ + @JsonProperty + private String accessKey = ""; + + @JsonProperty + private String secretKey = ""; + + public String getAccessKey() + { + return accessKey; + } + + public String getSecretKey() + { + return secretKey; + } +} diff --git a/server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java b/server/src/main/java/io/druid/segment/loading/DataSegmentPusherUtil.java similarity index 93% rename from server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java rename to server/src/main/java/io/druid/segment/loading/DataSegmentPusherUtil.java index 349194836d9..b64f26806ca 100644 --- a/server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java +++ b/server/src/main/java/io/druid/segment/loading/DataSegmentPusherUtil.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; - -import org.joda.time.format.ISODateTimeFormat; +package io.druid.segment.loading; import com.google.common.base.Joiner; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; +import org.joda.time.format.ISODateTimeFormat; /** */ diff --git a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPuller.java similarity index 67% rename from server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPuller.java index fd0c93264ab..d19c4cef793 100644 --- a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPuller.java @@ -1,8 +1,28 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.google.common.io.Closeables; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.utils.CompressionUtils; +import com.google.inject.Inject; +import io.druid.common.utils.CompressionUtils; +import io.druid.timeline.DataSegment; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -17,6 +37,7 @@ public class HdfsDataSegmentPuller implements DataSegmentPuller { private final Configuration config; + @Inject public HdfsDataSegmentPuller(final Configuration config) { this.config = config; diff --git a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusher.java similarity index 74% rename from server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusher.java index e232bbb8f08..eb620698689 100644 --- a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusher.java @@ -1,14 +1,34 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; import com.google.common.io.OutputSupplier; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -28,6 +48,7 @@ public class HdfsDataSegmentPusher implements DataSegmentPusher private final Configuration hadoopConfig; private final ObjectMapper jsonMapper; + @Inject public HdfsDataSegmentPusher( HdfsDataSegmentPusherConfig config, Configuration hadoopConfig, diff --git a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusherConfig.java b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusherConfig.java similarity index 75% rename from server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusherConfig.java rename to server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusherConfig.java index b27d03672bc..e329fdeb4c4 100644 --- a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusherConfig.java +++ b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusherConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import org.skife.config.Config; +import com.fasterxml.jackson.annotation.JsonProperty; /** */ public abstract class HdfsDataSegmentPusherConfig { - @Config("druid.pusher.hdfs.storageDirectory") - public abstract String getStorageDirectory(); + @JsonProperty + public String storageDirectory = ""; + + public String getStorageDirectory() + { + return storageDirectory; + } } diff --git a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPuller.java similarity index 94% rename from server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/LocalDataSegmentPuller.java index 8cdb8e0a7a2..fd29d9192e9 100644 --- a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPuller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.google.common.io.Files; import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.timeline.DataSegment; import java.io.File; import java.io.IOException; diff --git a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java similarity index 92% rename from server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java index 1493b162572..a252337499e 100644 --- a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Files; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; import java.io.File; import java.io.IOException; @@ -40,6 +41,7 @@ public class LocalDataSegmentPusher implements DataSegmentPusher private final LocalDataSegmentPusherConfig config; private final ObjectMapper jsonMapper; + @Inject public LocalDataSegmentPusher( LocalDataSegmentPusherConfig config, ObjectMapper jsonMapper diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusherConfig.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusherConfig.java new file mode 100644 index 00000000000..823ffe38ee5 --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusherConfig.java @@ -0,0 +1,37 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.File; + +/** + */ +public class LocalDataSegmentPusherConfig +{ + @JsonProperty + public File storageDirectory; + + public File getStorageDirectory() + { + return storageDirectory; + } +} diff --git a/server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java b/server/src/main/java/io/druid/segment/loading/MMappedQueryableIndexFactory.java similarity index 61% rename from server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java rename to server/src/main/java/io/druid/segment/loading/MMappedQueryableIndexFactory.java index d2cef9b76b3..717b0b513a0 100644 --- a/server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java +++ b/server/src/main/java/io/druid/segment/loading/MMappedQueryableIndexFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.metamx.common.logger.Logger; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.v1.IndexIO; +import io.druid.segment.IndexIO; +import io.druid.segment.QueryableIndex; import org.apache.commons.io.FileUtils; import java.io.File; @@ -37,25 +37,6 @@ public class MMappedQueryableIndexFactory implements QueryableIndexFactory public QueryableIndex factorize(File parentDir) throws SegmentLoadingException { try { - if (! IndexIO.canBeMapped(parentDir)) { - File canBeMappedDir = new File(parentDir, "forTheMapping"); - if (canBeMappedDir.exists()) { - FileUtils.deleteDirectory(canBeMappedDir); - } - canBeMappedDir.mkdirs(); - - IndexIO.storeLatest(IndexIO.readIndex(parentDir), canBeMappedDir); - if (! IndexIO.canBeMapped(canBeMappedDir)) { - throw new SegmentLoadingException("WTF!? newly written file[%s] cannot be mapped!?", canBeMappedDir); - } - for (File file : canBeMappedDir.listFiles()) { - if (! file.renameTo(new File(parentDir, file.getName()))) { - throw new SegmentLoadingException("Couldn't rename[%s] to [%s]", canBeMappedDir, parentDir); - } - } - FileUtils.deleteDirectory(canBeMappedDir); - } - return IndexIO.loadIndex(parentDir); } catch (IOException e) { diff --git a/server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java b/server/src/main/java/io/druid/segment/loading/OmniSegmentLoader.java similarity index 52% rename from server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java rename to server/src/main/java/io/druid/segment/loading/OmniSegmentLoader.java index 987939bb069..31efc1b0ea9 100644 --- a/server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java +++ b/server/src/main/java/io/druid/segment/loading/OmniSegmentLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,77 +17,52 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.common.IAE; import com.metamx.common.ISE; +import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.QueryableIndexSegment; -import com.metamx.druid.index.Segment; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.Segment; +import io.druid.timeline.DataSegment; import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; import java.util.Iterator; import java.util.List; -import java.util.Set; +import java.util.Map; /** */ -public class SingleSegmentLoader implements SegmentLoader +public class OmniSegmentLoader implements SegmentLoader { - private static final Logger log = new Logger(SingleSegmentLoader.class); + private static final Logger log = new Logger(OmniSegmentLoader.class); - private final DataSegmentPuller dataSegmentPuller; + private final Map pullers; private final QueryableIndexFactory factory; + private final SegmentLoaderConfig config; private final List locations; @Inject - public SingleSegmentLoader( - DataSegmentPuller dataSegmentPuller, + public OmniSegmentLoader( + Map pullers, QueryableIndexFactory factory, SegmentLoaderConfig config ) { - this.dataSegmentPuller = dataSegmentPuller; + this.pullers = pullers; this.factory = factory; + this.config = config; - final ImmutableList.Builder locBuilder = ImmutableList.builder(); - - // This is a really, really stupid way of getting this information. Splitting on commas and bars is error-prone - // We should instead switch it up to be a JSON Array of JSON Object or something and cool stuff like that - // But, that'll have to wait for some other day. - for (String dirSpec : config.getCacheDirectory().split(",")) { - String[] dirSplit = dirSpec.split("\\|"); - if (dirSplit.length == 1) { - locBuilder.add(new StorageLocation(new File(dirSplit[0]), config.getServerMaxSize())); - } - else if (dirSplit.length == 2) { - final Long maxSize = Longs.tryParse(dirSplit[1]); - if (maxSize == null) { - throw new IAE("Size of a local segment storage location must be an integral number, got[%s]", dirSplit[1]); - } - locBuilder.add(new StorageLocation(new File(dirSplit[0]), maxSize)); - } - else { - throw new ISE( - "Unknown segment storage location[%s]=>[%s], config[%s].", - dirSplit.length, dirSpec, config.getCacheDirectory() - ); - } + this.locations = Lists.newArrayList(); + for (StorageLocationConfig locationConfig : config.getLocations()) { + locations.add(new StorageLocation(locationConfig.getPath(), locationConfig.getMaxSize())); } - locations = locBuilder.build(); - - Preconditions.checkArgument(locations.size() > 0, "Must have at least one segment cache directory."); - log.info("Using storage locations[%s]", locations); } @Override @@ -116,6 +91,7 @@ public class SingleSegmentLoader implements SegmentLoader return new QueryableIndexSegment(segment.getIdentifier(), index); } + @Override public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException { StorageLocation loc = findStorageLocationIfLoaded(segment); @@ -141,7 +117,7 @@ public class SingleSegmentLoader implements SegmentLoader log.debug("Unable to make parent file[%s]", storageDir); } - dataSegmentPuller.getSegmentFiles(segment, storageDir); + getPuller(segment.getLoadSpec()).getSegmentFiles(segment, storageDir); loc.addSegment(segment); retVal = storageDir; @@ -158,6 +134,10 @@ public class SingleSegmentLoader implements SegmentLoader @Override public void cleanup(DataSegment segment) throws SegmentLoadingException { + if (!config.isDeleteOnRemove()) { + return; + } + StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { @@ -176,71 +156,15 @@ public class SingleSegmentLoader implements SegmentLoader } } - static class StorageLocation + private DataSegmentPuller getPuller(Map loadSpec) throws SegmentLoadingException { - private final File path; - private final long maxSize; - private final Set segments; + String type = MapUtils.getString(loadSpec, "type"); + DataSegmentPuller loader = pullers.get(type); - private volatile long currSize = 0; - - StorageLocation( - File path, - long maxSize - ) - { - this.path = path; - this.maxSize = maxSize; - - this.segments = Sets.newHashSet(); + if (loader == null) { + throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, pullers.keySet()); } - File getPath() - { - return path; - } - - Long getMaxSize() - { - return maxSize; - } - - synchronized void addSegment(DataSegment segment) - { - if (segments.add(segment)) { - currSize += segment.getSize(); - } - } - - synchronized void removeSegment(DataSegment segment) - { - if (segments.remove(segment)) { - currSize -= segment.getSize(); - } - } - - boolean canHandle(long size) - { - return available() >= size; - } - - synchronized long available() - { - return maxSize - currSize; - } - - StorageLocation mostEmpty(StorageLocation other) - { - return available() > other.available() ? this : other; - } - - @Override - public String toString() - { - return "StorageLocation{" + - "path=" + path + - ", maxSize=" + maxSize + - '}'; - } + return loader; } } diff --git a/server/src/main/java/com/metamx/druid/loading/QueryableIndexFactory.java b/server/src/main/java/io/druid/segment/loading/QueryableIndexFactory.java similarity index 85% rename from server/src/main/java/com/metamx/druid/loading/QueryableIndexFactory.java rename to server/src/main/java/io/druid/segment/loading/QueryableIndexFactory.java index 535649baea4..cf802b56e41 100644 --- a/server/src/main/java/com/metamx/druid/loading/QueryableIndexFactory.java +++ b/server/src/main/java/io/druid/segment/loading/QueryableIndexFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import com.metamx.druid.index.QueryableIndex; +import io.druid.segment.QueryableIndex; import java.io.File; -import java.io.IOException; /** */ diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentKiller.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentKiller.java similarity index 94% rename from server/src/main/java/com/metamx/druid/loading/S3DataSegmentKiller.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentKiller.java index 63b815a7e85..314822679ae 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentKiller.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentKiller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPuller.java similarity index 96% rename from server/src/main/java/com/metamx/druid/loading/S3DataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentPuller.java index c6a82ffa3ff..49b7501ad51 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPuller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.google.common.base.Throwables; import com.google.common.io.ByteStreams; @@ -27,9 +27,9 @@ import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.common.s3.S3Utils; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.storage.s3.S3Utils; +import io.druid.timeline.DataSegment; import org.apache.commons.io.FileUtils; import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusher.java similarity index 94% rename from server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentPusher.java index a5462a83eb7..a1469446f31 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusher.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -25,11 +25,12 @@ import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Files; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.common.s3.S3Utils; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.utils.CompressionUtils; +import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.storage.s3.S3Utils; +import io.druid.timeline.DataSegment; import org.jets3t.service.ServiceException; import org.jets3t.service.acl.gs.GSAccessControlList; import org.jets3t.service.impl.rest.httpclient.RestS3Service; @@ -48,6 +49,7 @@ public class S3DataSegmentPusher implements DataSegmentPusher private final S3DataSegmentPusherConfig config; private final ObjectMapper jsonMapper; + @Inject public S3DataSegmentPusher( RestS3Service s3Client, S3DataSegmentPusherConfig config, diff --git a/indexing-hadoop/src/main/java/com/metamx/druid/indexer/updater/ZkUpdaterJobSpec.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusherConfig.java similarity index 64% rename from indexing-hadoop/src/main/java/com/metamx/druid/indexer/updater/ZkUpdaterJobSpec.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentPusherConfig.java index b79fab21747..b74d22597d7 100644 --- a/indexing-hadoop/src/main/java/com/metamx/druid/indexer/updater/ZkUpdaterJobSpec.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusherConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,34 +17,35 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.indexer.updater; +package io.druid.segment.loading; import com.fasterxml.jackson.annotation.JsonProperty; /** */ -public class ZkUpdaterJobSpec implements UpdaterJobSpec +public class S3DataSegmentPusherConfig { - @JsonProperty("zkHosts") - public String zkQuorum; + @JsonProperty + public String bucket = ""; - @JsonProperty("zkBasePath") - private String zkBasePath; + @JsonProperty + public String baseKey = ""; - public ZkUpdaterJobSpec() {} + @JsonProperty + public boolean disableAcl = false; - public String getZkQuorum() + public String getBucket() { - return zkQuorum; + return bucket; } - public String getZkBasePath() + public String getBaseKey() { - return zkBasePath; + return baseKey; } - public boolean postToZk() + public boolean getDisableAcl() { - return !(zkQuorum == null || zkBasePath == null); + return disableAcl; } } diff --git a/server/src/main/java/com/metamx/druid/loading/SegmentLoader.java b/server/src/main/java/io/druid/segment/loading/SegmentLoader.java similarity index 73% rename from server/src/main/java/com/metamx/druid/loading/SegmentLoader.java rename to server/src/main/java/io/druid/segment/loading/SegmentLoader.java index 75c7591a395..d4c050b6044 100644 --- a/server/src/main/java/com/metamx/druid/loading/SegmentLoader.java +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.Segment; +import io.druid.segment.Segment; +import io.druid.timeline.DataSegment; + +import java.io.File; /** */ public interface SegmentLoader { public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException; - public Segment getSegment(DataSegment loadSpec) throws SegmentLoadingException; + public Segment getSegment(DataSegment segment) throws SegmentLoadingException; + public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException; public void cleanup(DataSegment loadSpec) throws SegmentLoadingException; } diff --git a/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java new file mode 100644 index 00000000000..f4034865022 --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java @@ -0,0 +1,70 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hibernate.validator.constraints.NotEmpty; + +import java.io.File; +import java.util.List; + +/** + */ +public class SegmentLoaderConfig +{ + @JsonProperty + @NotEmpty + private List locations = null; + + @JsonProperty("deleteOnRemove") + private boolean deleteOnRemove = true; + + @JsonProperty + private File infoDir = null; + + public List getLocations() + { + return locations; + } + + public boolean isDeleteOnRemove() + { + return deleteOnRemove; + } + + public File getInfoDir() + { + if (infoDir == null) { + infoDir = new File(locations.get(0).getPath(), "info_dir"); + } + + return infoDir; + } + + @Override + public String toString() + { + return "SegmentLoaderConfig{" + + "locations=" + getLocations() + + ", deleteOnRemove=" + isDeleteOnRemove() + + ", infoDir=" + getInfoDir() + + '}'; + } +} diff --git a/server/src/main/java/io/druid/segment/loading/StorageLocation.java b/server/src/main/java/io/druid/segment/loading/StorageLocation.java new file mode 100644 index 00000000000..339ed88058f --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/StorageLocation.java @@ -0,0 +1,84 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.google.common.collect.Sets; +import io.druid.timeline.DataSegment; + +import java.io.File; +import java.util.Set; + +/** +*/ +class StorageLocation +{ + private final File path; + private final long maxSize; + private final Set segments; + + private volatile long currSize = 0; + + StorageLocation(File path, long maxSize) + { + this.path = path; + this.maxSize = maxSize; + + this.segments = Sets.newHashSet(); + } + + File getPath() + { + return path; + } + + long getMaxSize() + { + return maxSize; + } + + synchronized void addSegment(DataSegment segment) + { + if (segments.add(segment)) { + currSize += segment.getSize(); + } + } + + synchronized void removeSegment(DataSegment segment) + { + if (segments.remove(segment)) { + currSize -= segment.getSize(); + } + } + + boolean canHandle(long size) + { + return available() >= size; + } + + synchronized long available() + { + return maxSize - currSize; + } + + StorageLocation mostEmpty(StorageLocation other) + { + return available() > other.available() ? this : other; + } +} diff --git a/server/src/main/java/io/druid/segment/loading/StorageLocationConfig.java b/server/src/main/java/io/druid/segment/loading/StorageLocationConfig.java new file mode 100644 index 00000000000..2ab5920313d --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/StorageLocationConfig.java @@ -0,0 +1,58 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.File; + +/** + */ +public class StorageLocationConfig +{ + @JsonProperty + @NotNull + private File path = null; + + @JsonProperty + @Min(1) + private long maxSize = Long.MAX_VALUE; + + public File getPath() + { + return path; + } + + public long getMaxSize() + { + return maxSize; + } + + @Override + public String toString() + { + return "StorageLocationConfig{" + + "path=" + path + + ", maxSize=" + maxSize + + '}'; + } +} diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentConfig.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentConfig.java similarity index 71% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentConfig.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentConfig.java index 6e7976f824d..d9551a78782 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentConfig.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading.cassandra; +package io.druid.segment.loading.cassandra; -import org.skife.config.Config; -import org.skife.config.Default; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Cassandra Config @@ -29,11 +28,19 @@ import org.skife.config.Default; */ public abstract class CassandraDataSegmentConfig { - @Config("druid.pusher.cassandra.host") - @Default("") - public abstract String getHost(); + @JsonProperty + public String host = ""; - @Config("druid.pusher.cassandra.keyspace") - @Default("") - public abstract String getKeyspace(); + @JsonProperty + public String keyspace = ""; + + public String getKeyspace() + { + return keyspace; + } + + public String getHost() + { + return host; + } } diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPuller.java similarity index 91% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPuller.java index 87e6105f161..01f9264d3cc 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPuller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,25 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading.cassandra; - -import java.io.File; -import java.io.OutputStream; - -import org.apache.commons.io.FileUtils; +package io.druid.segment.loading.cassandra; import com.google.common.io.Files; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.loading.DataSegmentPuller; -import com.metamx.druid.loading.SegmentLoadingException; -import com.metamx.druid.utils.CompressionUtils; import com.netflix.astyanax.connectionpool.OperationResult; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; import com.netflix.astyanax.model.ColumnList; import com.netflix.astyanax.recipes.storage.ChunkedStorage; import com.netflix.astyanax.recipes.storage.ObjectMetadata; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.loading.DataSegmentPuller; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.timeline.DataSegment; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.OutputStream; /** * Cassandra Segment Puller @@ -48,6 +48,7 @@ public class CassandraDataSegmentPuller extends CassandraStorage implements Data private static final int CONCURRENCY = 10; private static final int BATCH_SIZE = 10; + @Inject public CassandraDataSegmentPuller(CassandraDataSegmentConfig config) { super(config); diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPusher.java similarity index 68% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPusher.java index 57bc72b9124..76b4ac52ddd 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPusher.java @@ -1,20 +1,40 @@ -package com.metamx.druid.loading.cassandra; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; +package io.druid.segment.loading.cassandra; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.loading.DataSegmentPusher; -import com.metamx.druid.loading.DataSegmentPusherUtil; -import com.metamx.druid.utils.CompressionUtils; import com.netflix.astyanax.MutationBatch; import com.netflix.astyanax.recipes.storage.ChunkedStorage; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.DataSegmentPusherUtil; +import io.druid.timeline.DataSegment; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; /** * Cassandra Segment Pusher @@ -28,6 +48,7 @@ public class CassandraDataSegmentPusher extends CassandraStorage implements Data private static final Joiner JOINER = Joiner.on("/").skipNulls(); private final ObjectMapper jsonMapper; + @Inject public CassandraDataSegmentPusher( CassandraDataSegmentConfig config, ObjectMapper jsonMapper) diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraStorage.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraStorage.java similarity index 97% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraStorage.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraStorage.java index e9d04609135..3a693575c18 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraStorage.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraStorage.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading.cassandra; +package io.druid.segment.loading.cassandra; import com.netflix.astyanax.AstyanaxContext; import com.netflix.astyanax.Keyspace; diff --git a/client/src/main/java/com/metamx/druid/http/ClientInfoResource.java b/server/src/main/java/io/druid/server/ClientInfoResource.java similarity index 94% rename from client/src/main/java/com/metamx/druid/http/ClientInfoResource.java rename to server/src/main/java/io/druid/server/ClientInfoResource.java index 8bcb8865e7c..782029f29b5 100644 --- a/client/src/main/java/com/metamx/druid/http/ClientInfoResource.java +++ b/server/src/main/java/io/druid/server/ClientInfoResource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.inject.Inject; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.InventoryView; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/client/src/main/java/com/metamx/druid/http/ClientQuerySegmentWalker.java b/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java similarity index 80% rename from client/src/main/java/com/metamx/druid/http/ClientQuerySegmentWalker.java rename to server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java index d053d6507a2..76d9d1aa838 100644 --- a/client/src/main/java/com/metamx/druid/http/ClientQuerySegmentWalker.java +++ b/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.google.common.base.Function; -import com.metamx.druid.Query; -import com.metamx.druid.client.CachingClusteredClient; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.MetricsEmittingQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.segment.SegmentDescriptor; +import com.google.inject.Inject; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.CachingClusteredClient; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.MetricsEmittingQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChest; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -39,19 +40,20 @@ import javax.annotation.Nullable; */ public class ClientQuerySegmentWalker implements QuerySegmentWalker { - private final QueryToolChestWarehouse warehouse; private final ServiceEmitter emitter; private final CachingClusteredClient baseClient; + private final QueryToolChestWarehouse warehouse; + @Inject public ClientQuerySegmentWalker( - QueryToolChestWarehouse warehouse, ServiceEmitter emitter, - CachingClusteredClient baseClient + CachingClusteredClient baseClient, + QueryToolChestWarehouse warehouse ) { - this.warehouse = warehouse; this.emitter = emitter; this.baseClient = baseClient; + this.warehouse = warehouse; } @Override diff --git a/client/src/main/java/com/metamx/druid/http/DirectClientQuerySegmentWalker.java b/server/src/main/java/io/druid/server/DirectClientQuerySegmentWalker.java similarity index 79% rename from client/src/main/java/com/metamx/druid/http/DirectClientQuerySegmentWalker.java rename to server/src/main/java/io/druid/server/DirectClientQuerySegmentWalker.java index e1cac8a90c3..224f03691ea 100644 --- a/client/src/main/java/com/metamx/druid/http/DirectClientQuerySegmentWalker.java +++ b/server/src/main/java/io/druid/server/DirectClientQuerySegmentWalker.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; -import com.metamx.druid.Query; -import com.metamx.druid.client.DirectDruidClient; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.segment.SegmentDescriptor; +import io.druid.client.DirectDruidClient; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; /** diff --git a/server/src/main/java/io/druid/server/DruidNode.java b/server/src/main/java/io/druid/server/DruidNode.java new file mode 100644 index 00000000000..4c8528bd6ee --- /dev/null +++ b/server/src/main/java/io/druid/server/DruidNode.java @@ -0,0 +1,121 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.common.utils.SocketUtil; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + */ +public class DruidNode +{ + private String hostNoPort; + + @JsonProperty("service") + @NotNull + private String serviceName = null; + + @JsonProperty + @NotNull + private String host = null; + + @JsonProperty + @Min(0) @Max(0xffff) + private int port = -1; + + @JsonCreator + public DruidNode( + @JsonProperty("service") String serviceName, + @JsonProperty("host") String host, + @JsonProperty("port") Integer port + ) + { + this.serviceName = serviceName; + + if (port == null) { + if (host == null) { + setHostAndPort(null, -1, null); + } + else if (host.contains(":")) { + final String[] hostParts = host.split(":"); + try { + setHostAndPort(host, Integer.parseInt(hostParts[1]), hostParts[0]); + } + catch (NumberFormatException e) { + setHostAndPort(host, -1, hostParts[0]); + } + } + else { + final int openPort = SocketUtil.findOpenPort(8080); + setHostAndPort(String.format("%s:%d", host, openPort), openPort, host); + } + } + else { + if (host == null || host.contains(":")) { + setHostAndPort(host, port, host == null ? null : host.split(":")[0]); + } + else { + setHostAndPort(String.format("%s:%d", host, port), port, host); + } + } + } + + private void setHostAndPort(String host, int port, String hostNoPort) + { + this.host = host; + this.port = port; + this.hostNoPort = hostNoPort; + } + + public String getServiceName() + { + return serviceName; + } + + public String getHost() + { + return host; + } + + public int getPort() + { + return port; + } + + public String getHostNoPort() + { + return hostNoPort; + } + + @Override + public String toString() + { + return "DruidNode{" + + "serviceName='" + serviceName + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + '}'; + } +} diff --git a/server/src/main/java/com/metamx/druid/DruidProcessingConfig.java b/server/src/main/java/io/druid/server/DruidProcessingConfig.java similarity index 93% rename from server/src/main/java/com/metamx/druid/DruidProcessingConfig.java rename to server/src/main/java/io/druid/server/DruidProcessingConfig.java index b497ae8f6ce..ada4eef609d 100644 --- a/server/src/main/java/com/metamx/druid/DruidProcessingConfig.java +++ b/server/src/main/java/io/druid/server/DruidProcessingConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.server; import com.metamx.common.concurrent.ExecutorServiceConfig; import org.skife.config.Config; diff --git a/client/src/main/java/com/metamx/druid/http/GuiceServletConfig.java b/server/src/main/java/io/druid/server/GuiceServletConfig.java similarity index 93% rename from client/src/main/java/com/metamx/druid/http/GuiceServletConfig.java rename to server/src/main/java/io/druid/server/GuiceServletConfig.java index fcfac0a66f8..b7a52a4eb6b 100644 --- a/client/src/main/java/com/metamx/druid/http/GuiceServletConfig.java +++ b/server/src/main/java/io/druid/server/GuiceServletConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.google.inject.Injector; import com.google.inject.servlet.GuiceServletContextListener; diff --git a/client/src/main/java/com/metamx/druid/http/QueryServlet.java b/server/src/main/java/io/druid/server/QueryServlet.java similarity index 92% rename from client/src/main/java/com/metamx/druid/http/QueryServlet.java rename to server/src/main/java/io/druid/server/QueryServlet.java index 8b871a49bcc..49d32ebe0dd 100644 --- a/client/src/main/java/com/metamx/druid/http/QueryServlet.java +++ b/server/src/main/java/io/druid/server/QueryServlet.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -25,17 +25,20 @@ import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; +import com.google.inject.Inject; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.logger.Logger; -import com.metamx.druid.Query; -import com.metamx.druid.query.segment.QuerySegmentWalker; import com.metamx.emitter.service.AlertEvent; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; - +import io.druid.guice.annotations.Json; +import io.druid.guice.annotations.Smile; +import io.druid.query.Query; +import io.druid.query.QuerySegmentWalker; +import io.druid.server.log.RequestLogger; +import org.eclipse.jetty.server.Request; import org.joda.time.DateTime; -import org.mortbay.jetty.Request; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -58,9 +61,10 @@ public class QueryServlet extends HttpServlet private final ServiceEmitter emitter; private final RequestLogger requestLogger; + @Inject public QueryServlet( - ObjectMapper jsonMapper, - ObjectMapper smileMapper, + @Json ObjectMapper jsonMapper, + @Smile ObjectMapper smileMapper, QuerySegmentWalker texasRanger, ServiceEmitter emitter, RequestLogger requestLogger diff --git a/client/src/main/java/com/metamx/druid/http/RequestLogLine.java b/server/src/main/java/io/druid/server/RequestLogLine.java similarity index 94% rename from client/src/main/java/com/metamx/druid/http/RequestLogLine.java rename to server/src/main/java/io/druid/server/RequestLogLine.java index 0dbecb03ec8..a52485ae279 100644 --- a/client/src/main/java/com/metamx/druid/http/RequestLogLine.java +++ b/server/src/main/java/io/druid/server/RequestLogLine.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; -import com.metamx.druid.Query; +import io.druid.query.Query; import org.joda.time.DateTime; import java.util.Arrays; diff --git a/server/src/main/java/io/druid/server/StatusResource.java b/server/src/main/java/io/druid/server/StatusResource.java new file mode 100644 index 00000000000..a2b30268d4d --- /dev/null +++ b/server/src/main/java/io/druid/server/StatusResource.java @@ -0,0 +1,103 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; + +/** + */ +@Path("/{a:status|health}") +public class StatusResource +{ + @GET + @Produces("application/json") + public Status doGet() + { + return new Status( + StatusResource.class.getPackage().getImplementationVersion(), + new Memory(Runtime.getRuntime()) + ); + } + + public static class Status { + final String version; + final Memory memory; + + public Status(String version, Memory memory) + { + this.version = version; + this.memory = memory; + } + + @JsonProperty + public String getVersion() + { + return version; + } + + @JsonProperty + public Memory getMemory() + { + return memory; + } + } + + public static class Memory { + final long maxMemory; + final long totalMemory; + final long freeMemory; + final long usedMemory; + + public Memory(Runtime runtime) { + maxMemory = runtime.maxMemory(); + totalMemory = runtime.totalMemory(); + freeMemory = runtime.freeMemory(); + usedMemory = totalMemory - freeMemory; + } + + @JsonProperty + public long getMaxMemory() + { + return maxMemory; + } + + @JsonProperty + public long getTotalMemory() + { + return totalMemory; + } + + @JsonProperty + public long getFreeMemory() + { + return freeMemory; + } + + @JsonProperty + public long getUsedMemory() + { + return usedMemory; + } + } +} diff --git a/server/src/main/java/io/druid/server/ZkPathsModule.java b/server/src/main/java/io/druid/server/ZkPathsModule.java new file mode 100644 index 00000000000..34631221353 --- /dev/null +++ b/server/src/main/java/io/druid/server/ZkPathsModule.java @@ -0,0 +1,33 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.google.inject.Binder; +import com.google.inject.Module; + +/** + */ +public class ZkPathsModule implements Module +{ + @Override + public void configure(Binder binder) + { + } +} diff --git a/client/src/main/java/com/metamx/druid/coordination/AbstractDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/AbstractDataSegmentAnnouncer.java similarity index 93% rename from client/src/main/java/com/metamx/druid/coordination/AbstractDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/AbstractDataSegmentAnnouncer.java index 1bb0b8114ee..ba73b74e994 100644 --- a/client/src/main/java/com/metamx/druid/coordination/AbstractDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/AbstractDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,8 +25,8 @@ import com.google.common.base.Throwables; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkPathsConfig; +import io.druid.curator.announcement.Announcer; +import io.druid.server.initialization.ZkPathsConfig; import org.apache.curator.utils.ZKPaths; /** diff --git a/client/src/main/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncer.java similarity index 83% rename from client/src/main/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncer.java index 46aedcee162..d0b24ea5653 100644 --- a/client/src/main/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; +import io.druid.curator.announcement.Announcer; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.utils.ZKPaths; import org.joda.time.DateTime; @@ -45,7 +47,7 @@ public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer { private static final Logger log = new Logger(BatchDataSegmentAnnouncer.class); - private final ZkDataSegmentAnnouncerConfig config; + private final BatchDataSegmentAnnouncerConfig config; private final Announcer announcer; private final ObjectMapper jsonMapper; private final String liveSegmentLocation; @@ -56,27 +58,29 @@ public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer private final Set availableZNodes = new ConcurrentSkipListSet(); private final Map segmentLookup = Maps.newConcurrentMap(); + @Inject public BatchDataSegmentAnnouncer( DruidServerMetadata server, - ZkDataSegmentAnnouncerConfig config, + BatchDataSegmentAnnouncerConfig config, + ZkPathsConfig zkPaths, Announcer announcer, ObjectMapper jsonMapper ) { - super(server, config, announcer, jsonMapper); - + super(server, zkPaths, announcer, jsonMapper); this.config = config; this.announcer = announcer; this.jsonMapper = jsonMapper; - this.liveSegmentLocation = ZKPaths.makePath(config.getLiveSegmentsPath(), server.getName()); + + this.liveSegmentLocation = ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), server.getName()); } @Override public void announceSegment(DataSegment segment) throws IOException { int newBytesLen = jsonMapper.writeValueAsBytes(segment).length; - if (newBytesLen > config.getMaxNumBytes()) { - throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes()); + if (newBytesLen > config.getMaxBytesPerNode()) { + throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxBytesPerNode()); } synchronized (lock) { @@ -85,17 +89,17 @@ public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer SegmentZNode availableZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString())); availableZNode.addSegment(segment); - log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath()); - announcer.announce(availableZNode.getPath(), availableZNode.getBytes()); - segmentLookup.put(segment, availableZNode); - availableZNodes.add(availableZNode); - } else { // update existing batch - Iterator iter = availableZNodes.iterator(); - boolean done = false; - while (iter.hasNext() && !done) { - SegmentZNode availableZNode = iter.next(); - if (availableZNode.getBytes().length + newBytesLen < config.getMaxNumBytes()) { - availableZNode.addSegment(segment); + log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath()); + announcer.announce(availableZNode.getPath(), availableZNode.getBytes()); + segmentLookup.put(segment, availableZNode); + availableZNodes.add(availableZNode); + } else { // update existing batch + Iterator iter = availableZNodes.iterator(); + boolean done = false; + while (iter.hasNext() && !done) { + SegmentZNode availableZNode = iter.next(); + if (availableZNode.getBytes().length + newBytesLen < config.getMaxBytesPerNode()) { + availableZNode.addSegment(segment); log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath()); announcer.update(availableZNode.getPath(), availableZNode.getBytes()); @@ -145,11 +149,11 @@ public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer for (DataSegment segment : segments) { int newBytesLen = jsonMapper.writeValueAsBytes(segment).length; - if (newBytesLen > config.getMaxNumBytes()) { - throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes()); + if (newBytesLen > config.getMaxBytesPerNode()) { + throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxBytesPerNode()); } - if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxNumBytes()) { + if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxBytesPerNode()) { segmentZNode.addSegments(batch); announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes()); diff --git a/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncerProvider.java b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncerProvider.java new file mode 100644 index 00000000000..e51de3a245c --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncerProvider.java @@ -0,0 +1,39 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JacksonInject; + +import javax.validation.constraints.NotNull; + +/** + */ +public class BatchDataSegmentAnnouncerProvider implements DataSegmentAnnouncerProvider +{ + @JacksonInject + @NotNull + private BatchDataSegmentAnnouncer batchAnnouncer = null; + + @Override + public DataSegmentAnnouncer get() + { + return batchAnnouncer; + } +} diff --git a/client/src/main/java/com/metamx/druid/coordination/DataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncer.java similarity index 89% rename from client/src/main/java/com/metamx/druid/coordination/DataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncer.java index 71eaaa37276..8993551c1b4 100644 --- a/client/src/main/java/com/metamx/druid/coordination/DataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.io.IOException; diff --git a/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncerProvider.java b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncerProvider.java new file mode 100644 index 00000000000..eeed92b045a --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncerProvider.java @@ -0,0 +1,35 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LegacyDataSegmentAnnouncerProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "legacy", value = LegacyDataSegmentAnnouncerProvider.class), + @JsonSubTypes.Type(name = "batch", value = BatchDataSegmentAnnouncerProvider.class) +}) +public interface DataSegmentAnnouncerProvider extends Provider +{ +} diff --git a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeHandler.java b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeHandler.java similarity index 87% rename from server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeHandler.java rename to server/src/main/java/io/druid/server/coordination/DataSegmentChangeHandler.java index bc88027bbb8..460e4b4256f 100644 --- a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeHandler.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeHandler.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeRequest.java b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeRequest.java similarity index 90% rename from server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeRequest.java rename to server/src/main/java/io/druid/server/coordination/DataSegmentChangeRequest.java index 596d49428f6..0c29d1e1c75 100644 --- a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeRequest.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeRequest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.client.DataSegment; /** */ diff --git a/client/src/main/java/com/metamx/druid/coordination/DruidServerMetadata.java b/server/src/main/java/io/druid/server/coordination/DruidServerMetadata.java similarity index 92% rename from client/src/main/java/com/metamx/druid/coordination/DruidServerMetadata.java rename to server/src/main/java/io/druid/server/coordination/DruidServerMetadata.java index 232df085e07..ecb7f33b2ea 100644 --- a/client/src/main/java/com/metamx/druid/coordination/DruidServerMetadata.java +++ b/server/src/main/java/io/druid/server/coordination/DruidServerMetadata.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -37,7 +37,8 @@ public class DruidServerMetadata @JsonProperty("name") String name, @JsonProperty("host") String host, @JsonProperty("maxSize") long maxSize, - @JsonProperty("type") String type, @JsonProperty("tier") String tier + @JsonProperty("type") String type, + @JsonProperty("tier") String tier ) { this.name = name; diff --git a/server/src/main/java/io/druid/server/coordination/LegacyDataSegmentAnnouncerProvider.java b/server/src/main/java/io/druid/server/coordination/LegacyDataSegmentAnnouncerProvider.java new file mode 100644 index 00000000000..f5a5dc40545 --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/LegacyDataSegmentAnnouncerProvider.java @@ -0,0 +1,46 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JacksonInject; + +import javax.validation.constraints.NotNull; +import java.util.Arrays; + +/** + */ +public class LegacyDataSegmentAnnouncerProvider implements DataSegmentAnnouncerProvider +{ + @JacksonInject + @NotNull + private SingleDataSegmentAnnouncer singleAnnouncer = null; + + @JacksonInject + @NotNull + private BatchDataSegmentAnnouncer batchAnnouncer = null; + + @Override + public DataSegmentAnnouncer get() + { + return new MultipleDataSegmentAnnouncerDataSegmentAnnouncer( + Arrays.asList(singleAnnouncer, batchAnnouncer) + ); + } +} diff --git a/client/src/main/java/com/metamx/druid/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java similarity index 69% rename from client/src/main/java/com/metamx/druid/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java index 053a3d85fac..59eb4312497 100644 --- a/client/src/main/java/com/metamx/druid/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.common.lifecycle.LifecycleStart; -import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import java.io.IOException; @@ -33,31 +28,15 @@ import java.io.IOException; */ public class MultipleDataSegmentAnnouncerDataSegmentAnnouncer implements DataSegmentAnnouncer { - private final Iterable dataSegmentAnnouncers; + private final Iterable dataSegmentAnnouncers; public MultipleDataSegmentAnnouncerDataSegmentAnnouncer( - Iterable dataSegmentAnnouncers + Iterable dataSegmentAnnouncers ) { this.dataSegmentAnnouncers = dataSegmentAnnouncers; } - @LifecycleStart - public void start() - { - for (AbstractDataSegmentAnnouncer dataSegmentAnnouncer : dataSegmentAnnouncers) { - dataSegmentAnnouncer.start(); - } - } - - @LifecycleStop - public void stop() - { - for (AbstractDataSegmentAnnouncer dataSegmentAnnouncer : dataSegmentAnnouncers) { - dataSegmentAnnouncer.stop(); - } - } - @Override public void announceSegment(DataSegment segment) throws IOException { diff --git a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestDrop.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java similarity index 92% rename from server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestDrop.java rename to server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java index 05d4121c039..9b0fc755fb3 100644 --- a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestDrop.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestLoad.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java similarity index 92% rename from server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestLoad.java rename to server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java index 64bb64fc3ed..2a66693b66f 100644 --- a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestLoad.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestNoop.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestNoop.java similarity index 91% rename from server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestNoop.java rename to server/src/main/java/io/druid/server/coordination/SegmentChangeRequestNoop.java index e72154fd81d..e713278b47c 100644 --- a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestNoop.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestNoop.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java b/server/src/main/java/io/druid/server/coordination/ServerManager.java similarity index 89% rename from server/src/main/java/com/metamx/druid/coordination/ServerManager.java rename to server/src/main/java/io/druid/server/coordination/ServerManager.java index 965cb4e3de6..ee7135db0bb 100644 --- a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java +++ b/server/src/main/java/io/druid/server/coordination/ServerManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,41 +17,43 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.collect.Ordering; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.FunctionalIterable; -import com.metamx.druid.Query; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.collect.CountingMap; -import com.metamx.druid.index.ReferenceCountingSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.loading.SegmentLoadingException; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.partition.PartitionHolder; -import com.metamx.druid.query.BySegmentQueryRunner; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.MetricsEmittingQueryRunner; -import com.metamx.druid.query.NoopQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.ReferenceCountingSegmentQueryRunner; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.segment.SegmentDescriptor; -import com.metamx.druid.query.segment.SpecificSegmentQueryRunner; -import com.metamx.druid.query.segment.SpecificSegmentSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.collections.CountingMap; +import io.druid.guice.annotations.Processing; +import io.druid.query.BySegmentQueryRunner; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.MetricsEmittingQueryRunner; +import io.druid.query.NoopQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerFactoryConglomerate; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChest; +import io.druid.query.ReferenceCountingSegmentQueryRunner; +import io.druid.query.SegmentDescriptor; +import io.druid.query.spec.QuerySegmentSpec; +import io.druid.query.spec.SpecificSegmentQueryRunner; +import io.druid.query.spec.SpecificSegmentSpec; +import io.druid.segment.ReferenceCountingSegment; +import io.druid.segment.Segment; +import io.druid.segment.loading.SegmentLoader; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; +import io.druid.timeline.partition.PartitionHolder; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -78,11 +80,12 @@ public class ServerManager implements QuerySegmentWalker private final CountingMap dataSourceSizes = new CountingMap(); private final CountingMap dataSourceCounts = new CountingMap(); + @Inject public ServerManager( SegmentLoader segmentLoader, QueryRunnerFactoryConglomerate conglomerate, ServiceEmitter emitter, - ExecutorService exec + @Processing ExecutorService exec ) { this.segmentLoader = segmentLoader; diff --git a/client/src/main/java/com/metamx/druid/coordination/SingleDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/SingleDataSegmentAnnouncer.java similarity index 90% rename from client/src/main/java/com/metamx/druid/coordination/SingleDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/SingleDataSegmentAnnouncer.java index 1757ccbe6af..37107e0dfdb 100644 --- a/client/src/main/java/com/metamx/druid/coordination/SingleDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/SingleDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkPathsConfig; +import io.druid.curator.announcement.Announcer; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.utils.ZKPaths; import java.io.IOException; @@ -36,6 +37,7 @@ public class SingleDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer private final ObjectMapper jsonMapper; private final String servedSegmentsLocation; + @Inject public SingleDataSegmentAnnouncer( DruidServerMetadata server, ZkPathsConfig config, diff --git a/server/src/main/java/com/metamx/druid/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java similarity index 89% rename from server/src/main/java/com/metamx/druid/coordination/ZkCoordinator.java rename to server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 39799c812ee..a55341a75a1 100644 --- a/server/src/main/java/com/metamx/druid/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.loading.SegmentLoadingException; import com.metamx.emitter.EmittingLogger; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -49,7 +51,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler private final Object lock = new Object(); private final ObjectMapper jsonMapper; - private final ZkCoordinatorConfig config; + private final SegmentLoaderConfig config; private final ZkPathsConfig zkPaths; private final DruidServerMetadata me; private final DataSegmentAnnouncer announcer; @@ -59,9 +61,10 @@ public class ZkCoordinator implements DataSegmentChangeHandler private volatile PathChildrenCache loadQueueCache; private volatile boolean started; + @Inject public ZkCoordinator( ObjectMapper jsonMapper, - ZkCoordinatorConfig config, + SegmentLoaderConfig config, ZkPathsConfig zkPaths, DruidServerMetadata me, DataSegmentAnnouncer announcer, @@ -100,21 +103,13 @@ public class ZkCoordinator implements DataSegmentChangeHandler ); try { - config.getSegmentInfoCacheDirectory().mkdirs(); + config.getInfoDir().mkdirs(); curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(servedSegmentsLocation).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(liveSegmentsLocation).ensure(curator.getZookeeperClient()); - if (config.isLoadFromSegmentCacheEnabled()) { - try { - loadCache(); - } - catch (Exception e) { - log.makeAlert(e, "Exception loading from cache") - .emit(); - } - } + loadCache(); loadQueueCache.getListenable().addListener( new PathChildrenCacheListener() @@ -198,7 +193,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler private void loadCache() { - File baseDir = config.getSegmentInfoCacheDirectory(); + File baseDir = config.getInfoDir(); if (!baseDir.exists()) { return; } @@ -213,7 +208,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler } else { log.warn("Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier()); - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } @@ -243,7 +238,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler throw new SegmentLoadingException(e, "Exception loading segment[%s]", segment.getIdentifier()); } - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.exists()) { try { jsonMapper.writeValue(segmentInfoCacheFile, segment); @@ -290,7 +285,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler continue; } - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.exists()) { try { jsonMapper.writeValue(segmentInfoCacheFile, segment); @@ -334,7 +329,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler try { serverManager.dropSegment(segment); - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } @@ -354,7 +349,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler for (DataSegment segment : segments) { serverManager.dropSegment(segment); - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } diff --git a/server/src/main/java/com/metamx/druid/http/InfoResource.java b/server/src/main/java/io/druid/server/http/InfoResource.java similarity index 87% rename from server/src/main/java/com/metamx/druid/http/InfoResource.java rename to server/src/main/java/io/druid/server/http/InfoResource.java index 8fd17f315de..670942d06cf 100644 --- a/server/src/main/java/com/metamx/druid/http/InfoResource.java +++ b/server/src/main/java/io/druid/server/http/InfoResource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import com.google.common.base.Function; import com.google.common.collect.Collections2; @@ -26,19 +26,19 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.rules.Rule; +import com.google.inject.Inject; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.db.DatabaseRuleManager; +import io.druid.db.DatabaseSegmentManager; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import javax.annotation.Nullable; -import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -83,8 +83,8 @@ public class InfoResource { return new ImmutableMap.Builder() .put("id", segment.getIdentifier()) - .put("dataSource", segment.getDimensions()) - .put("interval", segment.getInterval()) + .put("dataSource", segment.getDataSource()) + .put("interval", segment.getInterval().toString()) .put("version", segment.getVersion()) .put("size", segment.getSize()) .build(); @@ -143,9 +143,11 @@ public class InfoResource .build(); } return Response.ok( - Iterables.transform( - serverInventoryView.getInventory(), - simplifyClusterFn + Lists.newArrayList( + Iterables.transform( + serverInventoryView.getInventory(), + simplifyClusterFn + ) ) ).build(); } @@ -159,20 +161,22 @@ public class InfoResource { Response.ResponseBuilder builder = Response.status(Response.Status.OK); if (full != null) { - return builder.entity(serverInventoryView.getInventory()).build(); + return builder.entity(Lists.newArrayList(serverInventoryView.getInventory())).build(); } return builder.entity( - Iterables.transform( - serverInventoryView.getInventory(), - new Function() - { - @Override - public String apply(@Nullable DruidServer druidServer) - { - return druidServer.getName(); - } - } + Lists.newArrayList( + Iterables.transform( + serverInventoryView.getInventory(), + new Function() + { + @Override + public String apply(DruidServer druidServer) + { + return druidServer.getHost(); + } + } + ) ) ).build(); } @@ -259,44 +263,48 @@ public class InfoResource Response.ResponseBuilder builder = Response.status(Response.Status.OK); if (full != null) { return builder.entity( - Iterables.concat( - Iterables.transform( - serverInventoryView.getInventory(), - new Function>() - { - @Override - public Iterable apply(@Nullable DruidServer druidServer) - { - return druidServer.getSegments().values(); - } - } + Lists.newArrayList( + Iterables.concat( + Iterables.transform( + serverInventoryView.getInventory(), + new Function>() + { + @Override + public Iterable apply(@Nullable DruidServer druidServer) + { + return druidServer.getSegments().values(); + } + } + ) ) ) ).build(); } return builder.entity( - Iterables.concat( - Iterables.transform( - serverInventoryView.getInventory(), - new Function>() - { - @Override - public Iterable apply(@Nullable DruidServer druidServer) - { - return Collections2.transform( - druidServer.getSegments().values(), - new Function() - { - @Override - public String apply(@Nullable DataSegment segment) - { - return segment.getIdentifier(); - } - } - ); - } - } + Lists.newArrayList( + Iterables.concat( + Iterables.transform( + serverInventoryView.getInventory(), + new Function>() + { + @Override + public Iterable apply(@Nullable DruidServer druidServer) + { + return Collections2.transform( + druidServer.getSegments().values(), + new Function() + { + @Override + public String apply(@Nullable DataSegment segment) + { + return segment.getIdentifier(); + } + } + ); + } + } + ) ) ) ).build(); @@ -388,16 +396,18 @@ public class InfoResource } return builder.entity( - Iterables.transform( - getDataSources(), - new Function() - { - @Override - public String apply(@Nullable DruidDataSource dataSource) - { - return dataSource.getName(); - } - } + Lists.newArrayList( + Iterables.transform( + getDataSources(), + new Function() + { + @Override + public String apply(@Nullable DruidDataSource dataSource) + { + return dataSource.getName(); + } + } + ) ) ).build(); } diff --git a/server/src/main/resources/static/js/.Rhistory b/server/src/main/java/io/druid/server/http/MasterMain.java similarity index 100% rename from server/src/main/resources/static/js/.Rhistory rename to server/src/main/java/io/druid/server/http/MasterMain.java diff --git a/server/src/main/java/io/druid/server/http/MasterRedirectInfo.java b/server/src/main/java/io/druid/server/http/MasterRedirectInfo.java new file mode 100644 index 00000000000..3857ea04d63 --- /dev/null +++ b/server/src/main/java/io/druid/server/http/MasterRedirectInfo.java @@ -0,0 +1,66 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.http; + +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import io.druid.server.master.DruidMaster; + +import java.net.URL; + +/** +*/ +public class MasterRedirectInfo implements RedirectInfo +{ + private final DruidMaster master; + + @Inject + public MasterRedirectInfo(DruidMaster master) { + this.master = master; + } + + @Override + public boolean doLocal() + { + return master.isClusterMaster(); + } + + @Override + public URL getRedirectURL(String queryString, String requestURI) + { + try { + final String currentMaster = master.getCurrentMaster(); + if (currentMaster == null) { + return null; + } + + String location = String.format("http://%s%s", currentMaster, requestURI); + + if (queryString != null) { + location = String.format("%s?%s", location, queryString); + } + + return new URL(location); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } +} diff --git a/server/src/main/java/com/metamx/druid/http/MasterResource.java b/server/src/main/java/io/druid/server/http/MasterResource.java similarity index 93% rename from server/src/main/java/com/metamx/druid/http/MasterResource.java rename to server/src/main/java/io/druid/server/http/MasterResource.java index 9bb59d79d43..794e48a1c24 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterResource.java +++ b/server/src/main/java/io/druid/server/http/MasterResource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.LoadPeonCallback; +import com.google.inject.Inject; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.LoadPeonCallback; -import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; diff --git a/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java b/server/src/main/java/io/druid/server/http/MasterSegmentSettingsResource.java similarity index 56% rename from server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java rename to server/src/main/java/io/druid/server/http/MasterSegmentSettingsResource.java index ea14c587f78..e86311ccaec 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java +++ b/server/src/main/java/io/druid/server/http/MasterSegmentSettingsResource.java @@ -1,25 +1,26 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.http; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.master.MasterSegmentSettings; +package io.druid.server.http; + +import io.druid.common.config.JacksonConfigManager; +import io.druid.server.master.MasterSegmentSettings; import javax.inject.Inject; import javax.ws.rs.Consumes; diff --git a/server/src/main/java/com/metamx/druid/http/RedirectFilter.java b/server/src/main/java/io/druid/server/http/RedirectFilter.java similarity index 86% rename from server/src/main/java/com/metamx/druid/http/RedirectFilter.java rename to server/src/main/java/io/druid/server/http/RedirectFilter.java index f6f4c30e7cf..671a2cb207e 100644 --- a/server/src/main/java/com/metamx/druid/http/RedirectFilter.java +++ b/server/src/main/java/io/druid/server/http/RedirectFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.http.client.response.HttpResponseHandler; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -39,15 +39,13 @@ public class RedirectFilter implements Filter { private static final Logger log = new Logger(RedirectFilter.class); - private final HttpResponseHandler responseHandler; private final RedirectInfo redirectInfo; + @Inject public RedirectFilter( - HttpResponseHandler responseHandler, RedirectInfo redirectInfo ) { - this.responseHandler = responseHandler; this.redirectInfo = redirectInfo; } @@ -73,7 +71,13 @@ public class RedirectFilter implements Filter chain.doFilter(request, response); } else { URL url = redirectInfo.getRedirectURL(request.getQueryString(), request.getRequestURI()); - log.info("Forwarding request to [%s]", url); + log.debug("Forwarding request to [%s]", url); + + if (url == null) { + // We apparently have no master, so let's do a Service Unavailable + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY); response.setHeader("Location", url.toString()); diff --git a/server/src/main/java/com/metamx/druid/http/RedirectInfo.java b/server/src/main/java/io/druid/server/http/RedirectInfo.java similarity index 91% rename from server/src/main/java/com/metamx/druid/http/RedirectInfo.java rename to server/src/main/java/io/druid/server/http/RedirectInfo.java index d14839710fc..3efea33a146 100644 --- a/server/src/main/java/com/metamx/druid/http/RedirectInfo.java +++ b/server/src/main/java/io/druid/server/http/RedirectInfo.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import java.net.URL; diff --git a/server/src/main/java/com/metamx/druid/http/RedirectServlet.java b/server/src/main/java/io/druid/server/http/RedirectServlet.java similarity index 94% rename from server/src/main/java/com/metamx/druid/http/RedirectServlet.java rename to server/src/main/java/io/druid/server/http/RedirectServlet.java index 2befa9cd458..2226fed3d83 100644 --- a/server/src/main/java/com/metamx/druid/http/RedirectServlet.java +++ b/server/src/main/java/io/druid/server/http/RedirectServlet.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; import org.mortbay.jetty.servlet.DefaultServlet; @@ -38,6 +39,7 @@ public class RedirectServlet extends DefaultServlet private final RedirectInfo redirectInfo; + @Inject public RedirectServlet( RedirectInfo redirectInfo ) diff --git a/server/src/main/java/com/metamx/druid/http/SegmentToDrop.java b/server/src/main/java/io/druid/server/http/SegmentToDrop.java similarity index 94% rename from server/src/main/java/com/metamx/druid/http/SegmentToDrop.java rename to server/src/main/java/io/druid/server/http/SegmentToDrop.java index 44e9d4a0d33..e3fa092e610 100644 --- a/server/src/main/java/com/metamx/druid/http/SegmentToDrop.java +++ b/server/src/main/java/io/druid/server/http/SegmentToDrop.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/com/metamx/druid/http/SegmentToMove.java b/server/src/main/java/io/druid/server/http/SegmentToMove.java similarity index 94% rename from server/src/main/java/com/metamx/druid/http/SegmentToMove.java rename to server/src/main/java/io/druid/server/http/SegmentToMove.java index 85a6ab7227f..159afa0cda2 100644 --- a/server/src/main/java/com/metamx/druid/http/SegmentToMove.java +++ b/server/src/main/java/io/druid/server/http/SegmentToMove.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/io/druid/server/initialization/BatchDataSegmentAnnouncerConfig.java b/server/src/main/java/io/druid/server/initialization/BatchDataSegmentAnnouncerConfig.java new file mode 100644 index 00000000000..5df79357c77 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/BatchDataSegmentAnnouncerConfig.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; + +/** + */ +public class BatchDataSegmentAnnouncerConfig +{ + @JsonProperty + @Min(1) + private int segmentsPerNode = 50; + + @JsonProperty + @Max(1024 * 1024) + @Min(1024) + private long maxBytesPerNode = 512 * 1024; + + public int getSegmentsPerNode() + { + return segmentsPerNode; + } + + public long getMaxBytesPerNode() + { + return maxBytesPerNode; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/ConfigModule.java b/server/src/main/java/io/druid/server/initialization/ConfigModule.java new file mode 100644 index 00000000000..62644146a05 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/ConfigModule.java @@ -0,0 +1,50 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.config.Config; +import io.druid.guice.JsonConfigurator; +import io.druid.guice.LazySingleton; +import org.skife.config.ConfigurationObjectFactory; + +import javax.validation.Validation; +import javax.validation.Validator; +import java.util.Properties; + +/** + */ +public class ConfigModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator()); + binder.bind(JsonConfigurator.class).in(LazySingleton.class); + } + + @Provides @LazySingleton + public ConfigurationObjectFactory makeFactory(Properties props) + { + return Config.createFactory(props); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/CuratorDiscoveryConfig.java b/server/src/main/java/io/druid/server/initialization/CuratorDiscoveryConfig.java new file mode 100644 index 00000000000..e029112b183 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/CuratorDiscoveryConfig.java @@ -0,0 +1,40 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class CuratorDiscoveryConfig +{ + @JsonProperty + private String path = null; + + public String getPath() + { + return path; + } + + public boolean useDiscovery() + { + return path != null; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/EmitterModule.java b/server/src/main/java/io/druid/server/initialization/EmitterModule.java new file mode 100644 index 00000000000..c7b29d3af53 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/EmitterModule.java @@ -0,0 +1,142 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Binding; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import com.metamx.emitter.EmittingLogger; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.service.ServiceEmitter; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.annotations.Self; +import io.druid.server.DruidNode; + +import java.lang.annotation.Annotation; +import java.util.List; +import java.util.Properties; + +/** + */ +public class EmitterModule implements Module +{ + private static final Logger log = new Logger(EmitterModule.class); + private static final String EMITTER_PROPERTY = "druid.emitter"; + + private final Properties props; + + @Inject + public EmitterModule( + Properties props + ) + { + this.props = props; + } + + @Override + public void configure(Binder binder) + { + String emitterType = props.getProperty(EMITTER_PROPERTY, ""); + + binder.install(new LogEmitterModule()); + binder.install(new HttpEmitterModule()); + + binder.bind(Emitter.class).toProvider(new EmitterProvider(emitterType)).in(LazySingleton.class); + } + + @Provides + @ManageLifecycle + public ServiceEmitter getServiceEmitter(@Self Supplier configSupplier, Emitter emitter) + { + final DruidNode config = configSupplier.get(); + final ServiceEmitter retVal = new ServiceEmitter(config.getServiceName(), config.getHost(), emitter); + EmittingLogger.registerEmitter(retVal); + return retVal; + } + + private static class EmitterProvider implements Provider + { + private final String emitterType; + + private Emitter emitter = null; + + EmitterProvider( + String emitterType + ) + { + this.emitterType = emitterType; + } + + @Inject + public void inject(Injector injector) + { + final List> emitterBindings = injector.findBindingsByType(new TypeLiteral(){}); + + emitter = findEmitter(emitterType, emitterBindings); + + if (emitter == null) { + emitter = findEmitter(LogEmitterModule.EMITTER_TYPE, emitterBindings); + } + + if (emitter == null) { + List knownTypes = Lists.newArrayList(); + for (Binding binding : emitterBindings) { + final Annotation annotation = binding.getKey().getAnnotation(); + if (annotation != null) { + knownTypes.add(((Named) annotation).value()); + } + } + throw new ISE("Uknown emitter type[%s]=[%s], known types[%s]", EMITTER_PROPERTY, emitterType, knownTypes); + } + } + + private Emitter findEmitter(String emitterType, List> emitterBindings) + { + for (Binding binding : emitterBindings) { + if (Names.named(emitterType).equals(binding.getKey().getAnnotation())) { + return binding.getProvider().get(); + } + } + return null; + } + + + @Override + public Emitter get() + { + if (emitter == null) { + throw new ISE("Emitter was null, that's bad!"); + } + return emitter; + } + } +} diff --git a/server/src/main/java/io/druid/server/initialization/ExtensionsConfig.java b/server/src/main/java/io/druid/server/initialization/ExtensionsConfig.java new file mode 100644 index 00000000000..0973a249434 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/ExtensionsConfig.java @@ -0,0 +1,61 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + */ +public class ExtensionsConfig +{ + @JsonProperty + @NotNull + private List coordinates = ImmutableList.of(); + + @JsonProperty + @NotNull + private String localRepository = String.format("%s/%s", System.getProperty("user.home"), ".m2/repository"); + + @JsonProperty + @NotNull + private List remoteRepositories = ImmutableList.of( + "http://repo1.maven.org/maven2/", + "https://metamx.artifactoryonline.com/metamx/pub-libs-releases-local" + ); + + public List getCoordinates() + { + return coordinates; + } + + public String getLocalRepository() + { + return localRepository; + } + + public List getRemoteRepositories() + { + return remoteRepositories; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java b/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java new file mode 100644 index 00000000000..9141f7316b1 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +/** + */ +public class HttpEmitterConfig extends com.metamx.emitter.core.HttpEmitterConfig +{ + @JsonProperty + private Period timeOut = new Period("PT5M"); + + public Period getReadTimeout() + { + return timeOut; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java b/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java new file mode 100644 index 00000000000..fb460b7f3d0 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java @@ -0,0 +1,67 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.google.inject.util.Providers; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.HttpPostEmitter; +import com.metamx.http.client.HttpClientConfig; +import com.metamx.http.client.HttpClientInit; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; + +import javax.annotation.Nullable; +import javax.net.ssl.SSLContext; + +/** + */ +public class HttpEmitterModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter.http", HttpEmitterConfig.class); + + // Fix the injection of this if we want to enable ssl emission of events. + binder.bind(SSLContext.class).toProvider(Providers.of(null)).in(LazySingleton.class); + } + + @Provides @ManageLifecycle @Named("http") + public Emitter getEmitter(Supplier config, @Nullable SSLContext sslContext, Lifecycle lifecycle) + { + final HttpClientConfig.Builder builder = HttpClientConfig + .builder() + .withNumConnections(1) + .withReadTimeout(config.get().getReadTimeout().toStandardDuration()); + + if (sslContext != null) { + builder.withSslContext(sslContext); + } + + return new HttpPostEmitter(config.get(), HttpClientInit.createClient(builder.build(), lifecycle)); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/Initialization.java b/server/src/main/java/io/druid/server/initialization/Initialization.java new file mode 100644 index 00000000000..96b20ae077a --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/Initialization.java @@ -0,0 +1,245 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.io.ByteStreams; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import io.druid.guice.DruidSecondaryModule; +import io.druid.guice.annotations.Json; +import io.druid.guice.annotations.Smile; +import io.druid.initialization.DruidModule; +import io.tesla.aether.TeslaAether; +import io.tesla.aether.internal.DefaultTeslaAether; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.collection.CollectRequest; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyFilter; +import org.eclipse.aether.graph.DependencyNode; +import org.eclipse.aether.resolution.DependencyRequest; +import org.eclipse.aether.util.artifact.JavaScopes; +import org.eclipse.aether.util.filter.DependencyFilterUtils; + +import java.io.PrintStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.Set; + +/** + */ +public class Initialization +{ + private static final Logger log = new Logger(Initialization.class); + private static final Map loadersMap = Maps.newHashMap(); + + private static final Set exclusions = Sets.newHashSet( + "io.druid", + "com.metamx.druid" + ); + + public synchronized static List getFromExtensions(ExtensionsConfig config, Class clazz) + { + final TeslaAether aether = getAetherClient(config); + List retVal = Lists.newArrayList(); + + for (String coordinate : config.getCoordinates()) { + log.info("Loading extension[%s]", coordinate); + try { + ClassLoader loader = loadersMap.get(coordinate); + if (loader == null) { + final CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot(new Dependency(new DefaultArtifact(coordinate), JavaScopes.RUNTIME)); + DependencyRequest dependencyRequest = new DependencyRequest( + collectRequest, + DependencyFilterUtils.andFilter( + DependencyFilterUtils.classpathFilter(JavaScopes.RUNTIME), + new DependencyFilter() + { + @Override + public boolean accept(DependencyNode node, List parents) + { + if (accept(node.getArtifact())) { + return false; + } + + for (DependencyNode parent : parents) { + if (accept(parent.getArtifact())) { + return false; + } + } + + return true; + } + + private boolean accept(final Artifact artifact) + { + return exclusions.contains(artifact.getGroupId()); + } + } + ) + ); + + final List artifacts = aether.resolveArtifacts(dependencyRequest); + List urls = Lists.newArrayListWithExpectedSize(artifacts.size()); + for (Artifact artifact : artifacts) { + if (!exclusions.contains(artifact.getGroupId())) { + urls.add(artifact.getFile().toURI().toURL()); + } + else { + log.error("Skipped Artifact[%s]", artifact); + } + } + + for (URL url : urls) { + log.error("Added URL[%s]", url); + } + + loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader()); + loadersMap.put(coordinate, loader); + } + + final ServiceLoader serviceLoader = ServiceLoader.load(clazz, loader); + + for (T module : serviceLoader) { + log.info("Adding extension module[%s]", module.getClass()); + retVal.add(module); + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + return retVal; + } + + private static DefaultTeslaAether getAetherClient(ExtensionsConfig config) + { + /* + DefaultTeslaAether logs a bunch of stuff to System.out, which is annoying. We choose to disable that + unless debug logging is turned on. "Disabling" it, however, is kinda bass-ackwards. We copy out a reference + to the current System.out, and set System.out to a noop output stream. Then after DefaultTeslaAether has pulled + The reference we swap things back. + + This has implications for other things that are running in parallel to this. Namely, if anything else also grabs + a reference to System.out or tries to log to it while we have things adjusted like this, then they will also log + to nothingness. Fortunately, the code that calls this is single-threaded and shouldn't hopefully be running + alongside anything else that's grabbing System.out. But who knows. + */ + if (log.isTraceEnabled() || log.isDebugEnabled()) { + return new DefaultTeslaAether(config.getLocalRepository(), config.getRemoteRepositories()); + } + + PrintStream oldOut = System.out; + try { + System.setOut(new PrintStream(ByteStreams.nullOutputStream())); + return new DefaultTeslaAether(config.getLocalRepository(), config.getRemoteRepositories()); + } + finally { + System.setOut(oldOut); + } + } + + public static Injector makeInjectorWithModules(final Injector baseInjector, List modules) + { + ModuleList actualModules = new ModuleList(baseInjector); + actualModules.addModule(DruidSecondaryModule.class); + for (Object module : modules) { + actualModules.addModule(module); + } + + final ExtensionsConfig config = baseInjector.getInstance(ExtensionsConfig.class); + for (DruidModule module : Initialization.getFromExtensions(config, DruidModule.class)) { + actualModules.addModule(module); + } + + return Guice.createInjector(actualModules.getModules()); + } + + private static class ModuleList + { + private final Injector baseInjector; + private final ObjectMapper jsonMapper; + private final ObjectMapper smileMapper; + private final List modules; + + public ModuleList(Injector baseInjector) { + this.baseInjector = baseInjector; + this.jsonMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Json.class)); + this.smileMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Smile.class)); + this.modules = Lists.newArrayList(); + } + + private List getModules() + { + return Collections.unmodifiableList(modules); + } + + public void addModule(Object input) + { + if (input instanceof DruidModule) { + baseInjector.injectMembers(input); + modules.add(registerJacksonModules(((DruidModule) input))); + } + else if (input instanceof Module) { + baseInjector.injectMembers(input); + modules.add((Module) input); + } + else if (input instanceof Class) { + if (DruidModule.class.isAssignableFrom((Class) input)) { + modules.add(registerJacksonModules(baseInjector.getInstance((Class) input))); + } + else if (Module.class.isAssignableFrom((Class) input)) { + modules.add(baseInjector.getInstance((Class) input)); + return; + } + else { + throw new ISE("Class[%s] does not implement %s", input.getClass(), Module.class); + } + } + else { + throw new ISE("Unknown module type[%s]", input.getClass()); + } + } + + private DruidModule registerJacksonModules(DruidModule module) + { + for (com.fasterxml.jackson.databind.Module jacksonModule : module.getJacksonModules()) { + jsonMapper.registerModule(jacksonModule); + smileMapper.registerModule(jacksonModule); + } + return module; + } + } +} diff --git a/server/src/main/java/io/druid/server/initialization/JettyServerInitializer.java b/server/src/main/java/io/druid/server/initialization/JettyServerInitializer.java new file mode 100644 index 00000000000..44229e99dc7 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/JettyServerInitializer.java @@ -0,0 +1,30 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.inject.Injector; +import org.eclipse.jetty.server.Server; + +/** + */ +public interface JettyServerInitializer +{ + public void initialize(Server server, Injector injector); +} diff --git a/server/src/main/java/io/druid/server/initialization/JettyServerModule.java b/server/src/main/java/io/druid/server/initialization/JettyServerModule.java new file mode 100644 index 00000000000..80c6fb7cc4b --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/JettyServerModule.java @@ -0,0 +1,175 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.google.inject.Binder; +import com.google.inject.ConfigurationException; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import com.google.inject.Scopes; +import com.google.inject.name.Names; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.common.logger.Logger; +import com.sun.jersey.api.core.DefaultResourceConfig; +import com.sun.jersey.api.core.ResourceConfig; +import com.sun.jersey.guice.JerseyServletModule; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.spi.container.servlet.WebConfig; +import io.druid.guice.Jerseys; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.annotations.JSR311Resource; +import io.druid.guice.annotations.Self; +import io.druid.server.DruidNode; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.nio.SelectChannelConnector; +import org.eclipse.jetty.util.thread.QueuedThreadPool; + +import javax.servlet.ServletException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + */ +public class JettyServerModule extends JerseyServletModule +{ + private static final Logger log = new Logger(JettyServerModule.class); + + private final JettyServerInitializer initializer; + private final List> resources = Lists.newArrayList(); + + public JettyServerModule( + JettyServerInitializer initializer + ) + { + this.initializer = initializer; + } + + public JettyServerModule addResource(Class resource) + { + resources.add(resource); + return this; + } + + @Override + protected void configureServlets() + { + Binder binder = binder(); + + JsonConfigProvider.bind(binder, "druid.server.http", ServerConfig.class); + + binder.bind(GuiceContainer.class).to(DruidGuiceContainer.class); + binder.bind(DruidGuiceContainer.class).in(Scopes.SINGLETON); + serve("/*").with(DruidGuiceContainer.class); + + for (Class resource : resources) { + Jerseys.addResource(binder, resource); + binder.bind(resource).in(LazySingleton.class); + } + + binder.bind(Key.get(Server.class, Names.named("ForTheEagerness"))).to(Server.class).asEagerSingleton(); + } + + public static class DruidGuiceContainer extends GuiceContainer + { + private final Set> resources; + + @Inject + public DruidGuiceContainer( + Injector injector, + @JSR311Resource Set> resources + ) + { + super(injector); + this.resources = resources; + } + + @Override + protected ResourceConfig getDefaultResourceConfig( + Map props, WebConfig webConfig + ) throws ServletException + { + return new DefaultResourceConfig(resources); + } + } + + @Provides @LazySingleton + public Server getServer(Injector injector, Lifecycle lifecycle, @Self DruidNode node, ServerConfig config) + { + final Server server = makeJettyServer(node, config); + try { + initializer.initialize(server, injector); + } + catch (ConfigurationException e) { + throw new ProvisionException(Iterables.getFirst(e.getErrorMessages(), null).getMessage()); + } + + + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + server.start(); + } + + @Override + public void stop() + { + try { + server.stop(); + } + catch (Exception e) { + log.warn(e, "Unable to stop Jetty server."); + } + } + } + ); + return server; + } + + private static Server makeJettyServer(@Self DruidNode node, ServerConfig config) + { + final QueuedThreadPool threadPool = new QueuedThreadPool(); + threadPool.setMinThreads(config.getNumThreads()); + threadPool.setMaxThreads(config.getNumThreads()); + + final Server server = new Server(); + server.setThreadPool(threadPool); + + SelectChannelConnector connector = new SelectChannelConnector(); + connector.setPort(node.getPort()); + connector.setMaxIdleTime(Ints.checkedCast(config.getMaxIdleTime().toStandardDuration().getMillis())); + connector.setStatsOn(true); + + server.setConnectors(new Connector[]{connector}); + + return server; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java b/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java new file mode 100644 index 00000000000..4759cf57758 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java @@ -0,0 +1,51 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.LoggingEmitter; +import com.metamx.emitter.core.LoggingEmitterConfig; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.ManageLifecycle; + +/** + */ +public class LogEmitterModule implements Module +{ + public static final String EMITTER_TYPE = "logging"; + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter.logging", LoggingEmitterConfig.class); + } + + @Provides @ManageLifecycle @Named(EMITTER_TYPE) + public Emitter makeEmitter(Supplier config, ObjectMapper jsonMapper) + { + return new LoggingEmitter(config.get(), jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/PropertiesModule.java b/server/src/main/java/io/druid/server/initialization/PropertiesModule.java new file mode 100644 index 00000000000..3aad005dd68 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/PropertiesModule.java @@ -0,0 +1,88 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.common.io.Closeables; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.metamx.common.logger.Logger; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Properties; + +/** + */ +public class PropertiesModule implements Module +{ + private static final Logger log = new Logger(PropertiesModule.class); + + private final String propertiesFile; + + public PropertiesModule(String propertiesFile) + { + this.propertiesFile = propertiesFile; + } + + @Override + public void configure(Binder binder) + { + final Properties fileProps = new Properties(); + Properties systemProps = System.getProperties(); + + Properties props = new Properties(fileProps); + props.putAll(systemProps); + + InputStream stream = ClassLoader.getSystemResourceAsStream(propertiesFile); + try { + if (stream == null) { + File workingDirectoryFile = new File(systemProps.getProperty("druid.properties.file", propertiesFile)); + if (workingDirectoryFile.exists()) { + stream = new BufferedInputStream(new FileInputStream(workingDirectoryFile)); + } + } + + if (stream != null) { + log.info("Loading properties from %s", propertiesFile); + try { + fileProps.load(new InputStreamReader(stream, Charsets.UTF_8)); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } + } + catch (FileNotFoundException e) { + log.wtf(e, "This can only happen if the .exists() call lied. That's f'd up."); + } + finally { + Closeables.closeQuietly(stream); + } + + binder.bind(Properties.class).toInstance(props); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/ServerConfig.java b/server/src/main/java/io/druid/server/initialization/ServerConfig.java new file mode 100644 index 00000000000..6188e9681f5 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/ServerConfig.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + */ +public class ServerConfig +{ + @JsonProperty + @Min(1) + private int numThreads = 10; + + @JsonProperty + @NotNull + private Period maxIdleTime = new Period("PT5m"); + + public int getNumThreads() + { + return numThreads; + } + + public Period getMaxIdleTime() + { + return maxIdleTime; + } +} diff --git a/client/src/main/java/com/metamx/druid/initialization/ZkPathsConfig.java b/server/src/main/java/io/druid/server/initialization/ZkPathsConfig.java similarity index 95% rename from client/src/main/java/com/metamx/druid/initialization/ZkPathsConfig.java rename to server/src/main/java/io/druid/server/initialization/ZkPathsConfig.java index 065829fd9ea..2169177b16c 100644 --- a/client/src/main/java/com/metamx/druid/initialization/ZkPathsConfig.java +++ b/server/src/main/java/io/druid/server/initialization/ZkPathsConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.initialization; +package io.druid.server.initialization; import org.apache.curator.utils.ZKPaths; import org.skife.config.Config; -import org.skife.config.Default; public abstract class ZkPathsConfig { diff --git a/client/src/main/java/com/metamx/druid/http/EmittingRequestLogger.java b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java similarity index 74% rename from client/src/main/java/com/metamx/druid/http/EmittingRequestLogger.java rename to server/src/main/java/io/druid/server/log/EmittingRequestLogger.java index 600eceae6f9..70df84450a6 100644 --- a/client/src/main/java/com/metamx/druid/http/EmittingRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2013 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,38 +17,35 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.Query; -import com.metamx.emitter.core.Emitter; import com.metamx.emitter.core.Event; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.query.Query; +import io.druid.server.RequestLogLine; import org.joda.time.DateTime; import java.util.Map; public class EmittingRequestLogger implements RequestLogger { - - final String service; - final String host; - final Emitter emitter; + final ServiceEmitter emitter; final String feed; - public EmittingRequestLogger(String service, String host, Emitter emitter, String feed) + public EmittingRequestLogger(ServiceEmitter emitter, String feed) { this.emitter = emitter; - this.host = host; - this.service = service; this.feed = feed; } @Override public void log(final RequestLogLine requestLogLine) throws Exception { - emitter.emit(new RequestLogEvent(service, host, feed, requestLogLine)); + emitter.emit(new RequestLogEventBuilder(feed, requestLogLine)); } public static class RequestLogEvent implements Event @@ -119,4 +116,26 @@ public class EmittingRequestLogger implements RequestLogger return true; } } + + private class RequestLogEventBuilder implements ServiceEventBuilder + { + private final String feed; + private final RequestLogLine requestLogLine; + + public RequestLogEventBuilder( + String feed, + RequestLogLine requestLogLine + ) + { + this.feed = feed; + this.requestLogLine = requestLogLine; + } + + + @Override + public Event build(String service, String host) + { + return new RequestLogEvent(service, host, feed, requestLogLine); + } + } } diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java new file mode 100644 index 00000000000..5509b7c38be --- /dev/null +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java @@ -0,0 +1,55 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.metamx.emitter.service.ServiceEmitter; + +import javax.validation.constraints.NotNull; + +/** + */ +@JsonTypeName("emitter") +public class EmittingRequestLoggerProvider implements RequestLoggerProvider +{ + @JsonProperty + @NotNull + private String feed = null; + + @JacksonInject + @NotNull + private ServiceEmitter emitter = null; + + @Inject + public void injectMe(Injector injector) + { + System.out.println("YAYAYAYAYAYA!!!"); + } + + @Override + public RequestLogger get() + { + return new EmittingRequestLogger(emitter, feed); + } +} diff --git a/client/src/main/java/com/metamx/druid/http/FileRequestLogger.java b/server/src/main/java/io/druid/server/log/FileRequestLogger.java similarity index 96% rename from client/src/main/java/com/metamx/druid/http/FileRequestLogger.java rename to server/src/main/java/io/druid/server/log/FileRequestLogger.java index 32b245e6c60..ba450ce9c5f 100644 --- a/client/src/main/java/com/metamx/druid/http/FileRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/FileRequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; @@ -25,6 +25,7 @@ import com.google.common.io.Closeables; import com.metamx.common.concurrent.ScheduledExecutors; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import io.druid.server.RequestLogLine; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.MutableDateTime; diff --git a/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java new file mode 100644 index 00000000000..c698e55daf7 --- /dev/null +++ b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java @@ -0,0 +1,56 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.metamx.common.concurrent.ScheduledExecutorFactory; +import io.druid.guice.annotations.Json; + +import javax.validation.constraints.NotNull; +import java.io.File; + +/** + */ +@JsonTypeName("file") +public class FileRequestLoggerProvider implements RequestLoggerProvider +{ + @JsonProperty + @NotNull + private File dir = null; + + @JacksonInject + @NotNull + private ScheduledExecutorFactory factory = null; + + + @JacksonInject + @NotNull + @Json + private ObjectMapper jsonMapper = null; + + @Override + public RequestLogger get() + { + return new FileRequestLogger(jsonMapper, factory.create(1, "RequestLogger-%s"), dir); + } +} diff --git a/client/src/main/java/com/metamx/druid/http/NoopRequestLogger.java b/server/src/main/java/io/druid/server/log/NoopRequestLogger.java similarity index 86% rename from client/src/main/java/com/metamx/druid/http/NoopRequestLogger.java rename to server/src/main/java/io/druid/server/log/NoopRequestLogger.java index ddc422c91be..1088a13032a 100644 --- a/client/src/main/java/com/metamx/druid/http/NoopRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/NoopRequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; + +import io.druid.server.RequestLogLine; /** */ @@ -26,6 +28,6 @@ public class NoopRequestLogger implements RequestLogger @Override public void log(RequestLogLine requestLogLine) throws Exception { - // do nothing + // This is a no op! } } diff --git a/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java new file mode 100644 index 00000000000..86d41241ab2 --- /dev/null +++ b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java @@ -0,0 +1,31 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +/** + */ +public class NoopRequestLoggerProvider implements RequestLoggerProvider +{ + @Override + public RequestLogger get() + { + return new NoopRequestLogger(); + } +} diff --git a/client/src/main/java/com/metamx/druid/http/RequestLogger.java b/server/src/main/java/io/druid/server/log/RequestLogger.java similarity index 87% rename from client/src/main/java/com/metamx/druid/http/RequestLogger.java rename to server/src/main/java/io/druid/server/log/RequestLogger.java index 7720938c4bc..2d4c938252b 100644 --- a/client/src/main/java/com/metamx/druid/http/RequestLogger.java +++ b/server/src/main/java/io/druid/server/log/RequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; + +import io.druid.server.RequestLogLine; /** */ diff --git a/server/src/main/java/io/druid/server/log/RequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/RequestLoggerProvider.java new file mode 100644 index 00000000000..236c2452034 --- /dev/null +++ b/server/src/main/java/io/druid/server/log/RequestLoggerProvider.java @@ -0,0 +1,32 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +/** + * A Marker interface for things that can provide a RequestLogger. This can be combined with jackson polymorphic serde + * to provide new RequestLogger implementations as plugins. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopRequestLoggerProvider.class) +public interface RequestLoggerProvider extends Provider +{ +} diff --git a/server/src/main/java/com/metamx/druid/master/BalancerSegmentHolder.java b/server/src/main/java/io/druid/server/master/BalancerSegmentHolder.java similarity index 88% rename from server/src/main/java/com/metamx/druid/master/BalancerSegmentHolder.java rename to server/src/main/java/io/druid/server/master/BalancerSegmentHolder.java index 7d62968e4b9..7eb01282b48 100644 --- a/server/src/main/java/com/metamx/druid/master/BalancerSegmentHolder.java +++ b/server/src/main/java/io/druid/server/master/BalancerSegmentHolder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; /** */ diff --git a/indexing-service/src/main/java/com/metamx/druid/indexing/worker/config/WorkerConfig.java b/server/src/main/java/io/druid/server/master/BalancerStrategy.java similarity index 55% rename from indexing-service/src/main/java/com/metamx/druid/indexing/worker/config/WorkerConfig.java rename to server/src/main/java/io/druid/server/master/BalancerStrategy.java index deca87c56f5..0fb10eb2b6b 100644 --- a/indexing-service/src/main/java/com/metamx/druid/indexing/worker/config/WorkerConfig.java +++ b/server/src/main/java/io/druid/server/master/BalancerStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,30 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.indexing.worker.config; +package io.druid.server.master; -import org.skife.config.Config; -import org.skife.config.Default; +import io.druid.timeline.DataSegment; -/** - */ -public abstract class WorkerConfig +import java.util.List; + +public interface BalancerStrategy { - @Config("druid.host") - public abstract String getHost(); + public ServerHolder findNewSegmentHomeBalancer(final DataSegment proposalSegment, final List serverHolders); - @Config("druid.worker.ip") - public abstract String getIp(); + public ServerHolder findNewSegmentHomeReplicator(final DataSegment proposalSegment, final List serverHolders); - @Config("druid.worker.version") - public abstract String getVersion(); + public BalancerSegmentHolder pickSegmentToMove(final List serverHolders); - @Config("druid.worker.masterService") - public abstract String getMasterService(); - - @Config("druid.worker.capacity") - public int getCapacity() - { - return Runtime.getRuntime().availableProcessors() - 1; - } + public void emitStats(String tier, MasterStats stats, List serverHolderList); } diff --git a/server/src/main/java/io/druid/server/master/BalancerStrategyFactory.java b/server/src/main/java/io/druid/server/master/BalancerStrategyFactory.java new file mode 100644 index 00000000000..abdb9e88020 --- /dev/null +++ b/server/src/main/java/io/druid/server/master/BalancerStrategyFactory.java @@ -0,0 +1,26 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; + +import org.joda.time.DateTime; + +public interface BalancerStrategyFactory +{ + public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp); +} diff --git a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java b/server/src/main/java/io/druid/server/master/CostBalancerStrategy.java similarity index 89% rename from server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java rename to server/src/main/java/io/druid/server/master/CostBalancerStrategy.java index 0da949b9a7c..06d5aa0836d 100644 --- a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java +++ b/server/src/main/java/io/druid/server/master/CostBalancerStrategy.java @@ -1,32 +1,30 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.Pair; -import com.metamx.druid.client.DataSegment; import com.metamx.emitter.EmittingLogger; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; -import java.util.Comparator; import java.util.List; public class CostBalancerStrategy implements BalancerStrategy diff --git a/server/src/main/java/io/druid/server/master/CostBalancerStrategyFactory.java b/server/src/main/java/io/druid/server/master/CostBalancerStrategyFactory.java new file mode 100644 index 00000000000..832ee851277 --- /dev/null +++ b/server/src/main/java/io/druid/server/master/CostBalancerStrategyFactory.java @@ -0,0 +1,31 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; + +import org.joda.time.DateTime; + +public class CostBalancerStrategyFactory implements BalancerStrategyFactory +{ + + @Override + public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) + { + return new CostBalancerStrategy(referenceTimestamp); + } +} diff --git a/server/src/main/java/com/metamx/druid/master/DruidCluster.java b/server/src/main/java/io/druid/server/master/DruidCluster.java similarity index 95% rename from server/src/main/java/com/metamx/druid/master/DruidCluster.java rename to server/src/main/java/io/druid/server/master/DruidCluster.java index d7ef284ede0..dd9d0c33bc5 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidCluster.java +++ b/server/src/main/java/io/druid/server/master/DruidCluster.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Ordering; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; import java.util.Map; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMaster.java b/server/src/main/java/io/druid/server/master/DruidMaster.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/DruidMaster.java rename to server/src/main/java/io/druid/server/master/DruidMaster.java index fa1731be37e..937c1c6bfa1 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMaster.java +++ b/server/src/main/java/io/druid/server/master/DruidMaster.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.base.Function; import com.google.common.base.Predicate; @@ -28,6 +28,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closeables; +import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.Pair; import com.metamx.common.concurrent.ScheduledExecutorFactory; @@ -36,23 +37,25 @@ import com.metamx.common.guava.Comparators; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.ServerInventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.initialization.ZkPathsConfig; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.ServerInventoryView; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.common.config.JacksonConfigManager; +import io.druid.concurrent.Execs; +import io.druid.db.DatabaseRuleManager; +import io.druid.db.DatabaseSegmentManager; +import io.druid.guice.ManageLifecycle; +import io.druid.segment.IndexIO; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.framework.recipes.leader.LeaderLatchListener; +import org.apache.curator.framework.recipes.leader.Participant; import org.apache.curator.utils.ZKPaths; import org.joda.time.DateTime; import org.joda.time.Duration; @@ -70,6 +73,7 @@ import java.util.concurrent.atomic.AtomicReference; /** */ +@ManageLifecycle public class DruidMaster { public static final String MASTER_OWNER_NODE = "_MASTER"; @@ -96,6 +100,7 @@ public class DruidMaster private final AtomicReference leaderLatch; private volatile AtomicReference segmentSettingsAtomicReference; + @Inject public DruidMaster( DruidMasterConfig config, ZkPathsConfig zkPaths, @@ -155,8 +160,8 @@ public class DruidMaster this.exec = scheduledExecutorFactory.create(1, "Master-Exec--%d"); - this.leaderLatch = new AtomicReference(null); - this.segmentSettingsAtomicReference= new AtomicReference(null); + this.leaderLatch = new AtomicReference<>(null); + this.segmentSettingsAtomicReference= new AtomicReference<>(null); this.loadManagementPeons = loadQueuePeonMap; } @@ -210,16 +215,6 @@ public class DruidMaster return loadStatus; } - public int lookupSegmentLifetime(DataSegment segment) - { - return serverInventoryView.lookupSegmentLifetime(segment); - } - - public void decrementRemovedSegmentsLifetime() - { - serverInventoryView.decrementRemovedSegmentsLifetime(); - } - public void removeSegment(DataSegment segment) { log.info("Removing Segment[%s]", segment); @@ -240,7 +235,17 @@ public class DruidMaster { try { final LeaderLatch latch = leaderLatch.get(); - return latch == null ? null : latch.getLeader().getId(); + + if (latch == null) { + return null; + } + + Participant participant = latch.getLeader(); + if (participant.isLeader()) { + return participant.getId(); + } + + return null; } catch (Exception e) { throw Throwables.propagate(e); @@ -726,45 +731,28 @@ public class DruidMaster SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(cluster); // Stop peons for servers that aren't there anymore. - for (String name : Sets.difference( - loadManagementPeons.keySet(), - Sets.newHashSet( - Iterables.transform( - servers, - new Function() - { - @Override - public String apply(@Nullable DruidServer input) - { - return input.getName(); - } - } - ) - ) - )) { + final Set disdappearedServers = Sets.newHashSet(loadManagementPeons.keySet()); + for (DruidServer server : servers) { + disdappearedServers.remove(server.getName()); + } + for (String name : disdappearedServers) { log.info("Removing listener for server[%s] which is no longer there.", name); LoadQueuePeon peon = loadManagementPeons.remove(name); peon.stop(); } - decrementRemovedSegmentsLifetime(); - return params.buildFromExisting() .withDruidCluster(cluster) .withDatabaseRuleManager(databaseRuleManager) .withLoadManagementPeons(loadManagementPeons) .withSegmentReplicantLookup(segmentReplicantLookup) .withBalancerReferenceTimestamp(DateTime.now()) - .withMasterSegmentSettings( - segmentSettingsAtomicReference.get() - ) + .withMasterSegmentSettings(segmentSettingsAtomicReference.get()) .build(); } }, new DruidMasterRuleRunner( - DruidMaster.this, - config.getReplicantLifetime(), - config.getReplicantThrottleLimit() + DruidMaster.this, config.getReplicantLifetime(), config.getReplicantThrottleLimit() ), new DruidMasterCleanup(DruidMaster.this), new DruidMasterBalancer(DruidMaster.this), diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java b/server/src/main/java/io/druid/server/master/DruidMasterBalancer.java similarity index 79% rename from server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java rename to server/src/main/java/io/druid/server/master/DruidMasterBalancer.java index bb547bae8d8..28c966b4162 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterBalancer.java @@ -1,31 +1,31 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.guava.Comparators; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; import com.metamx.emitter.EmittingLogger; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.Comparator; @@ -156,27 +156,33 @@ public class DruidMasterBalancer implements DruidMasterHelper (toServer.getSegment(segmentName) == null) && new ServerHolder(toServer, toPeon).getAvailableSize() > segmentToMove.getSize()) { log.info("Moving [%s] from [%s] to [%s]", segmentName, fromServerName, toServerName); + + LoadPeonCallback callback = null; try { + currentlyMovingSegments.get(toServer.getTier()).put(segmentName, segment); + callback = new LoadPeonCallback() + { + @Override + protected void execute() + { + Map movingSegments = currentlyMovingSegments.get(toServer.getTier()); + if (movingSegments != null) { + movingSegments.remove(segmentName); + } + } + }; master.moveSegment( fromServerName, toServerName, segmentToMove.getIdentifier(), - new LoadPeonCallback() - { - @Override - protected void execute() - { - Map movingSegments = currentlyMovingSegments.get(toServer.getTier()); - if (movingSegments != null) { - movingSegments.remove(segmentName); - } - } - } + callback ); - currentlyMovingSegments.get(toServer.getTier()).put(segmentName, segment); } catch (Exception e) { log.makeAlert(e, String.format("[%s] : Moving exception", segmentName)).emit(); + if (callback != null) { + callback.execute(); + } } } else { currentlyMovingSegments.get(toServer.getTier()).remove(segmentName); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterCleanup.java b/server/src/main/java/io/druid/server/master/DruidMasterCleanup.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/DruidMasterCleanup.java rename to server/src/main/java/io/druid/server/master/DruidMasterCleanup.java index 6ba31a77249..e31b1ae45c7 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterCleanup.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterCleanup.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.guava.Comparators; import com.metamx.common.logger.Logger; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java b/server/src/main/java/io/druid/server/master/DruidMasterConfig.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java rename to server/src/main/java/io/druid/server/master/DruidMasterConfig.java index 3e2b2719870..d279b91d5c1 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import org.joda.time.Duration; import org.skife.config.Config; @@ -54,12 +54,6 @@ public abstract class DruidMasterConfig return false; } - @Config("druid.master.merger.service") - public String getMergerServiceName() - { - return null; - } - @Config("druid.master.replicant.lifetime") @Default("15") public abstract int getReplicantLifetime(); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterHelper.java b/server/src/main/java/io/druid/server/master/DruidMasterHelper.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/DruidMasterHelper.java rename to server/src/main/java/io/druid/server/master/DruidMasterHelper.java index d218fa46f28..860a8546e7a 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterHelper.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; /** */ diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java b/server/src/main/java/io/druid/server/master/DruidMasterLogger.java similarity index 96% rename from server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java rename to server/src/main/java/io/druid/server/master/DruidMasterLogger.java index 1b226933a52..49a771177d3 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.collect.CountingMap; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.collections.CountingMap; +import io.druid.timeline.DataSegment; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuleRunner.java b/server/src/main/java/io/druid/server/master/DruidMasterRuleRunner.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/DruidMasterRuleRunner.java rename to server/src/main/java/io/druid/server/master/DruidMasterRuleRunner.java index 04a3ce55da6..6f23976f127 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuleRunner.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterRuleRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.master.rules.Rule; import com.metamx.emitter.EmittingLogger; +import io.druid.db.DatabaseRuleManager; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java b/server/src/main/java/io/druid/server/master/DruidMasterRuntimeParams.java similarity index 97% rename from server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java rename to server/src/main/java/io/druid/server/master/DruidMasterRuntimeParams.java index 0dfe9afca4f..9f89ad1f05b 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterRuntimeParams.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.metamx.common.guava.Comparators; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.emitter.service.ServiceEmitter; +import io.druid.client.DruidDataSource; +import io.druid.db.DatabaseRuleManager; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.Collection; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentInfoLoader.java b/server/src/main/java/io/druid/server/master/DruidMasterSegmentInfoLoader.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/DruidMasterSegmentInfoLoader.java rename to server/src/main/java/io/druid/server/master/DruidMasterSegmentInfoLoader.java index 2158328c1d8..b80eaf8b42d 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentInfoLoader.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterSegmentInfoLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java b/server/src/main/java/io/druid/server/master/DruidMasterSegmentMerger.java similarity index 96% rename from server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java rename to server/src/main/java/io/druid/server/master/DruidMasterSegmentMerger.java index d178cf1a8ad..0ca304ffe91 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterSegmentMerger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -32,12 +32,12 @@ import com.metamx.common.ISE; import com.metamx.common.Pair; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.logger.Logger; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.NoneShardSpec; +import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/master/LoadPeonCallback.java b/server/src/main/java/io/druid/server/master/LoadPeonCallback.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/LoadPeonCallback.java rename to server/src/main/java/io/druid/server/master/LoadPeonCallback.java index 309a9d26b1e..50a513961e9 100644 --- a/server/src/main/java/com/metamx/druid/master/LoadPeonCallback.java +++ b/server/src/main/java/io/druid/server/master/LoadPeonCallback.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; /** */ diff --git a/server/src/main/java/com/metamx/druid/master/LoadQueuePeon.java b/server/src/main/java/io/druid/server/master/LoadQueuePeon.java similarity index 97% rename from server/src/main/java/com/metamx/druid/master/LoadQueuePeon.java rename to server/src/main/java/io/druid/server/master/LoadQueuePeon.java index 2b5c96cb187..ece30058a3f 100644 --- a/server/src/main/java/com/metamx/druid/master/LoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/master/LoadQueuePeon.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; @@ -25,12 +25,12 @@ import com.google.common.collect.Collections2; import com.google.common.collect.Lists; import com.metamx.common.ISE; import com.metamx.common.guava.Comparators; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.coordination.DataSegmentChangeRequest; -import com.metamx.druid.coordination.SegmentChangeRequestDrop; -import com.metamx.druid.coordination.SegmentChangeRequestLoad; -import com.metamx.druid.coordination.SegmentChangeRequestNoop; import com.metamx.emitter.EmittingLogger; +import io.druid.server.coordination.DataSegmentChangeRequest; +import io.druid.server.coordination.SegmentChangeRequestDrop; +import io.druid.server.coordination.SegmentChangeRequestLoad; +import io.druid.server.coordination.SegmentChangeRequestNoop; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorWatcher; import org.apache.curator.utils.ZKPaths; diff --git a/server/src/main/java/com/metamx/druid/master/LoadQueueTaskMaster.java b/server/src/main/java/io/druid/server/master/LoadQueueTaskMaster.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/LoadQueueTaskMaster.java rename to server/src/main/java/io/druid/server/master/LoadQueueTaskMaster.java index 2547127bc5a..1d1d114d6dc 100644 --- a/server/src/main/java/com/metamx/druid/master/LoadQueueTaskMaster.java +++ b/server/src/main/java/io/druid/server/master/LoadQueueTaskMaster.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; import org.apache.curator.framework.CuratorFramework; -import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; /** @@ -35,6 +35,7 @@ public class LoadQueueTaskMaster private final ScheduledExecutorService peonExec; private final DruidMasterConfig config; + @Inject public LoadQueueTaskMaster( CuratorFramework curator, ObjectMapper jsonMapper, diff --git a/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java b/server/src/main/java/io/druid/server/master/MasterSegmentSettings.java similarity index 80% rename from server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java rename to server/src/main/java/io/druid/server/master/MasterSegmentSettings.java index 1bafb38ecc8..a31f9b8dce8 100644 --- a/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java +++ b/server/src/main/java/io/druid/server/master/MasterSegmentSettings.java @@ -1,22 +1,22 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/com/metamx/druid/master/MasterStats.java b/server/src/main/java/io/druid/server/master/MasterStats.java similarity index 95% rename from server/src/main/java/com/metamx/druid/master/MasterStats.java rename to server/src/main/java/io/druid/server/master/MasterStats.java index 7cdbe510395..ed23b9f0b79 100644 --- a/server/src/main/java/com/metamx/druid/master/MasterStats.java +++ b/server/src/main/java/io/druid/server/master/MasterStats.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; -import com.metamx.druid.collect.CountingMap; +import io.druid.collections.CountingMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; diff --git a/server/src/main/java/com/metamx/druid/master/MergerWhitelist.java b/server/src/main/java/io/druid/server/master/MergerWhitelist.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/MergerWhitelist.java rename to server/src/main/java/io/druid/server/master/MergerWhitelist.java index 8bfa7293b00..e84d91eead9 100644 --- a/server/src/main/java/com/metamx/druid/master/MergerWhitelist.java +++ b/server/src/main/java/io/druid/server/master/MergerWhitelist.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategy.java b/server/src/main/java/io/druid/server/master/RandomBalancerStrategy.java similarity index 58% rename from server/src/main/java/com/metamx/druid/master/RandomBalancerStrategy.java rename to server/src/main/java/io/druid/server/master/RandomBalancerStrategy.java index d953b69b3db..cc8e62afec6 100644 --- a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategy.java +++ b/server/src/main/java/io/druid/server/master/RandomBalancerStrategy.java @@ -1,25 +1,25 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.List; import java.util.Random; diff --git a/server/src/main/java/io/druid/server/master/RandomBalancerStrategyFactory.java b/server/src/main/java/io/druid/server/master/RandomBalancerStrategyFactory.java new file mode 100644 index 00000000000..e2f4641ed56 --- /dev/null +++ b/server/src/main/java/io/druid/server/master/RandomBalancerStrategyFactory.java @@ -0,0 +1,30 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; + +import org.joda.time.DateTime; + +public class RandomBalancerStrategyFactory implements BalancerStrategyFactory +{ + @Override + public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) + { + return new RandomBalancerStrategy(); + } +} diff --git a/server/src/main/java/com/metamx/druid/master/ReplicationThrottler.java b/server/src/main/java/io/druid/server/master/ReplicationThrottler.java similarity index 98% rename from server/src/main/java/com/metamx/druid/master/ReplicationThrottler.java rename to server/src/main/java/io/druid/server/master/ReplicationThrottler.java index fc2774374bd..9584bf46f8c 100644 --- a/server/src/main/java/com/metamx/druid/master/ReplicationThrottler.java +++ b/server/src/main/java/io/druid/server/master/ReplicationThrottler.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java b/server/src/main/java/io/druid/server/master/ReservoirSegmentSampler.java similarity index 50% rename from server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java rename to server/src/main/java/io/druid/server/master/ReservoirSegmentSampler.java index 4db994b821a..b50cfc8ace0 100644 --- a/server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java +++ b/server/src/main/java/io/druid/server/master/ReservoirSegmentSampler.java @@ -1,25 +1,25 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.List; import java.util.Random; diff --git a/server/src/main/java/com/metamx/druid/master/SegmentReplicantLookup.java b/server/src/main/java/io/druid/server/master/SegmentReplicantLookup.java similarity index 95% rename from server/src/main/java/com/metamx/druid/master/SegmentReplicantLookup.java rename to server/src/main/java/io/druid/server/master/SegmentReplicantLookup.java index 560787247f5..d455a89c5f9 100644 --- a/server/src/main/java/com/metamx/druid/master/SegmentReplicantLookup.java +++ b/server/src/main/java/io/druid/server/master/SegmentReplicantLookup.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Table; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; import java.util.Map; diff --git a/server/src/main/java/com/metamx/druid/master/ServerHolder.java b/server/src/main/java/io/druid/server/master/ServerHolder.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/ServerHolder.java rename to server/src/main/java/io/druid/server/master/ServerHolder.java index 5fba424f429..1a6d40b57b5 100644 --- a/server/src/main/java/com/metamx/druid/master/ServerHolder.java +++ b/server/src/main/java/io/druid/server/master/ServerHolder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/master/rules/DropRule.java b/server/src/main/java/io/druid/server/master/rules/DropRule.java similarity index 81% rename from server/src/main/java/com/metamx/druid/master/rules/DropRule.java rename to server/src/main/java/io/druid/server/master/rules/DropRule.java index 5806031d7c6..f9040cb46b7 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/DropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/DropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.MasterStats; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterRuntimeParams; +import io.druid.server.master.MasterStats; +import io.druid.timeline.DataSegment; /** * DropRules indicate when segments should be completely removed from the cluster. diff --git a/server/src/main/java/com/metamx/druid/master/rules/IntervalDropRule.java b/server/src/main/java/io/druid/server/master/rules/IntervalDropRule.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/rules/IntervalDropRule.java rename to server/src/main/java/io/druid/server/master/rules/IntervalDropRule.java index 1363c88b9da..f5b570ca626 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/IntervalDropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/IntervalDropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/master/rules/IntervalLoadRule.java b/server/src/main/java/io/druid/server/master/rules/IntervalLoadRule.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/rules/IntervalLoadRule.java rename to server/src/main/java/io/druid/server/master/rules/IntervalLoadRule.java index 0cfe603e2b1..436c2e98e0a 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/IntervalLoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/IntervalLoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/master/rules/LoadRule.java b/server/src/main/java/io/druid/server/master/rules/LoadRule.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/rules/LoadRule.java rename to server/src/main/java/io/druid/server/master/rules/LoadRule.java index b133af1799e..6caafeb15c3 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/LoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.google.common.collect.Lists; import com.google.common.collect.MinMaxPriorityQueue; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.master.BalancerStrategy; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.LoadPeonCallback; -import com.metamx.druid.master.MasterStats; -import com.metamx.druid.master.ReplicationThrottler; -import com.metamx.druid.master.ServerHolder; import com.metamx.emitter.EmittingLogger; +import io.druid.server.master.BalancerStrategy; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterRuntimeParams; +import io.druid.server.master.LoadPeonCallback; +import io.druid.server.master.MasterStats; +import io.druid.server.master.ReplicationThrottler; +import io.druid.server.master.ServerHolder; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.ArrayList; diff --git a/server/src/main/java/com/metamx/druid/master/rules/PeriodDropRule.java b/server/src/main/java/io/druid/server/master/rules/PeriodDropRule.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/rules/PeriodDropRule.java rename to server/src/main/java/io/druid/server/master/rules/PeriodDropRule.java index 5fa6affcf9e..7a490bccf91 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/PeriodDropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/PeriodDropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/main/java/com/metamx/druid/master/rules/PeriodLoadRule.java b/server/src/main/java/io/druid/server/master/rules/PeriodLoadRule.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/rules/PeriodLoadRule.java rename to server/src/main/java/io/druid/server/master/rules/PeriodLoadRule.java index 632b6acbc67..e75734a0737 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/PeriodLoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/PeriodLoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/main/java/com/metamx/druid/master/rules/Rule.java b/server/src/main/java/io/druid/server/master/rules/Rule.java similarity index 86% rename from server/src/main/java/com/metamx/druid/master/rules/Rule.java rename to server/src/main/java/io/druid/server/master/rules/Rule.java index 1c77a0ebc8f..c576dc3a9fd 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/Rule.java +++ b/server/src/main/java/io/druid/server/master/rules/Rule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.MasterStats; - +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterRuntimeParams; +import io.druid.server.master.MasterStats; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; /** diff --git a/server/src/main/java/com/metamx/druid/master/rules/RuleMap.java b/server/src/main/java/io/druid/server/master/rules/RuleMap.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/rules/RuleMap.java rename to server/src/main/java/io/druid/server/master/rules/RuleMap.java index 8354a5e110f..a8f80ee1b2d 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/RuleMap.java +++ b/server/src/main/java/io/druid/server/master/rules/RuleMap.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.google.common.collect.Lists; diff --git a/server/src/main/java/com/metamx/druid/master/rules/SizeDropRule.java b/server/src/main/java/io/druid/server/master/rules/SizeDropRule.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/rules/SizeDropRule.java rename to server/src/main/java/io/druid/server/master/rules/SizeDropRule.java index 0bd9f94cd00..f7ad948a54b 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/SizeDropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/SizeDropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Range; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; /** diff --git a/server/src/main/java/com/metamx/druid/master/rules/SizeLoadRule.java b/server/src/main/java/io/druid/server/master/rules/SizeLoadRule.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/rules/SizeLoadRule.java rename to server/src/main/java/io/druid/server/master/rules/SizeLoadRule.java index 421432ec6b9..df33cb0f50d 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/SizeLoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/SizeLoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Range; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; /** diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManagerConfig.java b/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java similarity index 59% rename from server/src/main/java/com/metamx/druid/db/DatabaseRuleManagerConfig.java rename to server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java index 41e21a0943b..9de045854de 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManagerConfig.java +++ b/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,24 +17,29 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.server.metrics; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.metrics.MonitorSchedulerConfig; import org.joda.time.Duration; -import org.skife.config.Config; -import org.skife.config.Default; +import org.joda.time.Period; /** */ -public abstract class DatabaseRuleManagerConfig +public class DruidMonitorSchedulerConfig extends MonitorSchedulerConfig { - @Config("druid.database.ruleTable") - public abstract String getRuleTable(); + @JsonProperty + private Period emissionPeriod = new Period("PT1M"); - @Config("druid.database.defaultDatasource") - @Default("_default") - public abstract String getDefaultDatasource(); + @JsonProperty + public Period getEmissionPeriod() + { + return emissionPeriod; + } - @Config("druid.database.rules.poll.duration") - @Default("PT1M") - public abstract Duration getRulesPollDuration(); + @Override + public Duration getEmitterPeriod() + { + return emissionPeriod.toStandardDuration(); + } } diff --git a/server/src/main/java/io/druid/server/metrics/MetricsModule.java b/server/src/main/java/io/druid/server/metrics/MetricsModule.java new file mode 100644 index 00000000000..ee7dd20c2f8 --- /dev/null +++ b/server/src/main/java/io/druid/server/metrics/MetricsModule.java @@ -0,0 +1,122 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.metrics; + +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Names; +import com.metamx.common.logger.Logger; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.metrics.Monitor; +import com.metamx.metrics.MonitorScheduler; +import io.druid.concurrent.Execs; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.JsonConfigurator; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Sets up the {@link MonitorScheduler} to monitor things on a regular schedule. {@link Monitor}s must be explicitly + * bound in order to be loaded. + */ +public class MetricsModule implements Module +{ + private static final Logger log = new Logger(MetricsModule.class); + + private final List> monitors = new CopyOnWriteArrayList>(); + public boolean configured = false; + + public MetricsModule register(Class monitorClazz) + { + synchronized (monitors) { + Preconditions.checkState(!configured, "Cannot register monitor[%s] after configuration.", monitorClazz); + } + monitors.add(monitorClazz); + return this; + } + + @Inject + public void setProperties(Properties props, JsonConfigurator configurator) + { + final MonitorsConfig config = configurator.configurate( + props, + "druid.monitoring", + MonitorsConfig.class + ); + + for (Class monitorClazz : config.getMonitors()) { + register(monitorClazz); + } + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.monitoring", DruidMonitorSchedulerConfig.class); + + for (Class monitor : monitors) { + binder.bind(monitor).in(LazySingleton.class); + } + + // Instantiate eagerly so that we get everything registered and put into the Lifecycle + binder.bind(Key.get(MonitorScheduler.class, Names.named("ForTheEagerness"))) + .to(MonitorScheduler.class) + .asEagerSingleton(); + } + + @Provides + @ManageLifecycle + public MonitorScheduler getMonitorScheduler( + Supplier config, + ServiceEmitter emitter, + Injector injector + ) + { + List monitors = Lists.newArrayList(); + + for (Key key : injector.getBindings().keySet()) { + if (Monitor.class.isAssignableFrom(key.getTypeLiteral().getRawType())) { + final Monitor monitor = (Monitor) injector.getInstance(key); + + log.info("Adding monitor[%s]", monitor); + + monitors.add(monitor); + } + } + + return new MonitorScheduler( + config.get(), + Execs.scheduledSingleThreaded("MonitorScheduler-%s"), + emitter, + monitors + ); + } +} diff --git a/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java new file mode 100644 index 00000000000..1bf1e78bbac --- /dev/null +++ b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.metrics; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; +import com.metamx.metrics.Monitor; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + */ +public class MonitorsConfig +{ + @JsonProperty("monitors") + @NotNull + private List> monitors = Lists.newArrayList(); + + public List> getMonitors() + { + return monitors; + } + + @Override + public String toString() + { + return "MonitorsConfig{" + + "monitors=" + monitors + + '}'; + } +} diff --git a/server/src/main/java/com/metamx/druid/metrics/ServerMonitor.java b/server/src/main/java/io/druid/server/metrics/ServerMonitor.java similarity index 60% rename from server/src/main/java/com/metamx/druid/metrics/ServerMonitor.java rename to server/src/main/java/io/druid/server/metrics/ServerMonitor.java index 7db7e3ba922..684068fa453 100644 --- a/server/src/main/java/com/metamx/druid/metrics/ServerMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/ServerMonitor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,52 +17,53 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.metrics; +package io.druid.server.metrics; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.coordination.DruidServerMetadata; -import com.metamx.druid.coordination.ServerManager; +import com.google.inject.Inject; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.metrics.AbstractMonitor; +import io.druid.client.DruidServerConfig; +import io.druid.server.coordination.ServerManager; import java.util.Map; public class ServerMonitor extends AbstractMonitor { - private final DruidServerMetadata druidServer; + private final DruidServerConfig serverConfig; private final ServerManager serverManager; + @Inject public ServerMonitor( - DruidServerMetadata druidServer, + DruidServerConfig serverConfig, ServerManager serverManager ) { - this.druidServer = druidServer; + this.serverConfig = serverConfig; this.serverManager = serverManager; } @Override public boolean doMonitor(ServiceEmitter emitter) { - emitter.emit(new ServiceMetricEvent.Builder().build("server/segment/max", druidServer.getMaxSize())); + emitter.emit(new ServiceMetricEvent.Builder().build("server/segment/max", serverConfig.getMaxSize())); for (Map.Entry entry : serverManager.getDataSourceSizes().entrySet()) { String dataSource = entry.getKey(); long used = entry.getValue(); - emitter.emit( - new ServiceMetricEvent.Builder() - .setUser1(dataSource) - .build("server/segment/used", used) - ); + final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setUser1(dataSource) + .setUser2(serverConfig.getTier()); + + emitter.emit(builder.build("server/segment/used", used)); + emitter.emit(builder.build("server/segment/usedPercent", used / (double) serverConfig.getMaxSize())); } + for (Map.Entry entry : serverManager.getDataSourceCounts().entrySet()) { String dataSource = entry.getKey(); long count = entry.getValue(); - emitter.emit( - new ServiceMetricEvent.Builder() - .setUser1(dataSource) - .build("server/segment/count", count) - ); + final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setUser1(dataSource) + .setUser2(serverConfig.getTier()); + + emitter.emit(builder.build("server/segment/count", count)); } return true; diff --git a/client/src/main/java/com/metamx/druid/sql/SQLRunner.java b/server/src/main/java/io/druid/server/sql/SQLRunner.java similarity index 85% rename from client/src/main/java/com/metamx/druid/sql/SQLRunner.java rename to server/src/main/java/io/druid/server/sql/SQLRunner.java index e2111060b01..561b10f2618 100644 --- a/client/src/main/java/com/metamx/druid/sql/SQLRunner.java +++ b/server/src/main/java/io/druid/server/sql/SQLRunner.java @@ -1,4 +1,23 @@ -package com.metamx.druid.sql; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.sql; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -8,17 +27,17 @@ import com.google.common.base.Joiner; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.Closeables; -import com.metamx.druid.Druids; -import com.metamx.druid.Query; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.input.Row; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.dimension.DimensionSpec; -import com.metamx.druid.query.group.GroupByQuery; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; -import com.metamx.druid.sql.antlr4.DruidSQLLexer; -import com.metamx.druid.sql.antlr4.DruidSQLParser; +import io.druid.data.input.Row; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.Query; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.timeseries.TimeseriesResultValue; +import io.druid.sql.antlr4.DruidSQLLexer; +import io.druid.sql.antlr4.DruidSQLParser; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; diff --git a/indexing-common/src/main/java/com/metamx/druid/common/s3/S3Utils.java b/server/src/main/java/io/druid/storage/s3/S3Utils.java similarity index 97% rename from indexing-common/src/main/java/com/metamx/druid/common/s3/S3Utils.java rename to server/src/main/java/io/druid/storage/s3/S3Utils.java index 84ce35df947..044b3fa76c9 100644 --- a/indexing-common/src/main/java/com/metamx/druid/common/s3/S3Utils.java +++ b/server/src/main/java/io/druid/storage/s3/S3Utils.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.common.s3; +package io.druid.storage.s3; import com.google.common.base.Throwables; import com.metamx.common.logger.Logger; diff --git a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java new file mode 100644 index 00000000000..4dd8c1002a0 --- /dev/null +++ b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java @@ -0,0 +1,61 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.timeline.partition; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import io.druid.data.input.InputRow; + +import java.util.Map; + +public class LinearShardSpec implements ShardSpec +{ + private int partitionNum; + + @JsonCreator + public LinearShardSpec( + @JsonProperty("partitionNum") Integer partitionNum + ) + { + this.partitionNum = Preconditions.checkNotNull(partitionNum, "Must set partitionNum on LinearShardSpec"); + } + + @JsonProperty("partitionNum") + @Override + public int getPartitionNum() { + return partitionNum; + } + + @Override + public PartitionChunk createChunk(T obj) { + return new LinearPartitionChunk(partitionNum, obj); + } + + @Override + public boolean isInChunk(Map dimensions) { + return true; + } + + @Override + public boolean isInChunk(InputRow inputRow) { + return true; + } +} diff --git a/client/src/main/java/com/metamx/druid/shard/NumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java similarity index 90% rename from client/src/main/java/com/metamx/druid/shard/NumberedShardSpec.java rename to server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java index 325446cb25b..e0e09c9fa76 100644 --- a/client/src/main/java/com/metamx/druid/shard/NumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.NumberedPartitionChunk; -import com.metamx.druid.partition.PartitionChunk; +import io.druid.data.input.InputRow; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/shard/SingleDimensionShardSpec.java b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/shard/SingleDimensionShardSpec.java rename to server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java index 98ff7f2eae1..e6b41f5a62a 100644 --- a/client/src/main/java/com/metamx/druid/shard/SingleDimensionShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.partition.StringPartitionChunk; +import io.druid.data.input.InputRow; import java.util.List; import java.util.Map; diff --git a/server/src/test/java/com/metamx/druid/master/LoadQueuePeonTester.java b/server/src/test/java/com/metamx/druid/master/LoadQueuePeonTester.java deleted file mode 100644 index 366cde87e1d..00000000000 --- a/server/src/test/java/com/metamx/druid/master/LoadQueuePeonTester.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.metamx.druid.master; - -import com.metamx.druid.client.DataSegment; - -import java.util.concurrent.ConcurrentSkipListSet; - -public class LoadQueuePeonTester extends LoadQueuePeon -{ - private final ConcurrentSkipListSet segmentsToLoad = new ConcurrentSkipListSet(); - - public LoadQueuePeonTester() - { - super(null, null, null, null, null); - } - - @Override - public void loadSegment( - DataSegment segment, - LoadPeonCallback callback - ) - { - segmentsToLoad.add(segment); - } - - public ConcurrentSkipListSet getSegmentsToLoad() - { - return segmentsToLoad; - } -} diff --git a/server/src/test/java/io/druid/TestUtil.java b/server/src/test/java/io/druid/TestUtil.java new file mode 100644 index 00000000000..e184c77998f --- /dev/null +++ b/server/src/test/java/io/druid/TestUtil.java @@ -0,0 +1,41 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.guice.ServerModule; +import io.druid.jackson.DefaultObjectMapper; + +import java.util.List; + +/** + */ +public class TestUtil +{ + public static final ObjectMapper MAPPER = new DefaultObjectMapper(); + + static { + final List list = new ServerModule().getJacksonModules(); + for (Module module : list) { + MAPPER.registerModule(module); + } + } +} diff --git a/client/src/test/java/com/metamx/druid/client/DataSegmentTest.java b/server/src/test/java/io/druid/client/DataSegmentTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/client/DataSegmentTest.java rename to server/src/test/java/io/druid/client/DataSegmentTest.java index 63f3326a92e..c79a965d47e 100644 --- a/client/src/test/java/com/metamx/druid/client/DataSegmentTest.java +++ b/server/src/test/java/io/druid/client/DataSegmentTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,11 +25,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.shard.NoneShardSpec; -import com.metamx.druid.shard.SingleDimensionShardSpec; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; +import io.druid.timeline.partition.SingleDimensionShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; diff --git a/client/src/test/java/com/metamx/druid/client/cache/ByteCountingLRUMapTest.java b/server/src/test/java/io/druid/client/cache/ByteCountingLRUMapTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/client/cache/ByteCountingLRUMapTest.java rename to server/src/test/java/io/druid/client/cache/ByteCountingLRUMapTest.java index fac858d2192..5edf6a669e6 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/ByteCountingLRUMapTest.java +++ b/server/src/test/java/io/druid/client/cache/ByteCountingLRUMapTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import org.junit.Assert; import org.junit.Before; diff --git a/client/src/test/java/com/metamx/druid/client/cache/MapCacheTest.java b/server/src/test/java/io/druid/client/cache/MapCacheTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/client/cache/MapCacheTest.java rename to server/src/test/java/io/druid/client/cache/MapCacheTest.java index 23a3bd1d641..fbff6f9fe2f 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MapCacheTest.java +++ b/server/src/test/java/io/druid/client/cache/MapCacheTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.primitives.Ints; import org.junit.Assert; diff --git a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBenchmark.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java similarity index 81% rename from client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBenchmark.java rename to server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java index c63e98e8988..7ef6047d9ac 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBenchmark.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java @@ -1,4 +1,23 @@ -package com.metamx.druid.client.cache; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; import com.google.caliper.Param; import com.google.caliper.Runner; diff --git a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheTest.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java similarity index 99% rename from client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheTest.java rename to server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java index 23ca0ea9693..55c85f967f0 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheTest.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/client/src/test/java/com/metamx/druid/client/BatchServerInventoryViewTest.java b/server/src/test/java/io/druid/client/client/BatchServerInventoryViewTest.java similarity index 83% rename from client/src/test/java/com/metamx/druid/client/BatchServerInventoryViewTest.java rename to server/src/test/java/io/druid/client/client/BatchServerInventoryViewTest.java index 7690b0ab06b..38864822eb8 100644 --- a/client/src/test/java/com/metamx/druid/client/BatchServerInventoryViewTest.java +++ b/server/src/test/java/io/druid/client/client/BatchServerInventoryViewTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -26,13 +26,16 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.ISE; -import com.metamx.druid.coordination.BatchDataSegmentAnnouncer; -import com.metamx.druid.coordination.DruidServerMetadata; -import com.metamx.druid.curator.PotentiallyGzippedCompressionProvider; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.client.BatchServerInventoryView; +import io.druid.client.DruidServer; +import io.druid.curator.PotentiallyGzippedCompressionProvider; +import io.druid.curator.announcement.Announcer; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DruidServerMetadata; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import junit.framework.Assert; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -45,7 +48,6 @@ import org.junit.Before; import org.junit.Test; import java.util.Set; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** @@ -93,30 +95,20 @@ public class BatchServerInventoryViewTest "type", "tier" ), - new ZkDataSegmentAnnouncerConfig() + new BatchDataSegmentAnnouncerConfig() { - @Override - public String getZkBasePath() - { - return testBasePath; - } - @Override public int getSegmentsPerNode() { return 50; } - + }, + new ZkPathsConfig() + { @Override - public long getMaxNumBytes() + public String getZkBasePath() { - return 100000; - } - - @Override - public String getAnnouncerType() - { - return "batch"; + return testBasePath; } }, announcer, @@ -130,20 +122,6 @@ public class BatchServerInventoryViewTest } batchServerInventoryView = new BatchServerInventoryView( - new ServerInventoryViewConfig() - { - @Override - public int getRemovedSegmentLifetime() - { - return 0; - } - - @Override - public String getAnnouncerType() - { - return "batch"; - } - }, new ZkPathsConfig() { @Override @@ -153,7 +131,6 @@ public class BatchServerInventoryViewTest } }, cf, - Executors.newSingleThreadExecutor(), jsonMapper ); diff --git a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java b/server/src/test/java/io/druid/client/selector/ServerSelectorTest.java similarity index 89% rename from client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java rename to server/src/test/java/io/druid/client/selector/ServerSelectorTest.java index 9009ec3d246..d66648beb68 100644 --- a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java +++ b/server/src/test/java/io/druid/client/selector/ServerSelectorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,22 +17,22 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.selector; +package io.druid.client.selector; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.SettableFuture; -import com.metamx.druid.Druids; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DirectDruidClient; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.ReflectionQueryToolChestWarehouse; -import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.http.client.HttpClient; import com.metamx.http.client.Request; import com.metamx.http.client.RequestBuilder; +import io.druid.client.DirectDruidClient; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.ReflectionQueryToolChestWarehouse; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpMethod; diff --git a/client/src/test/java/com/metamx/druid/curator/CuratorTestBase.java b/server/src/test/java/io/druid/curator/CuratorTestBase.java similarity index 54% rename from client/src/test/java/com/metamx/druid/curator/CuratorTestBase.java rename to server/src/test/java/io/druid/curator/CuratorTestBase.java index dcc0cfda60e..a850d756ea3 100644 --- a/client/src/test/java/com/metamx/druid/curator/CuratorTestBase.java +++ b/server/src/test/java/io/druid/curator/CuratorTestBase.java @@ -1,4 +1,23 @@ -package com.metamx.druid.curator; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator; import com.google.common.io.Closeables; import org.apache.curator.framework.CuratorFramework; diff --git a/client/src/test/java/com/metamx/druid/curator/announcement/AnnouncerTest.java b/server/src/test/java/io/druid/curator/announcement/AnnouncerTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/curator/announcement/AnnouncerTest.java rename to server/src/test/java/io/druid/curator/announcement/AnnouncerTest.java index 9f4276e164a..528b0fd77f4 100644 --- a/client/src/test/java/com/metamx/druid/curator/announcement/AnnouncerTest.java +++ b/server/src/test/java/io/druid/curator/announcement/AnnouncerTest.java @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.announcement; +package io.druid.curator.announcement; import com.google.common.collect.Sets; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.curator.CuratorTestBase; +import io.druid.concurrent.Execs; +import io.druid.curator.CuratorTestBase; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorEvent; import org.apache.curator.framework.api.CuratorEventType; @@ -114,8 +114,6 @@ public class AnnouncerTest extends CuratorTestBase @Test public void testSessionKilled() throws Exception { - final ExecutorService exec = Execs.singleThreaded("test-announcer-sanity-%s"); - curator.start(); Announcer announcer = new Announcer(curator, exec); try { diff --git a/client/src/test/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerTest.java b/server/src/test/java/io/druid/curator/inventory/CuratorInventoryManagerTest.java similarity index 88% rename from client/src/test/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerTest.java rename to server/src/test/java/io/druid/curator/inventory/CuratorInventoryManagerTest.java index da72db06048..1c13475912c 100644 --- a/client/src/test/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerTest.java +++ b/server/src/test/java/io/druid/curator/inventory/CuratorInventoryManagerTest.java @@ -1,10 +1,28 @@ -package com.metamx.druid.curator.inventory; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.inventory; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.curator.CuratorTestBase; +import io.druid.concurrent.Execs; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorEvent; import org.apache.curator.framework.api.CuratorEventType; @@ -21,7 +39,7 @@ import java.util.concurrent.ExecutorService; /** */ -public class CuratorInventoryManagerTest extends CuratorTestBase +public class CuratorInventoryManagerTest extends io.druid.curator.CuratorTestBase { private ExecutorService exec; diff --git a/server/src/test/java/com/metamx/druid/db/DatabaseSegmentManagerTest.java b/server/src/test/java/io/druid/db/DatabaseSegmentManagerTest.java similarity index 85% rename from server/src/test/java/com/metamx/druid/db/DatabaseSegmentManagerTest.java rename to server/src/test/java/io/druid/db/DatabaseSegmentManagerTest.java index 751cad29a13..267f86a8a56 100644 --- a/server/src/test/java/com/metamx/druid/db/DatabaseSegmentManagerTest.java +++ b/server/src/test/java/io/druid/db/DatabaseSegmentManagerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,52 +17,38 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.db; +import com.google.common.base.Suppliers; import com.google.common.collect.Maps; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.jackson.DefaultObjectMapper; import org.easymock.EasyMock; -import org.joda.time.Duration; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.skife.jdbi.v2.DBI; +import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.tweak.HandleCallback; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; /** */ public class DatabaseSegmentManagerTest { private DatabaseSegmentManager manager; - private DBI dbi; + private IDBI dbi; private List> testRows; @Before public void setUp() throws Exception { - dbi = EasyMock.createMock(DBI.class); + dbi = EasyMock.createMock(IDBI.class); manager = new DatabaseSegmentManager( new DefaultObjectMapper(), - EasyMock.createMock(ScheduledExecutorService.class), - new DatabaseSegmentManagerConfig() - { - @Override - public String getSegmentTable() - { - return null; - } - - @Override - public Duration getPollDuration() - { - return null; - } - }, + Suppliers.ofInstance(new DatabaseSegmentManagerConfig()), + Suppliers.ofInstance(DbTablesConfig.fromBase("test")), dbi ); @@ -108,10 +94,7 @@ public class DatabaseSegmentManagerTest + "\"twitterstream_2012-01-05T00:00:00.000Z_2012-01-06T00:00:00.000Z_2012-01-06T22:19:12.565Z\"}" ); - testRows = Arrays.>asList( - map1, - map2 - ); + testRows = Arrays.>asList(map1, map2); } @After @@ -128,5 +111,6 @@ public class DatabaseSegmentManagerTest manager.start(); manager.poll(); + manager.stop(); } } diff --git a/server/src/test/java/com/metamx/druid/loading/CacheTestSegmentLoader.java b/server/src/test/java/io/druid/segment/loading/CacheTestSegmentLoader.java similarity index 83% rename from server/src/test/java/com/metamx/druid/loading/CacheTestSegmentLoader.java rename to server/src/test/java/io/druid/segment/loading/CacheTestSegmentLoader.java index ebf6ae09b10..42f16fd21c1 100644 --- a/server/src/test/java/com/metamx/druid/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/io/druid/segment/loading/CacheTestSegmentLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.metamx.common.MapUtils; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.Segment; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import java.io.File; @@ -77,6 +77,12 @@ public class CacheTestSegmentLoader implements SegmentLoader }; } + @Override + public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException + { + throw new UnsupportedOperationException(); + } + @Override public void cleanup(DataSegment loadSpec) throws SegmentLoadingException { diff --git a/server/src/test/java/com/metamx/druid/loading/DataSegmentPusherUtilTest.java b/server/src/test/java/io/druid/segment/loading/DataSegmentPusherUtilTest.java similarity index 51% rename from server/src/test/java/com/metamx/druid/loading/DataSegmentPusherUtilTest.java rename to server/src/test/java/io/druid/segment/loading/DataSegmentPusherUtilTest.java index 2337f882098..d58c210eb4b 100644 --- a/server/src/test/java/com/metamx/druid/loading/DataSegmentPusherUtilTest.java +++ b/server/src/test/java/io/druid/segment/loading/DataSegmentPusherUtilTest.java @@ -1,9 +1,28 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/loading/SingleSegmentLoaderTest.java b/server/src/test/java/io/druid/segment/loading/StorageLocationTest.java similarity index 59% rename from server/src/test/java/com/metamx/druid/loading/SingleSegmentLoaderTest.java rename to server/src/test/java/io/druid/segment/loading/StorageLocationTest.java index f82311609ac..ca499d041e5 100644 --- a/server/src/test/java/com/metamx/druid/loading/SingleSegmentLoaderTest.java +++ b/server/src/test/java/io/druid/segment/loading/StorageLocationTest.java @@ -1,7 +1,26 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; @@ -11,13 +30,13 @@ import java.util.Arrays; /** */ -public class SingleSegmentLoaderTest +public class StorageLocationTest { @Test public void testStorageLocation() throws Exception { long expectedAvail = 1000l; - SingleSegmentLoader.StorageLocation loc = new SingleSegmentLoader.StorageLocation(new File("/tmp"), expectedAvail); + StorageLocation loc = new StorageLocation(new File("/tmp"), expectedAvail); verifyLoc(expectedAvail, loc); @@ -46,7 +65,7 @@ public class SingleSegmentLoaderTest verifyLoc(expectedAvail, loc); } - private void verifyLoc(long maxSize, SingleSegmentLoader.StorageLocation loc) + private void verifyLoc(long maxSize, StorageLocation loc) { Assert.assertEquals(maxSize, loc.available()); for (int i = 0; i <= maxSize; ++i) { diff --git a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestDropTest.java b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestDropTest.java similarity index 90% rename from server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestDropTest.java rename to server/src/test/java/io/druid/server/coordination/SegmentChangeRequestDropTest.java index 92026a533e6..1d87114032e 100644 --- a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestDropTest.java +++ b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestDropTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.shard.NoneShardSpec; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestLoadTest.java b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestLoadTest.java similarity index 90% rename from server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestLoadTest.java rename to server/src/test/java/io/druid/server/coordination/SegmentChangeRequestLoadTest.java index 62688974180..fb2745df042 100644 --- a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestLoadTest.java +++ b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestLoadTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.shard.NoneShardSpec; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/coordination/ServerManagerTest.java b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/coordination/ServerManagerTest.java rename to server/src/test/java/io/druid/server/coordination/ServerManagerTest.java index f2cd54f4551..f3483487c1d 100644 --- a/server/src/test/java/com/metamx/druid/coordination/ServerManagerTest.java +++ b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; @@ -35,36 +35,37 @@ import com.metamx.common.guava.Sequences; import com.metamx.common.guava.Yielder; import com.metamx.common.guava.YieldingAccumulator; import com.metamx.common.guava.YieldingSequenceBase; -import com.metamx.druid.Druids; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.ReferenceCountingSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.loading.SegmentLoadingException; -import com.metamx.druid.metrics.NoopServiceEmitter; -import com.metamx.druid.query.ConcatQueryRunner; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.NoopQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.granularity.QueryGranularity; +import io.druid.query.ConcatQueryRunner; +import io.druid.query.Druids; +import io.druid.query.NoopQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerFactoryConglomerate; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.search.SearchResultValue; +import io.druid.query.search.search.SearchQuery; +import io.druid.segment.IndexIO; +import io.druid.segment.QueryableIndex; +import io.druid.segment.ReferenceCountingSegment; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.segment.loading.SegmentLoader; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.server.metrics.NoopServiceEmitter; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Iterator; @@ -114,6 +115,12 @@ public class ServerManagerTest ); } + @Override + public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException + { + throw new UnsupportedOperationException(); + } + @Override public void cleanup(DataSegment segment) throws SegmentLoadingException { diff --git a/server/src/test/java/com/metamx/druid/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java similarity index 81% rename from server/src/test/java/com/metamx/druid/coordination/ZkCoordinatorTest.java rename to server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index 746e7035cbb..e1d78741b7c 100644 --- a/server/src/test/java/com/metamx/druid/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,24 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.curator.CuratorTestBase; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.loading.CacheTestSegmentLoader; -import com.metamx.druid.metrics.NoopServiceEmitter; -import com.metamx.druid.query.NoopQueryRunnerFactoryConglomerate; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.concurrent.Execs; +import io.druid.curator.CuratorTestBase; +import io.druid.curator.announcement.Announcer; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.NoopQueryRunnerFactoryConglomerate; +import io.druid.segment.IndexIO; +import io.druid.segment.loading.CacheTestSegmentLoader; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.server.metrics.NoopServiceEmitter; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.After; import org.junit.Assert; @@ -54,7 +55,7 @@ public class ZkCoordinatorTest extends CuratorTestBase private ZkCoordinator zkCoordinator; private ServerManager serverManager; private DataSegmentAnnouncer announcer; - private File cacheDir; + private File infoDir; private final ObjectMapper jsonMapper = new DefaultObjectMapper(); private static final Logger log = new Logger(ZkCoordinatorTest.class); @@ -64,12 +65,12 @@ public class ZkCoordinatorTest extends CuratorTestBase setupServerAndCurator(); curator.start(); try { - cacheDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest"); - cacheDir.mkdirs(); - for (File file : cacheDir.listFiles()) { + infoDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest"); + infoDir.mkdirs(); + for (File file : infoDir.listFiles()) { file.delete(); } - log.info("Creating tmp test files in [%s]", cacheDir); + log.info("Creating tmp test files in [%s]", infoDir); } catch (IOException e) { throw new RuntimeException(e); @@ -99,12 +100,11 @@ public class ZkCoordinatorTest extends CuratorTestBase zkCoordinator = new ZkCoordinator( jsonMapper, - new ZkCoordinatorConfig() - { + new SegmentLoaderConfig(){ @Override - public File getSegmentInfoCacheDirectory() + public File getInfoDir() { - return cacheDir; + return infoDir; } }, zkPaths, @@ -155,8 +155,8 @@ public class ZkCoordinatorTest extends CuratorTestBase deleteSegmentFromCache(segment); } - Assert.assertEquals(0, cacheDir.listFiles().length); - Assert.assertTrue(cacheDir.delete()); + Assert.assertEquals(0, infoDir.listFiles().length); + Assert.assertTrue(infoDir.delete()); } private DataSegment makeSegment(String dataSource, String version, Interval interval) @@ -165,7 +165,7 @@ public class ZkCoordinatorTest extends CuratorTestBase dataSource, interval, version, - ImmutableMap.of("version", version, "interval", interval, "cacheDir", cacheDir), + ImmutableMap.of("version", version, "interval", interval, "cacheDir", infoDir), Arrays.asList("dim1", "dim2", "dim3"), Arrays.asList("metric1", "metric2"), new NoneShardSpec(), @@ -176,12 +176,12 @@ public class ZkCoordinatorTest extends CuratorTestBase private void writeSegmentToCache(final DataSegment segment) throws IOException { - if (!cacheDir.exists()) { - cacheDir.mkdir(); + if (!infoDir.exists()) { + infoDir.mkdir(); } File segmentInfoCacheFile = new File( - cacheDir, + infoDir, segment.getIdentifier() ); try { @@ -197,7 +197,7 @@ public class ZkCoordinatorTest extends CuratorTestBase private void deleteSegmentFromCache(final DataSegment segment) throws IOException { File segmentInfoCacheFile = new File( - cacheDir, + infoDir, segment.getIdentifier() ); if (segmentInfoCacheFile.exists()) { @@ -209,8 +209,8 @@ public class ZkCoordinatorTest extends CuratorTestBase private void checkCache(List segments) throws IOException { - Assert.assertTrue(cacheDir.exists()); - File[] files = cacheDir.listFiles(); + Assert.assertTrue(infoDir.exists()); + File[] files = infoDir.listFiles(); List sortedFiles = Lists.newArrayList(files); Collections.sort(sortedFiles); diff --git a/client/src/test/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncerTest.java b/server/src/test/java/io/druid/server/coordination/coordination/BatchDataSegmentAnnouncerTest.java similarity index 91% rename from client/src/test/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncerTest.java rename to server/src/test/java/io/druid/server/coordination/coordination/BatchDataSegmentAnnouncerTest.java index 608fe850a74..245a3573c5f 100644 --- a/client/src/test/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncerTest.java +++ b/server/src/test/java/io/druid/server/coordination/coordination/BatchDataSegmentAnnouncerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,11 +25,14 @@ import com.google.common.base.Joiner; import com.google.common.base.Throwables; import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.PotentiallyGzippedCompressionProvider; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.curator.PotentiallyGzippedCompressionProvider; +import io.druid.curator.announcement.Announcer; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DruidServerMetadata; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import junit.framework.Assert; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -92,30 +95,20 @@ public class BatchDataSegmentAnnouncerTest "type", "tier" ), - new ZkDataSegmentAnnouncerConfig() + new BatchDataSegmentAnnouncerConfig() { - @Override - public String getZkBasePath() - { - return testBasePath; - } - @Override public int getSegmentsPerNode() { return 50; } - + }, + new ZkPathsConfig() + { @Override - public long getMaxNumBytes() + public String getZkBasePath() { - return 100000; - } - - @Override - public String getAnnouncerType() - { - return "batch"; + return testBasePath; } }, announcer, diff --git a/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java b/server/src/test/java/io/druid/server/master/DruidMasterBalancerProfiler.java similarity index 85% rename from server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java rename to server/src/test/java/io/druid/server/master/DruidMasterBalancerProfiler.java index 482274c0b28..9830f3c71ef 100644 --- a/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterBalancerProfiler.java @@ -1,23 +1,23 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.utils; +package io.druid.server.master; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; @@ -25,26 +25,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.master.DruidCluster; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterBalancerTester; -import com.metamx.druid.master.DruidMasterRuleRunner; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.LoadPeonCallback; -import com.metamx.druid.master.LoadQueuePeon; -import com.metamx.druid.master.LoadQueuePeonTester; -import com.metamx.druid.master.MasterSegmentSettings; -import com.metamx.druid.master.ReplicationThrottler; -import com.metamx.druid.master.SegmentReplicantLookup; -import com.metamx.druid.master.ServerHolder; -import com.metamx.druid.master.rules.PeriodLoadRule; -import com.metamx.druid.master.rules.Rule; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; +import io.druid.client.DruidServer; +import io.druid.db.DatabaseRuleManager; +import io.druid.server.master.rules.PeriodLoadRule; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.easymock.EasyMock; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterBalancerTest.java index 8513c157de3..a284bbe9931 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTest.java @@ -1,31 +1,31 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.joda.time.DateTime; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTester.java b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTester.java similarity index 63% rename from server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTester.java rename to server/src/test/java/io/druid/server/master/DruidMasterBalancerTester.java index 3b5865fb361..e2afe47c909 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTester.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTester.java @@ -1,7 +1,26 @@ -package com.metamx.druid.master; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +package io.druid.server.master; + +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; public class DruidMasterBalancerTester extends DruidMasterBalancer { diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java b/server/src/test/java/io/druid/server/master/DruidMasterRuleRunnerTest.java similarity index 98% rename from server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterRuleRunnerTest.java index 27dc46636ac..937430d5d15 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterRuleRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -25,17 +25,17 @@ import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.master.rules.IntervalDropRule; -import com.metamx.druid.master.rules.IntervalLoadRule; -import com.metamx.druid.master.rules.Rule; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.client.DruidServer; +import io.druid.db.DatabaseRuleManager; +import io.druid.segment.IndexIO; +import io.druid.server.master.rules.IntervalDropRule; +import io.druid.server.master.rules.IntervalLoadRule; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.easymock.EasyMock; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java b/server/src/test/java/io/druid/server/master/DruidMasterSegmentMergerTest.java similarity index 98% rename from server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterSegmentMergerTest.java index 04caa13a8a4..37392ac2e9a 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterSegmentMergerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.shard.LinearShardSpec; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.LinearShardSpec; import junit.framework.Assert; import org.joda.time.Interval; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java b/server/src/test/java/io/druid/server/master/DruidMasterTest.java similarity index 89% rename from server/src/test/java/com/metamx/druid/master/DruidMasterTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterTest.java index 80993ed62f2..a1334718e94 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.MapMaker; import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.SingleServerInventoryView; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.metrics.NoopServiceEmitter; +import io.druid.client.DruidServer; +import io.druid.client.SingleServerInventoryView; +import io.druid.curator.inventory.InventoryManagerConfig; +import io.druid.db.DatabaseSegmentManager; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.server.metrics.NoopServiceEmitter; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.easymock.EasyMock; import org.joda.time.Duration; @@ -94,12 +94,6 @@ public class DruidMasterTest return null; } - @Override - public String getMergerServiceName() - { - return ""; - } - @Override public int getReplicantLifetime() { diff --git a/server/src/test/java/io/druid/server/master/LoadQueuePeonTester.java b/server/src/test/java/io/druid/server/master/LoadQueuePeonTester.java new file mode 100644 index 00000000000..a8e53d1504d --- /dev/null +++ b/server/src/test/java/io/druid/server/master/LoadQueuePeonTester.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; + +import io.druid.timeline.DataSegment; + +import java.util.concurrent.ConcurrentSkipListSet; + +public class LoadQueuePeonTester extends LoadQueuePeon +{ + private final ConcurrentSkipListSet segmentsToLoad = new ConcurrentSkipListSet(); + + public LoadQueuePeonTester() + { + super(null, null, null, null, null); + } + + @Override + public void loadSegment( + DataSegment segment, + LoadPeonCallback callback + ) + { + segmentsToLoad.add(segment); + } + + public ConcurrentSkipListSet getSegmentsToLoad() + { + return segmentsToLoad; + } +} diff --git a/server/src/test/java/com/metamx/druid/master/ReservoirSegmentSamplerTest.java b/server/src/test/java/io/druid/server/master/ReservoirSegmentSamplerTest.java similarity index 88% rename from server/src/test/java/com/metamx/druid/master/ReservoirSegmentSamplerTest.java rename to server/src/test/java/io/druid/server/master/ReservoirSegmentSamplerTest.java index 7b768e11172..9561e5feda7 100644 --- a/server/src/test/java/com/metamx/druid/master/ReservoirSegmentSamplerTest.java +++ b/server/src/test/java/io/druid/server/master/ReservoirSegmentSamplerTest.java @@ -1,29 +1,30 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.joda.time.DateTime; diff --git a/server/src/test/java/com/metamx/druid/master/rules/PeriodDropRuleTest.java b/server/src/test/java/io/druid/server/master/rules/PeriodDropRuleTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/master/rules/PeriodDropRuleTest.java rename to server/src/test/java/io/druid/server/master/rules/PeriodDropRuleTest.java index c6bceb08e5e..3b4967a1c7d 100644 --- a/server/src/test/java/com/metamx/druid/master/rules/PeriodDropRuleTest.java +++ b/server/src/test/java/io/druid/server/master/rules/PeriodDropRuleTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/test/java/com/metamx/druid/master/rules/PeriodLoadRuleTest.java b/server/src/test/java/io/druid/server/master/rules/PeriodLoadRuleTest.java similarity index 93% rename from server/src/test/java/com/metamx/druid/master/rules/PeriodLoadRuleTest.java rename to server/src/test/java/io/druid/server/master/rules/PeriodLoadRuleTest.java index 3944d96ecb9..de10902c6f2 100644 --- a/server/src/test/java/com/metamx/druid/master/rules/PeriodLoadRuleTest.java +++ b/server/src/test/java/io/druid/server/master/rules/PeriodLoadRuleTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/test/java/com/metamx/druid/metrics/NoopServiceEmitter.java b/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java similarity index 92% rename from server/src/test/java/com/metamx/druid/metrics/NoopServiceEmitter.java rename to server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java index f980bc4885c..8a9a5944099 100644 --- a/server/src/test/java/com/metamx/druid/metrics/NoopServiceEmitter.java +++ b/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.metrics; +package io.druid.server.metrics; import com.metamx.emitter.core.Event; import com.metamx.emitter.service.ServiceEmitter; diff --git a/client/src/test/java/com/metamx/druid/shard/NumberedShardSpecTest.java b/server/src/test/java/io/druid/server/shard/NumberedShardSpecTest.java similarity index 85% rename from client/src/test/java/com/metamx/druid/shard/NumberedShardSpecTest.java rename to server/src/test/java/io/druid/server/shard/NumberedShardSpecTest.java index 15a9c898922..cc553324562 100644 --- a/client/src/test/java/com/metamx/druid/shard/NumberedShardSpecTest.java +++ b/server/src/test/java/io/druid/server/shard/NumberedShardSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.server.shard; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.partition.PartitionChunk; +import io.druid.TestUtil; +import io.druid.timeline.partition.NumberedShardSpec; +import io.druid.timeline.partition.PartitionChunk; +import io.druid.timeline.partition.ShardSpec; import junit.framework.Assert; import org.junit.Test; @@ -35,9 +36,8 @@ public class NumberedShardSpecTest @Test public void testSerdeRoundTrip() throws Exception { - final ObjectMapper jsonMapper = new DefaultObjectMapper(); - final ShardSpec spec = jsonMapper.readValue( - jsonMapper.writeValueAsBytes(new NumberedShardSpec(1, 2)), + final ShardSpec spec = TestUtil.MAPPER.readValue( + TestUtil.MAPPER.writeValueAsBytes(new NumberedShardSpec(1, 2)), ShardSpec.class ); Assert.assertEquals(1, spec.getPartitionNum()); @@ -47,8 +47,7 @@ public class NumberedShardSpecTest @Test public void testSerdeBackwardsCompat() throws Exception { - final ObjectMapper jsonMapper = new DefaultObjectMapper(); - final ShardSpec spec = jsonMapper.readValue( + final ShardSpec spec = TestUtil.MAPPER.readValue( "{\"type\": \"numbered\", \"partitions\": 2, \"partitionNum\": 1}", ShardSpec.class ); diff --git a/client/src/test/java/com/metamx/druid/shard/SingleDimensionShardSpecTest.java b/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java similarity index 94% rename from client/src/test/java/com/metamx/druid/shard/SingleDimensionShardSpecTest.java rename to server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java index 217b18d9be7..e262bab1f98 100644 --- a/client/src/test/java/com/metamx/druid/shard/SingleDimensionShardSpecTest.java +++ b/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.server.shard; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -26,12 +26,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.metamx.common.Pair; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedInputRow; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.timeline.partition.SingleDimensionShardSpec; import org.junit.Assert; import org.junit.Test; -import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/services/pom.xml b/services/pom.xml index de247ffba4b..0f202976a75 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -1,7 +1,7 @@