mirror of https://github.com/apache/druid.git
Some things that didn't get committed with the merge for some reason!?
This commit is contained in:
parent
5fa944dd26
commit
8097450d8c
|
@ -22,7 +22,6 @@ package com.metamx.druid.index.v1;
|
||||||
import com.google.common.base.Function;
|
import com.google.common.base.Function;
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import com.google.common.base.Splitter;
|
import com.google.common.base.Splitter;
|
||||||
import com.google.common.base.Stopwatch;
|
|
||||||
import com.google.common.collect.Iterables;
|
import com.google.common.collect.Iterables;
|
||||||
import com.google.common.collect.Iterators;
|
import com.google.common.collect.Iterators;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
|
|
@ -1,10 +1,10 @@
|
||||||
package com.metamx.druid.indexer.data;
|
package com.metamx.druid.indexer.data;
|
||||||
|
|
||||||
import java.nio.ByteBuffer;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonSubTypes;
|
import com.fasterxml.jackson.annotation.JsonSubTypes;
|
||||||
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
import com.fasterxml.jackson.annotation.JsonTypeInfo;
|
||||||
|
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class)
|
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class)
|
||||||
@JsonSubTypes(value = {
|
@JsonSubTypes(value = {
|
||||||
@JsonSubTypes.Type(name = "protobuf", value = ProtoBufInputRowParser.class),
|
@JsonSubTypes.Type(name = "protobuf", value = ProtoBufInputRowParser.class),
|
||||||
|
|
|
@ -1,14 +1,5 @@
|
||||||
package com.metamx.druid.indexer.data;
|
package com.metamx.druid.indexer.data;
|
||||||
|
|
||||||
import static com.google.protobuf.DescriptorProtos.FileDescriptorSet;
|
|
||||||
import static com.google.protobuf.Descriptors.Descriptor;
|
|
||||||
import static com.google.protobuf.Descriptors.FileDescriptor;
|
|
||||||
|
|
||||||
import java.io.InputStream;
|
|
||||||
import java.nio.ByteBuffer;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.base.Throwables;
|
import com.google.common.base.Throwables;
|
||||||
|
@ -20,6 +11,15 @@ import com.google.protobuf.InvalidProtocolBufferException;
|
||||||
import com.metamx.common.exception.FormattedException;
|
import com.metamx.common.exception.FormattedException;
|
||||||
import com.metamx.druid.input.InputRow;
|
import com.metamx.druid.input.InputRow;
|
||||||
|
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import static com.google.protobuf.DescriptorProtos.FileDescriptorSet;
|
||||||
|
import static com.google.protobuf.Descriptors.Descriptor;
|
||||||
|
import static com.google.protobuf.Descriptors.FileDescriptor;
|
||||||
|
|
||||||
public class ProtoBufInputRowParser implements ByteBufferInputRowParser
|
public class ProtoBufInputRowParser implements ByteBufferInputRowParser
|
||||||
{
|
{
|
||||||
|
|
||||||
|
|
|
@ -1,18 +1,16 @@
|
||||||
package com.metamx.druid.indexer.data;
|
package com.metamx.druid.indexer.data;
|
||||||
|
|
||||||
import static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ONE;
|
import com.metamx.druid.input.InputRow;
|
||||||
import static org.junit.Assert.assertEquals;
|
import org.joda.time.DateTime;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
import com.google.protobuf.ByteString;
|
import static com.metamx.druid.indexer.data.ProtoTestEventWrapper.ProtoTestEvent.EventCategory.CATEGORY_ONE;
|
||||||
import org.joda.time.DateTime;
|
import static org.junit.Assert.assertEquals;
|
||||||
import org.junit.Test;
|
|
||||||
|
|
||||||
import com.metamx.druid.input.InputRow;
|
|
||||||
|
|
||||||
public class ProtoBufInputRowParserTest {
|
public class ProtoBufInputRowParserTest {
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import com.google.common.base.Objects;
|
import com.google.common.base.Objects;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.metamx.druid.indexing.common.task.TaskResource;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Represents the status of a task from the perspective of the coordinator. The task may be ongoing
|
* Represents the status of a task from the perspective of the coordinator. The task may be ongoing
|
||||||
|
|
|
@ -62,7 +62,6 @@ import java.util.Properties;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.Callable;
|
import java.util.concurrent.Callable;
|
||||||
import java.util.concurrent.Executors;
|
import java.util.concurrent.Executors;
|
||||||
import java.util.concurrent.ExecutorService;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Runs tasks in separate processes using the "internal peon" verb.
|
* Runs tasks in separate processes using the "internal peon" verb.
|
||||||
|
@ -146,12 +145,12 @@ public class ForkingTaskRunner implements TaskRunner, TaskLogStreamer
|
||||||
throw new IllegalStateException("Task has been shut down!");
|
throw new IllegalStateException("Task has been shut down!");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (taskInfo == null) {
|
if (taskWorkItem == null) {
|
||||||
log.makeAlert("WTF?! TaskInfo disappeared!").addData("task", task.getId()).emit();
|
log.makeAlert("WTF?! TaskInfo disappeared!").addData("task", task.getId()).emit();
|
||||||
throw new ISE("TaskInfo disappeared for task[%s]!", task.getId());
|
throw new ISE("TaskInfo disappeared for task[%s]!", task.getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (taskInfo.processHolder != null) {
|
if (taskWorkItem.processHolder != null) {
|
||||||
log.makeAlert("WTF?! TaskInfo already has a processHolder")
|
log.makeAlert("WTF?! TaskInfo already has a processHolder")
|
||||||
.addData("task", task.getId())
|
.addData("task", task.getId())
|
||||||
.emit();
|
.emit();
|
||||||
|
|
|
@ -20,7 +20,6 @@
|
||||||
package com.metamx.druid.indexing.coordinator;
|
package com.metamx.druid.indexing.coordinator;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import com.google.api.client.util.Maps;
|
|
||||||
import com.google.common.base.Charsets;
|
import com.google.common.base.Charsets;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
import com.google.common.base.Optional;
|
import com.google.common.base.Optional;
|
||||||
|
@ -29,6 +28,7 @@ import com.google.common.base.Stopwatch;
|
||||||
import com.google.common.base.Supplier;
|
import com.google.common.base.Supplier;
|
||||||
import com.google.common.base.Throwables;
|
import com.google.common.base.Throwables;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
import com.google.common.io.InputSupplier;
|
import com.google.common.io.InputSupplier;
|
||||||
import com.google.common.primitives.Ints;
|
import com.google.common.primitives.Ints;
|
||||||
|
@ -272,7 +272,8 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer
|
||||||
runningTasks.put(
|
runningTasks.put(
|
||||||
task.getId(),
|
task.getId(),
|
||||||
new RemoteTaskRunnerWorkItem(
|
new RemoteTaskRunnerWorkItem(
|
||||||
task, SettableFuture.<TaskStatus>create(),
|
task,
|
||||||
|
SettableFuture.<TaskStatus>create(),
|
||||||
worker
|
worker
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
@ -680,7 +681,7 @@ public class RemoteTaskRunner implements TaskRunner, TaskLogStreamer
|
||||||
if (zkWorker != null) {
|
if (zkWorker != null) {
|
||||||
try {
|
try {
|
||||||
List<String> tasksToFail = Lists.newArrayList(
|
List<String> tasksToFail = Lists.newArrayList(
|
||||||
cf.getChildren().forPath(JOINER.join(config.getIndexerTaskPath(), worker.getHost()))
|
cf.getChildren().forPath(JOINER.join(zkPaths.getIndexerTaskPath(), worker.getHost()))
|
||||||
);
|
);
|
||||||
log.info("[%s]: Found %d tasks assigned", worker.getHost(), tasksToFail.size());
|
log.info("[%s]: Found %d tasks assigned", worker.getHost(), tasksToFail.size());
|
||||||
|
|
||||||
|
|
|
@ -26,7 +26,6 @@ import com.google.common.base.Throwables;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
import com.google.common.collect.Maps;
|
import com.google.common.collect.Maps;
|
||||||
import com.google.common.collect.Sets;
|
import com.google.common.collect.Sets;
|
||||||
import com.metamx.druid.indexing.common.TaskStatus;
|
|
||||||
import com.metamx.druid.indexing.common.task.Task;
|
import com.metamx.druid.indexing.common.task.Task;
|
||||||
import com.metamx.druid.indexing.worker.TaskAnnouncement;
|
import com.metamx.druid.indexing.worker.TaskAnnouncement;
|
||||||
import com.metamx.druid.indexing.worker.Worker;
|
import com.metamx.druid.indexing.worker.Worker;
|
||||||
|
|
|
@ -30,7 +30,6 @@ import com.metamx.common.lifecycle.LifecycleStart;
|
||||||
import com.metamx.common.lifecycle.LifecycleStop;
|
import com.metamx.common.lifecycle.LifecycleStop;
|
||||||
import com.metamx.common.logger.Logger;
|
import com.metamx.common.logger.Logger;
|
||||||
import com.metamx.druid.curator.announcement.Announcer;
|
import com.metamx.druid.curator.announcement.Announcer;
|
||||||
import com.metamx.druid.indexing.common.TaskStatus;
|
|
||||||
import com.metamx.druid.indexing.coordinator.config.RemoteTaskRunnerConfig;
|
import com.metamx.druid.indexing.coordinator.config.RemoteTaskRunnerConfig;
|
||||||
import com.metamx.druid.initialization.ZkPathsConfig;
|
import com.metamx.druid.initialization.ZkPathsConfig;
|
||||||
import org.apache.curator.framework.CuratorFramework;
|
import org.apache.curator.framework.CuratorFramework;
|
||||||
|
|
|
@ -35,7 +35,6 @@ import com.metamx.common.logger.Logger;
|
||||||
import com.metamx.druid.input.InputRow;
|
import com.metamx.druid.input.InputRow;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.LinkedBlockingQueue;
|
import java.util.concurrent.LinkedBlockingQueue;
|
||||||
|
|
|
@ -25,8 +25,6 @@ import com.metamx.common.guava.Yielder;
|
||||||
import com.metamx.common.guava.YieldingAccumulator;
|
import com.metamx.common.guava.YieldingAccumulator;
|
||||||
import com.metamx.common.guava.YieldingSequenceBase;
|
import com.metamx.common.guava.YieldingSequenceBase;
|
||||||
|
|
||||||
import java.io.Closeable;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*/
|
*/
|
||||||
public class ReferenceCountingSequence<T> extends YieldingSequenceBase<T>
|
public class ReferenceCountingSequence<T> extends YieldingSequenceBase<T>
|
||||||
|
|
|
@ -19,14 +19,12 @@
|
||||||
|
|
||||||
package com.metamx.druid.master;
|
package com.metamx.druid.master;
|
||||||
|
|
||||||
import com.google.common.collect.MinMaxPriorityQueue;
|
|
||||||
import com.metamx.common.Pair;
|
import com.metamx.common.Pair;
|
||||||
import com.metamx.druid.client.DataSegment;
|
import com.metamx.druid.client.DataSegment;
|
||||||
import com.metamx.emitter.EmittingLogger;
|
import com.metamx.emitter.EmittingLogger;
|
||||||
import org.joda.time.DateTime;
|
import org.joda.time.DateTime;
|
||||||
import org.joda.time.Interval;
|
import org.joda.time.Interval;
|
||||||
|
|
||||||
import java.util.Comparator;
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public class CostBalancerStrategy implements BalancerStrategy
|
public class CostBalancerStrategy implements BalancerStrategy
|
||||||
|
|
Loading…
Reference in New Issue