Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reformat code according to Druid Java and Scala style

This commit is contained in:
Li Zhanhui 2016-01-18 16:37:03 +08:00
parent 0b627dde71
commit da5d7b9376
3 changed files with 513 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
extensions/druid-rocketmq/pom.xml
View File

@ -27,26 +27,21 @@
<version>0.9.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<artifactId>druid-rocketmq</artifactId>
<properties>
<rocketmq.version>3.2.6</rocketmq.version>
</properties>
<artifactId>druid-rocketmq</artifactId>
<dependencies>
<dependency>
<groupId>com.alibaba.rocketmq</groupId>
<artifactId>rocketmq-client</artifactId>
<version>${rocketmq.version}</version>
</dependency>
<dependency>
<groupId>io.druid</groupId>
<artifactId>druid-api</artifactId>
</dependency>
</dependencies>
</project>

9
extensions/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQDruidModule.java
View File

@ -27,10 +27,12 @@ import io.druid.initialization.DruidModule;
import java.util.List;
public class RocketMQDruidModule implements DruidModule {
public class RocketMQDruidModule implements DruidModule
{
@Override
public List<? extends Module> getJacksonModules() {
public List<? extends Module> getJacksonModules()
{
return ImmutableList.of(
new SimpleModule("RocketMQFirehoseModule")
.registerSubtypes(
@ -40,7 +42,8 @@ public class RocketMQDruidModule implements DruidModule {
}
@Override
public void configure(Binder binder) {
public void configure(Binder binder)
{
}
}

153
extensions/druid-rocketmq/src/main/java/io/druid/firehose/rocketmq/RocketMQFirehoseFactory.java
View File

@ -48,7 +48,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.CountDownLatch;
public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputRowParser> {
public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputRowParser>
{
private static final Logger LOGGER = new Logger(RocketMQFirehoseFactory.class);
@ -94,10 +95,13 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
private static final int DEFAULT_PULL_BATCH_SIZE = 32;
@JsonCreator
public RocketMQFirehoseFactory(@JsonProperty("consumerProps") Properties consumerProps,
public RocketMQFirehoseFactory(
@JsonProperty("consumerProps") Properties consumerProps,
@JsonProperty("consumerGroup") String consumerGroup,
@JsonProperty("feed") List<String> feed,
@JsonProperty("pullBatchSize") String pullBatchSize) {
@JsonProperty("pullBatchSize") String pullBatchSize
)
{
this.consumerProps = consumerProps;
this.pullBatchSize = pullBatchSize;
for (Map.Entry<Object, Object> configItem : this.consumerProps.entrySet()) {
@ -109,9 +113,11 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
/**
* Check if there are locally pending messages to consume.
*
* @return true if there are some; false otherwise.
*/
private boolean hasMessagesPending() {
private boolean hasMessagesPending()
{
for (ConcurrentHashMap.Entry<MessageQueue, ConcurrentSkipListSet<MessageExt>> entry : messageQueueTreeSetMap.entrySet()) {
if (!entry.getValue().isEmpty()) {
@ -123,7 +129,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
@Override
public Firehose connect(ByteBufferInputRowParser byteBufferInputRowParser) throws IOException, ParseException {
public Firehose connect(ByteBufferInputRowParser byteBufferInputRowParser) throws IOException, ParseException
{
Set<String> newDimExclus = Sets.union(
byteBufferInputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(),
@ -170,15 +177,18 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
defaultMQPullConsumer.setMessageQueueListener(druidMessageQueueListener);
defaultMQPullConsumer.start();
pullMessageService.start();
} catch (MQClientException e) {
}
catch (MQClientException e) {
LOGGER.error("Failed to start DefaultMQPullConsumer", e);
throw new IOException("Failed to start RocketMQ client", e);
}
return new Firehose() {
return new Firehose()
{
@Override
public boolean hasMore() {
public boolean hasMore()
{
boolean hasMore = false;
DruidPullRequest earliestPullRequest = null;
@ -194,7 +204,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
DEFAULT_PULL_BATCH_SIZE : Integer.parseInt(pullBatchSize);
DruidPullRequest newPullRequest = new DruidPullRequest(messageQueue, null, offset,
batchSize, !hasMessagesPending());
batchSize, !hasMessagesPending()
);
// notify pull message service to pull messages from brokers.
pullMessageService.putRequest(newPullRequest);
@ -203,7 +214,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
if (null == earliestPullRequest) {
earliestPullRequest = newPullRequest;
}
} catch (MQClientException e) {
}
catch (MQClientException e) {
LOGGER.error("Failed to fetch consume offset for queue: {}", entry.getKey());
}
}
@ -215,7 +227,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
try {
earliestPullRequest.getCountDownLatch().await();
hasMore = true;
} catch (InterruptedException e) {
}
catch (InterruptedException e) {
LOGGER.error("CountDownLatch await got interrupted", e);
}
}
@ -223,7 +236,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
@Override
public InputRow nextRow() {
public InputRow nextRow()
{
for (Map.Entry<MessageQueue, ConcurrentSkipListSet<MessageExt>> entry : messageQueueTreeSetMap.entrySet()) {
if (!entry.getValue().isEmpty()) {
MessageExt message = entry.getValue().pollFirst();
@ -242,10 +256,13 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
@Override
public Runnable commit() {
return new Runnable() {
public Runnable commit()
{
return new Runnable()
{
@Override
public void run() {
public void run()
{
OffsetStore offsetStore = defaultMQPullConsumer.getOffsetStore();
Set<MessageQueue> updated = new HashSet<>();
// calculate offsets according to consuming windows.
@ -271,7 +288,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
@Override
public void close() throws IOException {
public void close() throws IOException
{
defaultMQPullConsumer.shutdown();
pullMessageService.shutdown(false);
}
@ -282,7 +300,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
/**
* Pull request.
*/
final class DruidPullRequest {
final class DruidPullRequest
{
private final MessageQueue messageQueue;
private final String tag;
private final long nextBeginOffset;
@ -290,11 +309,14 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
private final boolean longPull;
private final CountDownLatch countDownLatch;
public DruidPullRequest(final MessageQueue messageQueue,
public DruidPullRequest(
final MessageQueue messageQueue,
final String tag,
final long nextBeginOffset,
final int pullBatchSize,
final boolean useLongPull) {
final boolean useLongPull
)
{
this.messageQueue = messageQueue;
this.tag = (null == tag ? "*" : tag);
this.nextBeginOffset = nextBeginOffset;
@ -303,27 +325,33 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
countDownLatch = new CountDownLatch(1);
}
public MessageQueue getMessageQueue() {
public MessageQueue getMessageQueue()
{
return messageQueue;
}
public long getNextBeginOffset() {
public long getNextBeginOffset()
{
return nextBeginOffset;
}
public String getTag() {
public String getTag()
{
return tag;
}
public int getPullBatchSize() {
public int getPullBatchSize()
{
return pullBatchSize;
}
public boolean isLongPull() {
public boolean isLongPull()
{
return longPull;
}
public CountDownLatch getCountDownLatch() {
public CountDownLatch getCountDownLatch()
{
return countDownLatch;
}
}
@ -331,21 +359,24 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
/**
* Pull message service for druid.
*
* <p/>
* <strong>Note: this is a single thread service.</strong>
*/
final class DruidPullMessageService extends ServiceThread {
final class DruidPullMessageService extends ServiceThread
{
private volatile List<DruidPullRequest> requestsWrite = new ArrayList<>();
private volatile List<DruidPullRequest> requestsRead = new ArrayList<>();
private final DefaultMQPullConsumer defaultMQPullConsumer;
public DruidPullMessageService(final DefaultMQPullConsumer defaultMQPullConsumer) {
public DruidPullMessageService(final DefaultMQPullConsumer defaultMQPullConsumer)
{
this.defaultMQPullConsumer = defaultMQPullConsumer;
}
public void putRequest(final DruidPullRequest request) {
public void putRequest(final DruidPullRequest request)
{
synchronized (this) {
this.requestsWrite.add(request);
if (!hasNotified) {
@ -355,21 +386,24 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
}
private void swapRequests() {
private void swapRequests()
{
List<DruidPullRequest> tmp = requestsWrite;
requestsWrite = requestsRead;
requestsRead = tmp;
}
@Override
public String getServiceName() {
public String getServiceName()
{
return getClass().getSimpleName();
}
/**
* Core message pulling logic code goes here.
*/
private void doPull() {
private void doPull()
{
for (DruidPullRequest pullRequest : requestsRead) {
PullResult pullResult;
try {
@ -378,7 +412,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
pullRequest.getMessageQueue(),
pullRequest.getTag(),
pullRequest.getNextBeginOffset(),
pullRequest.getPullBatchSize());
pullRequest.getPullBatchSize()
);
} else {
pullResult = defaultMQPullConsumer.pullBlockIfNotFound(
pullRequest.getMessageQueue(),
@ -392,8 +427,10 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
case FOUND:
// Handle pull result.
if (!messageQueueTreeSetMap.keySet().contains(pullRequest.getMessageQueue())) {
messageQueueTreeSetMap.putIfAbsent(pullRequest.getMessageQueue(),
new ConcurrentSkipListSet<>(new MessageComparator()));
messageQueueTreeSetMap.putIfAbsent(
pullRequest.getMessageQueue(),
new ConcurrentSkipListSet<>(new MessageComparator())
);
}
messageQueueTreeSetMap.get(pullRequest.getMessageQueue()).addAll(pullResult.getMsgFoundList());
break;
@ -403,16 +440,20 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
break;
case OFFSET_ILLEGAL:
LOGGER.error("Bad Pull Request: Offset is illegal. Offset used: {}",
pullRequest.getNextBeginOffset());
LOGGER.error(
"Bad Pull Request: Offset is illegal. Offset used: {}",
pullRequest.getNextBeginOffset()
);
break;
default:
break;
}
} catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
}
catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) {
LOGGER.error("Failed to pull message from broker.", e);
} finally {
}
finally {
pullRequest.getCountDownLatch().countDown();
}
@ -424,7 +465,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
* Thread looping entry.
*/
@Override
public void run() {
public void run()
{
LOGGER.info(getServiceName() + " starts.");
while (!isStoped()) {
waitForRunning(0);
@ -434,7 +476,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
// in case this service is shutdown gracefully without interruption.
try {
Thread.sleep(10);
} catch (InterruptedException e) {
}
catch (InterruptedException e) {
LOGGER.error("", e);
}
@ -447,7 +490,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
@Override
protected void onWaitEnd() {
protected void onWaitEnd()
{
swapRequests();
}
}
@ -456,9 +500,11 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
/**
* Compare messages pulled from same message queue according to queue offset.
*/
static final class MessageComparator implements Comparator<MessageExt> {
static final class MessageComparator implements Comparator<MessageExt>
{
@Override
public int compare(MessageExt lhs, MessageExt rhs) {
public int compare(MessageExt lhs, MessageExt rhs)
{
return Long.compare(lhs.getQueueOffset(), lhs.getQueueOffset());
}
}
@ -467,7 +513,8 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
/**
* Handle message queues re-balance operations.
*/
final class DruidMessageQueueListener implements MessageQueueListener {
final class DruidMessageQueueListener implements MessageQueueListener
{
private final Set<String> topics;
@ -475,16 +522,20 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
private final DefaultMQPullConsumer defaultMQPullConsumer;
public DruidMessageQueueListener(final Set<String> topics,
public DruidMessageQueueListener(
final Set<String> topics,
final ConcurrentHashMap<String, Set<MessageQueue>> topicQueueMap,
final DefaultMQPullConsumer defaultMQPullConsumer) {
final DefaultMQPullConsumer defaultMQPullConsumer
)
{
this.topics = topics;
this.topicQueueMap = topicQueueMap;
this.defaultMQPullConsumer = defaultMQPullConsumer;
}
@Override
public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {
public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided)
{
if (topics.contains(topic)) {
topicQueueMap.put(topic, mqDivided);
@ -506,10 +557,12 @@ public class RocketMQFirehoseFactory implements FirehoseFactory<ByteBufferInputR
}
if (LOGGER.isDebugEnabled() && stringBuilder.length() > 2) {
LOGGER.debug(String.format("%s@%s is consuming the following message queues: %s",
LOGGER.debug(String.format(
"%s@%s is consuming the following message queues: %s",
defaultMQPullConsumer.getClientIP(),
defaultMQPullConsumer.getInstanceName(),
stringBuilder.substring(0, stringBuilder.length() - 2) /*Remove the trailing comma*/));
stringBuilder.substring(0, stringBuilder.length() - 2) /*Remove the trailing comma*/
));
}
}