From 4f70bbc6874fc3409003119954aeacd7b516fb42 Mon Sep 17 00:00:00 2001 From: Jae Hyeon Bae Date: Mon, 4 Feb 2013 10:27:09 -0800 Subject: [PATCH 1/2] adding linear shard spec and ACL for s3 bucket --- .../metamx/druid/shard/LinearShardSpec.java | 49 ++++++++++ .../com/metamx/druid/shard/ShardSpec.java | 3 +- .../druid/partition/LinearPartitionChunk.java | 91 +++++++++++++++++++ .../metamx/druid/loading/S3SegmentPusher.java | 9 +- 4 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java create mode 100644 common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java diff --git a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java b/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java new file mode 100644 index 00000000000..249cb002bd2 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java @@ -0,0 +1,49 @@ +package com.metamx.druid.shard; + +import com.metamx.druid.input.InputRow; +import com.metamx.druid.partition.LinearPartitionChunk; +import com.metamx.druid.partition.PartitionChunk; +import org.codehaus.jackson.annotate.JsonProperty; + +import java.util.Map; + +/** + * Created with IntelliJ IDEA. + * User: jbae + * Date: 2/4/13 + * Time: 10:22 AM + * To change this template use File | Settings | File Templates. + */ +public class LinearShardSpec implements ShardSpec { + private int partitionNum; + + public LinearShardSpec() { + this(-1); + } + + public LinearShardSpec(int partitionNum) { + this.partitionNum = partitionNum; + } + + @JsonProperty("partitionNum") + @Override + public int getPartitionNum() { + return partitionNum; + } + + + @Override + public PartitionChunk createChunk(T obj) { + return new LinearPartitionChunk(partitionNum, obj); + } + + @Override + public boolean isInChunk(Map dimensions) { + return true; + } + + @Override + public boolean isInChunk(InputRow inputRow) { + return true; + } +} \ No newline at end of file diff --git a/client/src/main/java/com/metamx/druid/shard/ShardSpec.java b/client/src/main/java/com/metamx/druid/shard/ShardSpec.java index c0127b055e9..df3eb29004a 100644 --- a/client/src/main/java/com/metamx/druid/shard/ShardSpec.java +++ b/client/src/main/java/com/metamx/druid/shard/ShardSpec.java @@ -32,7 +32,8 @@ import java.util.Map; @JsonTypeInfo(use=JsonTypeInfo.Id.NAME, property="type", include=JsonTypeInfo.As.PROPERTY) @JsonSubTypes({ @JsonSubTypes.Type(name="single", value=SingleDimensionShardSpec.class), - @JsonSubTypes.Type(name="none", value=NoneShardSpec.class) + @JsonSubTypes.Type(name="none", value=NoneShardSpec.class), + @JsonSubTypes.Type(name="linear", value=LinearShardSpec.class) }) public interface ShardSpec { diff --git a/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java b/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java new file mode 100644 index 00000000000..fecc541fd84 --- /dev/null +++ b/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java @@ -0,0 +1,91 @@ +package com.metamx.druid.partition; + +/** + * Created with IntelliJ IDEA. + * User: jbae + * Date: 2/4/13 + * Time: 10:24 AM + * To change this template use File | Settings | File Templates. + */ +public class LinearPartitionChunk implements PartitionChunk +{ + private final int chunkNumber; + private final T object; + + public static LinearPartitionChunk make(int chunkNumber, T obj) + { + return new LinearPartitionChunk(chunkNumber, obj); + } + + + public LinearPartitionChunk( + int chunkNumber, + T object + ) + { + this.chunkNumber = chunkNumber; + this.object = object; + } + + @Override + public T getObject() + { + return object; + } + + @Override + public boolean abuts(PartitionChunk chunk) + { + return true; // always complete + } + + @Override + public boolean isStart() + { + return true; // always complete + } + + @Override + + public boolean isEnd() + { + return true; // always complete + } + + @Override + public int getChunkNumber() + { + return chunkNumber; + } + + @Override + public int compareTo(PartitionChunk chunk) + { + if (chunk instanceof LinearPartitionChunk) { + LinearPartitionChunk linearChunk = (LinearPartitionChunk) chunk; + + return chunkNumber - chunk.getChunkNumber(); + } + throw new IllegalArgumentException("Cannot compare against something that is not a LinearPartitionChunk."); + } + + @Override + @SuppressWarnings("unchecked") + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + return compareTo((LinearPartitionChunk) o) == 0; + } + + @Override + public int hashCode() + { + return chunkNumber; + } +} \ No newline at end of file diff --git a/server/src/main/java/com/metamx/druid/loading/S3SegmentPusher.java b/server/src/main/java/com/metamx/druid/loading/S3SegmentPusher.java index f4099154d2d..7a517a6422d 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3SegmentPusher.java +++ b/server/src/main/java/com/metamx/druid/loading/S3SegmentPusher.java @@ -30,14 +30,11 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.codehaus.jackson.map.ObjectMapper; import org.jets3t.service.S3ServiceException; +import org.jets3t.service.acl.gs.GSAccessControlList; import org.jets3t.service.impl.rest.httpclient.RestS3Service; import org.jets3t.service.model.S3Object; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; +import java.io.*; import java.security.NoSuchAlgorithmException; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -106,6 +103,7 @@ public class S3SegmentPusher implements SegmentPusher final String outputBucket = config.getBucket(); toPush.setBucketName(outputBucket); toPush.setKey(outputKey + "/index.zip"); + toPush.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); log.info("Pushing %s.", toPush); s3Client.putObject(outputBucket, toPush); @@ -124,6 +122,7 @@ public class S3SegmentPusher implements SegmentPusher S3Object descriptorObject = new S3Object(descriptorFile); descriptorObject.setBucketName(outputBucket); descriptorObject.setKey(outputKey + "/descriptor.json"); + descriptorObject.setAcl(GSAccessControlList.REST_CANNED_BUCKET_OWNER_FULL_CONTROL); log.info("Pushing %s", descriptorObject); s3Client.putObject(outputBucket, descriptorObject); From 6708029043c0771c645191c3a63f130ffd98dda7 Mon Sep 17 00:00:00 2001 From: Eric Tschetter Date: Mon, 11 Feb 2013 13:13:52 -0600 Subject: [PATCH 2/2] 1) Remove class level comments auto-created by IntelliJ --- .../main/java/com/metamx/druid/shard/LinearShardSpec.java | 7 ------- .../com/metamx/druid/partition/LinearPartitionChunk.java | 7 ------- 2 files changed, 14 deletions(-) diff --git a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java b/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java index 249cb002bd2..97f22ae6660 100644 --- a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java +++ b/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java @@ -7,13 +7,6 @@ import org.codehaus.jackson.annotate.JsonProperty; import java.util.Map; -/** - * Created with IntelliJ IDEA. - * User: jbae - * Date: 2/4/13 - * Time: 10:22 AM - * To change this template use File | Settings | File Templates. - */ public class LinearShardSpec implements ShardSpec { private int partitionNum; diff --git a/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java b/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java index fecc541fd84..530029239f2 100644 --- a/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java +++ b/common/src/main/java/com/metamx/druid/partition/LinearPartitionChunk.java @@ -1,12 +1,5 @@ package com.metamx.druid.partition; -/** - * Created with IntelliJ IDEA. - * User: jbae - * Date: 2/4/13 - * Time: 10:24 AM - * To change this template use File | Settings | File Templates. - */ public class LinearPartitionChunk implements PartitionChunk { private final int chunkNumber;