From 4cb3ca00bc88e60eeebc925e511b992a448a98f5 Mon Sep 17 00:00:00 2001 From: xvrl Date: Tue, 5 Mar 2013 14:15:58 -0800 Subject: [PATCH 1/2] support for javascript dimfilters --- .../metamx/druid/query/filter/DimFilter.java | 3 +- .../query/filter/DimFilterCacheHelper.java | 1 + .../query/filter/JavaScriptDimFilter.java | 48 +++++++ .../com/metamx/druid/index/brita/Filters.java | 5 + .../druid/index/brita/JavaScriptFilter.java | 127 ++++++++++++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java create mode 100644 server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java b/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java index 8da47da465c..5099f95b3d2 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java +++ b/client/src/main/java/com/metamx/druid/query/filter/DimFilter.java @@ -33,7 +33,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; @JsonSubTypes.Type(name="selector", value=SelectorDimFilter.class), @JsonSubTypes.Type(name="extraction", value=ExtractionDimFilter.class), @JsonSubTypes.Type(name="regex", value=RegexDimFilter.class), - @JsonSubTypes.Type(name="search", value=SearchQueryDimFilter.class) + @JsonSubTypes.Type(name="search", value=SearchQueryDimFilter.class), + @JsonSubTypes.Type(name="javascript", value=JavaScriptDimFilter.class) }) public interface DimFilter { diff --git a/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java b/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java index 67f35a9bab9..8497a53b206 100644 --- a/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java +++ b/client/src/main/java/com/metamx/druid/query/filter/DimFilterCacheHelper.java @@ -34,6 +34,7 @@ class DimFilterCacheHelper static final byte EXTRACTION_CACHE_ID = 0x4; static final byte REGEX_CACHE_ID = 0x5; static final byte SEARCH_QUERY_TYPE_ID = 0x6; + static final byte JAVASCRIPT_CACHE_ID = 0x7; static byte[] computeCacheKey(byte cacheIdKey, List filters) { diff --git a/client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java b/client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java new file mode 100644 index 00000000000..847d13c9bd4 --- /dev/null +++ b/client/src/main/java/com/metamx/druid/query/filter/JavaScriptDimFilter.java @@ -0,0 +1,48 @@ +package com.metamx.druid.query.filter; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Charsets; + +import java.nio.ByteBuffer; + +public class JavaScriptDimFilter implements DimFilter +{ + private final String dimension; + private final String function; + + @JsonCreator + public JavaScriptDimFilter( + @JsonProperty("dimension") String dimension, + @JsonProperty("function") String function + ) + { + this.dimension = dimension; + this.function = function; + } + + @JsonProperty + public String getDimension() + { + return dimension; + } + + @JsonProperty + public String getFunction() + { + return function; + } + + @Override + public byte[] getCacheKey() + { + final byte[] dimensionBytes = dimension.getBytes(Charsets.UTF_8); + final byte[] functionBytes = function.getBytes(Charsets.UTF_8); + + return ByteBuffer.allocate(1 + dimensionBytes.length + functionBytes.length) + .put(DimFilterCacheHelper.JAVASCRIPT_CACHE_ID) + .put(dimensionBytes) + .put(functionBytes) + .array(); + } +} diff --git a/server/src/main/java/com/metamx/druid/index/brita/Filters.java b/server/src/main/java/com/metamx/druid/index/brita/Filters.java index 0e4e4c4e8ea..f7c390c44a9 100644 --- a/server/src/main/java/com/metamx/druid/index/brita/Filters.java +++ b/server/src/main/java/com/metamx/druid/index/brita/Filters.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.metamx.druid.query.filter.AndDimFilter; import com.metamx.druid.query.filter.DimFilter; import com.metamx.druid.query.filter.ExtractionDimFilter; +import com.metamx.druid.query.filter.JavaScriptDimFilter; import com.metamx.druid.query.filter.NotDimFilter; import com.metamx.druid.query.filter.OrDimFilter; import com.metamx.druid.query.filter.RegexDimFilter; @@ -84,6 +85,10 @@ public class Filters final SearchQueryDimFilter searchQueryFilter = (SearchQueryDimFilter) dimFilter; filter = new SearchQueryFilter(searchQueryFilter.getDimension(), searchQueryFilter.getQuery()); + } else if (dimFilter instanceof JavaScriptDimFilter) { + final JavaScriptDimFilter javaScriptDimFilter = (JavaScriptDimFilter) dimFilter; + + filter = new JavaScriptFilter(javaScriptDimFilter.getDimension(), javaScriptDimFilter.getFunction()); } return filter; diff --git a/server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java b/server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java new file mode 100644 index 00000000000..f17de0b8d85 --- /dev/null +++ b/server/src/main/java/com/metamx/druid/index/brita/JavaScriptFilter.java @@ -0,0 +1,127 @@ +package com.metamx.druid.index.brita; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.metamx.common.guava.FunctionalIterable; +import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; +import org.mozilla.javascript.Context; +import org.mozilla.javascript.Function; +import org.mozilla.javascript.ScriptableObject; + +import javax.annotation.Nullable; + + +public class JavaScriptFilter implements Filter +{ + private final JavaScriptPredicate predicate; + private final String dimension; + + public JavaScriptFilter(String dimension, final String script) + { + this.dimension = dimension; + this.predicate = new JavaScriptPredicate(script); + } + + @Override + public ImmutableConciseSet goConcise(final BitmapIndexSelector selector) + { + final Context cx = Context.enter(); + try { + ImmutableConciseSet conciseSet = ImmutableConciseSet.union( + FunctionalIterable.create(selector.getDimensionValues(dimension)) + .filter(new Predicate() + { + @Override + public boolean apply(@Nullable String input) + { + return predicate.applyInContext(cx, input); + } + }) + .transform( + new com.google.common.base.Function() + { + @Override + public ImmutableConciseSet apply(@Nullable String input) + { + return selector.getConciseInvertedIndex(dimension, input); + } + } + ) + ); + return conciseSet; + } finally { + Context.exit(); + } + } + + @Override + public ValueMatcher makeMatcher(ValueMatcherFactory factory) + { + // suboptimal, since we need create one context per call to predicate.apply() + return factory.makeValueMatcher(dimension, predicate); + } + + static class JavaScriptPredicate implements Predicate { + final ScriptableObject scope; + final Function fnApply; + final String script; + + public JavaScriptPredicate(final String script) { + Preconditions.checkNotNull(script, "script must not be null"); + this.script = script; + + final Context cx = Context.enter(); + try { + cx.setOptimizationLevel(9); + scope = cx.initStandardObjects(); + + fnApply = cx.compileFunction(scope, script, "script", 1, null); + } finally { + Context.exit(); + } + } + + @Override + public boolean apply(final String input) + { + // one and only one context per thread + final Context cx = Context.enter(); + try { + return applyInContext(cx, input); + } finally { + Context.exit(); + } + + } + + public boolean applyInContext(Context cx, String input) + { + return Context.toBoolean(fnApply.call(cx, scope, scope, new String[]{input})); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + JavaScriptPredicate that = (JavaScriptPredicate) o; + + if (!script.equals(that.script)) { + return false; + } + + return true; + } + + @Override + public int hashCode() + { + return script.hashCode(); + } + } +} From 055a98a89c0a0093db78f07ce54cfbc2b8d78fc9 Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Thu, 14 Mar 2013 10:37:40 -0700 Subject: [PATCH 2/2] Merger: Fix VersionConverterTask version selection by making SegmentInsertAction more flexible --- .../common/actions/SegmentInsertAction.java | 27 +++++++++++++++++-- .../common/task/VersionConverterTask.java | 3 ++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/merger/src/main/java/com/metamx/druid/merger/common/actions/SegmentInsertAction.java b/merger/src/main/java/com/metamx/druid/merger/common/actions/SegmentInsertAction.java index 5354e14878c..a04a426e045 100644 --- a/merger/src/main/java/com/metamx/druid/merger/common/actions/SegmentInsertAction.java +++ b/merger/src/main/java/com/metamx/druid/merger/common/actions/SegmentInsertAction.java @@ -1,5 +1,6 @@ package com.metamx.druid.merger.common.actions; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.base.Predicate; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableSet; @@ -18,14 +19,25 @@ import java.util.Set; public class SegmentInsertAction implements TaskAction { + @JsonIgnore private final Set segments; + @JsonIgnore + private final boolean allowOlderVersions; + + public SegmentInsertAction(Set segments) + { + this(segments, false); + } + @JsonCreator public SegmentInsertAction( - @JsonProperty("segments") Set segments + @JsonProperty("segments") Set segments, + @JsonProperty("allowOlderVersions") boolean allowOlderVersions ) { this.segments = ImmutableSet.copyOf(segments); + this.allowOlderVersions = allowOlderVersions; } @JsonProperty @@ -34,6 +46,17 @@ public class SegmentInsertAction implements TaskAction return segments; } + @JsonProperty + public boolean isAllowOlderVersions() + { + return allowOlderVersions; + } + + public SegmentInsertAction withAllowOlderVersions(boolean _allowOlderVersions) + { + return new SegmentInsertAction(segments, _allowOlderVersions); + } + public TypeReference getReturnTypeReference() { return new TypeReference() {}; @@ -42,7 +65,7 @@ public class SegmentInsertAction implements TaskAction @Override public Void perform(Task task, TaskActionToolbox toolbox) { - if(!toolbox.taskLockCoversSegments(task, segments, false)) { + if(!toolbox.taskLockCoversSegments(task, segments, allowOlderVersions)) { throw new ISE("Segments not covered by locks for task: %s", task.getId()); } diff --git a/merger/src/main/java/com/metamx/druid/merger/common/task/VersionConverterTask.java b/merger/src/main/java/com/metamx/druid/merger/common/task/VersionConverterTask.java index c5db8aba959..4f4bd02b734 100644 --- a/merger/src/main/java/com/metamx/druid/merger/common/task/VersionConverterTask.java +++ b/merger/src/main/java/com/metamx/druid/merger/common/task/VersionConverterTask.java @@ -236,7 +236,8 @@ public class VersionConverterTask extends AbstractTask DataSegment updatedSegment = segment.withVersion(String.format("%s_v%s", segment.getVersion(), outVersion)); updatedSegment = toolbox.getSegmentPusher().push(outLocation, updatedSegment); - toolbox.getTaskActionClient().submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment))); + toolbox.getTaskActionClient() + .submit(new SegmentInsertAction(Sets.newHashSet(updatedSegment)).withAllowOlderVersions(true)); } else { log.info("Conversion failed."); }