Merge pull request #2691 from metamx/lookupExtrFn

Add ExtractionFn to LookupExtractor bridge

docs/content/querying/dimensionspecs.md

@@ -308,6 +308,26 @@ This allows distinguishing between a null dimension and a lookup resulting in a
 For example, specifying `{"":"bar","bat":"baz"}` with dimension values `[null, "foo", "bat"]` and replacing missing values with `"oof"` will yield results of `["bar", "oof", "baz"]`.
 Omitting the empty string key will cause the missing value to take over. For example, specifying `{"bat":"baz"}` with dimension values `[null, "foo", "bat"]` and replacing missing values with `"oof"` will yield results of `["oof", "oof", "baz"]`.
 
+### Registered Lookup Extraction Function
+
+While it is recommended that the [lookup dimension spec](#lookup-dimensionspecs) be used whenever possible, any lookup that is registered for use as a lookup dimension spec can be used as a dimension extraction.
+
+The specification for dimension extraction using dimension specification named lookups is formatted as per the following example:
+
+```json
+{
+  "type":"registeredLookup",
+  "lookup":"some_lookup_name",
+  "retainMissingValue":true,
+  "injective":false
+}
+```
+
+All the flags for [lookup extraction function](#lookup-extraction-function) apply here as well.
+
+In general, the dimension specification should be used. This dimension **extraction** implementation is made available for testing, validation, and transitioning from dimension extraction to the dimension specification style lookups.
+There is also a chance that a feature uses dimension extraction in such a way that it is not applied to dimension specification lookups. Such a scenario should be brought to the attention of the development mailing list.
+
 ### Cascade Extraction Function
 
 Provides chained execution of extraction functions.

processing/src/main/java/io/druid/query/extraction/ExtractionFn.java

@@ -22,6 +22,7 @@ package io.druid.query.extraction;
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import io.druid.query.lookup.LookupExtractionFn;
+import io.druid.query.lookup.RegisteredLookupExtractionFn;
 
 /**
  */
@@ -35,6 +36,7 @@ import io.druid.query.lookup.LookupExtractionFn;
     @JsonSubTypes.Type(name = "timeFormat", value = TimeFormatExtractionFn.class),
     @JsonSubTypes.Type(name = "identity", value = IdentityExtractionFn.class),
     @JsonSubTypes.Type(name = "lookup", value = LookupExtractionFn.class),
+    @JsonSubTypes.Type(name = "registeredLookup", value = RegisteredLookupExtractionFn.class),
     @JsonSubTypes.Type(name = "substring", value = SubstringDimExtractionFn.class),
     @JsonSubTypes.Type(name = "cascade", value = CascadeExtractionFn.class),
     @JsonSubTypes.Type(name = "stringFormat", value = StringFormatExtractionFn.class),

processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java

@@ -37,6 +37,10 @@ public class LookupExtractionFn extends FunctionalExtraction
 {
   private final LookupExtractor lookup;
   private final boolean optimize;
+  // Thes are retained for auto generated hashCode and Equals
+  private final boolean retainMissingValue;
+  private final String replaceMissingValueWith;
+  private final boolean injective;
 
   @JsonCreator
   public LookupExtractionFn(
@@ -63,6 +67,9 @@ public class LookupExtractionFn extends FunctionalExtraction
     );
     this.lookup = lookup;
     this.optimize = optimize == null ? true : optimize;
+    this.retainMissingValue = retainMissingValue;
+    this.injective = injective;
+    this.replaceMissingValueWith = replaceMissingValueWith;
   }
 
 
@@ -121,7 +128,7 @@ public class LookupExtractionFn extends FunctionalExtraction
     if (this == o) {
       return true;
     }
-    if (!(o instanceof LookupExtractionFn)) {
+    if (o == null || getClass() != o.getClass()) {
       return false;
     }
 
@@ -130,15 +137,29 @@ public class LookupExtractionFn extends FunctionalExtraction
     if (isOptimize() != that.isOptimize()) {
       return false;
     }
-    return getLookup().equals(that.getLookup());
+    if (isRetainMissingValue() != that.isRetainMissingValue()) {
+      return false;
+    }
+    if (isInjective() != that.isInjective()) {
+      return false;
+    }
+    if (getLookup() != null ? !getLookup().equals(that.getLookup()) : that.getLookup() != null) {
+      return false;
+    }
+    return getReplaceMissingValueWith() != null
+           ? getReplaceMissingValueWith().equals(that.getReplaceMissingValueWith())
+           : that.getReplaceMissingValueWith() == null;
 
   }
 
   @Override
   public int hashCode()
   {
-    int result = getLookup().hashCode();
+    int result = getLookup() != null ? getLookup().hashCode() : 0;
     result = 31 * result + (isOptimize() ? 1 : 0);
+    result = 31 * result + (isRetainMissingValue() ? 1 : 0);
+    result = 31 * result + (getReplaceMissingValueWith() != null ? getReplaceMissingValueWith().hashCode() : 0);
+    result = 31 * result + (isInjective() ? 1 : 0);
     return result;
   }
 }

processing/src/main/java/io/druid/query/lookup/RegisteredLookupExtractionFn.java

@@ -0,0 +1,167 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import io.druid.query.extraction.ExtractionFn;
+
+import javax.annotation.Nullable;
+
+public class RegisteredLookupExtractionFn implements ExtractionFn
+{
+  private final LookupExtractionFn delegate;
+  private final String name;
+
+  RegisteredLookupExtractionFn(LookupExtractionFn delegate, String name)
+  {
+    this.delegate = delegate;
+    this.name = name;
+  }
+
+  @JsonCreator
+  public static RegisteredLookupExtractionFn create(
+      @JacksonInject LookupReferencesManager manager,
+      @JsonProperty("lookup") String lookup,
+      @JsonProperty("retainMissingValue") final boolean retainMissingValue,
+      @Nullable @JsonProperty("replaceMissingValueWith") final String replaceMissingValueWith,
+      @JsonProperty("injective") final boolean injective,
+      @JsonProperty("optimize") Boolean optimize
+  )
+  {
+    Preconditions.checkArgument(lookup != null, "`lookup` required");
+    final LookupExtractorFactory factory = manager.get(lookup);
+    Preconditions.checkNotNull(factory, "lookup [%s] not found", lookup);
+    return new RegisteredLookupExtractionFn(
+        new LookupExtractionFn(
+            factory.get(),
+            retainMissingValue,
+            replaceMissingValueWith,
+            injective,
+            optimize
+        ),
+        lookup
+    );
+  }
+
+  @JsonProperty("lookup")
+  public String getLookup()
+  {
+    return name;
+  }
+
+  @JsonProperty("retainMissingValue")
+  public boolean isRetainMissingValue()
+  {
+    return delegate.isRetainMissingValue();
+  }
+
+  @JsonProperty("replaceMissingValueWith")
+  public String getReplaceMissingValueWith()
+  {
+    return delegate.getReplaceMissingValueWith();
+  }
+
+  @JsonProperty("injective")
+  public boolean isInjective()
+  {
+    return delegate.isInjective();
+  }
+
+  @JsonProperty("optimize")
+  public boolean isOptimize()
+  {
+    return delegate.isOptimize();
+  }
+
+  @Override
+  public byte[] getCacheKey()
+  {
+    return delegate.getCacheKey();
+  }
+
+  @Override
+  public String apply(Object value)
+  {
+    return delegate.apply(value);
+  }
+
+  @Override
+  public String apply(String value)
+  {
+    return delegate.apply(value);
+  }
+
+  @Override
+  public String apply(long value)
+  {
+    return delegate.apply(value);
+  }
+
+  @Override
+  public boolean preservesOrdering()
+  {
+    return delegate.preservesOrdering();
+  }
+
+  @Override
+  public ExtractionType getExtractionType()
+  {
+    return delegate.getExtractionType();
+  }
+
+  @Override
+  public String toString()
+  {
+    return "RegisteredLookupExtractionFn{" +
+           "delegate=" + delegate +
+           ", name='" + name + '\'' +
+           '}';
+  }
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    RegisteredLookupExtractionFn that = (RegisteredLookupExtractionFn) o;
+
+    if (!delegate.equals(that.delegate)) {
+      return false;
+    }
+    return name.equals(that.name);
+
+  }
+
+  @Override
+  public int hashCode()
+  {
+    int result = delegate.hashCode();
+    result = 31 * result + name.hashCode();
+    return result;
+  }
+}

processing/src/test/java/io/druid/query/lookup/RegisteredLookupExtractionFnTest.java

@@ -0,0 +1,256 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import io.druid.jackson.DefaultObjectMapper;
+import io.druid.query.extraction.MapLookupExtractor;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import javax.annotation.Nullable;
+import java.util.Arrays;
+import java.util.Map;
+
+public class RegisteredLookupExtractionFnTest
+{
+  private static Map<String, String> MAP = ImmutableMap.of(
+      "foo", "bar",
+      "bat", "baz"
+  );
+  private static final LookupExtractor LOOKUP_EXTRACTOR = new MapLookupExtractor(MAP, true);
+  private static final String LOOKUP_NAME = "some lookup";
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  @Test
+  public void testSimpleDelegation()
+  {
+    final LookupReferencesManager manager = EasyMock.createStrictMock(LookupReferencesManager.class);
+    managerReturnsMap(manager);
+    EasyMock.replay(manager);
+    final RegisteredLookupExtractionFn fn = RegisteredLookupExtractionFn.create(
+        manager,
+        LOOKUP_NAME,
+        true,
+        null,
+        true,
+        false
+    );
+    EasyMock.verify(manager);
+    for (String orig : Arrays.asList("", "foo", "bat")) {
+      Assert.assertEquals(LOOKUP_EXTRACTOR.apply(orig), fn.apply(orig));
+    }
+    Assert.assertEquals("not in the map", fn.apply("not in the map"));
+  }
+
+
+  @Test
+  public void testMissingDelegation()
+  {
+    final LookupReferencesManager manager = EasyMock.createStrictMock(LookupReferencesManager.class);
+    EasyMock.expect(manager.get(EasyMock.eq(LOOKUP_NAME))).andReturn(null).once();
+    EasyMock.replay(manager);
+
+    expectedException.expectMessage("lookup [some lookup] not found");
+    try {
+      RegisteredLookupExtractionFn.create(
+          manager,
+          LOOKUP_NAME,
+          true,
+          null,
+          true,
+          false
+      );
+    }
+    finally {
+      EasyMock.verify(manager);
+    }
+  }
+
+  @Test
+  public void testNullLookup()
+  {
+    expectedException.expectMessage("`lookup` required");
+    RegisteredLookupExtractionFn.create(
+        null,
+        null,
+        true,
+        null,
+        true,
+        false
+    );
+  }
+
+  @Test
+  public void testSerDe() throws Exception
+  {
+    final ObjectMapper mapper = new DefaultObjectMapper();
+
+    final LookupReferencesManager manager = EasyMock.createStrictMock(LookupReferencesManager.class);
+    managerReturnsMap(manager);
+    EasyMock.replay(manager);
+    final RegisteredLookupExtractionFn fn = RegisteredLookupExtractionFn.create(
+        manager,
+        LOOKUP_NAME,
+        true,
+        null,
+        true,
+        false
+    );
+    EasyMock.verify(manager);
+
+    final TypeReference<Map<String, Object>> typeReference = new TypeReference<Map<String, Object>>()
+    {
+    };
+    final Map<String, Object> result = mapper.readValue(mapper.writeValueAsString(fn), typeReference);
+    Assert.assertEquals(mapper.convertValue(fn, typeReference), result);
+    Assert.assertEquals(LOOKUP_NAME, result.get("lookup"));
+    Assert.assertEquals(true, result.get("retainMissingValue"));
+    Assert.assertEquals(true, result.get("injective"));
+    Assert.assertNull(result.get("replaceMissingValueWith"));
+    Assert.assertEquals(false, result.get("optimize"));
+  }
+
+  @Test
+  public void testEquals()
+  {
+    final LookupReferencesManager manager = EasyMock.createStrictMock(LookupReferencesManager.class);
+    managerReturnsMap(manager);
+    EasyMock.replay(manager);
+    final RegisteredLookupExtractionFn fn = RegisteredLookupExtractionFn.create(
+        manager,
+        LOOKUP_NAME,
+        false,
+        "something",
+        true,
+        false
+    );
+    Assert.assertEquals(
+        fn,
+        RegisteredLookupExtractionFn.create(
+            manager,
+            LOOKUP_NAME,
+            false,
+            "something",
+            true,
+            false
+        )
+    );
+    Assert.assertNotEquals(
+        fn,
+        RegisteredLookupExtractionFn.create(
+            manager,
+            LOOKUP_NAME,
+            true,
+            null,
+            true,
+            false
+        )
+    );
+
+    Assert.assertNotEquals(
+        fn,
+        RegisteredLookupExtractionFn.create(
+            manager,
+            LOOKUP_NAME,
+            false,
+            "something else",
+            true,
+            false
+        )
+    );
+
+
+    Assert.assertNotEquals(
+        fn,
+        RegisteredLookupExtractionFn.create(
+            manager,
+            LOOKUP_NAME,
+            false,
+            "something",
+            false,
+            false
+        )
+    );
+
+    Assert.assertNotEquals(
+        fn,
+        RegisteredLookupExtractionFn.create(
+            manager,
+            LOOKUP_NAME,
+            false,
+            "something",
+            true,
+            true
+        )
+    );
+
+
+    Assert.assertNotEquals(
+        fn,
+        RegisteredLookupExtractionFn.create(
+            manager,
+            LOOKUP_NAME,
+            false,
+            null,
+            true,
+            false
+        )
+    );
+    EasyMock.verify(manager);
+  }
+
+  private void managerReturnsMap(LookupReferencesManager manager)
+  {
+    EasyMock.expect(manager.get(EasyMock.eq(LOOKUP_NAME))).andReturn(new LookupExtractorFactory()
+    {
+      @Override
+      public boolean start()
+      {
+        return false;
+      }
+
+      @Override
+      public boolean close()
+      {
+        return false;
+      }
+
+      @Override
+      public boolean replaces(@Nullable LookupExtractorFactory other)
+      {
+        return false;
+      }
+
+      @Override
+      public LookupExtractor get()
+      {
+        return LOOKUP_EXTRACTOR;
+      }
+    }).anyTimes();
+  }
+}