Add some new expression functions and macros. (#4442)

* Add some new expression functions and macros.

See misc/math-expr.md for the list of added functions, except for
"like", which previously existed but was not documented.

* Add easymock to datasketches tests.

* Add easymock to distinctcount tests.

* Add easymock to virtual-columns tests.

* Code review comments.

* Clean up code a bit.

* Add easymock to scan-query tests.

* Rework ExprMacros that have multiple impls.

* Improve test coverage.

common/src/main/java/io/druid/math/expr/Function.java

@@ -19,6 +19,7 @@
 
 package io.druid.math.expr;
 
+import com.google.common.base.Strings;
 import io.druid.java.util.common.IAE;
 import org.joda.time.DateTime;
 import org.joda.time.format.DateTimeFormat;
@@ -41,7 +42,7 @@ interface Function
     public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
     {
       if (args.size() != 1) {
-        throw new IAE("function '%s' needs 1 argument", name());
+        throw new IAE("Function[%s] needs 1 argument", name());
       }
       Expr expr = args.get(0);
       return eval(expr.eval(bindings));
@@ -56,7 +57,7 @@ interface Function
     public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
     {
       if (args.size() != 2) {
-        throw new IAE("function '%s' needs 2 arguments", name());
+        throw new IAE("Function[%s] needs 2 arguments", name());
       }
       Expr expr1 = args.get(0);
       Expr expr2 = args.get(1);
@@ -242,6 +243,27 @@ interface Function
     }
   }
 
+  class Div extends DoubleParamMath
+  {
+    @Override
+    public String name()
+    {
+      return "div";
+    }
+
+    @Override
+    protected ExprEval eval(final long x, final long y)
+    {
+      return ExprEval.of(x / y);
+    }
+
+    @Override
+    protected ExprEval eval(final double x, final double y)
+    {
+      return ExprEval.of((long) (x / y));
+    }
+  }
+
   class Exp extends SingleParamMath
   {
     @Override
@@ -686,7 +708,7 @@ interface Function
     public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
     {
       if (args.size() != 3) {
-        throw new IAE("function 'if' needs 3 arguments");
+        throw new IAE("Function[%s] needs 3 arguments", name());
       }
 
       ExprEval x = args.get(0).eval(bindings);
@@ -694,6 +716,70 @@ interface Function
     }
   }
 
+  /**
+   * "Searched CASE" function, similar to {@code CASE WHEN boolean_expr THEN result [ELSE else_result] END} in SQL.
+   */
+  class CaseSearchedFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "case_searched";
+    }
+
+    @Override
+    public ExprEval apply(final List<Expr> args, final Expr.ObjectBinding bindings)
+    {
+      if (args.size() < 2) {
+        throw new IAE("Function[%s] must have at least 2 arguments", name());
+      }
+
+      for (int i = 0; i < args.size(); i += 2) {
+        if (i == args.size() - 1) {
+          // ELSE else_result.
+          return args.get(i).eval(bindings);
+        } else if (args.get(i).eval(bindings).asBoolean()) {
+          // Matching WHEN boolean_expr THEN result
+          return args.get(i + 1).eval(bindings);
+        }
+      }
+
+      return ExprEval.of(null);
+    }
+  }
+
+  /**
+   * "Simple CASE" function, similar to {@code CASE expr WHEN value THEN result [ELSE else_result] END} in SQL.
+   */
+  class CaseSimpleFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "case_simple";
+    }
+
+    @Override
+    public ExprEval apply(final List<Expr> args, final Expr.ObjectBinding bindings)
+    {
+      if (args.size() < 3) {
+        throw new IAE("Function[%s] must have at least 3 arguments", name());
+      }
+
+      for (int i = 1; i < args.size(); i += 2) {
+        if (i == args.size() - 1) {
+          // ELSE else_result.
+          return args.get(i).eval(bindings);
+        } else if (new BinEqExpr("==", args.get(0), args.get(i)).eval(bindings).asBoolean()) {
+          // Matching WHEN value THEN result
+          return args.get(i + 1).eval(bindings);
+        }
+      }
+
+      return ExprEval.of(null);
+    }
+  }
+
   class CastFunc extends DoubleParam
   {
     @Override
@@ -728,7 +814,7 @@ interface Function
     public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
     {
       if (args.size() != 1 && args.size() != 2) {
-        throw new IAE("function '%s' needs 1 or 2 arguments", name());
+        throw new IAE("Function[%s] needs 1 or 2 arguments", name());
       }
       ExprEval value = args.get(0).eval(bindings);
       if (value.type() != ExprType.STRING) {
@@ -786,10 +872,178 @@ interface Function
     public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
     {
       if (args.size() != 2) {
-        throw new IAE("function 'nvl' needs 2 arguments");
+        throw new IAE("Function[%s] needs 2 arguments", name());
       }
       final ExprEval eval = args.get(0).eval(bindings);
       return eval.isNull() ? args.get(1).eval(bindings) : eval;
     }
   }
+
+  class ConcatFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "concat";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() == 0) {
+        return ExprEval.of(null);
+      } else {
+        // Pass first argument in to the constructor to provide StringBuilder a little extra sizing hint.
+        final StringBuilder builder = new StringBuilder(Strings.nullToEmpty(args.get(0).eval(bindings).asString()));
+        for (int i = 1; i < args.size(); i++) {
+          final String s = args.get(i).eval(bindings).asString();
+          if (s != null) {
+            builder.append(s);
+          }
+        }
+        return ExprEval.of(builder.toString());
+      }
+    }
+  }
+
+  class StrlenFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "strlen";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() != 1) {
+        throw new IAE("Function[%s] needs 1 argument", name());
+      }
+
+      final String arg = args.get(0).eval(bindings).asString();
+      return arg == null ? ExprEval.of(0) : ExprEval.of(arg.length());
+    }
+  }
+
+  class SubstringFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "substring";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() != 3) {
+        throw new IAE("Function[%s] needs 3 arguments", name());
+      }
+
+      final String arg = args.get(0).eval(bindings).asString();
+
+      if (arg == null) {
+        return ExprEval.of(null);
+      }
+
+      // Behaves like SubstringDimExtractionFn, not SQL SUBSTRING
+      final int index = args.get(1).eval(bindings).asInt();
+      final int length = args.get(2).eval(bindings).asInt();
+
+      if (index < arg.length()) {
+        if (length >= 0) {
+          return ExprEval.of(arg.substring(index, Math.min(index + length, arg.length())));
+        } else {
+          return ExprEval.of(arg.substring(index));
+        }
+      } else {
+        return ExprEval.of(null);
+      }
+    }
+  }
+
+  class ReplaceFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "replace";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() != 3) {
+        throw new IAE("Function[%s] needs 3 arguments", name());
+      }
+
+      final String arg = args.get(0).eval(bindings).asString();
+      final String pattern = args.get(1).eval(bindings).asString();
+      final String replacement = args.get(2).eval(bindings).asString();
+      return ExprEval.of(
+          Strings.nullToEmpty(arg).replace(Strings.nullToEmpty(pattern), Strings.nullToEmpty(replacement))
+      );
+    }
+  }
+
+  class TrimFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "trim";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() != 1) {
+        throw new IAE("Function[%s] needs 1 argument", name());
+      }
+
+      final String arg = args.get(0).eval(bindings).asString();
+      return ExprEval.of(Strings.nullToEmpty(arg).trim());
+    }
+  }
+
+  class LowerFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "lower";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() != 1) {
+        throw new IAE("Function[%s] needs 1 argument", name());
+      }
+
+      final String arg = args.get(0).eval(bindings).asString();
+      return ExprEval.of(Strings.nullToEmpty(arg).toLowerCase());
+    }
+  }
+
+  class UpperFunc implements Function
+  {
+    @Override
+    public String name()
+    {
+      return "upper";
+    }
+
+    @Override
+    public ExprEval apply(List<Expr> args, Expr.ObjectBinding bindings)
+    {
+      if (args.size() != 1) {
+        throw new IAE("Function[%s] needs 1 argument", name());
+      }
+
+      final String arg = args.get(0).eval(bindings).asString();
+      return ExprEval.of(Strings.nullToEmpty(arg).toUpperCase());
+    }
+  }
 }

common/src/test/java/io/druid/math/expr/FunctionTest.java

@@ -0,0 +1,111 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.math.expr;
+
+import com.google.common.collect.ImmutableMap;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class FunctionTest
+{
+  private final Expr.ObjectBinding bindings = Parser.withMap(
+      ImmutableMap.of(
+          "x", "foo",
+          "y", 2,
+          "z", 3.1
+      )
+  );
+
+  @Test
+  public void testCaseSimple()
+  {
+    assertExpr("case_simple(x,'baz','is baz','foo','is foo','is other')", "is foo");
+    assertExpr("case_simple(x,'baz','is baz','bar','is bar','is other')", "is other");
+    assertExpr("case_simple(y,2,'is 2',3,'is 3','is other')", "is 2");
+    assertExpr("case_simple(z,2,'is 2',3,'is 3','is other')", "is other");
+  }
+
+  @Test
+  public void testCaseSearched()
+  {
+    assertExpr("case_searched(x=='baz','is baz',x=='foo','is foo','is other')", "is foo");
+    assertExpr("case_searched(x=='baz','is baz',x=='bar','is bar','is other')", "is other");
+    assertExpr("case_searched(y==2,'is 2',y==3,'is 3','is other')", "is 2");
+    assertExpr("case_searched(z==2,'is 2',z==3,'is 3','is other')", "is other");
+  }
+
+  @Test
+  public void testConcat()
+  {
+    assertExpr("concat(x,' ',y)", "foo 2");
+    assertExpr("concat(x,' ',nonexistent,' ',y)", "foo  2");
+    assertExpr("concat(z)", "3.1");
+    assertExpr("concat()", null);
+  }
+
+  @Test
+  public void testReplace()
+  {
+    assertExpr("replace(x,'oo','ab')", "fab");
+    assertExpr("replace(x,x,'ab')", "ab");
+    assertExpr("replace(x,'oo',y)", "f2");
+  }
+
+  @Test
+  public void testSubstring()
+  {
+    assertExpr("substring(x,0,2)", "fo");
+    assertExpr("substring(x,1,2)", "oo");
+    assertExpr("substring(x,y,1)", "o");
+    assertExpr("substring(x,0,-1)", "foo");
+    assertExpr("substring(x,0,100)", "foo");
+  }
+
+  @Test
+  public void testStrlen()
+  {
+    assertExpr("strlen(x)", 3L);
+    assertExpr("strlen(nonexistent)", 0L);
+  }
+
+  @Test
+  public void testTrim()
+  {
+    assertExpr("trim(concat(' ',x,' '))", "foo");
+  }
+
+  @Test
+  public void testLower()
+  {
+    assertExpr("lower('FOO')", "foo");
+  }
+
+  @Test
+  public void testUpper()
+  {
+    assertExpr("upper(x)", "FOO");
+  }
+
+  private void assertExpr(final String expression, final Object expectedResult)
+  {
+    final Expr expr = Parser.parse(expression, ExprMacroTable.nil());
+    Assert.assertEquals(expression, expectedResult, expr.eval(bindings).value());
+  }
+}

docs/content/misc/math-expr.md

@@ -21,16 +21,48 @@ For logical operators, a number is true if and only if it is positive (0 or minu
 
 Also, the following built-in functions are supported.
 
+## General functions
+
 |name|description|
 |----|-----------|
 |cast|cast(expr,'LONG' or 'DOUBLE' or 'STRING') returns expr with specified type. exception can be thrown |
 |if|if(predicate,then,else) returns 'then' if 'predicate' evaluates to a positive number, otherwise it returns 'else' |
 |nvl|nvl(expr,expr-for-null) returns 'expr-for-null' if 'expr' is null (or empty string for string type) |
 |like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`|
+|case_searched|case_searched(expr1, result1, \[\[expr2, result2, ...\], else-result\])|
+|case_simple|case_simple(expr, value1, result1, \[\[value2, result2, ...\], else-result\])|
+
+## String functions
+
+|name|description|
+|----|-----------|
+|concat|concatenate a list of strings|
+|like|like(expr, pattern[, escape]) is equivalent to SQL `expr LIKE pattern`|
+|lookup|lookup(expr, lookup-name) looks up expr in a registered [query-time lookup](lookups.html)|
+|regexp_extract|regexp_extract(expr, pattern[, index]) applies a regular expression pattern and extracts a capture group index, or null if there is no match. If index is unspecified or zero, returns the substring that matched the pattern.|
+|replace|replace(expr, pattern, replacement) replaces pattern with replacement|
+|substring|substring(expr, index, length) behaves like java.lang.String's substring|
+|strlen|returns length of a string in UTF-16 code units|
+|trim|remove leading and trailing whitespace from a string|
+|lower|convert a string to lowercase|
+|upper|convert a string to uppercase|
+
+## Time functions
+
+|name|description|
+|----|-----------|
 |timestamp|timestamp(expr[,format-string]) parses string expr into date then returns milli-seconds from java epoch. without 'format-string' it's regarded as ISO datetime format |
 |unix_timestamp|same with 'timestamp' function but returns seconds instead |
+|timestamp_ceil|timestamp_ceil(expr, period, \[origin, \[timezone\]\]) rounds up a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
+|timestamp_floor|timestamp_floor(expr, period, \[origin, [timezone\]\]) rounds down a timestamp, returning it as a new timestamp. Period can be any ISO8601 period, like P3M (quarters) or PT12H (half-days). The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
+|timestamp_shift|timestamp_shift(expr, period, step, \[timezone\]) shifts a timestamp by a period (step times), returning it as a new timestamp. Period can be any ISO8601 period. Step may be negative. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00".|
+|timestamp_extract|timestamp_extract(expr, unit, \[timezone\]) extracts a time part from expr, returning it as a number. Unit can be EPOCH, SECOND, MINUTE, HOUR, DAY (day of month), DOW (day of week), DOY (day of year), WEEK (week of [week year](https://en.wikipedia.org/wiki/ISO_week_date)), MONTH (1 through 12), QUARTER (1 through 4), or YEAR. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00"|
+|timestamp_parse|timestamp_parse(string expr, \[pattern, [timezone\]\]) parses a string into a timestamp using a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00", and will be used as the time zone for strings that do not include a time zone offset. Pattern and time zone must be literals. Strings that cannot be parsed as timestamps will be returned as nulls.|
+|timestamp_format|timestamp_format(expr, \[pattern, \[timezone\]\]) formats a timestamp as a string with a given [Joda DateTimeFormat pattern](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html), or ISO8601 if the pattern is not provided. The time zone, if provided, should be a time zone name like "America/Los_Angeles" or offset like "-08:00". Pattern and time zone must be literals.|
 
-And built-in math functions. See javadoc of java.lang.Math for detailed explanation for each function.
+## Math functions
+
+See javadoc of java.lang.Math for detailed explanation for each function.
 
 |name|description|
 |----|-----------|
@@ -44,6 +76,7 @@ And built-in math functions. See javadoc of java.lang.Math for detailed explanat
 |copysign|copysign(x) would return the first floating-point argument with the sign of the second floating-point argument|
 |cos|cos(x) would return the trigonometric cosine of x|
 |cosh|cosh(x) would return the hyperbolic cosine of x|
+|div|div(x,y) is integer division of x by y|
 |exp|exp(x) would return Euler's number raised to the power of x|
 |expm1|expm1(x) would return e^x-1|
 |floor|floor(x) would return the largest (closest to positive infinity) double value that is less than or equal to x and is equal to a mathematical integer|

extensions-contrib/distinctcount/pom.xml

@@ -55,6 +55,11 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.easymock</groupId>
+            <artifactId>easymock</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>

extensions-contrib/scan-query/pom.xml

@@ -46,6 +46,11 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.druid</groupId>
       <artifactId>druid-processing</artifactId>

extensions-contrib/virtual-columns/pom.xml

@@ -59,6 +59,11 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.easymock</groupId>
+            <artifactId>easymock</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>io.druid</groupId>
             <artifactId>druid-processing</artifactId>

extensions-core/datasketches/pom.xml

@@ -108,6 +108,11 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>io.druid</groupId>
       <artifactId>druid-processing</artifactId>

processing/src/main/java/io/druid/query/expression/ExprUtils.java

@@ -19,7 +19,12 @@
 
 package io.druid.query.expression;
 
+import io.druid.java.util.common.IAE;
+import io.druid.java.util.common.granularity.PeriodGranularity;
 import io.druid.math.expr.Expr;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.Period;
 
 public class ExprUtils
 {
@@ -29,4 +34,42 @@ public class ExprUtils
   {
     return NIL_BINDINGS;
   }
+
+  public static DateTimeZone toTimeZone(final Expr timeZoneArg)
+  {
+    if (!timeZoneArg.isLiteral()) {
+      throw new IAE("Time zone must be a literal");
+    }
+
+    final Object literalValue = timeZoneArg.getLiteralValue();
+    return literalValue == null ? DateTimeZone.UTC : DateTimeZone.forID((String) literalValue);
+  }
+
+  public static PeriodGranularity toPeriodGranularity(
+      final Expr periodArg,
+      final Expr originArg,
+      final Expr timeZoneArg,
+      final Expr.ObjectBinding bindings
+  )
+  {
+    final Period period = new Period(periodArg.eval(bindings).asString());
+    final DateTime origin;
+    final DateTimeZone timeZone;
+
+    if (originArg == null) {
+      origin = null;
+    } else {
+      final Object value = originArg.eval(bindings).value();
+      origin = value != null ? new DateTime(value) : null;
+    }
+
+    if (timeZoneArg == null) {
+      timeZone = null;
+    } else {
+      final String value = timeZoneArg.eval(bindings).asString();
+      timeZone = value != null ? DateTimeZone.forID(value) : null;
+    }
+
+    return new PeriodGranularity(period, origin, timeZone);
+  }
 }

processing/src/main/java/io/druid/query/expression/LikeExprMacro.java

@@ -19,6 +19,7 @@
 
 package io.druid.query.expression;
 
+import com.google.common.base.Strings;
 import io.druid.java.util.common.IAE;
 import io.druid.math.expr.Expr;
 import io.druid.math.expr.ExprEval;
@@ -62,7 +63,7 @@ public class LikeExprMacro implements ExprMacroTable.ExprMacro
     }
 
     final LikeDimFilter.LikeMatcher likeMatcher = LikeDimFilter.LikeMatcher.from(
-        (String) patternExpr.getLiteralValue(),
+        Strings.nullToEmpty((String) patternExpr.getLiteralValue()),
         escapeChar
     );
 

processing/src/main/java/io/druid/query/expression/LookupExprMacro.java

@@ -0,0 +1,93 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import com.google.common.base.Strings;
+import com.google.inject.Inject;
+import io.druid.java.util.common.IAE;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import io.druid.query.lookup.LookupReferencesManager;
+import io.druid.query.lookup.RegisteredLookupExtractionFn;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class LookupExprMacro implements ExprMacroTable.ExprMacro
+{
+  private final LookupReferencesManager lookupReferencesManager;
+
+  @Inject
+  public LookupExprMacro(final LookupReferencesManager lookupReferencesManager)
+  {
+    this.lookupReferencesManager = lookupReferencesManager;
+  }
+
+  @Override
+  public String name()
+  {
+    return "lookup";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() != 2) {
+      throw new IAE("Function[%s] must have 2 arguments", name());
+    }
+
+    final Expr arg = args.get(0);
+    final Expr lookupExpr = args.get(1);
+
+    if (!lookupExpr.isLiteral() || lookupExpr.getLiteralValue() == null) {
+      throw new IAE("Function[%s] second argument must be a registered lookup name", name());
+    }
+
+    final String lookupName = lookupExpr.getLiteralValue().toString();
+    final RegisteredLookupExtractionFn extractionFn = new RegisteredLookupExtractionFn(
+        lookupReferencesManager,
+        lookupName,
+        false,
+        null,
+        false,
+        null
+    );
+
+    class LookupExpr implements Expr
+    {
+      @Nonnull
+      @Override
+      public ExprEval eval(final ObjectBinding bindings)
+      {
+        return ExprEval.of(extractionFn.apply(Strings.emptyToNull(arg.eval(bindings).asString())));
+      }
+
+      @Override
+      public void visit(final Visitor visitor)
+      {
+        arg.visit(visitor);
+        visitor.visit(this);
+      }
+    }
+
+    return new LookupExpr();
+  }
+}

processing/src/main/java/io/druid/query/expression/RegexpExtractExprMacro.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import com.google.common.base.Strings;
+import io.druid.java.util.common.IAE;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class RegexpExtractExprMacro implements ExprMacroTable.ExprMacro
+{
+  @Override
+  public String name()
+  {
+    return "regexp_extract";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 2 || args.size() > 3) {
+      throw new IAE("Function[%s] must have 2 to 3 arguments", name());
+    }
+
+    final Expr arg = args.get(0);
+    final Expr patternExpr = args.get(1);
+    final Expr indexExpr = args.size() > 2 ? args.get(2) : null;
+
+    if (!patternExpr.isLiteral() || (indexExpr != null && !indexExpr.isLiteral())) {
+      throw new IAE("Function[%s] pattern and index must be literals", name());
+    }
+
+    // Precompile the pattern.
+    final Pattern pattern = Pattern.compile(String.valueOf(patternExpr.getLiteralValue()));
+
+    final int index = indexExpr == null ? 0 : ((Number) indexExpr.getLiteralValue()).intValue();
+    class RegexpExtractExpr implements Expr
+    {
+      @Nonnull
+      @Override
+      public ExprEval eval(final ObjectBinding bindings)
+      {
+        final Matcher matcher = pattern.matcher(Strings.nullToEmpty(arg.eval(bindings).asString()));
+        final String retVal = matcher.find() ? matcher.group(index) : null;
+        return ExprEval.of(Strings.emptyToNull(retVal));
+      }
+
+      @Override
+      public void visit(final Visitor visitor)
+      {
+        arg.visit(visitor);
+        visitor.visit(this);
+      }
+    }
+    return new RegexpExtractExpr();
+  }
+}

processing/src/main/java/io/druid/query/expression/TimestampCeilExprMacro.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import io.druid.java.util.common.IAE;
+import io.druid.java.util.common.granularity.Granularity;
+import io.druid.java.util.common.granularity.PeriodGranularity;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import org.joda.time.DateTime;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class TimestampCeilExprMacro implements ExprMacroTable.ExprMacro
+{
+  @Override
+  public String name()
+  {
+    return "timestamp_ceil";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 2 || args.size() > 4) {
+      throw new IAE("Function[%s] must have 2 to 4 arguments", name());
+    }
+
+    if (args.stream().skip(1).allMatch(Expr::isLiteral)) {
+      return new TimestampCeilExpr(args);
+    } else {
+      return new TimestampCeilDynamicExpr(args);
+    }
+  }
+
+  private static class TimestampCeilExpr implements Expr
+  {
+    private final Expr arg;
+    private final Granularity granularity;
+
+    public TimestampCeilExpr(final List<Expr> args)
+    {
+      this.arg = args.get(0);
+      this.granularity = getGranularity(args, ExprUtils.nilBindings());
+    }
+
+    @Nonnull
+    @Override
+    public ExprEval eval(final ObjectBinding bindings)
+    {
+      return ExprEval.of(granularity.bucketEnd(new DateTime(arg.eval(bindings).asLong())).getMillis());
+    }
+
+    @Override
+    public void visit(final Visitor visitor)
+    {
+      arg.visit(visitor);
+      visitor.visit(this);
+    }
+  }
+
+  private static PeriodGranularity getGranularity(final List<Expr> args, final Expr.ObjectBinding bindings)
+  {
+    return ExprUtils.toPeriodGranularity(
+        args.get(1),
+        args.size() > 2 ? args.get(2) : null,
+        args.size() > 3 ? args.get(3) : null,
+        bindings
+    );
+  }
+
+  private static class TimestampCeilDynamicExpr implements Expr
+  {
+    private final List<Expr> args;
+
+    public TimestampCeilDynamicExpr(final List<Expr> args)
+    {
+      this.args = args;
+    }
+
+    @Nonnull
+    @Override
+    public ExprEval eval(final ObjectBinding bindings)
+    {
+      final PeriodGranularity granularity = getGranularity(args, bindings);
+      return ExprEval.of(granularity.bucketEnd(new DateTime(args.get(0).eval(bindings).asLong())).getMillis());
+    }
+
+    @Override
+    public void visit(final Visitor visitor)
+    {
+      for (Expr arg : args) {
+        arg.visit(visitor);
+      }
+      visitor.visit(this);
+    }
+  }
+}

processing/src/main/java/io/druid/query/expression/TimestampExtractExprMacro.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import io.druid.java.util.common.IAE;
+import io.druid.java.util.common.ISE;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.chrono.ISOChronology;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class TimestampExtractExprMacro implements ExprMacroTable.ExprMacro
+{
+  public enum Unit
+  {
+    EPOCH,
+    SECOND,
+    MINUTE,
+    HOUR,
+    DAY,
+    DOW,
+    DOY,
+    WEEK,
+    MONTH,
+    QUARTER,
+    YEAR
+  }
+
+  @Override
+  public String name()
+  {
+    return "timestamp_extract";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 2 || args.size() > 3) {
+      throw new IAE("Function[%s] must have 2 to 3 arguments", name());
+    }
+
+    if (!args.get(1).isLiteral() || args.get(1).getLiteralValue() == null) {
+      throw new IAE("Function[%s] unit arg must be literal", name());
+    }
+
+    if (args.size() > 2 && !args.get(2).isLiteral()) {
+      throw new IAE("Function[%s] timezone arg must be literal", name());
+    }
+
+    final Expr arg = args.get(0);
+    final Unit unit = Unit.valueOf(((String) args.get(1).getLiteralValue()).toUpperCase());
+    final DateTimeZone timeZone;
+
+    if (args.size() > 2) {
+      timeZone = ExprUtils.toTimeZone(args.get(2));
+    } else {
+      timeZone = DateTimeZone.UTC;
+    }
+
+    final ISOChronology chronology = ISOChronology.getInstance(timeZone);
+
+    class TimestampExtractExpr implements Expr
+    {
+      @Nonnull
+      @Override
+      public ExprEval eval(final ObjectBinding bindings)
+      {
+        final DateTime dateTime = new DateTime(arg.eval(bindings).asLong()).withChronology(chronology);
+        switch (unit) {
+          case EPOCH:
+            return ExprEval.of(dateTime.getMillis());
+          case SECOND:
+            return ExprEval.of(dateTime.secondOfMinute().get());
+          case MINUTE:
+            return ExprEval.of(dateTime.minuteOfHour().get());
+          case HOUR:
+            return ExprEval.of(dateTime.hourOfDay().get());
+          case DAY:
+            return ExprEval.of(dateTime.dayOfMonth().get());
+          case DOW:
+            return ExprEval.of(dateTime.dayOfWeek().get());
+          case DOY:
+            return ExprEval.of(dateTime.dayOfYear().get());
+          case WEEK:
+            return ExprEval.of(dateTime.weekOfWeekyear().get());
+          case MONTH:
+            return ExprEval.of(dateTime.monthOfYear().get());
+          case QUARTER:
+            return ExprEval.of((dateTime.monthOfYear().get() - 1) / 3 + 1);
+          case YEAR:
+            return ExprEval.of(dateTime.year().get());
+          default:
+            throw new ISE("Unhandled unit[%s]", unit);
+        }
+      }
+
+      @Override
+      public void visit(final Visitor visitor)
+      {
+        arg.visit(visitor);
+        visitor.visit(this);
+      }
+    }
+
+    return new TimestampExtractExpr();
+  }
+}

processing/src/main/java/io/druid/query/expression/TimestampFloorExprMacro.java

@@ -0,0 +1,117 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import io.druid.java.util.common.IAE;
+import io.druid.java.util.common.granularity.Granularity;
+import io.druid.java.util.common.granularity.PeriodGranularity;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import org.joda.time.DateTime;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class TimestampFloorExprMacro implements ExprMacroTable.ExprMacro
+{
+  @Override
+  public String name()
+  {
+    return "timestamp_floor";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 2 || args.size() > 4) {
+      throw new IAE("Function[%s] must have 2 to 4 arguments", name());
+    }
+
+    if (args.stream().skip(1).allMatch(Expr::isLiteral)) {
+      return new TimestampFloorExpr(args);
+    } else {
+      return new TimestampFloorDynamicExpr(args);
+    }
+  }
+
+  private static PeriodGranularity getGranularity(final List<Expr> args, final Expr.ObjectBinding bindings)
+  {
+    return ExprUtils.toPeriodGranularity(
+        args.get(1),
+        args.size() > 2 ? args.get(2) : null,
+        args.size() > 3 ? args.get(3) : null,
+        bindings
+    );
+  }
+
+  private static class TimestampFloorExpr implements Expr
+  {
+    private final Expr arg;
+    private final Granularity granularity;
+
+    public TimestampFloorExpr(final List<Expr> args)
+    {
+      this.arg = args.get(0);
+      this.granularity = getGranularity(args, ExprUtils.nilBindings());
+    }
+
+    @Nonnull
+    @Override
+    public ExprEval eval(final ObjectBinding bindings)
+    {
+      return ExprEval.of(granularity.bucketStart(new DateTime(arg.eval(bindings).asLong())).getMillis());
+    }
+
+    @Override
+    public void visit(final Visitor visitor)
+    {
+      arg.visit(visitor);
+      visitor.visit(this);
+    }
+  }
+
+  private static class TimestampFloorDynamicExpr implements Expr
+  {
+    private final List<Expr> args;
+
+    public TimestampFloorDynamicExpr(final List<Expr> args)
+    {
+      this.args = args;
+    }
+
+    @Nonnull
+    @Override
+    public ExprEval eval(final ObjectBinding bindings)
+    {
+      final PeriodGranularity granularity = getGranularity(args, bindings);
+      return ExprEval.of(granularity.bucketStart(new DateTime(args.get(0).eval(bindings).asLong())).getMillis());
+    }
+
+    @Override
+    public void visit(final Visitor visitor)
+    {
+      for (Expr arg : args) {
+        arg.visit(visitor);
+      }
+      visitor.visit(this);
+    }
+  }
+}

processing/src/main/java/io/druid/query/expression/TimestampFormatExprMacro.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import com.google.common.base.Preconditions;
+import io.druid.java.util.common.IAE;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class TimestampFormatExprMacro implements ExprMacroTable.ExprMacro
+{
+  @Override
+  public String name()
+  {
+    return "timestamp_format";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 1 || args.size() > 3) {
+      throw new IAE("Function[%s] must have 1 to 3 arguments", name());
+    }
+
+    final Expr arg = args.get(0);
+    final String formatString;
+    final DateTimeZone timeZone;
+
+    if (args.size() > 1) {
+      Preconditions.checkArgument(args.get(1).isLiteral(), "Function[%s] format arg must be a literal", name());
+      formatString = (String) args.get(1).getLiteralValue();
+    } else {
+      formatString = null;
+    }
+
+    if (args.size() > 2) {
+      timeZone = ExprUtils.toTimeZone(args.get(2));
+    } else {
+      timeZone = DateTimeZone.UTC;
+    }
+
+    final DateTimeFormatter formatter = formatString == null
+                                        ? ISODateTimeFormat.dateTime()
+                                        : DateTimeFormat.forPattern(formatString).withZone(timeZone);
+
+    class TimestampFormatExpr implements Expr
+    {
+      @Nonnull
+      @Override
+      public ExprEval eval(final ObjectBinding bindings)
+      {
+        return ExprEval.of(formatter.print(arg.eval(bindings).asLong()));
+      }
+
+      @Override
+      public void visit(final Visitor visitor)
+      {
+        arg.visit(visitor);
+        visitor.visit(this);
+      }
+    }
+
+    return new TimestampFormatExpr();
+  }
+}

processing/src/main/java/io/druid/query/expression/TimestampParseExprMacro.java

@@ -0,0 +1,89 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import io.druid.java.util.common.IAE;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class TimestampParseExprMacro implements ExprMacroTable.ExprMacro
+{
+  @Override
+  public String name()
+  {
+    return "timestamp_parse";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 1 || args.size() > 3) {
+      throw new IAE("Function[%s] must have 1 to 3 arguments", name());
+    }
+
+    final Expr arg = args.get(0);
+    final String formatString = args.size() > 1 ? (String) args.get(1).getLiteralValue() : null;
+    final DateTimeZone timeZone;
+
+    if (args.size() > 2 && args.get(2).getLiteralValue() != null) {
+      timeZone = DateTimeZone.forID((String) args.get(2).getLiteralValue());
+    } else {
+      timeZone = DateTimeZone.UTC;
+    }
+
+    final DateTimeFormatter formatter = formatString == null
+                                        ? ISODateTimeFormat.dateTimeParser()
+                                        : DateTimeFormat.forPattern(formatString).withZone(timeZone);
+
+    class TimestampParseExpr implements Expr
+    {
+      @Nonnull
+      @Override
+      public ExprEval eval(final ObjectBinding bindings)
+      {
+        try {
+          return ExprEval.of(formatter.parseDateTime(arg.eval(bindings).asString()).getMillis());
+        }
+        catch (IllegalArgumentException e) {
+          // Catch exceptions potentially thrown by formatter.parseDateTime. Our docs say that unparseable timestamps
+          // are returned as nulls.
+          return ExprEval.of(null);
+        }
+      }
+
+      @Override
+      public void visit(final Visitor visitor)
+      {
+        arg.visit(visitor);
+        visitor.visit(this);
+      }
+    }
+
+    return new TimestampParseExpr();
+  }
+}

processing/src/main/java/io/druid/query/expression/TimestampShiftExprMacro.java

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import io.druid.java.util.common.IAE;
+import io.druid.java.util.common.granularity.PeriodGranularity;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.ExprEval;
+import io.druid.math.expr.ExprMacroTable;
+import org.joda.time.Chronology;
+import org.joda.time.Period;
+import org.joda.time.chrono.ISOChronology;
+
+import javax.annotation.Nonnull;
+import java.util.List;
+
+public class TimestampShiftExprMacro implements ExprMacroTable.ExprMacro
+{
+  @Override
+  public String name()
+  {
+    return "timestamp_shift";
+  }
+
+  @Override
+  public Expr apply(final List<Expr> args)
+  {
+    if (args.size() < 3 || args.size() > 4) {
+      throw new IAE("Function[%s] must have 3 to 4 arguments", name());
+    }
+
+    if (args.stream().skip(1).allMatch(Expr::isLiteral)) {
+      return new TimestampShiftExpr(args);
+    } else {
+      // Use dynamic impl if any args are non-literal. Don't bother optimizing for the case where period is
+      // literal but step isn't.
+      return new TimestampShiftDynamicExpr(args);
+    }
+  }
+
+  private static PeriodGranularity getGranularity(final List<Expr> args, final Expr.ObjectBinding bindings)
+  {
+    return ExprUtils.toPeriodGranularity(
+        args.get(1),
+        null,
+        args.size() > 3 ? args.get(3) : null,
+        bindings
+    );
+  }
+
+  private static int getStep(final List<Expr> args, final Expr.ObjectBinding bindings)
+  {
+    return args.get(2).eval(bindings).asInt();
+  }
+
+  private static class TimestampShiftExpr implements Expr
+  {
+    private final Expr arg;
+    private final Chronology chronology;
+    private final Period period;
+    private final int step;
+
+    public TimestampShiftExpr(final List<Expr> args)
+    {
+      final PeriodGranularity granularity = getGranularity(args, ExprUtils.nilBindings());
+      arg = args.get(0);
+      period = granularity.getPeriod();
+      chronology = ISOChronology.getInstance(granularity.getTimeZone());
+      step = getStep(args, ExprUtils.nilBindings());
+    }
+
+    @Nonnull
+    @Override
+    public ExprEval eval(final ObjectBinding bindings)
+    {
+      return ExprEval.of(chronology.add(period, arg.eval(bindings).asLong(), step));
+    }
+
+    @Override
+    public void visit(final Visitor visitor)
+    {
+      arg.visit(visitor);
+      visitor.visit(this);
+    }
+  }
+
+  private static class TimestampShiftDynamicExpr implements Expr
+  {
+    private final List<Expr> args;
+
+    public TimestampShiftDynamicExpr(final List<Expr> args)
+    {
+      this.args = args;
+    }
+
+    @Nonnull
+    @Override
+    public ExprEval eval(final ObjectBinding bindings)
+    {
+      final PeriodGranularity granularity = getGranularity(args, bindings);
+      final Period period = granularity.getPeriod();
+      final Chronology chronology = ISOChronology.getInstance(granularity.getTimeZone());
+      final int step = getStep(args, bindings);
+      return ExprEval.of(chronology.add(period, args.get(0).eval(bindings).asLong(), step));
+    }
+
+    @Override
+    public void visit(final Visitor visitor)
+    {
+      for (Expr arg : args) {
+        arg.visit(visitor);
+      }
+      visitor.visit(this);
+    }
+  }
+}

processing/src/test/java/io/druid/query/expression/ExprMacroTest.java

@@ -0,0 +1,143 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.expression;
+
+import com.google.common.collect.ImmutableMap;
+import io.druid.math.expr.Expr;
+import io.druid.math.expr.Parser;
+import org.joda.time.DateTime;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class ExprMacroTest
+{
+  private static final Expr.ObjectBinding BINDINGS = Parser.withMap(
+      ImmutableMap.<String, Object>builder()
+          .put("t", new DateTime("2000-02-03T04:05:06").getMillis())
+          .put("tstr", "2000-02-03T04:05:06")
+          .put("tstr_sql", "2000-02-03 04:05:06")
+          .put("x", "foo")
+          .put("y", 2)
+          .put("z", 3.1)
+          .put("CityOfAngels", "America/Los_Angeles")
+          .build()
+  );
+
+  @Rule
+  public ExpectedException expectedException = ExpectedException.none();
+
+  @Test
+  public void testLike()
+  {
+    assertExpr("like(x, 'f%')", 1L);
+    assertExpr("like(x, 'f__')", 1L);
+    assertExpr("like(x, '%o%')", 1L);
+    assertExpr("like(x, 'b%')", 0L);
+    assertExpr("like(x, 'b__')", 0L);
+    assertExpr("like(x, '%x%')", 0L);
+    assertExpr("like(x, '')", 0L);
+  }
+
+  @Test
+  public void testLookup()
+  {
+    assertExpr("lookup(x, 'lookyloo')", "xfoo");
+  }
+
+  @Test
+  public void testLookupNotFound()
+  {
+    expectedException.expect(NullPointerException.class);
+    expectedException.expectMessage("Lookup [lookylook] not found");
+    assertExpr("lookup(x, 'lookylook')", null);
+  }
+
+  @Test
+  public void testRegexpExtract()
+  {
+    assertExpr("regexp_extract(x, 'f(.)')", "fo");
+    assertExpr("regexp_extract(x, 'f(.)', 0)", "fo");
+    assertExpr("regexp_extract(x, 'f(.)', 1)", "o");
+  }
+
+  @Test
+  public void testTimestampCeil()
+  {
+    assertExpr("timestamp_ceil(t, 'P1M')", new DateTime("2000-03-01").getMillis());
+    assertExpr("timestamp_ceil(t, 'P1D','','America/Los_Angeles')", new DateTime("2000-02-03T08").getMillis());
+    assertExpr("timestamp_ceil(t, 'P1D','',CityOfAngels)", new DateTime("2000-02-03T08").getMillis());
+    assertExpr("timestamp_ceil(t, 'P1D','1970-01-01T01','Etc/UTC')", new DateTime("2000-02-04T01").getMillis());
+  }
+
+  @Test
+  public void testTimestampFloor()
+  {
+    assertExpr("timestamp_floor(t, 'P1M')", new DateTime("2000-02-01").getMillis());
+    assertExpr("timestamp_floor(t, 'P1D','','America/Los_Angeles')", new DateTime("2000-02-02T08").getMillis());
+    assertExpr("timestamp_floor(t, 'P1D','',CityOfAngels)", new DateTime("2000-02-02T08").getMillis());
+    assertExpr("timestamp_floor(t, 'P1D','1970-01-01T01','Etc/UTC')", new DateTime("2000-02-03T01").getMillis());
+  }
+
+  @Test
+  public void testTimestampShift()
+  {
+    assertExpr("timestamp_shift(t, 'P1D', 2)", new DateTime("2000-02-05T04:05:06").getMillis());
+    assertExpr("timestamp_shift(t, 'P1D', 2, 'America/Los_Angeles')", new DateTime("2000-02-05T04:05:06").getMillis());
+    assertExpr("timestamp_shift(t, 'P1D', 2, CityOfAngels)", new DateTime("2000-02-05T04:05:06").getMillis());
+    assertExpr("timestamp_shift(t, 'P1D', 2, '-08:00')", new DateTime("2000-02-05T04:05:06").getMillis());
+  }
+
+  @Test
+  public void testTimestampExtract()
+  {
+    assertExpr("timestamp_extract(t, 'DAY')", 3L);
+    assertExpr("timestamp_extract(t, 'HOUR')", 4L);
+    assertExpr("timestamp_extract(t, 'DAY', 'America/Los_Angeles')", 2L);
+    assertExpr("timestamp_extract(t, 'HOUR', 'America/Los_Angeles')", 20L);
+  }
+
+  @Test
+  public void testTimestampParse()
+  {
+    assertExpr("timestamp_parse(tstr)", new DateTime("2000-02-03T04:05:06").getMillis());
+    assertExpr("timestamp_parse(tstr_sql)", null);
+    assertExpr("timestamp_parse(tstr_sql,'yyyy-MM-dd HH:mm:ss')", new DateTime("2000-02-03T04:05:06").getMillis());
+    assertExpr(
+        "timestamp_parse(tstr_sql,'yyyy-MM-dd HH:mm:ss','America/Los_Angeles')",
+        new DateTime("2000-02-03T04:05:06-08:00").getMillis()
+    );
+  }
+
+  @Test
+  public void testTimestampFormat()
+  {
+    assertExpr("timestamp_format(t)", "2000-02-03T04:05:06.000Z");
+    assertExpr("timestamp_format(t,'yyyy-MM-dd HH:mm:ss')", "2000-02-03 04:05:06");
+    assertExpr("timestamp_format(t,'yyyy-MM-dd HH:mm:ss','America/Los_Angeles')", "2000-02-02 20:05:06");
+  }
+
+  private void assertExpr(final String expression, final Object expectedResult)
+  {
+    final Expr expr = Parser.parse(expression, TestExprMacroTable.INSTANCE);
+    Assert.assertEquals(expression, expectedResult, expr.eval(BINDINGS).value());
+  }
+}

processing/src/test/java/io/druid/query/expression/TestExprMacroTable.java

@@ -20,7 +20,17 @@
 package io.druid.query.expression;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import io.druid.math.expr.ExprMacroTable;
+import io.druid.query.extraction.MapLookupExtractor;
+import io.druid.query.lookup.LookupExtractor;
+import io.druid.query.lookup.LookupExtractorFactory;
+import io.druid.query.lookup.LookupExtractorFactoryContainer;
+import io.druid.query.lookup.LookupIntrospectHandler;
+import io.druid.query.lookup.LookupReferencesManager;
+import org.easymock.EasyMock;
+
+import javax.annotation.Nullable;
 
 public class TestExprMacroTable extends ExprMacroTable
 {
@@ -30,8 +40,64 @@ public class TestExprMacroTable extends ExprMacroTable
   {
     super(
         ImmutableList.of(
-            new LikeExprMacro()
+            new LikeExprMacro(),
+            new LookupExprMacro(createTestLookupReferencesManager(ImmutableMap.of("foo", "xfoo"))),
+            new RegexpExtractExprMacro(),
+            new TimestampCeilExprMacro(),
+            new TimestampExtractExprMacro(),
+            new TimestampFloorExprMacro(),
+            new TimestampFormatExprMacro(),
+            new TimestampParseExprMacro(),
+            new TimestampShiftExprMacro()
         )
     );
   }
+
+  /**
+   * Returns a mock {@link LookupReferencesManager} that has one lookup, "lookyloo".
+   */
+  public static LookupReferencesManager createTestLookupReferencesManager(final ImmutableMap<String, String> theLookup)
+  {
+    final LookupReferencesManager lookupReferencesManager = EasyMock.createMock(LookupReferencesManager.class);
+    EasyMock.expect(lookupReferencesManager.get(EasyMock.eq("lookyloo"))).andReturn(
+        new LookupExtractorFactoryContainer(
+            "v0",
+            new LookupExtractorFactory()
+            {
+              @Override
+              public boolean start()
+              {
+                throw new UnsupportedOperationException();
+              }
+
+              @Override
+              public boolean close()
+              {
+                throw new UnsupportedOperationException();
+              }
+
+              @Override
+              public boolean replaces(@Nullable final LookupExtractorFactory other)
+              {
+                throw new UnsupportedOperationException();
+              }
+
+              @Override
+              public LookupIntrospectHandler getIntrospectHandler()
+              {
+                throw new UnsupportedOperationException();
+              }
+
+              @Override
+              public LookupExtractor get()
+              {
+                return new MapLookupExtractor(theLookup, false);
+              }
+            }
+        )
+    ).anyTimes();
+    EasyMock.expect(lookupReferencesManager.get(EasyMock.not(EasyMock.eq("lookyloo")))).andReturn(null).anyTimes();
+    EasyMock.replay(lookupReferencesManager);
+    return lookupReferencesManager;
+  }
 }

server/src/main/java/io/druid/guice/ExpressionModule.java

@@ -27,6 +27,13 @@ import io.druid.initialization.DruidModule;
 import io.druid.math.expr.ExprMacroTable;
 import io.druid.query.expression.GuiceExprMacroTable;
 import io.druid.query.expression.LikeExprMacro;
+import io.druid.query.expression.RegexpExtractExprMacro;
+import io.druid.query.expression.TimestampCeilExprMacro;
+import io.druid.query.expression.TimestampExtractExprMacro;
+import io.druid.query.expression.TimestampFloorExprMacro;
+import io.druid.query.expression.TimestampFormatExprMacro;
+import io.druid.query.expression.TimestampParseExprMacro;
+import io.druid.query.expression.TimestampShiftExprMacro;
 
 import java.util.List;
 
@@ -37,6 +44,13 @@ public class ExpressionModule implements DruidModule
   public static final List<Class<? extends ExprMacroTable.ExprMacro>> EXPR_MACROS =
       ImmutableList.<Class<? extends ExprMacroTable.ExprMacro>>builder()
           .add(LikeExprMacro.class)
+          .add(RegexpExtractExprMacro.class)
+          .add(TimestampCeilExprMacro.class)
+          .add(TimestampExtractExprMacro.class)
+          .add(TimestampFloorExprMacro.class)
+          .add(TimestampFormatExprMacro.class)
+          .add(TimestampParseExprMacro.class)
+          .add(TimestampShiftExprMacro.class)
           .build();
 
   @Override

server/src/main/java/io/druid/query/lookup/LookupModule.java

@@ -35,6 +35,7 @@ import com.google.inject.Binder;
 import com.google.inject.Inject;
 import io.druid.common.utils.ServletResourceUtils;
 import io.druid.curator.announcement.Announcer;
+import io.druid.guice.ExpressionModule;
 import io.druid.guice.Jerseys;
 import io.druid.guice.JsonConfigProvider;
 import io.druid.guice.LifecycleModule;
@@ -44,6 +45,7 @@ import io.druid.guice.annotations.Self;
 import io.druid.guice.annotations.Smile;
 import io.druid.initialization.DruidModule;
 import io.druid.java.util.common.logger.Logger;
+import io.druid.query.expression.LookupExprMacro;
 import io.druid.server.DruidNode;
 import io.druid.server.initialization.ZkPathsConfig;
 import io.druid.server.initialization.jetty.JettyBindings;
@@ -89,6 +91,7 @@ public class LookupModule implements DruidModule
     JsonConfigProvider.bind(binder, PROPERTY_BASE, LookupListeningAnnouncerConfig.class);
     Jerseys.addResource(binder, LookupListeningResource.class);
     Jerseys.addResource(binder, LookupIntrospectionResource.class);
+    ExpressionModule.addExprMacro(binder, LookupExprMacro.class);
     LifecycleModule.register(binder, LookupResourceListenerAnnouncer.class);
     // Nothing else starts this, so we bind it to get it to start
     binder.bind(LookupResourceListenerAnnouncer.class).in(ManageLifecycle.class);