SOLR-2690: New support for a "TZ" request param which overrides the TimeZone used when rounding Dates in DateMath expressions

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1329837 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2012-04-24 16:35:41 +00:00
parent 6f43ebfd90
commit d2875edd99
11 changed files with 463 additions and 20 deletions

View File

@ -274,6 +274,11 @@ New Features
* SOLR-3363: Consolidated Exceptions in Analysis Factories so they only throw
InitalizationExceptions (Chris Male)
* SOLR-2690: New support for a "TZ" request param which overrides the TimeZone
used when rounding Dates in DateMath expressions for the entire request
(all date range queries and date faceting is affected). The default TZ
is still UTC. (David Schlotfeldt, hossman)
Optimizations
----------------------

View File

@ -863,7 +863,7 @@ public class SimpleFacets {
}
final String gap = required.getFieldParam(f,FacetParams.FACET_DATE_GAP);
final DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.US);
final DateMathParser dmp = new DateMathParser();
final int minCount = params.getFieldInt(f,FacetParams.FACET_MINCOUNT, 0);
@ -1387,7 +1387,7 @@ public class SimpleFacets {
}
@Override
public Date parseAndAddGap(Date value, String gap) throws java.text.ParseException {
final DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.US);
final DateMathParser dmp = new DateMathParser();
dmp.setNow(value);
return dmp.parseMath(gap);
}

View File

@ -18,12 +18,15 @@
package org.apache.solr.request;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.TimeZoneUtils;
import java.io.Closeable;
import java.util.Date;
import java.util.TimeZone;
import java.util.LinkedList;
import java.util.List;
@ -34,6 +37,7 @@ public class SolrRequestInfo {
protected SolrQueryRequest req;
protected SolrQueryResponse rsp;
protected Date now;
protected TimeZone tz;
protected ResponseBuilder rb;
protected List<Closeable> closeHooks;
@ -79,7 +83,7 @@ public class SolrRequestInfo {
if (now != null) return now;
long ms = 0;
String nowStr = req.getParams().get("NOW");
String nowStr = req.getParams().get(CommonParams.NOW);
if (nowStr != null) {
ms = Long.parseLong(nowStr);
@ -91,6 +95,22 @@ public class SolrRequestInfo {
return now;
}
/** The TimeZone specified by the request, or null if none was specified */
public TimeZone getClientTimeZone() {
if (tz == null) {
String tzStr = req.getParams().get(CommonParams.TZ);
if (tzStr != null) {
tz = TimeZoneUtils.getTimeZone(tzStr);
if (null == tz) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Solr JVM does not support TZ: " + tzStr);
}
}
}
return tz;
}
public SolrQueryRequest getReq() {
return req;
}

View File

@ -105,12 +105,18 @@ public class DateField extends PrimitiveFieldType {
public static TimeZone UTC = TimeZone.getTimeZone("UTC");
/* :TODO: let Locale/TimeZone come from init args for rounding only */
/** TimeZone for DateMath (UTC) */
protected static final TimeZone MATH_TZ = UTC;
/** Locale for DateMath (Locale.US) */
protected static final Locale MATH_LOCALE = Locale.US;
/**
* No longer used
* @deprecated use DateMathParser.DEFAULT_MATH_TZ
* @see DateMathParser#DEFAULT_MATH_TZ
*/
protected static final TimeZone MATH_TZ = DateMathParser.DEFAULT_MATH_TZ;
/**
* No longer used
* @deprecated use DateMathParser.DEFAULT_MATH_LOCALE
* @see DateMathParser#DEFAULT_MATH_LOCALE
*/
protected static final Locale MATH_LOCALE = DateMathParser.DEFAULT_MATH_LOCALE;
/**
* Fixed TimeZone (UTC) needed for parsing/formating Dates in the
@ -146,7 +152,7 @@ public class DateField extends PrimitiveFieldType {
*/
public Date parseMath(Date now, String val) {
String math = null;
final DateMathParser p = new DateMathParser(MATH_TZ, MATH_LOCALE);
final DateMathParser p = new DateMathParser();
if (null != now) p.setNow(now);
@ -296,7 +302,7 @@ public class DateField extends PrimitiveFieldType {
*/
public Date parseMathLenient(Date now, String val, SolrQueryRequest req) {
String math = null;
final DateMathParser p = new DateMathParser(MATH_TZ, MATH_LOCALE);
final DateMathParser p = new DateMathParser();
if (null != now) p.setNow(now);

View File

@ -78,6 +78,13 @@ import java.util.regex.Pattern;
*
*/
public class DateMathParser {
public static TimeZone UTC = TimeZone.getTimeZone("UTC");
/** Default TimeZone for DateMath rounding (UTC) */
public static final TimeZone DEFAULT_MATH_TZ = UTC;
/** Default Locale for DateMath rounding (Locale.US) */
public static final Locale DEFAULT_MATH_LOCALE = Locale.US;
/**
* A mapping from (uppercased) String labels idenyifying time units,
@ -101,6 +108,10 @@ public class DateMathParser {
// because of complexity in rounding down to the nearest week
// arround a month/year boundry.
// (Not to mention: it's not clear what people would *expect*)
//
// If we consider adding some time of "week" support, then
// we probably need to change "Locale loc" to default to something
// from a param via SolrRequestInfo as well.
Map<String,Integer> units = new HashMap<String,Integer>(13);
units.put("YEAR", Calendar.YEAR);
@ -193,21 +204,53 @@ public class DateMathParser {
private Date now;
/**
* @param tz The TimeZone used for rounding (to determine when hours/days begin)
* @param l The Locale used for rounding (to determine when weeks begin)
* @see Calendar#getInstance(TimeZone,Locale)
* Default constructor that assumes UTC should be used for rounding unless
* otherwise specified in the SolrRequestInfo
*
* @see #DEFAULT_MATH_TZ
* @see #DEFAULT_MATH_LOCALE
*/
public DateMathParser(TimeZone tz, Locale l) {
zone = tz;
loc = l;
public DateMathParser() {
this(null, DEFAULT_MATH_LOCALE);
}
/** Redefines this instance's concept of "now" */
/**
* @param tz The TimeZone used for rounding (to determine when hours/days begin). If null, then this method defaults to the value dicated by the SolrRequestInfo if it
* exists -- otherwise it uses UTC.
* @param l The Locale used for rounding (to determine when weeks begin). If null, then this method defaults to en_US.
* @see #DEFAULT_MATH_TZ
* @see #DEFAULT_MATH_LOCALE
* @see Calendar#getInstance(TimeZone,Locale)
* @see SolrRequestInfo#getClientTimeZone
*/
public DateMathParser(TimeZone tz, Locale l) {
loc = (null != l) ? l : DEFAULT_MATH_LOCALE;
if (null == tz) {
SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
tz = (null != reqInfo) ? reqInfo.getClientTimeZone() : DEFAULT_MATH_TZ;
}
zone = (null != tz) ? tz : DEFAULT_MATH_TZ;
}
/**
* Defines this instance's concept of "now".
* @see #getNow
*/
public void setNow(Date n) {
now = n;
}
/** Returns a cloned of this instance's concept of "now" */
/**
* Returns a cloned of this instance's concept of "now".
*
* If setNow was never called (or if null was specified) then this method
* first defines 'now' as the value dictated by the SolrRequestInfo if it
* exists -- otherwise it uses a new Date instance at the moment getNow()
* is first called.
* @see #setNow
* @see SolrRequestInfo#getNow
*/
public Date getNow() {
if (now == null) {
SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();

View File

@ -0,0 +1,86 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util;
import java.util.TimeZone;
import java.util.Set;
import java.util.HashSet;
import java.util.Collections;
import java.util.Arrays;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
/**
* Simple utilities for working with TimeZones
* @see java.util.TimeZone
*/
public final class TimeZoneUtils {
private TimeZoneUtils() {
// :NOOP:
}
/**
* An immutable Set of all TimeZone IDs supported by the TimeZone class
* at the moment the TimeZoneUtils was initialized.
*
* @see TimeZone#getAvailableIDs
*/
public static final Set<String> KNOWN_TIMEZONE_IDS
= Collections.unmodifiableSet(new HashSet<String>
(Arrays.asList(TimeZone.getAvailableIDs())));
/**
* This method is provided as a replacement for TimeZone.getTimeZone but
* with out the anoying behavior of returning "GMT" for gibberish input.
* <p>
* This method will return null unless the input is either:
* </p>
* <ul>
* <li>Included in the set of known TimeZone IDs
* <li>A "CustomID" specified as a numeric offset from "GMT"</li>
* </ul>
*
* @param ID Either a TimeZone ID found in KNOWN_TIMEZONE_IDS, or a "CustomID" specified as a GMT offset.
* @return A TImeZone object corrisponding to the input, or null if no such TimeZone is supported.
* @see #KNOWN_TIMEZONE_IDS
* @see TimeZone
*/
public static final TimeZone getTimeZone(final String ID) {
if (null == ID) return null;
if (KNOWN_TIMEZONE_IDS.contains(ID)) return TimeZone.getTimeZone(ID);
Matcher matcher = CUSTOM_ID_REGEX.matcher(ID);
if (matcher.matches()) {
int hour = Integer.parseInt(matcher.group(1));
if (hour < 0 || 23 < hour) return null;
final String minStr = matcher.group(2);
if (null != minStr) {
int min = Integer.parseInt(minStr);
if (min < 0 || 59 < min) return null;
}
return TimeZone.getTimeZone(ID);
}
return null;
}
private static Pattern CUSTOM_ID_REGEX = Pattern.compile("GMT(?:\\+|\\-)(\\d{1,2})(?::?(\\d{2}))?");
}

View File

@ -690,6 +690,19 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
assertQ("check count for near stuff",
req("q", "bday:[NOW-1MONTH TO NOW+2HOURS]"), "*[count(//doc)=4]");
assertQ("check counts using fixed NOW",
req("q", "bday:[NOW/DAY TO NOW/DAY+1DAY]",
"NOW", "205369736000" // 1976-07-04T23:08:56.235Z
),
"*[count(//doc)=1]");
assertQ("check counts using fixed NOW and TZ rounding",
req("q", "bday:[NOW/DAY TO NOW/DAY+1DAY]",
"TZ", "GMT-23",
"NOW", "205369736000" // 1976-07-04T23:08:56.235Z
),
"*[count(//doc)=0]");
}
public void testDateRoundtrip() {

View File

@ -22,16 +22,23 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.TimeZoneUtils;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.HashSet;
import java.util.TimeZone;
public class SimpleFacetsTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig.xml","schema.xml");
@ -1028,6 +1035,100 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
);
}
@Test
public void testDateFacetsWithTz() {
for (String field : new String[] { "a_tdt", "a_pdt"}) {
for (boolean rangeType : new boolean[] { true, false }) {
helpTestDateFacetsWithTz(field, rangeType);
}
}
}
private void helpTestDateFacetsWithTz(final String fieldName,
final boolean rangeMode) {
final String p = rangeMode ? "facet.range" : "facet.date";
final String b = rangeMode ? "facet_ranges" : "facet_dates";
final String f = fieldName;
final String c = (rangeMode ? "/lst[@name='counts']" : "");
final String pre = "//lst[@name='"+b+"']/lst[@name='"+f+"']" + c;
final String meta = pre + (rangeMode ? "/../" : "");
final String TZ = "America/Los_Angeles";
assumeTrue("Test requires JVM to know about about TZ: " + TZ,
TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(TZ));
assertQ("checking facet counts for fixed now, using TZ: " + TZ,
req( "q", "*:*"
,"rows", "0"
,"facet", "true"
,"NOW", "205078333000" // 1976-07-01T14:12:13.000Z
,"TZ", TZ
,p, f
,p+".start", "NOW/MONTH"
,p+".end", "NOW/MONTH+15DAYS"
,p+".gap", "+1DAY"
,p+".other", "all"
,p+".include", "lower"
)
// 15 days + pre+post+inner = 18
,"*[count("+pre+"/int)="+(rangeMode ? 15 : 18)+"]"
,pre+"/int[@name='1976-07-01T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-02T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-03T07:00:00Z'][.='1' ]"
,pre+"/int[@name='1976-07-04T07:00:00Z'][.='1' ]"
,pre+"/int[@name='1976-07-05T07:00:00Z'][.='1' ]"
,pre+"/int[@name='1976-07-06T07:00:00Z'][.='1' ]"
,pre+"/int[@name='1976-07-07T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-08T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-09T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-10T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-11T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-12T07:00:00Z'][.='1' ]"
,pre+"/int[@name='1976-07-13T07:00:00Z'][.='1' ]"
,pre+"/int[@name='1976-07-14T07:00:00Z'][.='0']"
,pre+"/int[@name='1976-07-15T07:00:00Z'][.='1' ]"
//
,meta+"/int[@name='before' ][.='2']"
,meta+"/int[@name='after' ][.='1']"
,meta+"/int[@name='between'][.='7']"
);
// NOTE: the counts should all be zero, what we really care about
// is that the computed lower bounds take into account DST change
assertQ("checking facet counts arround DST change for TZ: " + TZ,
req( "q", "*:*"
,"rows", "0"
,"facet", "true"
,"NOW", "1288606136000" // 2010-11-01T10:08:56.235Z
,"TZ", TZ
,p, f
,p+".start", "NOW/MONTH"
,p+".end", "NOW/MONTH+15DAYS"
,p+".gap", "+1DAY"
,p+".other", "all"
,p+".include", "lower"
)
// 15 days + pre+post+inner = 18
,"*[count("+pre+"/int)="+(rangeMode ? 15 : 18)+"]"
,pre+"/int[@name='2010-11-01T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-02T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-03T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-04T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-05T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-06T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-07T07:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-08T08:00:00Z'][.='0']" // BOOM!
,pre+"/int[@name='2010-11-09T08:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-10T08:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-11T08:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-12T08:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-13T08:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-14T08:00:00Z'][.='0']"
,pre+"/int[@name='2010-11-15T08:00:00Z'][.='0']"
);
}
@Test
public void testNumericRangeFacetsTrieFloat() {
helpTestFractionalNumberRangeFacets("range_facet_f");

View File

@ -26,6 +26,9 @@ import java.util.Calendar;
import java.util.Date;
import java.util.TimeZone;
import java.util.Locale;
import java.util.Arrays;
import java.util.Set;
import java.util.HashSet;
import java.util.Map;
import java.util.HashMap;
@ -255,7 +258,57 @@ public class DateMathParserTest extends LuceneTestCase {
}
public void testParseMathTz() throws Exception {
final String PLUS_TZS = "America/Los_Angeles";
final String NEG_TZS = "Europe/Paris";
assumeTrue("Test requires JVM to know about about TZ: " + PLUS_TZS,
TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(PLUS_TZS));
assumeTrue("Test requires JVM to know about about TZ: " + NEG_TZS,
TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(NEG_TZS));
// US, Positive Offset with DST
TimeZone tz = TimeZone.getTimeZone(PLUS_TZS);
DateMathParser p = new DateMathParser(tz, Locale.US);
p.setNow(parser.parse("2001-07-04T12:08:56.235"));
// No-Op
assertMath("2001-07-04T12:08:56.235", p, "");
assertMath("2001-07-04T12:08:56.000", p, "/SECOND");
assertMath("2001-07-04T12:08:00.000", p, "/MINUTE");
assertMath("2001-07-04T12:00:00.000", p, "/HOUR");
assertMath("2001-07-04T07:00:00.000", p, "/DAY");
assertMath("2001-07-01T07:00:00.000", p, "/MONTH");
// no DST in jan
assertMath("2001-01-01T08:00:00.000", p, "/YEAR");
// no DST in nov 2001
assertMath("2001-11-04T08:00:00.000", p, "+4MONTH/DAY");
// yes DST in nov 2010
assertMath("2010-11-04T07:00:00.000", p, "+9YEAR+4MONTH/DAY");
// France, Negative Offset with DST
tz = TimeZone.getTimeZone(NEG_TZS);
p = new DateMathParser(tz, Locale.US);
p.setNow(parser.parse("2001-07-04T12:08:56.235"));
assertMath("2001-07-04T12:08:56.000", p, "/SECOND");
assertMath("2001-07-04T12:08:00.000", p, "/MINUTE");
assertMath("2001-07-04T12:00:00.000", p, "/HOUR");
assertMath("2001-07-03T22:00:00.000", p, "/DAY");
assertMath("2001-06-30T22:00:00.000", p, "/MONTH");
// no DST in dec
assertMath("2000-12-31T23:00:00.000", p, "/YEAR");
// no DST in nov
assertMath("2001-11-03T23:00:00.000", p, "+4MONTH/DAY");
}
public void testParseMathExceptions() throws Exception {
DateMathParser p = new DateMathParser(UTC, Locale.US);

View File

@ -0,0 +1,102 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.util;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import java.util.Set;
import java.util.HashSet;
import java.util.Random;
import java.util.TimeZone;
import java.util.Locale;
public class TimeZoneUtilsTest extends LuceneTestCase {
public void testValidIds() throws Exception {
final Set<String> idsTested = new HashSet<String>();
// brain dead: anything the JVM supports, should work
for (String validId : TimeZone.getAvailableIDs()) {
assertTrue(validId + " not found in list of known ids",
TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(validId));
final TimeZone expected = TimeZone.getTimeZone(validId);
final TimeZone actual = TimeZoneUtils.getTimeZone(validId);
assertEquals(validId, expected, actual);
idsTested.add(validId);
}
assertEquals("TimeZone.getAvailableIDs vs TimeZoneUtils.KNOWN_TIMEZONE_IDS",
TimeZoneUtils.KNOWN_TIMEZONE_IDS.size(), idsTested.size());
}
public void testCustom() throws Exception {
for (String input : new String[] {"GMT+08","GMT+8",
"GMT-0800","GMT-08:00",
"GMT+23", "GMT+2300"}) {
assertEquals(input,
TimeZone.getTimeZone(input),
TimeZoneUtils.getTimeZone(input));
}
}
public void testInvalidInput() throws Exception {
final String giberish = "giberish";
assumeFalse("This test assumes that " + giberish + " is not a valid tz id",
TimeZoneUtils.KNOWN_TIMEZONE_IDS.contains(giberish));
assertNull(giberish, TimeZoneUtils.getTimeZone(giberish));
for (String malformed : new String[] {"GMT+72", "GMT0800",
"GMT+2400" , "GMT+24:00",
"GMT+11-30" , "GMT+11:-30",
"GMT+0080" , "GMT+00:80"}) {
assertNull(malformed, TimeZoneUtils.getTimeZone(malformed));
}
}
public void testRandom() throws Exception {
final String ONE_DIGIT = "%1d";
final String TWO_DIGIT = "%02d";
final Random r = random();
final int iters = atLeast(r, 50);
for (int i = 0; i <= iters; i++) {
int hour = _TestUtil.nextInt(r, 0, 23);
int min = _TestUtil.nextInt(r, 0, 59);
String hours = String.format(Locale.US,
(r.nextBoolean() ? ONE_DIGIT : TWO_DIGIT),
hour);
String mins = String.format(Locale.US, TWO_DIGIT, min);
String input = "GMT" + (r.nextBoolean()?"+":"-")
+ hours + (r.nextBoolean() ? "" : ((r.nextBoolean()?":":"") + mins));
assertEquals(input,
TimeZone.getTimeZone(input),
TimeZoneUtils.getTimeZone(input));
}
}
}

View File

@ -25,6 +25,20 @@ import java.util.Locale;
*/
public interface CommonParams {
/**
* Override for the concept of "NOW" to be used throughout this request,
* expressed as milliseconds since epoch. This is primarily used in
* distributed search to ensure consistent time values are used across
* multiple sub-requests.
*/
public static final String NOW = "NOW";
/**
* Specifies the TimeZone used by the client for the purposes of
* any DateMath rounding that may take place when executing the request
*/
public static final String TZ = "TZ";
/** the query type - which query handler should handle the request */
public static final String QT ="qt";