Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-1729: get NOW from SolrRequestInfo thread local + propagate to shards 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1045010 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2010-12-13 03:29:14 +00:00
parent a586a6d741
commit 8c3300e5d5
18 changed files with 189 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
solr/CHANGES.txt
View File

@ -309,6 +309,12 @@ New Features
* SOLR-2237: Added StempelPolishStemFilterFactory to contrib/analysis-extras (rmuir)
* SOLR-1729: Evaluation of NOW for date math is done only once per request for
consistency, and is also propagated to shards in distributed search.
Adding a parameter NOW=<time_in_ms> to the request will override the
current time. (Peter Sturge, yonik)
Optimizations
----------------------

12
solr/src/java/org/apache/solr/core/QuerySenderListener.java
View File

@ -17,6 +17,8 @@
package org.apache.solr.core;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocIterator;
@ -40,15 +42,18 @@ class QuerySenderListener extends AbstractSolrEventListener {
final SolrIndexSearcher searcher = newSearcher;
log.info("QuerySenderListener sending requests to " + newSearcher);
for (NamedList nlst : (List<NamedList>)args.get("queries")) {
SolrQueryRequest req = null;
try {
// bind the request to a particular searcher (the newSearcher)
NamedList params = addEventParms(currentSearcher, nlst);
LocalSolrQueryRequest req = new LocalSolrQueryRequest(core,params) {
req = new LocalSolrQueryRequest(core,params) {
@Override public SolrIndexSearcher getSearcher() { return searcher; }
@Override public void close() { }
};
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp);
// Retrieve the Document instances (not just the ids) to warm
@ -65,11 +70,12 @@ class QuerySenderListener extends AbstractSolrEventListener {
}
}
req.close();
} catch (Exception e) {
// do nothing... we want to continue with the other requests.
// the failure should have already been logged.
} finally {
if (req != null) req.close();
SolrRequestInfo.clearRequestInfo();
}
}
log.info("QuerySenderListener done.");

11
solr/src/java/org/apache/solr/handler/component/ResponseBuilder.java
View File

@ -23,6 +23,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.RTimer;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.QParser;
@ -67,6 +68,16 @@ public class ResponseBuilder
public List<SearchComponent> components;
SolrRequestInfo requestInfo;
public ResponseBuilder(SolrQueryRequest req, SolrQueryResponse rsp, List<SearchComponent> components)
{
this.req = req;
this.rsp = rsp;
this.components = components;
this.requestInfo = SolrRequestInfo.getRequestInfo();
}
//////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////
//// Distributed Search section

13
solr/src/java/org/apache/solr/handler/component/SearchHandler.java
View File

@ -192,10 +192,11 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
{
// int sleep = req.getParams().getInt("sleep",0);
// if (sleep > 0) {log.error("SLEEPING for " + sleep); Thread.sleep(sleep);}
ResponseBuilder rb = new ResponseBuilder();
rb.req = req;
rb.rsp = rsp;
rb.components = components;
ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
if (rb.requestInfo != null) {
rb.requestInfo.setResponseBuilder(rb);
}
boolean dbg = req.getParams().getBool(CommonParams.DEBUG_QUERY, false);
rb.setDebug(dbg);
if (dbg == false){//if it's true, we are doing everything anyway.
@ -290,6 +291,10 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
params.remove("indent");
params.remove(CommonParams.HEADER_ECHO_PARAMS);
params.set(ShardParams.IS_SHARD, true); // a sub (shard) request
if (rb.requestInfo != null) {
// we could try and detect when this is needed, but it could be tricky
params.set("NOW", Long.toString(rb.requestInfo.getNOW().getTime()));
}
String shardHandler = req.getParams().get(ShardParams.SHARDS_QT);
if (shardHandler == null) {
params.remove(CommonParams.QT);

9
solr/src/java/org/apache/solr/request/SimpleFacets.java
View File

@ -73,8 +73,6 @@ public class SimpleFacets {
protected SimpleOrderedMap facetResponse;
public final Date NOW = new Date();
// per-facet values
SolrParams localParams; // localParams on this particular facet command
String facetValue; // the field to or query to facet on (minus local params)
@ -822,7 +820,7 @@ public class SimpleFacets {
= required.getFieldParam(f,FacetParams.FACET_DATE_START);
final Date start;
try {
start = ft.parseMath(NOW, startS);
start = ft.parseMath(null, startS);
} catch (SolrException e) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
@ -832,7 +830,7 @@ public class SimpleFacets {
= required.getFieldParam(f,FacetParams.FACET_DATE_END);
Date end; // not final, hardend may change this
try {
end = ft.parseMath(NOW, endS);
end = ft.parseMath(null, endS);
} catch (SolrException e) {
throw new SolrException
(SolrException.ErrorCode.BAD_REQUEST,
@ -847,7 +845,6 @@ public class SimpleFacets {
final String gap = required.getFieldParam(f,FacetParams.FACET_DATE_GAP);
final DateMathParser dmp = new DateMathParser(ft.UTC, Locale.US);
dmp.setNow(NOW);
final int minCount = params.getFieldInt(f,FacetParams.FACET_MINCOUNT, 0);
@ -1006,7 +1003,7 @@ public class SimpleFacets {
"Unable to range facet on tried field of unexpected type:" + f);
}
} else if (ft instanceof DateField) {
calc = new DateRangeEndpointCalculator(sf, NOW);
calc = new DateRangeEndpointCalculator(sf, null);
} else if (ft instanceof SortableIntField) {
calc = new IntegerRangeEndpointCalculator(sf);
} else if (ft instanceof SortableLongField) {

91
solr/src/java/org/apache/solr/request/SolrRequestInfo.java Executable file
View File

@ -0,0 +1,91 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.request;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.response.SolrQueryResponse;
import java.util.Date;
public class SolrRequestInfo {
protected final static ThreadLocal<SolrRequestInfo> threadLocal = new ThreadLocal<SolrRequestInfo>();
protected SolrQueryRequest req;
protected SolrQueryResponse rsp;
protected Date now;
protected ResponseBuilder rb;
public static SolrRequestInfo getRequestInfo() {
return threadLocal.get();
}
public static void setRequestInfo(SolrRequestInfo info) {
// TODO: temporary sanity check... this can be changed to just an assert in the future
SolrRequestInfo prev = threadLocal.get();
if (prev != null) {
SolrCore.log.error("Previous SolrRequestInfo was not closed! req=" + prev.req.getOriginalParams().toString());
}
assert prev == null;
threadLocal.set(info);
}
public static void clearRequestInfo() {
threadLocal.remove();
}
public SolrRequestInfo(SolrQueryRequest req, SolrQueryResponse rsp) {
this.req = req;
this.rsp = rsp;
}
public Date getNOW() {
if (now != null) return now;
long ms = 0;
String nowStr = req.getParams().get("NOW");
if (nowStr != null) {
ms = Long.parseLong(nowStr);
} else {
ms = req.getStartTime();
}
now = new Date(ms);
return now;
}
public SolrQueryRequest getReq() {
return req;
}
public SolrQueryResponse getRsp() {
return rsp;
}
/** May return null if the request handler is not based on SearchHandler */
public ResponseBuilder getResponseBuilder() {
return rb;
}
public void setResponseBuilder(ResponseBuilder rb) {
this.rb = rb;
}
}

14
solr/src/java/org/apache/solr/spelling/SpellCheckCollator.java
View File

@ -76,20 +76,18 @@ public class SpellCheckCollator {
if (verifyCandidateWithQuery) {
tryNo++;
ResponseBuilder checkResponse = new ResponseBuilder();
checkResponse.setQparser(ultimateResponse.getQparser());
checkResponse.setFilters(ultimateResponse.getFilters());
checkResponse.setQueryString(collationQueryStr);
checkResponse.components = Arrays.asList(new SearchComponent[] { queryComponent });
ModifiableSolrParams params = new ModifiableSolrParams(ultimateResponse.req.getParams());
params.set(CommonParams.Q, collationQueryStr);
params.remove(CommonParams.START);
params.set(CommonParams.FL, "id");
params.set(CommonParams.ROWS, "0");
// creating a request here... make sure to close it!
checkResponse.req = new LocalSolrQueryRequest(ultimateResponse.req.getCore(), params);
checkResponse.rsp = new SolrQueryResponse();
ResponseBuilder checkResponse = new ResponseBuilder(new LocalSolrQueryRequest(ultimateResponse.req.getCore(), params),new SolrQueryResponse(), Arrays.asList(new SearchComponent[] { queryComponent }));
checkResponse.setQparser(ultimateResponse.getQparser());
checkResponse.setFilters(ultimateResponse.getFilters());
checkResponse.setQueryString(collationQueryStr);
checkResponse.components = Arrays.asList(new SearchComponent[] { queryComponent });
try {
queryComponent.prepare(checkResponse);

13
solr/src/java/org/apache/solr/util/DateMathParser.java
View File

@ -17,6 +17,9 @@
package org.apache.solr.util;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import java.util.Date;
import java.util.Calendar;
import java.util.TimeZone;
@ -198,7 +201,6 @@ public class DateMathParser {
public DateMathParser(TimeZone tz, Locale l) {
zone = tz;
loc = l;
setNow(new Date());
}
/** Redefines this instance's concept of "now" */
@ -208,6 +210,15 @@ public class DateMathParser {
/** Returns a cloned of this instance's concept of "now" */
public Date getNow() {
if (now == null) {
SolrRequestInfo reqInfo = SolrRequestInfo.getRequestInfo();
if (reqInfo == null) {
// fall back to current time if no request info set
now = new Date();
} else {
now = reqInfo.getNOW();
}
}
return (Date) now.clone();
}

13
solr/src/java/org/apache/solr/util/TestHarness.java
View File

@ -28,6 +28,7 @@ import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.XmlUpdateRequestHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
@ -328,8 +329,12 @@ public class TestHarness {
*/
public String query(String handler, SolrQueryRequest req) throws IOException, Exception {
try {
SolrQueryResponse rsp = queryAndResponse(handler, req);
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
core.execute(core.getRequestHandler(handler),req,rsp);
if (rsp.getException() != null) {
throw rsp.getException();
}
StringWriter sw = new StringWriter(32000);
QueryResponseWriter responseWriter = core.getQueryResponseWriter(req);
responseWriter.write(sw,req,rsp);
@ -339,10 +344,12 @@ public class TestHarness {
return sw.toString();
} finally {
req.close();
SolrRequestInfo.clearRequestInfo();
}
}
/** It is the users responsibility to close the request object when done with it */
/** It is the users responsibility to close the request object when done with it.
* This method does not set/clear SolrRequestInfo */
public SolrQueryResponse queryAndResponse(String handler, SolrQueryRequest req) throws Exception {
SolrQueryResponse rsp = new SolrQueryResponse();
core.execute(core.getRequestHandler(handler),req,rsp);

4
solr/src/test/org/apache/solr/TestDistributedSearch.java
View File

@ -130,6 +130,10 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
query("q","matchesnothing","fl","*,score");
// test that a single NOW value is propagated to all shards... if that is true
// then the primary sort should always be a tie and then the secondary should always decide
query("q","{!func}ms(NOW)", "sort","score desc,"+i1+" desc","fl","id");
query("q","*:*", "rows",100, "facet","true", "facet.field",t1);
query("q","*:*", "rows",100, "facet","true", "facet.field",t1, "facet.limit",-1, "facet.sort","count");
query("q","*:*", "rows",100, "facet","true", "facet.field",t1, "facet.limit",-1, "facet.sort","count", "facet.mincount",2);

5
solr/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java
View File

@ -153,10 +153,7 @@ public class SpellCheckComponentTest extends SolrTestCaseJ4 {
request = req("qt", "spellCheckCompRH", "q", "*:*", "spellcheck.q", "ttle",
"spellcheck", "true", "spellcheck.dictionary", "default",
"spellcheck.reload", "true");
ResponseBuilder rb = new ResponseBuilder();
rb.req = request;
rb.rsp = new SolrQueryResponse();
rb.components = new ArrayList(h.getCore().getSearchComponents().values());
ResponseBuilder rb = new ResponseBuilder(request, new SolrQueryResponse(), new ArrayList(h.getCore().getSearchComponents().values()));
checker.prepare(rb);
try {

13
solr/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
View File

@ -170,12 +170,6 @@ public class TermVectorComponentTest extends SolrTestCaseJ4 {
TermVectorComponent tvComp = (TermVectorComponent) core.getSearchComponent("tvComponent");
assertTrue("tvComp is null and it shouldn't be", tvComp != null);
ModifiableSolrParams params = new ModifiableSolrParams();
ResponseBuilder rb = new ResponseBuilder();
rb.stage = ResponseBuilder.STAGE_GET_FIELDS;
rb.shards = new String[]{"localhost:0", "localhost:1", "localhost:2", "localhost:3"};//we don't actually call these, since we are going to invoke distributedProcess directly
rb.resultIds = new HashMap<Object, ShardDoc>();
rb.components = new ArrayList<SearchComponent>();
rb.components.add(tvComp);
params.add(CommonParams.Q, "id:0");
params.add(CommonParams.QT, "tvrh");
params.add(TermVectorParams.TF, "true");
@ -183,7 +177,12 @@ public class TermVectorComponentTest extends SolrTestCaseJ4 {
params.add(TermVectorParams.OFFSETS, "true");
params.add(TermVectorParams.POSITIONS, "true");
params.add(TermVectorComponent.COMPONENT_NAME, "true");
rb.req = new LocalSolrQueryRequest(core, params);
ResponseBuilder rb = new ResponseBuilder(new LocalSolrQueryRequest(core, params), new SolrQueryResponse(), (List)Arrays.asList(tvComp));
rb.stage = ResponseBuilder.STAGE_GET_FIELDS;
rb.shards = new String[]{"localhost:0", "localhost:1", "localhost:2", "localhost:3"};//we don't actually call these, since we are going to invoke distributedProcess directly
rb.resultIds = new HashMap<Object, ShardDoc>();
rb.outgoing = new ArrayList<ShardRequest>();
//one doc per shard, but make sure there are enough docs to go around
for (int i = 0; i < rb.shards.length; i++){

4
solr/src/test/org/apache/solr/search/function/TestFunctionQuery.java
View File

@ -318,6 +318,10 @@ public class TestFunctionQuery extends SolrTestCaseJ4 {
assertQ(req("fl","*,score","q", "{!func}ms(2009-08-31T12:10:10.125Z/SECOND,2009-08-31T12:10:10.124Z/SECOND)", "fq","id:1"), "//float[@name='score']='0.0'");
// test that we can specify "NOW"
assertQ(req("fl","*,score","q", "{!func}ms(NOW)", "NOW","1000"), "//float[@name='score']='1000.0'");
for (int i=100; i<112; i++) {
assertU(adoc("id",""+i, "text","batman"));
}

1
solr/src/test/test-files/solr/conf/schema12.xml
View File

@ -543,6 +543,7 @@
<dynamicField name="*_sw" type="text_sw" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_i" type="int" indexed="true" stored="true"/>
<dynamicField name="*_is" type="int" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_s" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_ss" type="string" indexed="true" stored="true" multiValued="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="true"/>

11
solr/src/webapp/src/org/apache/solr/client/solrj/embedded/EmbeddedSolrServer.java
View File

@ -41,6 +41,7 @@ import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.BinaryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.DocIterator;
@ -148,6 +149,8 @@ public class EmbeddedSolrServer extends SolrServer
req = _parser.buildRequestFrom( core, params, request.getContentStreams() );
req.getContext().put( "path", path );
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
core.execute( handler, req, rsp );
if( rsp.getException() != null ) {
throw new SolrServerException( rsp.getException() );
@ -230,11 +233,9 @@ public class EmbeddedSolrServer extends SolrServer
throw new SolrServerException( ex );
}
finally {
try {
if (req != null) req.close();
} finally {
core.close();
}
if (req != null) req.close();
core.close();
SolrRequestInfo.clearRequestInfo();
}
}

3
solr/src/webapp/src/org/apache/solr/servlet/DirectSolrConnection.java
View File

@ -36,6 +36,7 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.IndexSchema;
@ -185,6 +186,7 @@ public class DirectSolrConnection
try {
req = parser.buildRequestFrom( core, params, streams );
SolrQueryResponse rsp = new SolrQueryResponse();
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
core.execute( handler, req, rsp );
if( rsp.getException() != null ) {
throw rsp.getException();
@ -199,6 +201,7 @@ public class DirectSolrConnection
if (req != null) {
req.close();
}
SolrRequestInfo.clearRequestInfo();
}
}

4
solr/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -237,6 +237,7 @@ public class SolrDispatchFilter implements Filter
* QueryResponseWriter is selected and we get the correct
* Content-Type)
*/
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
this.execute( req, handler, solrReq, solrRsp );
HttpCacheHeaderUtil.checkHttpCachingVeto(solrRsp, resp, reqMethod);
// add info to http headers
@ -250,7 +251,7 @@ public class SolrDispatchFilter implements Filter
log.log(Level.WARNING, "exception adding response header log information", cce);
}*/
QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
writeResponse(solrRsp, response, responseWriter, solrReq, reqMethod);
writeResponse(solrRsp, response, responseWriter, solrReq, reqMethod);
}
return; // we are done with a valid handler
}
@ -278,6 +279,7 @@ public class SolrDispatchFilter implements Filter
if (core != null) {
core.close();
}
SolrRequestInfo.clearRequestInfo();
}
}

4
solr/src/webapp/src/org/apache/solr/servlet/SolrServlet.java
View File

@ -20,6 +20,8 @@ package org.apache.solr.servlet;
import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import org.apache.solr.request.SolrRequestInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -82,6 +84,7 @@ public class SolrServlet extends HttpServlet {
log.warn("Unknown Request Handler '" + solrReq.getQueryType() +"' :" + solrReq);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,"Unknown Request Handler '" + solrReq.getQueryType() + "'", true);
}
SolrRequestInfo.setRequestInfo(new SolrRequestInfo(solrReq, solrRsp));
core.execute(handler, solrReq, solrRsp );
if (solrRsp.getException() == null) {
QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
@ -105,6 +108,7 @@ public class SolrServlet extends HttpServlet {
} finally {
// This releases the IndexReader associated with the request
solrReq.close();
SolrRequestInfo.clearRequestInfo();
}
}