Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-127: HTTP Caching awareness 

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@630037 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter 2008-02-21 22:44:19 +00:00
parent fe9dcf82cb
commit 8cf3175518
14 changed files with 1328 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGES.txt
View File

@ -191,6 +191,12 @@ New Features
(ryan)
38. SOLR-478: Added ability to get back unique key information from the LukeRequestHandler. (gsingers)
39. SOLR-127: HTTP Caching awareness. Solr now recognizes HTTP Request
headers related to HTTP Caching (see RFC 2616 sec13) and will respond
with "304 Not Modified" when appropriate. New options have been added
to solrconfig.xml to influence this behavior.
(Thomas Peuss via hossman)
Changes in runtime behavior

7
client/java/solrj/src/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
View File

@ -45,6 +45,12 @@ public class JettySolrRunner
{
this.init( context, port );
}
public JettySolrRunner( String context, int port, String solrConfigFilename )
{
this.init( context, port );
dispatchFilter.setInitParameter("solrconfig-filename", solrConfigFilename);
}
// public JettySolrRunner( String context, String home, String dataDir, int port, boolean log )
// {
@ -88,6 +94,7 @@ public class JettySolrRunner
{
if( server.isRunning() ) {
server.stop();
server.join();
}
}

36
example/solr/conf/solrconfig.xml
View File

@ -264,6 +264,42 @@
<requestDispatcher handleSelect="true" >
<!--Make sure your system has some authentication before enabling remote streaming! -->
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<!-- Set HTTP caching related parameters (for proxy caches and clients).
To get the behaviour of Solr 1.2 (ie: no caching related headers)
use the never304="true" option and do not specify a value for
<cacheControl>
-->
<!-- <httpCaching never304="true"> -->
<httpCaching lastModifiedFrom="openTime"
etagSeed="Solr">
<!-- lastModFrom="openTime" is the default, the Last-Modified value
(and validation against If-Modified-Since requests) will all be
relative to when the current Searcher was opened.
You can change it to lastModFrom="dirLastMod" if you want the
value to exactly corrispond to when the physical index was last
modified.
etagSeed="..." is an option you can change to force the ETag
header (and validation against If-None-Match requests) to be
differnet even if the index has not changed (ie: when making
significant changes to your config file)
lastModifiedFrom and etagSeed are both ignored if you use the
never304="true" option.
-->
<!-- If you include a <cacheControl> directive, it will be used to
generate a Cache-Control header, as well as an Expires header
if the value contains "max-age="
By default, no Cache-Control header is generated.
You can use the <cacheControl> option even if you have set
never304="true"
-->
<!-- <cacheControl>max-age=30, public</cacheControl> -->
</httpCaching>
</requestDispatcher>

83
src/java/org/apache/solr/core/SolrConfig.java
View File

@ -33,6 +33,10 @@ import javax.xml.parsers.ParserConfigurationException;
import java.util.Collection;
import java.util.HashSet;
import java.util.StringTokenizer;
import java.util.logging.Logger;
import java.util.logging.Level;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.IOException;
import java.io.InputStream;
@ -117,6 +121,8 @@ public class SolrConfig extends Config {
hashDocSetMaxSize= getInt("//HashDocSet/@maxSize",3000);
pingQueryParams = readPingQueryParams(this);
httpCachingConfig = new HttpCachingConfig(this);
Config.log.info("Loaded SolrConfig: " + file);
// TODO -- at solr 2.0. this should go away
@ -146,6 +152,11 @@ public class SolrConfig extends Config {
public final SolrIndexConfig defaultIndexConfig;
public final SolrIndexConfig mainIndexConfig;
private final HttpCachingConfig httpCachingConfig;
public HttpCachingConfig getHttpCachingConfig() {
return httpCachingConfig;
}
// ping query request parameters
@Deprecated
private final NamedList pingQueryParams;
@ -175,4 +186,76 @@ public class SolrConfig extends Config {
public SolrQueryRequest getPingQueryRequest(SolrCore core) {
return new LocalSolrQueryRequest(core, pingQueryParams);
}
public static class HttpCachingConfig {
/** config xpath prefix for getting HTTP Caching options */
private final static String CACHE_PRE
= "requestDispatcher/httpCaching/";
/** For extracting Expires "ttl" from <cacheControl> config */
private final static Pattern MAX_AGE
= Pattern.compile("\\bmax-age=(\\d+)");
public static enum LastModFrom {
OPENTIME, DIRLASTMOD, BOGUS;
/** Input must not be null */
public static LastModFrom parse(final String s) {
try {
return valueOf(s.toUpperCase());
} catch (Exception e) {
log.log(Level.WARNING,
"Unrecognized value for lastModFrom: " + s, e);
return BOGUS;
}
}
}
private final boolean never304;
private final String etagSeed;
private final String cacheControlHeader;
private final Integer maxAge;
private final LastModFrom lastModFrom;
private HttpCachingConfig(SolrConfig conf) {
never304 = conf.getBool(CACHE_PRE+"@never304", false);
etagSeed = conf.get(CACHE_PRE+"@etagSeed", "Solr");
lastModFrom = LastModFrom.parse(conf.get(CACHE_PRE+"@lastModFrom",
"openTime"));
cacheControlHeader = conf.get(CACHE_PRE+"cacheControl",null);
Integer tmp = null; // maxAge
if (null != cacheControlHeader) {
try {
final Matcher ttlMatcher = MAX_AGE.matcher(cacheControlHeader);
final String ttlStr = ttlMatcher.find() ? ttlMatcher.group(1) : null;
tmp = (null != ttlStr && !"".equals(ttlStr))
? Integer.valueOf(ttlStr)
: null;
} catch (Exception e) {
log.log(Level.WARNING,
"Ignoring exception while attempting to " +
"extract max-age from cacheControl config: " +
cacheControlHeader, e);
}
}
maxAge = tmp;
}
public boolean isNever304() { return never304; }
public String getEtagSeed() { return etagSeed; }
/** null if no Cache-Control header */
public String getCacheControlHeader() { return cacheControlHeader; }
/** null if no max age limitation */
public Integer getMaxAge() { return maxAge; }
public LastModFrom getLastModFrom() { return lastModFrom; }
}
}

4
src/java/org/apache/solr/search/SolrIndexSearcher.java
View File

@ -1395,7 +1395,6 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
}
}
/**
* return the named generic cache
*/
@ -1419,6 +1418,9 @@ public class SolrIndexSearcher extends Searcher implements SolrInfoMBean {
return cache==null ? null : cache.put(key,val);
}
public long getOpenTime() {
return openTime;
}
/////////////////////////////////////////////////////////////////////
// SolrInfoMBean stuff: Statistics and Module Info

180
src/test/org/apache/solr/servlet/CacheHeaderTest.java Normal file
View File

@ -0,0 +1,180 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.servlet;
import java.util.Date;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.util.DateUtil;
/**
* A test case for the several HTTP cache headers emitted by Solr
*/
public class CacheHeaderTest extends CacheHeaderTestBase {
@Override public String getSolrConfigFilename() { return "solrconfig.xml"; }
protected void doLastModified(String method) throws Exception {
// We do a first request to get the last modified
// This must result in a 200 OK response
HttpMethodBase get = getSelectMethod(method);
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Got no response code 200 in initial request", 200, get
.getStatusCode());
Header head = get.getResponseHeader("Last-Modified");
assertNotNull("We got no Last-Modified header", head);
Date lastModified = DateUtil.parseDate(head.getValue());
// If-Modified-Since tests
get = getSelectMethod(method);
get.addRequestHeader("If-Modified-Since", DateUtil.formatDate(new Date()));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Expected 304 NotModified response with current date", 304,
get.getStatusCode());
get = getSelectMethod(method);
get.addRequestHeader("If-Modified-Since", DateUtil.formatDate(new Date(
lastModified.getTime() - 10000)));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Expected 200 OK response with If-Modified-Since in the past",
200, get.getStatusCode());
// If-Unmodified-Since tests
get = getSelectMethod(method);
get.addRequestHeader("If-Unmodified-Since", DateUtil.formatDate(new Date(
lastModified.getTime() - 10000)));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"Expected 412 Precondition failed with If-Unmodified-Since in the past",
412, get.getStatusCode());
get = getSelectMethod(method);
get
.addRequestHeader("If-Unmodified-Since", DateUtil
.formatDate(new Date()));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"Expected 200 OK response with If-Unmodified-Since and current date",
200, get.getStatusCode());
}
// test ETag
protected void doETag(String method) throws Exception {
HttpMethodBase get = getSelectMethod(method);
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Got no response code 200 in initial request", 200, get
.getStatusCode());
Header head = get.getResponseHeader("ETag");
assertNotNull("We got no ETag in the response", head);
assertTrue("Not a valid ETag", head.getValue().startsWith("\"")
&& head.getValue().endsWith("\""));
String etag = head.getValue();
// If-None-Match tests
// we set a non matching ETag
get = getSelectMethod(method);
get.addRequestHeader("If-None-Match", "\"xyz123456\"");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"If-None-Match: Got no response code 200 in response to non matching ETag",
200, get.getStatusCode());
// now we set matching ETags
get = getSelectMethod(method);
get.addRequestHeader("If-None-Match", "\"xyz1223\"");
get.addRequestHeader("If-None-Match", "\"1231323423\", \"1211211\", "
+ etag);
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("If-None-Match: Got no response 304 to matching ETag", 304,
get.getStatusCode());
// we now set the special star ETag
get = getSelectMethod(method);
get.addRequestHeader("If-None-Match", "*");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("If-None-Match: Got no response 304 for star ETag", 304, get
.getStatusCode());
// If-Match tests
// we set a non matching ETag
get = getSelectMethod(method);
get.addRequestHeader("If-Match", "\"xyz123456\"");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"If-Match: Got no response code 412 in response to non matching ETag",
412, get.getStatusCode());
// now we set matching ETags
get = getSelectMethod(method);
get.addRequestHeader("If-Match", "\"xyz1223\"");
get.addRequestHeader("If-Match", "\"1231323423\", \"1211211\", " + etag);
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("If-Match: Got no response 200 to matching ETag", 200, get
.getStatusCode());
// now we set the special star ETag
get = getSelectMethod(method);
get.addRequestHeader("If-Match", "*");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("If-Match: Got no response 200 to star ETag", 200, get
.getStatusCode());
}
protected void doCacheControl(String method) throws Exception {
if ("POST".equals(method)) {
HttpMethodBase m = getSelectMethod(method);
getClient().executeMethod(m);
checkResponseBody(method, m);
Header head = m.getResponseHeader("Cache-Control");
assertNull("We got a cache-control header in response to POST", head);
head=m.getResponseHeader("Expires");
assertNull("We got an Expires header in response to POST", head);
} else {
HttpMethodBase m = getSelectMethod(method);
getClient().executeMethod(m);
checkResponseBody(method, m);
Header head = m.getResponseHeader("Cache-Control");
assertNotNull("We got no cache-control header", head);
head=m.getResponseHeader("Expires");
assertNotNull("We got no Expires header in response",head);
}
}
}

148
src/test/org/apache/solr/servlet/CacheHeaderTestBase.java Normal file
View File

@ -0,0 +1,148 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.servlet;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.HeadMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.solr.client.solrj.SolrExampleTestBase;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
public abstract class CacheHeaderTestBase extends SolrExampleTestBase {
@Override public String getSolrHome() { return "solr/"; }
abstract public String getSolrConfigFilename();
public String getSolrConfigFile() { return getSolrHome()+"conf/"+getSolrConfigFilename(); }
CommonsHttpSolrServer server;
JettySolrRunner jetty;
static final int port = 8985; // not 8983
static final String context = "/example";
@Override
public void setUp() throws Exception {
super.setUp();
jetty = new JettySolrRunner(context, port, getSolrConfigFilename());
jetty.start();
server = this.createNewSolrServer();
}
@Override
public void tearDown() throws Exception {
super.tearDown();
jetty.stop(); // stop the server
}
@Override
protected SolrServer getSolrServer() {
return server;
}
@Override
protected CommonsHttpSolrServer createNewSolrServer() {
try {
// setup the server...
String url = "http://localhost:" + port + context;
CommonsHttpSolrServer s = new CommonsHttpSolrServer(url);
s.setConnectionTimeout(5);
s.setDefaultMaxConnectionsPerHost(100);
s.setMaxTotalConnections(100);
return s;
} catch (Exception ex) {
throw new RuntimeException(ex);
}
}
protected HttpMethodBase getSelectMethod(String method) {
HttpMethodBase m = null;
if ("GET".equals(method)) {
m = new GetMethod(server.getBaseURL() + "/select");
} else if ("HEAD".equals(method)) {
m = new HeadMethod(server.getBaseURL() + "/select");
} else if ("POST".equals(method)) {
m = new PostMethod(server.getBaseURL() + "/select");
}
m.setQueryString(new NameValuePair[] { new NameValuePair("q", "solr"),
new NameValuePair("qt", "standard") });
return m;
}
protected HttpClient getClient() {
return server.getHttpClient();
}
protected void checkResponseBody(String method, HttpMethodBase resp)
throws Exception {
String responseBody = resp.getResponseBodyAsString();
if ("GET".equals(method)) {
switch (resp.getStatusCode()) {
case 200:
assertTrue("Response body was empty for method " + method,
responseBody != null && responseBody.length() > 0);
break;
case 304:
assertTrue("Response body was not empty for method " + method,
responseBody == null || responseBody.length() == 0);
break;
case 412:
assertTrue("Response body was not empty for method " + method,
responseBody == null || responseBody.length() == 0);
break;
default:
System.err.println(responseBody);
assertEquals("Unknown request response", 0, resp.getStatusCode());
}
}
if ("HEAD".equals(method)) {
assertTrue("Response body was not empty for method " + method,
responseBody == null || responseBody.length() == 0);
}
}
// The tests
public void testLastModified() throws Exception {
doLastModified("GET");
doLastModified("HEAD");
}
public void testEtag() throws Exception {
doETag("GET");
doETag("HEAD");
}
public void testCacheControl() throws Exception {
doCacheControl("GET");
doCacheControl("HEAD");
doCacheControl("POST");
}
protected abstract void doCacheControl(String method) throws Exception;
protected abstract void doETag(String method) throws Exception;
protected abstract void doLastModified(String method) throws Exception;
}

158
src/test/org/apache/solr/servlet/NoCacheHeaderTest.java Normal file
View File

@ -0,0 +1,158 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.servlet;
import java.util.Date;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.util.DateUtil;
/**
* A test case for the several HTTP cache headers emitted by Solr
*/
public class NoCacheHeaderTest extends CacheHeaderTestBase {
@Override public String getSolrConfigFilename() { return "solrconfig-nocache.xml"; }
// The tests
public void testLastModified() throws Exception {
doLastModified("GET");
doLastModified("HEAD");
}
public void testEtag() throws Exception {
doETag("GET");
doETag("HEAD");
}
public void testCacheControl() throws Exception {
doCacheControl("GET");
doCacheControl("HEAD");
doCacheControl("POST");
}
protected void doLastModified(String method) throws Exception {
// We do a first request to get the last modified
// This must result in a 200 OK response
HttpMethodBase get = getSelectMethod(method);
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Got no response code 200 in initial request", 200, get
.getStatusCode());
Header head = get.getResponseHeader("Last-Modified");
assertNull("We got a Last-Modified header", head);
// If-Modified-Since tests
get = getSelectMethod(method);
get.addRequestHeader("If-Modified-Since", DateUtil.formatDate(new Date()));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Expected 200 with If-Modified-Since header. We should never get a 304 here", 200,
get.getStatusCode());
get = getSelectMethod(method);
get.addRequestHeader("If-Modified-Since", DateUtil.formatDate(new Date(System.currentTimeMillis()-10000)));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Expected 200 with If-Modified-Since header. We should never get a 304 here",
200, get.getStatusCode());
// If-Unmodified-Since tests
get = getSelectMethod(method);
get.addRequestHeader("If-Unmodified-Since", DateUtil.formatDate(new Date(System.currentTimeMillis()-10000)));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"Expected 200 with If-Unmodified-Since header. We should never get a 304 here",
200, get.getStatusCode());
get = getSelectMethod(method);
get
.addRequestHeader("If-Unmodified-Since", DateUtil
.formatDate(new Date()));
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"Expected 200 with If-Unmodified-Since header. We should never get a 304 here",
200, get.getStatusCode());
}
// test ETag
protected void doETag(String method) throws Exception {
HttpMethodBase get = getSelectMethod(method);
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("Got no response code 200 in initial request", 200, get
.getStatusCode());
Header head = get.getResponseHeader("ETag");
assertNull("We got an ETag in the response", head);
// If-None-Match tests
// we set a non matching ETag
get = getSelectMethod(method);
get.addRequestHeader("If-None-Match", "\"xyz123456\"");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"If-None-Match: Got no response code 200 in response to non matching ETag",
200, get.getStatusCode());
// we now set the special star ETag
get = getSelectMethod(method);
get.addRequestHeader("If-None-Match", "*");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("If-None-Match: Got no response 200 for star ETag", 200, get
.getStatusCode());
// If-Match tests
// we set a non matching ETag
get = getSelectMethod(method);
get.addRequestHeader("If-Match", "\"xyz123456\"");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals(
"If-Match: Got no response code 200 in response to non matching ETag",
200, get.getStatusCode());
// now we set the special star ETag
get = getSelectMethod(method);
get.addRequestHeader("If-Match", "*");
getClient().executeMethod(get);
checkResponseBody(method, get);
assertEquals("If-Match: Got no response 200 to star ETag", 200, get
.getStatusCode());
}
protected void doCacheControl(String method) throws Exception {
HttpMethodBase m = getSelectMethod(method);
getClient().executeMethod(m);
checkResponseBody(method, m);
Header head = m.getResponseHeader("Cache-Control");
assertNull("We got a cache-control header in response", head);
head = m.getResponseHeader("Expires");
assertNull("We got an Expires header in response", head);
}
}

324
src/test/test-files/solr/conf/solrconfig-nocache.xml Normal file
View File

@ -0,0 +1,324 @@
<?xml version="1.0" ?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!-- $Id$
$Source$
$Name$
-->
<config>
<!-- Used to specify an alternate directory to hold all index data.
It defaults to "index" if not present, and should probably
not be changed if replication is in use. -->
<!--
<indexDir>index</indexDir>
-->
<indexDefaults>
<!-- Values here affect all index writers and act as a default
unless overridden. -->
<useCompoundFile>false</useCompoundFile>
<mergeFactor>10</mergeFactor>
<maxBufferedDocs>1000</maxBufferedDocs>
<maxMergeDocs>2147483647</maxMergeDocs>
<maxFieldLength>10000</maxFieldLength>
<!-- these are global... can't currently override per index -->
<writeLockTimeout>1000</writeLockTimeout>
<commitLockTimeout>10000</commitLockTimeout>
<lockType>single</lockType>
</indexDefaults>
<mainIndex>
<!-- lucene options specific to the main on-disk lucene index -->
<useCompoundFile>false</useCompoundFile>
<mergeFactor>10</mergeFactor>
<maxBufferedDocs>1000</maxBufferedDocs>
<maxMergeDocs>2147483647</maxMergeDocs>
<maxFieldLength>10000</maxFieldLength>
<unlockOnStartup>true</unlockOnStartup>
</mainIndex>
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met
<autoCommit>
<maxDocs>10000</maxDocs>
<maxTime>3600000</maxTime>
</autoCommit>
-->
<!-- represents a lower bound on the frequency that commits may
occur (in seconds). NOTE: not yet implemented
<commitIntervalLowerBound>0</commitIntervalLowerBound>
-->
<!-- The RunExecutableListener executes an external command.
exe - the name of the executable to run
dir - dir to use as the current working directory. default="."
wait - the calling thread waits until the executable returns. default="true"
args - the arguments to pass to the program. default=nothing
env - environment variables to set. default=nothing
-->
<!-- A postCommit event is fired after every commit
<listener event="postCommit" class="solr.RunExecutableListener">
<str name="exe">/var/opt/resin3/__PORT__/scripts/solr/snapshooter</str>
<str name="dir">/var/opt/resin3/__PORT__</str>
<bool name="wait">true</bool>
<arr name="args"> <str>arg1</str> <str>arg2</str> </arr>
<arr name="env"> <str>MYVAR=val1</str> </arr>
</listener>
-->
</updateHandler>
<query>
<!-- Maximum number of clauses in a boolean query... can affect
range or wildcard queries that expand to big boolean
queries. An exception is thrown if exceeded.
-->
<maxBooleanClauses>1024</maxBooleanClauses>
<!-- Cache specification for Filters or DocSets - unordered set of *all* documents
that match a particular query.
-->
<filterCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="256"/>
<queryResultCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="1024"/>
<documentCache
class="solr.search.LRUCache"
size="512"
initialSize="512"
autowarmCount="0"/>
<!-- If true, stored fields that are not requested will be loaded lazily.
-->
<enableLazyFieldLoading>true</enableLazyFieldLoading>
<!--
<cache name="myUserCache"
class="solr.search.LRUCache"
size="4096"
initialSize="1024"
autowarmCount="1024"
regenerator="MyRegenerator"
/>
-->
<useFilterForSortedQuery>true</useFilterForSortedQuery>
<queryResultWindowSize>10</queryResultWindowSize>
<!-- set maxSize artificially low to exercise both types of sets -->
<HashDocSet maxSize="3" loadFactor="0.75"/>
<!-- boolToFilterOptimizer converts boolean clauses with zero boost
into cached filters if the number of docs selected by the clause exceeds
the threshold (represented as a fraction of the total index)
-->
<boolTofilterOptimizer enabled="false" cacheSize="32" threshold=".05"/>
<!-- a newSearcher event is fired whenever a new searcher is being prepared
and there is a current searcher handling requests (aka registered). -->
<!-- QuerySenderListener takes an array of NamedList and executes a
local query request for each NamedList in sequence. -->
<!--
<listener event="newSearcher" class="solr.QuerySenderListener">
<arr name="queries">
<lst> <str name="q">solr</str> <str name="start">0</str> <str name="rows">10</str> </lst>
<lst> <str name="q">rocks</str> <str name="start">0</str> <str name="rows">10</str> </lst>
</arr>
</listener>
-->
<!-- a firstSearcher event is fired whenever a new searcher is being
prepared but there is no current registered searcher to handle
requests or to gain prewarming data from. -->
<!--
<listener event="firstSearcher" class="solr.QuerySenderListener">
<arr name="queries">
<lst> <str name="q">fast_warm</str> <str name="start">0</str> <str name="rows">10</str> </lst>
</arr>
</listener>
-->
</query>
<!-- An alternate set representation that uses an integer hash to store filters (sets of docids).
If the set cardinality <= maxSize elements, then HashDocSet will be used instead of the bitset
based HashBitset. -->
<!-- requestHandler plugins... incoming queries will be dispatched to the
correct handler based on the qt (query type) param matching the
name of registered handlers.
The "standard" request handler is the default and will be used if qt
is not specified in the request.
-->
<requestHandler name="standard" class="solr.StandardRequestHandler"/>
<requestHandler name="dismaxOldStyleDefaults"
class="solr.DisMaxRequestHandler" >
<!-- for historic reasons, DisMaxRequestHandler will use all of
it's init params as "defaults" if there is no "defaults" list
specified
-->
<float name="tie">0.01</float>
<str name="qf">
text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0
</str>
<str name="pf">
text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5
</str>
<str name="bf">
ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3
</str>
<str name="mm">
3&lt;-1 5&lt;-2 6&lt;90%
</str>
<int name="ps">100</int>
</requestHandler>
<requestHandler name="dismax" class="solr.DisMaxRequestHandler" >
<lst name="defaults">
<str name="q.alt">*:*</str>
<float name="tie">0.01</float>
<str name="qf">
text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0
</str>
<str name="pf">
text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5
</str>
<str name="bf">
ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3
</str>
<str name="mm">
3&lt;-1 5&lt;-2 6&lt;90%
</str>
<int name="ps">100</int>
</lst>
</requestHandler>
<requestHandler name="old" class="solr.tst.OldRequestHandler" >
<int name="myparam">1000</int>
<float name="ratio">1.4142135</float>
<arr name="myarr"><int>1</int><int>2</int></arr>
<str>foo</str>
</requestHandler>
<requestHandler name="oldagain" class="solr.tst.OldRequestHandler" >
<lst name="lst1"> <str name="op">sqrt</str> <int name="val">2</int> </lst>
<lst name="lst2"> <str name="op">log</str> <float name="val">10</float> </lst>
</requestHandler>
<requestHandler name="test" class="solr.tst.TestRequestHandler" />
<!-- test query parameter defaults -->
<requestHandler name="defaults" class="solr.StandardRequestHandler">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<!-- test query parameter defaults -->
<requestHandler name="lazy" class="solr.StandardRequestHandler" startup="lazy">
<lst name="defaults">
<int name="rows">4</int>
<bool name="hl">true</bool>
<str name="hl.fl">text,name,subject,title,whitetok</str>
</lst>
</requestHandler>
<requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
<requestHandler name="/update/csv" class="solr.CSVRequestHandler" startup="lazy" />
<!-- test elevation -->
<searchComponent name="elevate" class="org.apache.solr.handler.component.QueryElevationComponent" >
<str name="queryFieldType">string</str>
<str name="config-file">elevate.xml</str>
</searchComponent>
<requestHandler name="/elevate" class="org.apache.solr.handler.component.SearchHandler">
<lst name="defaults">
<str name="echoParams">explicit</str>
</lst>
<arr name="last-components">
<str>elevate</str>
</arr>
</requestHandler>
<highlighting>
<!-- Configure the standard fragmenter -->
<fragmenter name="gap" class="org.apache.solr.highlight.GapFragmenter" default="true">
<lst name="defaults">
<int name="hl.fragsize">100</int>
</lst>
</fragmenter>
<fragmenter name="regex" class="org.apache.solr.highlight.RegexFragmenter">
<lst name="defaults">
<int name="hl.fragsize">70</int>
</lst>
</fragmenter>
<!-- Configure the standard formatter -->
<formatter name="html" class="org.apache.solr.highlight.HtmlFormatter" default="true">
<lst name="defaults">
<str name="hl.simple.pre"><![CDATA[<em>]]></str>
<str name="hl.simple.post"><![CDATA[</em>]]></str>
</lst>
</formatter>
</highlighting>
<!-- enable streaming for testing... -->
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />
<httpCaching never304="true" />
</requestDispatcher>
<admin>
<defaultQuery>solr</defaultQuery>
<gettableFiles>solrconfig.xml scheam.xml admin-extra.html</gettableFiles>
</admin>
<!-- test getting system property -->
<propTest attr1="${solr.test.sys.prop1}-$${literal}"
attr2="${non.existent.sys.prop:default-from-config}">prefix-${solr.test.sys.prop2}-suffix</propTest>
</config>

3
src/test/test-files/solr/conf/solrconfig.xml
View File

@ -309,6 +309,9 @@
<!-- enable streaming for testing... -->
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048" />
<httpCaching lastModifiedFrom="openTime" etagSeed="Solr" never304="false">
<cacheControl>max-age=30, public</cacheControl>
</httpCaching>
</requestDispatcher>
<admin>

42
src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -44,6 +44,8 @@ import org.apache.solr.request.QueryResponseWriter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryResponse;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
import org.apache.solr.servlet.cache.Method;
/**
* This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml
@ -51,13 +53,14 @@ import org.apache.solr.request.SolrRequestHandler;
public class SolrDispatchFilter implements Filter
{
final Logger log = Logger.getLogger(SolrDispatchFilter.class.getName());
protected SolrCore singlecore;
protected MultiCore multicore;
protected SolrRequestParsers parsers;
protected boolean handleSelect = false;
protected String pathPrefix = null; // strip this from the beginning of a path
protected String abortErrorMessage = null;
protected String solrConfigFilename = null;
public void init(FilterConfig config) throws ServletException
{
@ -67,6 +70,7 @@ public class SolrDispatchFilter implements Filter
try {
// web.xml configuration
this.pathPrefix = config.getInitParameter( "path-prefix" );
this.solrConfigFilename = config.getInitParameter("solrconfig-filename");
// Find a valid solr core
SolrCore core = null;
@ -87,7 +91,11 @@ public class SolrDispatchFilter implements Filter
core = multicore.getDefaultCore();
}
else {
singlecore = new SolrCore( null, null, new SolrConfig(), null );
if (this.solrConfigFilename==null) {
singlecore = new SolrCore( null, null, new SolrConfig(), null );
} else {
singlecore = new SolrCore( null, null, new SolrConfig(this.solrConfigFilename), null);
}
core = singlecore;
}
@ -168,6 +176,7 @@ public class SolrDispatchFilter implements Filter
if( request instanceof HttpServletRequest) {
SolrQueryRequest solrReq = null;
HttpServletRequest req = (HttpServletRequest)request;
HttpServletResponse resp = (HttpServletResponse)response;
try {
String path = req.getServletPath();
if( req.getPathInfo() != null ) {
@ -233,7 +242,29 @@ public class SolrDispatchFilter implements Filter
if( solrReq == null ) {
solrReq = parsers.parse( core, path, req );
}
final SolrConfig conf = core.getSolrConfig();
final Method reqMethod = Method.getMethod(req.getMethod());
if (Method.POST != reqMethod) {
HttpCacheHeaderUtil.setCacheControlHeader(conf, resp);
}
// unless we have been explicitly told not to, do cache validation
if (!conf.getHttpCachingConfig().isNever304()) {
// if we've confirmed cache validation, return immediately
if (HttpCacheHeaderUtil.doCacheHeaderValidation(solrReq,
req,resp)) {
return;
}
}
SolrQueryResponse solrRsp = new SolrQueryResponse();
/* even for HEAD requests, we need to execute the handler to
* ensure we don't get an error (and to make sure the correct
* QueryResponseWriter is selectedand we get the correct
* Content-Type)
*/
this.execute( req, handler, solrReq, solrRsp );
if( solrRsp.getException() != null ) {
sendError( (HttpServletResponse)response, solrRsp.getException() );
@ -243,6 +274,11 @@ public class SolrDispatchFilter implements Filter
// Now write it out
QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq);
response.setContentType(responseWriter.getContentType(solrReq, solrRsp));
if (Method.HEAD == Method.getMethod(req.getMethod())) {
// nothing to write out, waited this long just to get ContentType
return;
}
PrintWriter out = response.getWriter();
responseWriter.write(out, solrReq, solrRsp);
return;
@ -303,7 +339,7 @@ public class SolrDispatchFilter implements Filter
}
}
res.sendError( code, ex.getMessage() + trace );
}
}
//---------------------------------------------------------------------
//---------------------------------------------------------------------

4
src/webapp/src/org/apache/solr/servlet/SolrRequestParsers.java
View File

@ -348,7 +348,7 @@ class StandardRequestParser implements SolrRequestParser
final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
{
String method = req.getMethod().toUpperCase();
if( "GET".equals( method ) ) {
if( "GET".equals( method ) || "HEAD".equals( method )) {
return new ServletSolrParams(req);
}
if( "POST".equals( method ) ) {
@ -367,7 +367,7 @@ class StandardRequestParser implements SolrRequestParser
}
return raw.parseParamsAndFillStreams(req, streams);
}
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Unsuported method: "+method );
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Unsupported method: "+method );
}
}

298
src/webapp/src/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java vendored Normal file
View File

@ -0,0 +1,298 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.servlet.cache;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.WeakHashMap;
import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrConfig.HttpCachingConfig.LastModFrom;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.commons.codec.binary.Base64;
public final class HttpCacheHeaderUtil {
public static void sendNotModified(HttpServletResponse res)
throws IOException {
res.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
}
public static void sendPreconditionFailed(HttpServletResponse res)
throws IOException {
res.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED);
}
/**
* Weak Ref based cache for keeping track of core specific etagSeed
* and the last computed etag.
*
* @see #calcEtag
*/
private static Map<SolrCore, EtagCacheVal> etagCoreCache
= new WeakHashMap<SolrCore, EtagCacheVal>();
/** @see #etagCoreCache */
private static class EtagCacheVal {
private final String etagSeed;
private String etagCache = null;
private long indexVersionCache=-1;
public EtagCacheVal(final String etagSeed) {
this.etagSeed = etagSeed;
}
public String calcEtag(final long currentIndexVersion) {
if (currentIndexVersion != indexVersionCache) {
indexVersionCache=currentIndexVersion;
etagCache = "\""
+ new String(Base64.encodeBase64((Long.toHexString
(Long.reverse(indexVersionCache))
+ etagSeed).getBytes()))
+ "\"";
}
return etagCache;
}
}
/**
* Calculates a tag for the ETag header.
*
* @param solrReq
* @return a tag
*/
public static String calcEtag(final SolrQueryRequest solrReq) {
final SolrCore core = solrReq.getCore();
final long currentIndexVersion
= solrReq.getSearcher().getReader().getVersion();
EtagCacheVal etagCache = etagCoreCache.get(core);
if (null == etagCache) {
final String etagSeed
= core.getSolrConfig().getHttpCachingConfig().getEtagSeed();
etagCache = new EtagCacheVal(etagSeed);
etagCoreCache.put(core, etagCache);
}
return etagCache.calcEtag(currentIndexVersion);
}
/**
* Checks if one of the tags in the list equals the given etag.
*
* @param headerList
* the ETag header related header elements
* @param etag
* the ETag to compare with
* @return true if the etag is found in one of the header elements - false
* otherwise
*/
public static boolean isMatchingEtag(final List<String> headerList,
final String etag) {
for (String header : headerList) {
final String[] headerEtags = header.split(",");
for (String s : headerEtags) {
s = s.trim();
if (s.equals(etag) || "*".equals(s)) {
return true;
}
}
}
return false;
}
/**
* Calculate the appropriate last-modified time for Solr relative the current request.
*
* @param solrReq
* @return the timestamp to use as a last modified time.
*/
public static long calcLastModified(final SolrQueryRequest solrReq) {
final SolrCore core = solrReq.getCore();
final SolrIndexSearcher searcher = solrReq.getSearcher();
final LastModFrom lastModFrom
= core.getSolrConfig().getHttpCachingConfig().getLastModFrom();
long lastMod;
try {
// assume default, change if needed (getOpenTime() should be fast)
lastMod =
LastModFrom.DIRLASTMOD == lastModFrom
? IndexReader.lastModified(searcher.getReader().directory())
: searcher.getOpenTime();
} catch (IOException e) {
// we're pretty freaking screwed if this happens
throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
// Get the time where the searcher has been opened
// We get rid of the milliseconds because the HTTP header has only
// second granularity
return lastMod - (lastMod % 1000L);
}
/**
* Set the Cache-Control HTTP header (and Expires if needed)
* based on the SolrConfig.
*/
public static void setCacheControlHeader(final SolrConfig conf,
final HttpServletResponse resp) {
final String cc = conf.getHttpCachingConfig().getCacheControlHeader();
if (null != cc) {
resp.setHeader("Cache-Control", cc);
}
Integer maxAge = conf.getHttpCachingConfig().getMaxAge();
if (null != maxAge) {
resp.setDateHeader("Expires", System.currentTimeMillis()
+ (maxAge * 1000));
}
return;
}
/**
* Sets HTTP Response cache validator headers appropriately and
* validates the HTTP Request against these using any conditional
* request headers.
*
* If the request contains conditional headers, and those headers
* indicate a match with the current known state of the system, this
* method will return "true" indicating that a 304 Status code can be
* returned, and no further processing is needed.
*
*
* @return true if the request contains conditional headers, and those
* headers indicate a match with the current known state of the
* system -- indicating that a 304 Status code can be returned to
* the client, and no further request processing is needed.
*/
public static boolean doCacheHeaderValidation(final SolrQueryRequest solrReq,
final HttpServletRequest req,
final HttpServletResponse resp)
throws IOException {
final Method reqMethod=Method.getMethod(req.getMethod());
final long lastMod = HttpCacheHeaderUtil.calcLastModified(solrReq);
final String etag = HttpCacheHeaderUtil.calcEtag(solrReq);
resp.setDateHeader("Last-Modified", lastMod);
resp.setHeader("ETag", etag);
if (checkETagValidators(req, resp, reqMethod, etag)) {
return true;
}
if (checkLastModValidators(req, resp, lastMod)) {
return true;
}
return false;
}
/**
* Check for etag related conditional headers and set status
*
* @return true if no request processing is necessary and HTTP response status has been set, false otherwise.
* @throws IOException
*/
@SuppressWarnings("unchecked")
public static boolean checkETagValidators(final HttpServletRequest req,
final HttpServletResponse resp,
final Method reqMethod,
final String etag)
throws IOException {
// First check If-None-Match because this is the common used header
// element by HTTP clients
final List<String> ifNoneMatchList = Collections.list(req
.getHeaders("If-None-Match"));
if (ifNoneMatchList.size() > 0 && isMatchingEtag(ifNoneMatchList, etag)) {
if (reqMethod == Method.GET || reqMethod == Method.HEAD) {
sendNotModified(resp);
} else {
sendPreconditionFailed(resp);
}
return true;
}
// Check for If-Match headers
final List<String> ifMatchList = Collections.list(req
.getHeaders("If-Match"));
if (ifMatchList.size() > 0 && !isMatchingEtag(ifMatchList, etag)) {
sendPreconditionFailed(resp);
return true;
}
return false;
}
/**
* Check for modify time related conditional headers and set status
*
* @return true if no request processing is necessary and HTTP response status has been set, false otherwise.
* @throws IOException
*/
public static boolean checkLastModValidators(final HttpServletRequest req,
final HttpServletResponse resp,
final long lastMod)
throws IOException {
try {
// First check for If-Modified-Since because this is the common
// used header by HTTP clients
final long modifiedSince = req.getDateHeader("If-Modified-Since");
if (modifiedSince != -1L && lastMod <= modifiedSince) {
// Send a "not-modified"
sendNotModified(resp);
return true;
}
final long unmodifiedSince = req.getDateHeader("If-Unmodified-Since");
if (unmodifiedSince != -1L && lastMod > unmodifiedSince) {
// Send a "precondition failed"
sendPreconditionFailed(resp);
return true;
}
} catch (IllegalArgumentException iae) {
// one of our date headers was not formated properly, ignore it
/* NOOP */
}
return false;
}
}

41
src/webapp/src/org/apache/solr/servlet/cache/Method.java vendored Normal file
View File

@ -0,0 +1,41 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.servlet.cache;
public enum Method {
GET("GET"), POST("POST"), HEAD("HEAD"), OTHER("");
private final String method;
Method(String method) {
this.method = method.intern();
}
public static Method getMethod(String method) {
method = method.toUpperCase().intern();
for (Method m : Method.values()) {
// we can use == because we interned the String objects
if (m.method==method) {
return m;
}
}
return OTHER;
}
}