Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-4265: Fix decoding of GET/POST parameters for servlet containers with non-UTF-8 URL parsing (Tomcat) 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1429534 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Uwe Schindler 2013-01-06 14:55:11 +00:00
parent a347753992
commit cf8a0ca50b
18 changed files with 374 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lucene/tools/forbiddenApis/servlet-api.txt Normal file
View File

@ -0,0 +1,26 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# These methods from the Servlet API should not be used, because they are
# either broken and slow in some environments (e.g., Jetty's UTF-8 readers),
# or the parsing of request parameters is not using the correct encoding
# without extra configuration in the servlet container:
javax.servlet.ServletRequest#getReader()
javax.servlet.ServletRequest#getParameter(java.lang.String)
javax.servlet.ServletRequest#getParameterMap()
javax.servlet.ServletRequest#getParameterNames()
javax.servlet.ServletRequest#getParameterValues(java.lang.String)
javax.servlet.ServletResponse#getWriter()

9
solr/CHANGES.txt
View File

@ -195,6 +195,15 @@ New Features
that can be set to false to not filter. Its useful when there is already a spatial
filter query but you also need to sort or boost by distance. (David Smiley)
* SOLR-4265: Solr now parses request parameters (in URL or sent with POST using
content-type application/x-www-form-urlencoded) in its dispatcher code. It no
longer relies on special configuration settings in Tomcat or other web containers
to enable UTF-8 encoding, which is mandatory for correct Solr behaviour. Also
the maximum length of x-www-form-urlencoded POST parameters can now be configured
through the requestDispatcher/requestParsers/@formdataUploadLimitInKB setting in
solrconfig.xml (defaults to 2 MiB). Solr now works out of the box with
e.g. Tomcat, JBoss,... (Uwe Schindler, Dawid Weiss, Alex Rocher)
Optimizations
----------------------

3
solr/build.xml
View File

@ -247,9 +247,12 @@
<include name="jdk-deprecated.txt" />
<include name="commons-io.txt" />
<include name="executors.txt" />
<include name="servlet-api.txt" />
</apiFileSet>
<fileset dir="${basedir}/build">
<include name="**/*.class" />
<!-- violates the servlet-api restrictions, but it is safe to do so in this test: -->
<exclude name="solr-solrj/classes/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest$DebugServlet.class"/>
</fileset>
</forbidden-apis>
</target>

40
solr/core/src/java/org/apache/solr/request/ServletSolrParams.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.request;
import javax.servlet.ServletRequest;
import org.apache.solr.common.params.MultiMapSolrParams;
/**
*
*/
public class ServletSolrParams extends MultiMapSolrParams {
public ServletSolrParams(ServletRequest req) {
super(req.getParameterMap());
}
@Override
public String get(String name) {
String[] arr = map.get(name);
if (arr==null) return null;
String s = arr[0];
if (s.length()==0) return null; // screen out blank parameters
return s;
}
}

27
solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java
View File

@ -19,7 +19,8 @@ package org.apache.solr.servlet;
import java.io.InputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
@ -30,7 +31,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.solr.core.CoreContainer;
/**
* A simple servlet to load the Solr Admin UI
*
@ -42,15 +42,15 @@ public final class LoadAdminUiServlet extends HttpServlet {
public void doGet(HttpServletRequest request,
HttpServletResponse response)
throws IOException {
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
// This attribute is set by the SolrDispatchFilter
CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
PrintWriter out = response.getWriter();
InputStream in = getServletContext().getResourceAsStream("/admin.html");
if(in != null) {
if(in != null && cores != null) {
try {
// This attribute is set by the SolrDispatchFilter
CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
Writer out = new OutputStreamWriter(response.getOutputStream(), "UTF-8");
String html = IOUtils.toString(in, "UTF-8");
@ -63,19 +63,14 @@ public final class LoadAdminUiServlet extends HttpServlet {
StringEscapeUtils.escapeJavaScript(cores.getAdminPath())
};
out.println( StringUtils.replaceEach(html, search, replace) );
out.write( StringUtils.replaceEach(html, search, replace) );
out.flush();
} finally {
IOUtils.closeQuietly(in);
}
} else {
out.println("solr");
response.sendError(404);
}
}
@Override
public void doPost(HttpServletRequest request,
HttpServletResponse response)
throws IOException {
doGet(request, response);
}
}

5
solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java
View File

@ -62,9 +62,4 @@ public class RedirectServlet extends HttpServlet{
res.setHeader("Location", destination);
}
@Override
public void doPost(HttpServletRequest req, HttpServletResponse res)
throws ServletException,IOException {
doGet(req,res);
}
}

30
solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java
View File

@ -17,13 +17,13 @@
package org.apache.solr.servlet;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@ -45,6 +45,8 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
@ -53,7 +55,6 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.ContentStreamHandlerBase;
import org.apache.solr.request.ServletSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
import org.apache.solr.request.SolrRequestHandler;
@ -66,7 +67,6 @@ import org.apache.solr.servlet.cache.Method;
import org.apache.solr.util.FastWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
/**
* This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml
@ -82,19 +82,9 @@ public class SolrDispatchFilter implements Filter
protected String pathPrefix = null; // strip this from the beginning of a path
protected String abortErrorMessage = null;
protected final Map<SolrConfig, SolrRequestParsers> parsers = new WeakHashMap<SolrConfig, SolrRequestParsers>();
protected final SolrRequestParsers adminRequestParser;
private static final Charset UTF8 = Charset.forName("UTF-8");
public SolrDispatchFilter() {
try {
adminRequestParser = new SolrRequestParsers(new Config(null,"solr",new InputSource(new ByteArrayInputStream("<root/>".getBytes("UTF-8"))),"") );
} catch (Exception e) {
//unlikely
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,e);
}
}
@Override
public void init(FilterConfig config) throws ServletException
{
@ -180,14 +170,14 @@ public class SolrDispatchFilter implements Filter
// Check for the core admin page
if( path.equals( cores.getAdminPath() ) ) {
handler = cores.getMultiCoreHandler();
solrReq = adminRequestParser.parse(null,path, req);
solrReq = SolrRequestParsers.DEFAULT.parse(null,path, req);
handleAdminRequest(req, response, handler, solrReq);
return;
}
// Check for the core admin collections url
if( path.equals( "/admin/collections" ) ) {
handler = cores.getCollectionsHandler();
solrReq = adminRequestParser.parse(null,path, req);
solrReq = SolrRequestParsers.DEFAULT.parse(null,path, req);
handleAdminRequest(req, response, handler, solrReq);
return;
}
@ -475,7 +465,15 @@ public class SolrDispatchFilter implements Filter
core = cores.getCore(""); // default core
}
if(req==null) {
req = new SolrQueryRequestBase(core,new ServletSolrParams(request)) {};
final SolrParams solrParams;
if (request instanceof HttpServletRequest) {
// use GET parameters if available:
solrParams = SolrRequestParsers.parseQueryString(((HttpServletRequest) request).getQueryString());
} else {
// we have no params at all, use empty ones:
solrParams = new MapSolrParams(Collections.<String,String>emptyMap());
}
req = new SolrQueryRequestBase(core, solrParams) {};
}
QueryResponseWriter writer = core.getQueryResponseWriter(req);
writeResponse(solrResp, response, writer, req, Method.GET);

215
solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java
View File

@ -32,6 +32,8 @@ import java.util.Locale;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.input.BoundedInputStream;
import javax.servlet.http.HttpServletRequest;
@ -46,7 +48,6 @@ import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.core.Config;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.ServletSolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrQueryRequestBase;
@ -57,47 +58,63 @@ public class SolrRequestParsers
// Should these constants be in a more public place?
public static final String MULTIPART = "multipart";
public static final String FORMDATA = "formdata";
public static final String RAW = "raw";
public static final String SIMPLE = "simple";
public static final String STANDARD = "standard";
private HashMap<String, SolrRequestParser> parsers;
private boolean enableRemoteStreams = false;
private boolean handleSelect = true;
private final HashMap<String, SolrRequestParser> parsers =
new HashMap<String, SolrRequestParser>();
private final boolean enableRemoteStreams;
private StandardRequestParser standard;
private boolean handleSelect = true;
/** Default instance for e.g. admin requests. Limits to 2 MB uploads and does not allow remote streams. */
public static final SolrRequestParsers DEFAULT = new SolrRequestParsers();
/**
* Pass in an xml configuration. A null configuration will enable
* everythign with maximum values.
* everything with maximum values.
*/
public SolrRequestParsers( Config globalConfig )
{
long uploadLimitKB = 1048; // 2MB default
public SolrRequestParsers( Config globalConfig ) {
final int multipartUploadLimitKB, formUploadLimitKB;
if( globalConfig == null ) {
uploadLimitKB = Long.MAX_VALUE;
multipartUploadLimitKB = formUploadLimitKB = Integer.MAX_VALUE;
enableRemoteStreams = true;
handleSelect = true;
}
else {
uploadLimitKB = globalConfig.getInt(
"requestDispatcher/requestParsers/@multipartUploadLimitInKB", (int)uploadLimitKB );
} else {
multipartUploadLimitKB = globalConfig.getInt(
"requestDispatcher/requestParsers/@multipartUploadLimitInKB", 2048 );
formUploadLimitKB = globalConfig.getInt(
"requestDispatcher/requestParsers/@formdataUploadLimitInKB", 2048 );
enableRemoteStreams = globalConfig.getBool(
"requestDispatcher/requestParsers/@enableRemoteStreaming", false );
// Let this filter take care of /select?xxx format
handleSelect = globalConfig.getBool(
"requestDispatcher/@handleSelect", handleSelect );
"requestDispatcher/@handleSelect", true );
}
MultipartRequestParser multi = new MultipartRequestParser( uploadLimitKB );
init(multipartUploadLimitKB, formUploadLimitKB);
}
private SolrRequestParsers() {
enableRemoteStreams = false;
handleSelect = false;
init(2048, 2048);
}
private void init( int multipartUploadLimitKB, int formUploadLimitKB) {
MultipartRequestParser multi = new MultipartRequestParser( multipartUploadLimitKB );
RawRequestParser raw = new RawRequestParser();
standard = new StandardRequestParser( multi, raw );
FormDataRequestParser formdata = new FormDataRequestParser( formUploadLimitKB );
standard = new StandardRequestParser( multi, raw, formdata );
// I don't see a need to have this publicly configured just yet
// adding it is trivial
parsers = new HashMap<String, SolrRequestParser>();
parsers.put( MULTIPART, multi );
parsers.put( FORMDATA, formdata );
parsers.put( RAW, raw );
parsers.put( SIMPLE, new SimpleRequestParser() );
parsers.put( STANDARD, standard );
@ -175,23 +192,31 @@ public class SolrRequestParsers
return q;
}
/**
* Given a url-encoded query string (UTF-8), map it into solr params
*/
public static MultiMapSolrParams parseQueryString(String queryString) {
Map<String,String[]> map = new HashMap<String, String[]>();
parseQueryString(queryString, "UTF-8", map);
return new MultiMapSolrParams(map);
}
/**
* Given a standard query string map it into solr params
* Given a url-encoded query string, map it into the given map
* @param queryString as given from URL
* @param charset to be used to decode %-encoding
* @param map place all parameters in this map
*/
public static MultiMapSolrParams parseQueryString(String queryString)
{
Map<String,String[]> map = new HashMap<String, String[]>();
static void parseQueryString(String queryString, String charset, Map<String,String[]> map) {
if( queryString != null && queryString.length() > 0 ) {
try {
for( String kv : queryString.split( "&" ) ) {
int idx = kv.indexOf( '=' );
if( idx > 0 ) {
String name = URLDecoder.decode( kv.substring( 0, idx ), "UTF-8");
String value = URLDecoder.decode( kv.substring( idx+1 ), "UTF-8");
if( idx >= 0 ) {
String name = URLDecoder.decode( kv.substring( 0, idx ), charset);
String value = URLDecoder.decode( kv.substring( idx+1 ), charset);
MultiMapSolrParams.addParam( name, value, map );
}
else {
} else {
String name = URLDecoder.decode( kv, "UTF-8" );
MultiMapSolrParams.addParam( name, "", map );
}
@ -201,7 +226,6 @@ public class SolrRequestParsers
throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, uex );
}
}
return new MultiMapSolrParams( map );
}
public boolean isHandleSelect() {
@ -228,7 +252,7 @@ interface SolrRequestParser
//-----------------------------------------------------------------
/**
* The simple parser just uses the params directly
* The simple parser just uses the params directly, does not support POST URL-encoded forms
*/
class SimpleRequestParser implements SolrRequestParser
{
@ -236,7 +260,7 @@ class SimpleRequestParser implements SolrRequestParser
public SolrParams parseParamsAndFillStreams(
final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
{
return new ServletSolrParams(req);
return SolrRequestParsers.parseQueryString(req.getQueryString());
}
}
@ -298,13 +322,6 @@ class RawRequestParser implements SolrRequestParser
public SolrParams parseParamsAndFillStreams(
final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
{
// The javadocs for HttpServletRequest are clear that req.getReader() should take
// care of any character encoding issues. BUT, there are problems while running on
// some servlet containers: including Tomcat 5 and resin.
//
// Rather than return req.getReader(), this uses the default ContentStreamBase method
// that checks for charset definitions in the ContentType.
streams.add( new HttpRequestContentStream( req ) );
return SolrRequestParsers.parseQueryString( req.getQueryString() );
}
@ -317,9 +334,9 @@ class RawRequestParser implements SolrRequestParser
*/
class MultipartRequestParser implements SolrRequestParser
{
private long uploadLimitKB;
private final int uploadLimitKB;
public MultipartRequestParser( long limit )
public MultipartRequestParser( int limit )
{
uploadLimitKB = limit;
}
@ -343,7 +360,7 @@ class MultipartRequestParser implements SolrRequestParser
// Create a new file upload handler
ServletFileUpload upload = new ServletFileUpload(factory);
upload.setSizeMax( uploadLimitKB*1024 );
upload.setSizeMax( ((long) uploadLimitKB) * 1024L );
// Parse the request
List items = upload.parseRequest(req);
@ -367,6 +384,97 @@ class MultipartRequestParser implements SolrRequestParser
}
/**
* Extract application/x-www-form-urlencoded form data for POST requests
*/
class FormDataRequestParser implements SolrRequestParser
{
private final int uploadLimitKB;
public FormDataRequestParser( int limit )
{
uploadLimitKB = limit;
}
@Override
public SolrParams parseParamsAndFillStreams(
final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
{
if (!isFormData(req)) {
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
}
String charset = ContentStreamBase.getCharsetFromContentType(req.getContentType());
if (charset == null) charset = "UTF-8";
final Map<String,String[]> map = new HashMap<String, String[]>();
// also add possible URL parameters and include into the map (parsed using UTF-8):
final String qs = req.getQueryString();
if (qs != null) {
SolrRequestParsers.parseQueryString(qs, "UTF-8", map);
}
// may be -1, so we check again later. But if its already greater we can stop processing!
final long totalLength = req.getContentLength();
final long maxLength = ((long) uploadLimitKB) * 1024L;
if (totalLength > maxLength) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
}
// get query String from request body, using the charset given in content-type:
final InputStream in;
try {
in = req.getInputStream();
} catch (IllegalStateException ise) {
throw (SolrException) getParameterIncompatibilityException().initCause(ise);
}
try {
final String data = IOUtils.toString(new BoundedInputStream(in, maxLength), charset);
// if there is remaining data in the underlying stream, throw exception:
if (in.read() != -1) {
// read remaining data and throw away:
while (IOUtils.skip(in, 1024L) > 0);
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content exceeds upload limit of " + uploadLimitKB + " KB");
}
if (data.length() == 0 && totalLength > 0L) {
throw getParameterIncompatibilityException();
}
SolrRequestParsers.parseQueryString(data, charset, map);
} finally {
IOUtils.closeQuietly(in);
}
return new MultiMapSolrParams(map);
}
private SolrException getParameterIncompatibilityException() {
return new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Solr requires that request parameters sent using application/x-www-form-urlencoded " +
"content-type can be read through the request input stream. Unfortunately, the " +
"stream was empty / not available. This may be caused by another servlet filter calling " +
"ServletRequest.getParameter*() before SolrDispatchFilter, please remove it."
);
}
public boolean isFormData(HttpServletRequest req) {
String contentType = req.getContentType();
if (contentType != null) {
int idx = contentType.indexOf( ';' );
if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
contentType = contentType.substring( 0, idx );
}
contentType = contentType.trim();
if( "application/x-www-form-urlencoded".equalsIgnoreCase(contentType)) {
return true;
}
}
return false;
}
}
/**
* The default Logic
*/
@ -374,11 +482,13 @@ class StandardRequestParser implements SolrRequestParser
{
MultipartRequestParser multipart;
RawRequestParser raw;
FormDataRequestParser formdata;
StandardRequestParser( MultipartRequestParser multi, RawRequestParser raw )
StandardRequestParser(MultipartRequestParser multi, RawRequestParser raw, FormDataRequestParser formdata)
{
this.multipart = multi;
this.raw = raw;
this.formdata = formdata;
}
@Override
@ -386,22 +496,15 @@ class StandardRequestParser implements SolrRequestParser
final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
{
String method = req.getMethod().toUpperCase(Locale.ROOT);
if( "GET".equals( method ) || "HEAD".equals( method )) {
return new ServletSolrParams(req);
if ("GET".equals(method) || "HEAD".equals(method)) {
return SolrRequestParsers.parseQueryString(req.getQueryString());
}
if( "POST".equals( method ) ) {
String contentType = req.getContentType();
if( contentType != null ) {
int idx = contentType.indexOf( ';' );
if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
contentType = contentType.substring( 0, idx );
}
if( "application/x-www-form-urlencoded".equals( contentType.toLowerCase(Locale.ROOT) ) ) {
return new ServletSolrParams(req); // just get the params from parameterMap
}
if( ServletFileUpload.isMultipartContent(req) ) {
return multipart.parseParamsAndFillStreams(req, streams);
}
if ("POST".equals( method ) ) {
if (formdata.isFormData(req)) {
return formdata.parseParamsAndFillStreams(req, streams);
}
if (ServletFileUpload.isMultipartContent(req)) {
return multipart.parseParamsAndFillStreams(req, streams);
}
return raw.parseParamsAndFillStreams(req, streams);
}

59
solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java
View File

@ -18,21 +18,28 @@
package org.apache.solr.servlet;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URLEncoder;
import java.util.Date;
import java.util.List;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.noggit.CharArr;
import org.apache.noggit.JSONWriter;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.FastWriter;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
@ -46,7 +53,7 @@ import org.slf4j.LoggerFactory;
*/
public final class ZookeeperInfoServlet extends HttpServlet {
static final Logger log = LoggerFactory.getLogger(ZookeeperInfoServlet.class);
@Override
public void init() {
}
@ -54,28 +61,42 @@ public final class ZookeeperInfoServlet extends HttpServlet {
@Override
public void doGet(HttpServletRequest request,
HttpServletResponse response)
throws IOException {
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
throws ServletException,IOException {
// This attribute is set by the SolrDispatchFilter
CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
if (cores == null) {
throw new ServletException("Missing request attribute org.apache.solr.CoreContainer.");
}
final SolrParams params;
try {
params = SolrRequestParsers.DEFAULT.parse(null, request.getServletPath(), request).getParams();
} catch (Exception e) {
int code=500;
if (e instanceof SolrException) {
code = Math.min(599, Math.max(100, ((SolrException)e).code()));
}
response.sendError(code, e.toString());
return;
}
String path = request.getParameter("path");
String addr = request.getParameter("addr");
String path = params.get("path");
String addr = params.get("addr");
if (addr != null && addr.length() == 0) {
addr = null;
}
String detailS = request.getParameter("detail");
String detailS = params.get("detail");
boolean detail = detailS != null && detailS.equals("true");
String dumpS = request.getParameter("dump");
String dumpS = params.get("dump");
boolean dump = dumpS != null && dumpS.equals("true");
PrintWriter out = response.getWriter();
response.setCharacterEncoding("UTF-8");
response.setContentType("application/json");
Writer out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), IOUtils.CHARSET_UTF_8));
ZKPrinter printer = new ZKPrinter(response, out, cores.getZkController(), addr);
printer.detail = detail;
@ -86,12 +107,14 @@ public final class ZookeeperInfoServlet extends HttpServlet {
} finally {
printer.close();
}
out.flush();
}
@Override
public void doPost(HttpServletRequest request,
HttpServletResponse response)
throws IOException {
throws ServletException,IOException {
doGet(request, response);
}
@ -114,13 +137,13 @@ public final class ZookeeperInfoServlet extends HttpServlet {
boolean doClose; // close the client after done if we opened it
final HttpServletResponse response;
final PrintWriter out;
final Writer out;
SolrZkClient zkClient;
int level;
int maxData = 95;
public ZKPrinter(HttpServletResponse response, PrintWriter out, ZkController controller, String addr) throws IOException {
public ZKPrinter(HttpServletResponse response, Writer out, ZkController controller, String addr) throws IOException {
this.response = response;
this.out = out;
this.addr = addr;
@ -207,10 +230,10 @@ public final class ZookeeperInfoServlet extends HttpServlet {
}
json.endArray();
json.endObject();
out.println(chars.toString());
out.write(chars.toString());
}
void writeError(int code, String msg) {
void writeError(int code, String msg) throws IOException {
response.setStatus(code);
CharArr chars = new CharArr();
@ -227,7 +250,7 @@ public final class ZookeeperInfoServlet extends HttpServlet {
w.writeString(msg);
w.endObject();
out.println(chars.toString());
out.write(chars.toString());
}
@ -352,7 +375,7 @@ public final class ZookeeperInfoServlet extends HttpServlet {
json.write(v);
}
boolean printZnode(JSONWriter json, String path) {
boolean printZnode(JSONWriter json, String path) throws IOException {
try {
Stat stat = new Stat();
// Trickily, the call to zkClient.getData fills in the stat variable

104
solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
View File

@ -21,6 +21,7 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import java.io.ByteArrayInputStream;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URL;
@ -31,10 +32,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletInputStream;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.io.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.params.SolrParams;
@ -168,12 +171,11 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
@Test
public void testStandardParseParamsAndFillStreams() throws Exception
{
ArrayList<ContentStream> streams = new ArrayList<ContentStream>();
Map<String,String[]> params = new HashMap<String, String[]>();
params.put( "q", new String[] { "hello" } );
final String getParams = "qt=%C3%BC&dup=foo", postParams = "q=hello&d%75p=bar";
final byte[] postBytes = postParams.getBytes("UTF-8");
// Set up the expected behavior
String[] ct = new String[] {
final String[] ct = new String[] {
"application/x-www-form-urlencoded",
"Application/x-www-form-urlencoded",
"application/x-www-form-urlencoded; charset=utf-8",
@ -184,16 +186,102 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
HttpServletRequest request = createMock(HttpServletRequest.class);
expect(request.getMethod()).andReturn("POST").anyTimes();
expect(request.getContentType()).andReturn( contentType ).anyTimes();
expect(request.getParameterMap()).andReturn(params).anyTimes();
expect(request.getQueryString()).andReturn(getParams).anyTimes();
expect(request.getContentLength()).andReturn(postBytes.length).anyTimes();
expect(request.getInputStream()).andReturn(new ServletInputStream() {
private final ByteArrayInputStream in = new ByteArrayInputStream(postBytes);
@Override public int read() { return in.read(); }
});
replay(request);
MultipartRequestParser multipart = new MultipartRequestParser( 1000000 );
MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
RawRequestParser raw = new RawRequestParser();
StandardRequestParser standard = new StandardRequestParser( multipart, raw );
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
SolrParams p = standard.parseParamsAndFillStreams( request, streams );
SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
assertEquals( "contentType: "+contentType, "hello", p.get("q") );
assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
assertArrayEquals( "contentType: "+contentType, new String[]{"foo","bar"}, p.getParams("dup") );
}
}
@Test
public void testStandardFormdataUploadLimit() throws Exception
{
final int limitKBytes = 128;
final StringBuilder large = new StringBuilder("q=hello");
// grow exponentially to reach 128 KB limit:
while (large.length() <= limitKBytes * 1024) {
large.append('&').append(large);
}
HttpServletRequest request = createMock(HttpServletRequest.class);
expect(request.getMethod()).andReturn("POST").anyTimes();
expect(request.getContentType()).andReturn("application/x-www-form-urlencoded").anyTimes();
// we dont pass a content-length to let the security mechanism limit it:
expect(request.getContentLength()).andReturn(-1).anyTimes();
expect(request.getQueryString()).andReturn(null).anyTimes();
expect(request.getInputStream()).andReturn(new ServletInputStream() {
private final ByteArrayInputStream in = new ByteArrayInputStream(large.toString().getBytes("UTF-8"));
@Override public int read() { return in.read(); }
});
replay(request);
FormDataRequestParser formdata = new FormDataRequestParser( limitKBytes );
try {
formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
fail("should throw SolrException");
} catch (SolrException solre) {
assertTrue(solre.getMessage().contains("upload limit"));
assertEquals(400, solre.code());
}
}
@Test
public void testParameterIncompatibilityException1() throws Exception
{
HttpServletRequest request = createMock(HttpServletRequest.class);
expect(request.getMethod()).andReturn("POST").anyTimes();
expect(request.getContentType()).andReturn("application/x-www-form-urlencoded").anyTimes();
expect(request.getContentLength()).andReturn(100).anyTimes();
expect(request.getQueryString()).andReturn(null).anyTimes();
// we emulate Jetty that returns empty stream when parameters were parsed before:
expect(request.getInputStream()).andReturn(new ServletInputStream() {
@Override public int read() { return -1; }
});
replay(request);
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
try {
formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
fail("should throw SolrException");
} catch (SolrException solre) {
assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
assertEquals(500, solre.code());
}
}
@Test
public void testParameterIncompatibilityException2() throws Exception
{
HttpServletRequest request = createMock(HttpServletRequest.class);
expect(request.getMethod()).andReturn("POST").anyTimes();
expect(request.getContentType()).andReturn("application/x-www-form-urlencoded").anyTimes();
expect(request.getContentLength()).andReturn(100).anyTimes();
expect(request.getQueryString()).andReturn(null).anyTimes();
// we emulate Tomcat that throws IllegalStateException when parameters were parsed before:
expect(request.getInputStream()).andThrow(new IllegalStateException());
replay(request);
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
try {
formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
fail("should throw SolrException");
} catch (SolrException solre) {
assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
assertEquals(500, solre.code());
}
}
}

2
solr/example/example-DIH/solr/db/conf/solrconfig.xml
View File

@ -217,7 +217,7 @@
-->
<requestDispatcher handleSelect="true" >
<!--Make sure your system has some authentication before enabling remote streaming! -->
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
<!-- Set HTTP caching related parameters (for proxy caches and clients).

2
solr/example/example-DIH/solr/mail/conf/solrconfig.xml
View File

@ -249,7 +249,7 @@
-->
<requestDispatcher handleSelect="true" >
<!--Make sure your system has some authentication before enabling remote streaming! -->
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" />
<requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" />
<!-- Set HTTP caching related parameters (for proxy caches and clients).

2
solr/example/example-DIH/solr/rss/conf/solrconfig.xml
View File

@ -217,7 +217,7 @@
-->
<requestDispatcher handleSelect="true" >
<!--Make sure your system has some authentication before enabling remote streaming! -->
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
<!-- Set HTTP caching related parameters (for proxy caches and clients).

2
solr/example/example-DIH/solr/solr/conf/solrconfig.xml
View File

@ -217,7 +217,7 @@
-->
<requestDispatcher handleSelect="true" >
<!--Make sure your system has some authentication before enabling remote streaming! -->
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
<!-- Set HTTP caching related parameters (for proxy caches and clients).

2
solr/example/example-DIH/solr/tika/conf/solrconfig.xml
View File

@ -188,7 +188,7 @@
-->
<requestDispatcher handleSelect="true" >
<!--Make sure your system has some authentication before enabling remote streaming! -->
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
<!-- Set HTTP caching related parameters (for proxy caches and clients).

2
solr/example/multicore/core0/conf/solrconfig.xml
View File

@ -47,7 +47,7 @@
<requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" />
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
</requestDispatcher>
<requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />

2
solr/example/multicore/core1/conf/solrconfig.xml
View File

@ -47,7 +47,7 @@
<requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" />
<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
</requestDispatcher>
<requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />

10
solr/example/solr/collection1/conf/solrconfig.xml
View File

@ -657,9 +657,14 @@
enableRemoteStreaming - enables use of the stream.file
and stream.url parameters for specifying remote streams.
multipartUploadLimitInKB - specifies the max size of
multipartUploadLimitInKB - specifies the max size (in KiB) of
Multipart File Uploads that Solr will allow in a Request.
formdataUploadLimitInKB - specifies the max size (in KiB) of
form data (application/x-www-form-urlencoded) sent via
POST. You can use POST to pass request parameters not
fitting into the URL.
*** WARNING ***
The settings below authorize Solr to fetch remote files, You
should make sure your system has some authentication before
@ -667,7 +672,8 @@
-->
<requestParsers enableRemoteStreaming="true"
multipartUploadLimitInKB="2048000" />
multipartUploadLimitInKB="2048000"
formdataUploadLimitInKB="2048"/>
<!-- HTTP Caching