SOLR-4265: Fix decoding of GET/POST parameters for servlet containers with non-UTF-8 URL parsing (Tomcat)

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1429534 13f79535-47bb-0310-9956-ffa450edef68

lucene/tools/forbiddenApis/servlet-api.txt

@@ -0,0 +1,26 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# These methods from the Servlet API should not be used, because they are
+# either broken and slow in some environments (e.g., Jetty's UTF-8 readers),
+# or the parsing of request parameters is not using the correct encoding
+# without extra configuration in the servlet container:
+javax.servlet.ServletRequest#getReader()
+javax.servlet.ServletRequest#getParameter(java.lang.String) 
+javax.servlet.ServletRequest#getParameterMap() 
+javax.servlet.ServletRequest#getParameterNames() 
+javax.servlet.ServletRequest#getParameterValues(java.lang.String) 
+javax.servlet.ServletResponse#getWriter()
+

solr/CHANGES.txt

@@ -195,6 +195,15 @@ New Features
   that can be set to false to not filter. Its useful when there is already a spatial
   filter query but you also need to sort or boost by distance. (David Smiley)
 
+* SOLR-4265: Solr now parses request parameters (in URL or sent with POST using
+  content-type application/x-www-form-urlencoded) in its dispatcher code. It no
+  longer relies on special configuration settings in Tomcat or other web containers
+  to enable UTF-8 encoding, which is mandatory for correct Solr behaviour. Also
+  the maximum length of x-www-form-urlencoded POST parameters can now be configured
+  through the requestDispatcher/requestParsers/@formdataUploadLimitInKB setting in
+  solrconfig.xml (defaults to 2 MiB). Solr now works out of the box with
+  e.g. Tomcat, JBoss,...  (Uwe Schindler, Dawid Weiss, Alex Rocher)
+
 Optimizations
 ----------------------
 

solr/build.xml

@@ -247,9 +247,12 @@
         <include name="jdk-deprecated.txt" />
         <include name="commons-io.txt" />
         <include name="executors.txt" />
+        <include name="servlet-api.txt" />
       </apiFileSet>
       <fileset dir="${basedir}/build">
         <include name="**/*.class" />        
+        <!-- violates the servlet-api restrictions, but it is safe to do so in this test: -->
+        <exclude name="solr-solrj/classes/test/org/apache/solr/client/solrj/impl/BasicHttpSolrServerTest$DebugServlet.class"/>
       </fileset>
     </forbidden-apis>
   </target>

solr/core/src/java/org/apache/solr/request/ServletSolrParams.java

@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.request;
-
-import javax.servlet.ServletRequest;
-
-import org.apache.solr.common.params.MultiMapSolrParams;
-
-/**
- *
- */
-public class ServletSolrParams extends MultiMapSolrParams {
-  public ServletSolrParams(ServletRequest req) {
-    super(req.getParameterMap());
-  }
-
-  @Override
-  public String get(String name) {
-    String[] arr = map.get(name);
-    if (arr==null) return null;
-    String s = arr[0];
-    if (s.length()==0) return null;  // screen out blank parameters
-    return s;
-  }
-}

solr/core/src/java/org/apache/solr/servlet/LoadAdminUiServlet.java

@@ -19,7 +19,8 @@ package org.apache.solr.servlet;
 
 import java.io.InputStream;
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
 
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
@@ -30,7 +31,6 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.solr.core.CoreContainer;
 
-
 /**
  * A simple servlet to load the Solr Admin UI
  * 
@@ -42,15 +42,15 @@ public final class LoadAdminUiServlet extends HttpServlet {
   public void doGet(HttpServletRequest request,
                     HttpServletResponse response)
       throws IOException {
-    response.setCharacterEncoding("UTF-8");
+    // This attribute is set by the SolrDispatchFilter
-    response.setContentType("text/html");
+    CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
 
-    PrintWriter out = response.getWriter();
     InputStream in = getServletContext().getResourceAsStream("/admin.html");
-    if(in != null) {
+    if(in != null && cores != null) {
       try {
-        // This attribute is set by the SolrDispatchFilter
+        response.setCharacterEncoding("UTF-8");
-        CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
+        response.setContentType("text/html");
+        Writer out = new OutputStreamWriter(response.getOutputStream(), "UTF-8");
 
         String html = IOUtils.toString(in, "UTF-8");
 
@@ -63,19 +63,14 @@ public final class LoadAdminUiServlet extends HttpServlet {
             StringEscapeUtils.escapeJavaScript(cores.getAdminPath())
         };
         
-        out.println( StringUtils.replaceEach(html, search, replace) );
+        out.write( StringUtils.replaceEach(html, search, replace) );
+        out.flush();
       } finally {
         IOUtils.closeQuietly(in);
       }
     } else {
-      out.println("solr");
+      response.sendError(404);
     }
   }
 
-  @Override
-  public void doPost(HttpServletRequest request,
-                     HttpServletResponse response)
-      throws IOException {
-    doGet(request, response);
-  }
 }

solr/core/src/java/org/apache/solr/servlet/RedirectServlet.java

@@ -62,9 +62,4 @@ public class RedirectServlet extends HttpServlet{
     res.setHeader("Location", destination);
   }
 
-  @Override
-  public void doPost(HttpServletRequest req, HttpServletResponse res)
-          throws ServletException,IOException {
-    doGet(req,res);
-  }
 }

solr/core/src/java/org/apache/solr/servlet/SolrDispatchFilter.java

@@ -17,13 +17,13 @@
 
 package org.apache.solr.servlet;
 
-import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.io.StringWriter;
 import java.io.Writer;
 import java.nio.charset.Charset;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -45,6 +45,8 @@ import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.cloud.ZkNodeProps;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.SimpleOrderedMap;
@@ -53,7 +55,6 @@ import org.apache.solr.core.CoreContainer;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.handler.ContentStreamHandlerBase;
-import org.apache.solr.request.ServletSolrParams;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequestBase;
 import org.apache.solr.request.SolrRequestHandler;
@@ -66,7 +67,6 @@ import org.apache.solr.servlet.cache.Method;
 import org.apache.solr.util.FastWriter;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.xml.sax.InputSource;
 
 /**
  * This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml
@@ -82,19 +82,9 @@ public class SolrDispatchFilter implements Filter
   protected String pathPrefix = null; // strip this from the beginning of a path
   protected String abortErrorMessage = null;
   protected final Map<SolrConfig, SolrRequestParsers> parsers = new WeakHashMap<SolrConfig, SolrRequestParsers>();
-  protected final SolrRequestParsers adminRequestParser;
   
   private static final Charset UTF8 = Charset.forName("UTF-8");
 
-  public SolrDispatchFilter() {
-    try {
-      adminRequestParser = new SolrRequestParsers(new Config(null,"solr",new InputSource(new ByteArrayInputStream("<root/>".getBytes("UTF-8"))),"") );
-    } catch (Exception e) {
-      //unlikely
-      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,e);
-    }
-  }
-
   @Override
   public void init(FilterConfig config) throws ServletException
   {
@@ -180,14 +170,14 @@ public class SolrDispatchFilter implements Filter
         // Check for the core admin page
         if( path.equals( cores.getAdminPath() ) ) {
           handler = cores.getMultiCoreHandler();
-          solrReq =  adminRequestParser.parse(null,path, req);
+          solrReq =  SolrRequestParsers.DEFAULT.parse(null,path, req);
           handleAdminRequest(req, response, handler, solrReq);
           return;
         }
         // Check for the core admin collections url
         if( path.equals( "/admin/collections" ) ) {
           handler = cores.getCollectionsHandler();
-          solrReq =  adminRequestParser.parse(null,path, req);
+          solrReq =  SolrRequestParsers.DEFAULT.parse(null,path, req);
           handleAdminRequest(req, response, handler, solrReq);
           return;
         }
@@ -475,7 +465,15 @@ public class SolrDispatchFilter implements Filter
         core = cores.getCore(""); // default core
       }
       if(req==null) {
-        req = new SolrQueryRequestBase(core,new ServletSolrParams(request)) {};
+        final SolrParams solrParams;
+        if (request instanceof HttpServletRequest) {
+          // use GET parameters if available:
+          solrParams = SolrRequestParsers.parseQueryString(((HttpServletRequest) request).getQueryString());
+        } else {
+          // we have no params at all, use empty ones:
+          solrParams = new MapSolrParams(Collections.<String,String>emptyMap());
+        }
+        req = new SolrQueryRequestBase(core, solrParams) {};
       }
       QueryResponseWriter writer = core.getQueryResponseWriter(req);
       writeResponse(solrResp, response, writer, req, Method.GET);

solr/core/src/java/org/apache/solr/servlet/SolrRequestParsers.java

@@ -32,6 +32,8 @@ import java.util.Locale;
 import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.BoundedInputStream;
 
 import javax.servlet.http.HttpServletRequest;
 
@@ -46,7 +48,6 @@ import org.apache.solr.common.util.ContentStream;
 import org.apache.solr.common.util.ContentStreamBase;
 import org.apache.solr.core.Config;
 import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.ServletSolrParams;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrQueryRequestBase;
 
@@ -57,47 +58,63 @@ public class SolrRequestParsers
   
   // Should these constants be in a more public place?
   public static final String MULTIPART = "multipart";
+  public static final String FORMDATA = "formdata";
   public static final String RAW = "raw";
   public static final String SIMPLE = "simple";
   public static final String STANDARD = "standard";
   
-  private HashMap<String, SolrRequestParser> parsers;
+  private final HashMap<String, SolrRequestParser> parsers =
-  private boolean enableRemoteStreams = false;
+      new HashMap<String, SolrRequestParser>();
-  private boolean handleSelect = true;
+  private final boolean enableRemoteStreams;
   private StandardRequestParser standard;
+  private boolean handleSelect = true;
   
+  /** Default instance for e.g. admin requests. Limits to 2 MB uploads and does not allow remote streams. */
+  public static final SolrRequestParsers DEFAULT = new SolrRequestParsers();
+
   /**
    * Pass in an xml configuration.  A null configuration will enable
-   * everythign with maximum values.
+   * everything with maximum values.
    */
-  public SolrRequestParsers( Config globalConfig )
+  public SolrRequestParsers( Config globalConfig ) {
-  {
+    final int multipartUploadLimitKB, formUploadLimitKB;
-    long uploadLimitKB = 1048;  // 2MB default
     if( globalConfig == null ) {
-      uploadLimitKB = Long.MAX_VALUE; 
+      multipartUploadLimitKB = formUploadLimitKB = Integer.MAX_VALUE; 
       enableRemoteStreams = true;
       handleSelect = true;
-    }
+    } else {
-    else {
+      multipartUploadLimitKB = globalConfig.getInt( 
-      uploadLimitKB = globalConfig.getInt( 
+          "requestDispatcher/requestParsers/@multipartUploadLimitInKB", 2048 );
-          "requestDispatcher/requestParsers/@multipartUploadLimitInKB", (int)uploadLimitKB );
+      
+      formUploadLimitKB = globalConfig.getInt( 
+          "requestDispatcher/requestParsers/@formdataUploadLimitInKB", 2048 );
       
       enableRemoteStreams = globalConfig.getBool( 
           "requestDispatcher/requestParsers/@enableRemoteStreaming", false ); 
   
       // Let this filter take care of /select?xxx format
       handleSelect = globalConfig.getBool( 
-          "requestDispatcher/@handleSelect", handleSelect ); 
+          "requestDispatcher/@handleSelect", true ); 
     }
-       
+    init(multipartUploadLimitKB, formUploadLimitKB);
-    MultipartRequestParser multi = new MultipartRequestParser( uploadLimitKB );
+  }
+  
+  private SolrRequestParsers() {
+    enableRemoteStreams = false;
+    handleSelect = false;
+    init(2048, 2048);
+  }
+
+  private void init( int multipartUploadLimitKB, int formUploadLimitKB) {       
+    MultipartRequestParser multi = new MultipartRequestParser( multipartUploadLimitKB );
     RawRequestParser raw = new RawRequestParser();
-    standard = new StandardRequestParser( multi, raw );
+    FormDataRequestParser formdata = new FormDataRequestParser( formUploadLimitKB );
+    standard = new StandardRequestParser( multi, raw, formdata );
     
     // I don't see a need to have this publicly configured just yet
     // adding it is trivial
-    parsers = new HashMap<String, SolrRequestParser>();
     parsers.put( MULTIPART, multi );
+    parsers.put( FORMDATA, formdata );
     parsers.put( RAW, raw );
     parsers.put( SIMPLE, new SimpleRequestParser() );
     parsers.put( STANDARD, standard );
@@ -175,23 +192,31 @@ public class SolrRequestParsers
     return q;
   }
   
+  /**
+   * Given a url-encoded query string (UTF-8), map it into solr params
+   */
+  public static MultiMapSolrParams parseQueryString(String queryString) {
+    Map<String,String[]> map = new HashMap<String, String[]>();
+    parseQueryString(queryString, "UTF-8", map);
+    return new MultiMapSolrParams(map);
+  }
 
   /**
-   * Given a standard query string map it into solr params
+   * Given a url-encoded query string, map it into the given map
+   * @param queryString as given from URL
+   * @param charset to be used to decode %-encoding
+   * @param map place all parameters in this map
    */
-  public static MultiMapSolrParams parseQueryString(String queryString) 
+  static void parseQueryString(String queryString, String charset, Map<String,String[]> map) {
-  {
-    Map<String,String[]> map = new HashMap<String, String[]>();
     if( queryString != null && queryString.length() > 0 ) {
       try {
         for( String kv : queryString.split( "&" ) ) {
           int idx = kv.indexOf( '=' );
-          if( idx > 0 ) {
+          if( idx >= 0 ) {
-            String name = URLDecoder.decode( kv.substring( 0, idx ), "UTF-8");
+            String name = URLDecoder.decode( kv.substring( 0, idx ), charset);
-            String value = URLDecoder.decode( kv.substring( idx+1 ), "UTF-8");
+            String value = URLDecoder.decode( kv.substring( idx+1 ), charset);
             MultiMapSolrParams.addParam( name, value, map );
-          }
+          } else {
-          else {
             String name = URLDecoder.decode( kv, "UTF-8" );
             MultiMapSolrParams.addParam( name, "", map );
           }
@@ -201,7 +226,6 @@ public class SolrRequestParsers
         throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, uex );
       }
     }
-    return new MultiMapSolrParams( map );
   }
 
   public boolean isHandleSelect() {
@@ -228,7 +252,7 @@ interface SolrRequestParser
 //-----------------------------------------------------------------
 
 /**
- * The simple parser just uses the params directly
+ * The simple parser just uses the params directly, does not support POST URL-encoded forms
  */
 class SimpleRequestParser implements SolrRequestParser
 {
@@ -236,7 +260,7 @@ class SimpleRequestParser implements SolrRequestParser
   public SolrParams parseParamsAndFillStreams( 
       final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
   {
-    return new ServletSolrParams(req);
+    return SolrRequestParsers.parseQueryString(req.getQueryString());
   }
 }
 
@@ -298,13 +322,6 @@ class RawRequestParser implements SolrRequestParser
   public SolrParams parseParamsAndFillStreams( 
       final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
   {
-    // The javadocs for HttpServletRequest are clear that req.getReader() should take
-    // care of any character encoding issues.  BUT, there are problems while running on
-    // some servlet containers: including Tomcat 5 and resin.
-    //
-    // Rather than return req.getReader(), this uses the default ContentStreamBase method
-    // that checks for charset definitions in the ContentType.
-    
     streams.add( new HttpRequestContentStream( req ) );
     return SolrRequestParsers.parseQueryString( req.getQueryString() );
   }
@@ -317,9 +334,9 @@ class RawRequestParser implements SolrRequestParser
  */
 class MultipartRequestParser implements SolrRequestParser
 {
-  private long uploadLimitKB;
+  private final int uploadLimitKB;
   
-  public MultipartRequestParser( long limit )
+  public MultipartRequestParser( int limit )
   {
     uploadLimitKB = limit;
   }
@@ -343,7 +360,7 @@ class MultipartRequestParser implements SolrRequestParser
 
     // Create a new file upload handler
     ServletFileUpload upload = new ServletFileUpload(factory);
-    upload.setSizeMax( uploadLimitKB*1024 );
+    upload.setSizeMax( ((long) uploadLimitKB) * 1024L );
 
     // Parse the request
     List items = upload.parseRequest(req);
@@ -367,6 +384,97 @@ class MultipartRequestParser implements SolrRequestParser
 }
 
 
+/**
+ * Extract application/x-www-form-urlencoded form data for POST requests
+ */
+class FormDataRequestParser implements SolrRequestParser
+{
+  private final int uploadLimitKB;
+  
+  public FormDataRequestParser( int limit )
+  {
+    uploadLimitKB = limit;
+  }
+  
+  @Override
+  public SolrParams parseParamsAndFillStreams( 
+      final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
+  {
+    if (!isFormData(req)) {
+      throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Not application/x-www-form-urlencoded content: "+req.getContentType() );
+    }
+    
+    String charset = ContentStreamBase.getCharsetFromContentType(req.getContentType());
+    if (charset == null) charset = "UTF-8";
+    
+    final Map<String,String[]> map = new HashMap<String, String[]>();
+    
+    // also add possible URL parameters and include into the map (parsed using UTF-8):
+    final String qs = req.getQueryString();
+    if (qs != null) {
+      SolrRequestParsers.parseQueryString(qs, "UTF-8", map);
+    }
+    
+    // may be -1, so we check again later. But if its already greater we can stop processing!
+    final long totalLength = req.getContentLength();
+    final long maxLength = ((long) uploadLimitKB) * 1024L;
+    if (totalLength > maxLength) {
+      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
+        totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
+    }
+    
+    // get query String from request body, using the charset given in content-type:
+    final InputStream in;
+    try {
+      in = req.getInputStream();
+    } catch (IllegalStateException ise) {
+      throw (SolrException) getParameterIncompatibilityException().initCause(ise);
+    }
+    try {
+      final String data = IOUtils.toString(new BoundedInputStream(in, maxLength), charset);
+      // if there is remaining data in the underlying stream, throw exception:
+      if (in.read() != -1) {
+        // read remaining data and throw away:
+        while (IOUtils.skip(in, 1024L) > 0);
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content exceeds upload limit of " + uploadLimitKB + " KB");
+      }
+      if (data.length() == 0 && totalLength > 0L) {
+        throw getParameterIncompatibilityException();
+      }
+      SolrRequestParsers.parseQueryString(data, charset, map);
+    } finally {
+      IOUtils.closeQuietly(in);
+    }
+    
+    return new MultiMapSolrParams(map);
+  }
+  
+  private SolrException getParameterIncompatibilityException() {
+    return new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+      "Solr requires that request parameters sent using application/x-www-form-urlencoded " +
+      "content-type can be read through the request input stream. Unfortunately, the " +
+      "stream was empty / not available. This may be caused by another servlet filter calling " +
+      "ServletRequest.getParameter*() before SolrDispatchFilter, please remove it."
+    );
+  }
+  
+  public boolean isFormData(HttpServletRequest req) {
+    String contentType = req.getContentType();
+    if (contentType != null) {
+      int idx = contentType.indexOf( ';' );
+      if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
+        contentType = contentType.substring( 0, idx );
+      }
+      contentType = contentType.trim();
+      if( "application/x-www-form-urlencoded".equalsIgnoreCase(contentType)) {
+        return true;
+      }
+    }
+    return false;
+  }
+}
+
+
 /**
  * The default Logic
  */
@@ -374,11 +482,13 @@ class StandardRequestParser implements SolrRequestParser
 {
   MultipartRequestParser multipart;
   RawRequestParser raw;
+  FormDataRequestParser formdata;
   
-  StandardRequestParser( MultipartRequestParser multi, RawRequestParser raw ) 
+  StandardRequestParser(MultipartRequestParser multi, RawRequestParser raw, FormDataRequestParser formdata) 
   {
     this.multipart = multi;
     this.raw = raw;
+    this.formdata = formdata;
   }
   
   @Override
@@ -386,22 +496,15 @@ class StandardRequestParser implements SolrRequestParser
       final HttpServletRequest req, ArrayList<ContentStream> streams ) throws Exception
   {
     String method = req.getMethod().toUpperCase(Locale.ROOT);
-    if( "GET".equals( method ) || "HEAD".equals( method )) {
+    if ("GET".equals(method) || "HEAD".equals(method)) {
-      return new ServletSolrParams(req);
+      return SolrRequestParsers.parseQueryString(req.getQueryString());
     }
-    if( "POST".equals( method ) ) {
+    if ("POST".equals( method ) ) {
-      String contentType = req.getContentType();
+      if (formdata.isFormData(req)) {
-      if( contentType != null ) {
+        return formdata.parseParamsAndFillStreams(req, streams);
-        int idx = contentType.indexOf( ';' );
+      }
-        if( idx > 0 ) { // remove the charset definition "; charset=utf-8"
+      if (ServletFileUpload.isMultipartContent(req)) {
-          contentType = contentType.substring( 0, idx );
+        return multipart.parseParamsAndFillStreams(req, streams);
-        }
-        if( "application/x-www-form-urlencoded".equals( contentType.toLowerCase(Locale.ROOT) ) ) {
-          return new ServletSolrParams(req); // just get the params from parameterMap
-        }
-        if( ServletFileUpload.isMultipartContent(req) ) {
-          return multipart.parseParamsAndFillStreams(req, streams);
-        }
       }
       return raw.parseParamsAndFillStreams(req, streams);
     }

solr/core/src/java/org/apache/solr/servlet/ZookeeperInfoServlet.java

@@ -18,21 +18,28 @@
 package org.apache.solr.servlet;
 
 import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
 import java.net.URLEncoder;
 import java.util.Date;
 import java.util.List;
 
+import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
 import org.apache.noggit.CharArr;
 import org.apache.noggit.JSONWriter;
 import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.core.CoreContainer;
+import org.apache.solr.util.FastWriter;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
 import org.slf4j.Logger;
@@ -46,7 +53,7 @@ import org.slf4j.LoggerFactory;
  */
 public final class ZookeeperInfoServlet extends HttpServlet {
   static final Logger log = LoggerFactory.getLogger(ZookeeperInfoServlet.class);
-
+  
   @Override
   public void init() {
   }
@@ -54,28 +61,42 @@ public final class ZookeeperInfoServlet extends HttpServlet {
   @Override
   public void doGet(HttpServletRequest request,
                     HttpServletResponse response)
-      throws IOException {
+      throws ServletException,IOException {
-    response.setCharacterEncoding("UTF-8");
-    response.setContentType("application/json");
-
     // This attribute is set by the SolrDispatchFilter
     CoreContainer cores = (CoreContainer) request.getAttribute("org.apache.solr.CoreContainer");
+    if (cores == null) {
+      throw new ServletException("Missing request attribute org.apache.solr.CoreContainer.");
+    }
+    
+    final SolrParams params;
+    try {
+      params = SolrRequestParsers.DEFAULT.parse(null, request.getServletPath(), request).getParams();
+    } catch (Exception e) {
+      int code=500;
+      if (e instanceof SolrException) {
+        code = Math.min(599, Math.max(100, ((SolrException)e).code()));
+      }
+      response.sendError(code, e.toString());
+      return;
+    }
 
-    String path = request.getParameter("path");
+    String path = params.get("path");
-    String addr = request.getParameter("addr");
+    String addr = params.get("addr");
 
     if (addr != null && addr.length() == 0) {
       addr = null;
     }
 
-    String detailS = request.getParameter("detail");
+    String detailS = params.get("detail");
     boolean detail = detailS != null && detailS.equals("true");
 
-    String dumpS = request.getParameter("dump");
+    String dumpS = params.get("dump");
     boolean dump = dumpS != null && dumpS.equals("true");
 
-    PrintWriter out = response.getWriter();
+    response.setCharacterEncoding("UTF-8");
+    response.setContentType("application/json");
 
+    Writer out = new FastWriter(new OutputStreamWriter(response.getOutputStream(), IOUtils.CHARSET_UTF_8));
 
     ZKPrinter printer = new ZKPrinter(response, out, cores.getZkController(), addr);
     printer.detail = detail;
@@ -86,12 +107,14 @@ public final class ZookeeperInfoServlet extends HttpServlet {
     } finally {
       printer.close();
     }
+    
+    out.flush();
   }
 
   @Override
   public void doPost(HttpServletRequest request,
                      HttpServletResponse response)
-      throws IOException {
+      throws ServletException,IOException {
     doGet(request, response);
   }
 
@@ -114,13 +137,13 @@ public final class ZookeeperInfoServlet extends HttpServlet {
     boolean doClose;  // close the client after done if we opened it
 
     final HttpServletResponse response;
-    final PrintWriter out;
+    final Writer out;
     SolrZkClient zkClient;
 
     int level;
     int maxData = 95;
 
-    public ZKPrinter(HttpServletResponse response, PrintWriter out, ZkController controller, String addr) throws IOException {
+    public ZKPrinter(HttpServletResponse response, Writer out, ZkController controller, String addr) throws IOException {
       this.response = response;
       this.out = out;
       this.addr = addr;
@@ -207,10 +230,10 @@ public final class ZookeeperInfoServlet extends HttpServlet {
       }
       json.endArray();
       json.endObject();
-      out.println(chars.toString());
+      out.write(chars.toString());
     }
 
-    void writeError(int code, String msg) {
+    void writeError(int code, String msg) throws IOException {
       response.setStatus(code);
 
       CharArr chars = new CharArr();
@@ -227,7 +250,7 @@ public final class ZookeeperInfoServlet extends HttpServlet {
       w.writeString(msg);
       w.endObject();
 
-      out.println(chars.toString());
+      out.write(chars.toString());
     }
 
 
@@ -352,7 +375,7 @@ public final class ZookeeperInfoServlet extends HttpServlet {
       json.write(v);
     }
 
-    boolean printZnode(JSONWriter json, String path) {
+    boolean printZnode(JSONWriter json, String path) throws IOException {
       try {
         Stat stat = new Stat();
         // Trickily, the call to zkClient.getData fills in the stat variable

solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java

@@ -21,6 +21,7 @@ import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 
+import java.io.ByteArrayInputStream;
 import java.net.HttpURLConnection;
 import java.net.SocketTimeoutException;
 import java.net.URL;
@@ -31,10 +32,12 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
+import javax.servlet.ServletInputStream;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MultiMapSolrParams;
 import org.apache.solr.common.params.SolrParams;
@@ -168,12 +171,11 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
   @Test
   public void testStandardParseParamsAndFillStreams() throws Exception
   {
-    ArrayList<ContentStream> streams = new ArrayList<ContentStream>();
+    final String getParams = "qt=%C3%BC&dup=foo", postParams = "q=hello&d%75p=bar";
-    Map<String,String[]> params = new HashMap<String, String[]>();
+    final byte[] postBytes = postParams.getBytes("UTF-8");
-    params.put( "q", new String[] { "hello" } );
     
     // Set up the expected behavior
-    String[] ct = new String[] {
+    final String[] ct = new String[] {
         "application/x-www-form-urlencoded",
         "Application/x-www-form-urlencoded",
         "application/x-www-form-urlencoded; charset=utf-8",
@@ -184,16 +186,102 @@ public class SolrRequestParserTest extends SolrTestCaseJ4 {
       HttpServletRequest request = createMock(HttpServletRequest.class);
       expect(request.getMethod()).andReturn("POST").anyTimes();
       expect(request.getContentType()).andReturn( contentType ).anyTimes();
-      expect(request.getParameterMap()).andReturn(params).anyTimes();
+      expect(request.getQueryString()).andReturn(getParams).anyTimes();
+      expect(request.getContentLength()).andReturn(postBytes.length).anyTimes();
+      expect(request.getInputStream()).andReturn(new ServletInputStream() {
+        private final ByteArrayInputStream in = new ByteArrayInputStream(postBytes);
+        @Override public int read() { return in.read(); }
+      });
       replay(request);
       
-      MultipartRequestParser multipart = new MultipartRequestParser( 1000000 );
+      MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
       RawRequestParser raw = new RawRequestParser();
-      StandardRequestParser standard = new StandardRequestParser( multipart, raw );
+      FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
+      StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
       
-      SolrParams p = standard.parseParamsAndFillStreams( request, streams );
+      SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
       
       assertEquals( "contentType: "+contentType, "hello", p.get("q") );
+      assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
+      assertArrayEquals( "contentType: "+contentType, new String[]{"foo","bar"}, p.getParams("dup") );
+    }
+  }
+  
+  @Test
+  public void testStandardFormdataUploadLimit() throws Exception
+  {
+    final int limitKBytes = 128;
+
+    final StringBuilder large = new StringBuilder("q=hello");
+    // grow exponentially to reach 128 KB limit:
+    while (large.length() <= limitKBytes * 1024) {
+      large.append('&').append(large);
+    }
+    HttpServletRequest request = createMock(HttpServletRequest.class);
+    expect(request.getMethod()).andReturn("POST").anyTimes();
+    expect(request.getContentType()).andReturn("application/x-www-form-urlencoded").anyTimes();
+    // we dont pass a content-length to let the security mechanism limit it:
+    expect(request.getContentLength()).andReturn(-1).anyTimes();
+    expect(request.getQueryString()).andReturn(null).anyTimes();
+    expect(request.getInputStream()).andReturn(new ServletInputStream() {
+      private final ByteArrayInputStream in = new ByteArrayInputStream(large.toString().getBytes("UTF-8"));
+      @Override public int read() { return in.read(); }
+    });
+    replay(request);
+    
+    FormDataRequestParser formdata = new FormDataRequestParser( limitKBytes );    
+    try {
+      formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
+      fail("should throw SolrException");
+    } catch (SolrException solre) {
+      assertTrue(solre.getMessage().contains("upload limit"));
+      assertEquals(400, solre.code());
+    }
+  }
+  
+  @Test
+  public void testParameterIncompatibilityException1() throws Exception
+  {
+    HttpServletRequest request = createMock(HttpServletRequest.class);
+    expect(request.getMethod()).andReturn("POST").anyTimes();
+    expect(request.getContentType()).andReturn("application/x-www-form-urlencoded").anyTimes();
+    expect(request.getContentLength()).andReturn(100).anyTimes();
+    expect(request.getQueryString()).andReturn(null).anyTimes();
+    // we emulate Jetty that returns empty stream when parameters were parsed before:
+    expect(request.getInputStream()).andReturn(new ServletInputStream() {
+      @Override public int read() { return -1; }
+    });
+    replay(request);
+    
+    FormDataRequestParser formdata = new FormDataRequestParser( 2048 );    
+    try {
+      formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
+      fail("should throw SolrException");
+    } catch (SolrException solre) {
+      assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
+      assertEquals(500, solre.code());
+    }
+  }
+  
+  @Test
+  public void testParameterIncompatibilityException2() throws Exception
+  {
+    HttpServletRequest request = createMock(HttpServletRequest.class);
+    expect(request.getMethod()).andReturn("POST").anyTimes();
+    expect(request.getContentType()).andReturn("application/x-www-form-urlencoded").anyTimes();
+    expect(request.getContentLength()).andReturn(100).anyTimes();
+    expect(request.getQueryString()).andReturn(null).anyTimes();
+    // we emulate Tomcat that throws IllegalStateException when parameters were parsed before:
+    expect(request.getInputStream()).andThrow(new IllegalStateException());
+    replay(request);
+    
+    FormDataRequestParser formdata = new FormDataRequestParser( 2048 );    
+    try {
+      formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
+      fail("should throw SolrException");
+    } catch (SolrException solre) {
+      assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
+      assertEquals(500, solre.code());
     }
   }
 }

solr/example/example-DIH/solr/db/conf/solrconfig.xml

@@ -217,7 +217,7 @@
     -->
   <requestDispatcher handleSelect="true" >
     <!--Make sure your system has some authentication before enabling remote streaming!  -->
-    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
         
     <!-- Set HTTP caching related parameters (for proxy caches and clients).
           

solr/example/example-DIH/solr/mail/conf/solrconfig.xml

@@ -249,7 +249,7 @@
     -->
   <requestDispatcher handleSelect="true" >
     <!--Make sure your system has some authentication before enabling remote streaming!  -->
-    <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" />
+    <requestParsers enableRemoteStreaming="true" multipartUploadLimitInKB="2048000" formdataUploadLimitInKB="2048" />
 
     <!-- Set HTTP caching related parameters (for proxy caches and clients).
 

solr/example/example-DIH/solr/rss/conf/solrconfig.xml

@@ -217,7 +217,7 @@
     -->
   <requestDispatcher handleSelect="true" >
     <!--Make sure your system has some authentication before enabling remote streaming!  -->
-    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
         
     <!-- Set HTTP caching related parameters (for proxy caches and clients).
           

solr/example/example-DIH/solr/solr/conf/solrconfig.xml

@@ -217,7 +217,7 @@
     -->
   <requestDispatcher handleSelect="true" >
     <!--Make sure your system has some authentication before enabling remote streaming!  -->
-    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
         
     <!-- Set HTTP caching related parameters (for proxy caches and clients).
           

solr/example/example-DIH/solr/tika/conf/solrconfig.xml

@@ -188,7 +188,7 @@
     -->
   <requestDispatcher handleSelect="true" >
     <!--Make sure your system has some authentication before enabling remote streaming!  -->
-    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
         
     <!-- Set HTTP caching related parameters (for proxy caches and clients).
           

solr/example/multicore/core0/conf/solrconfig.xml

@@ -47,7 +47,7 @@
   <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
 
   <requestDispatcher handleSelect="true" >
-    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
   </requestDispatcher>
   
   <requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />

solr/example/multicore/core1/conf/solrconfig.xml

@@ -47,7 +47,7 @@
   <requestHandler name="/replication" class="solr.ReplicationHandler" startup="lazy" /> 
 
   <requestDispatcher handleSelect="true" >
-    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
+    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" formdataUploadLimitInKB="2048" />
   </requestDispatcher>
   
   <requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />

solr/example/solr/collection1/conf/solrconfig.xml

@@ -657,9 +657,14 @@
          enableRemoteStreaming - enables use of the stream.file
          and stream.url parameters for specifying remote streams.
 
-         multipartUploadLimitInKB - specifies the max size of
+         multipartUploadLimitInKB - specifies the max size (in KiB) of
          Multipart File Uploads that Solr will allow in a Request.
          
+         formdataUploadLimitInKB - specifies the max size (in KiB) of
+         form data (application/x-www-form-urlencoded) sent via
+         POST. You can use POST to pass request parameters not
+         fitting into the URL.
+         
          *** WARNING ***
          The settings below authorize Solr to fetch remote files, You
          should make sure your system has some authentication before
@@ -667,7 +672,8 @@
 
       --> 
     <requestParsers enableRemoteStreaming="true" 
-                    multipartUploadLimitInKB="2048000" />
+                    multipartUploadLimitInKB="2048000"
+                    formdataUploadLimitInKB="2048"/>
 
     <!-- HTTP Caching