diff --git a/CHANGES.txt b/CHANGES.txt index d7337715fa0..82f9f15a086 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -60,6 +60,14 @@ New Features 7. SOLR-107: JAVA API: Change NamedList to use Java5 generics and implement Iterable (Ryan McKinley via yonik) + 8. SOLR-104: Support for "Update Plugins" -- RequestHandlers that want + access to streams of data for doing updates. ContentStreams can come + from the raw POST body, multi-part form data, or remote URLs. + Included in this change is a new SlrDispatchFilter that allows + RequestHandlers registered with names that begin with a "/" to be + accessed using a URL structure based on that name. + (Ryan McKinley via hossman) + Changes in runtime behavior 1. Highlighting using DisMax will only pick up terms from the main user query, not boost or filter queries (klaas). diff --git a/example/solr/conf/solrconfig.xml b/example/solr/conf/solrconfig.xml index dc4791e8155..cce47f6ad84 100755 --- a/example/solr/conf/solrconfig.xml +++ b/example/solr/conf/solrconfig.xml @@ -222,7 +222,9 @@ - + + + + + + + + + + + + + + solr diff --git a/src/webapp/WEB-INF/web.xml b/src/webapp/WEB-INF/web.xml index 9e937324e92..e1227ea35d3 100644 --- a/src/webapp/WEB-INF/web.xml +++ b/src/webapp/WEB-INF/web.xml @@ -30,7 +30,18 @@ "com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl"/> --> + + + SolrRequestFilter + org.apache.solr.servlet.SolrDispatchFilter + + + SolrRequestFilter + /* + + + SolrServer Solr diff --git a/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java b/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java new file mode 100644 index 00000000000..4e0e45041ea --- /dev/null +++ b/src/webapp/src/org/apache/solr/servlet/SolrDispatchFilter.java @@ -0,0 +1,160 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.servlet; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.logging.Logger; + +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.naming.NoInitialContextException; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.solr.core.Config; +import org.apache.solr.core.SolrConfig; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrException; +import org.apache.solr.request.QueryResponseWriter; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.request.SolrRequestHandler; + +/** + * This filter looks at the incoming URL maps them to handlers defined in solrconfig.xml + */ +public class SolrDispatchFilter implements Filter +{ + final Logger log = Logger.getLogger(SolrDispatchFilter.class.getName()); + + protected SolrCore core; + protected SolrRequestParsers parsers; + + public void init(FilterConfig config) throws ServletException + { + log.info("SolrDispatchFilter.init()"); + try { + Context c = new InitialContext(); + + /*** + System.out.println("Enumerating JNDI Context=" + c); + NamingEnumeration en = c.list("java:comp/env"); + while (en.hasMore()) { + NameClassPair ncp = en.next(); + System.out.println(" ENTRY:" + ncp); + } + System.out.println("JNDI lookup=" + c.lookup("java:comp/env/solr/home")); + ***/ + + String home = (String)c.lookup("java:comp/env/solr/home"); + if (home!=null) Config.setInstanceDir(home); + } catch (NoInitialContextException e) { + log.info("JNDI not configured for Solr (NoInitialContextEx)"); + } catch (NamingException e) { + log.info("No /solr/home in JNDI"); + } + + log.info("user.dir=" + System.getProperty("user.dir")); + core = SolrCore.getSolrCore(); + parsers = new SolrRequestParsers( core, SolrConfig.config ); + + log.info("SolrDispatchFilter.init() done"); + } + + public void destroy() { + core.close(); + } + + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException + { + if( request instanceof HttpServletRequest) { + HttpServletRequest req = (HttpServletRequest)request; + try { + String path = req.getServletPath(); + if( req.getPathInfo() != null ) { + // this lets you handle /update/commit when /update is a servlet + path += req.getPathInfo(); + } + int idx = path.indexOf( ':' ); + if( idx > 0 ) { + // save the portion after the ':' for a 'handler' path parameter + path = path.substring( 0, idx ); + } + + SolrRequestHandler handler = core.getRequestHandler( path ); + if( handler != null ) { + SolrQueryRequest solrReq = parsers.parse( path, req ); + SolrQueryResponse solrRsp = new SolrQueryResponse(); + core.execute( handler, solrReq, solrRsp ); + if( solrRsp.getException() != null ) { + sendError( (HttpServletResponse)response, solrRsp.getException() ); + return; + } + + // Now write it out + QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq); + response.setContentType(responseWriter.getContentType(solrReq, solrRsp)); + PrintWriter out = response.getWriter(); + responseWriter.write(out, solrReq, solrRsp); + return; + } + } + catch( Throwable ex ) { + sendError( (HttpServletResponse)response, ex ); + return; + } + } + + // Otherwise let the webapp handle the request + chain.doFilter(request, response); + } + + protected void sendError(HttpServletResponse res, Throwable ex) throws IOException + { + int code=500; + String trace = ""; + if( ex instanceof SolrException ) { + code = ((SolrException)ex).code(); + } + + // For any regular code, don't include the stack trace + if( code == 500 || code < 100 ) { + StringWriter sw = new StringWriter(); + ex.printStackTrace(new PrintWriter(sw)); + trace = "\n\n"+sw.toString(); + + SolrException.logOnce(log,null,ex ); + + // non standard codes have undefined results with various servers + if( code < 100 ) { + log.warning( "invalid return code: "+code ); + code = 500; + } + } + res.sendError( code, ex.getMessage() + trace ); + } +} diff --git a/src/webapp/src/org/apache/solr/servlet/SolrRequestParsers.java b/src/webapp/src/org/apache/solr/servlet/SolrRequestParsers.java new file mode 100644 index 00000000000..e07b16a3bb9 --- /dev/null +++ b/src/webapp/src/org/apache/solr/servlet/SolrRequestParsers.java @@ -0,0 +1,383 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.servlet; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; + +import javax.servlet.http.HttpServletRequest; +import javax.xml.xpath.XPathConstants; + +import org.apache.commons.fileupload.FileItem; +import org.apache.commons.fileupload.disk.DiskFileItemFactory; +import org.apache.commons.fileupload.servlet.ServletFileUpload; +import org.apache.solr.core.Config; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrException; +import org.apache.solr.request.ContentStream; +import org.apache.solr.request.MultiMapSolrParams; +import org.apache.solr.request.ServletSolrParams; +import org.apache.solr.request.SolrParams; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + + +public class SolrRequestParsers +{ + final Logger log = Logger.getLogger(SolrRequestParsers.class.getName()); + + // Should these constants be in a more public place? + public static final String MULTIPART = "multipart"; + public static final String RAW = "raw"; + public static final String SIMPLE = "simple"; + public static final String STANDARD = "standard"; + + private HashMap parsers; + private SolrCore core; + private boolean enableRemoteStreams = false; + private StandardRequestParser standard; + + public SolrRequestParsers( SolrCore core, Config config ) + { + this.core = core; + + long uploadLimitKB = 2000; // 2MB default + NodeList nodes = (NodeList)config.evaluate("requestParsers", XPathConstants.NODESET); + if( nodes!=null && nodes.getLength()>0 ) { + // only look at the first node. + NamedNodeMap attrs = nodes.item(0).getAttributes(); + Node node = attrs.getNamedItem( "enableRemoteStreaming" ); + if( node != null ) { + enableRemoteStreams = Boolean.parseBoolean( node.getTextContent() ); + } + node = attrs.getNamedItem( "multipartUploadLimitInKB" ); + if( node != null ) { + uploadLimitKB = Long.parseLong( node.getTextContent() ); + } + } + + MultipartRequestParser multi = new MultipartRequestParser( uploadLimitKB ); + RawRequestParser raw = new RawRequestParser(); + standard = new StandardRequestParser( multi, raw ); + + // I don't see a need to have this publically configured just yet + // adding it is trivial + parsers = new HashMap(); + parsers.put( MULTIPART, multi ); + parsers.put( RAW, raw ); + parsers.put( SIMPLE, new SimpleRequestParser() ); + parsers.put( STANDARD, standard ); + parsers.put( "", standard ); + } + + public SolrQueryRequest parse( String path, HttpServletRequest req ) throws Exception + { + SolrRequestParser parser = standard; + + // TODO -- in the future, we could pick a different parser based on the request + + // Pick the parer from the request... + ArrayList streams = new ArrayList(1); + SolrParams params = parser.parseParamsAndFillStreams( req, streams ); + SolrQueryRequest sreq = buildRequestFrom( params, streams ); + + // If there is some path left over, add it to the context + int idx = req.getServletPath().indexOf( ':' ); + if( idx > 0 ) { + sreq.getContext().put( "path", req.getServletPath().substring( idx+1 ) ); + } + return sreq; + } + + SolrQueryRequest buildRequestFrom( SolrParams params, List streams ) throws Exception + { + // Handle anything with a remoteURL + String[] strs = params.getParams( SolrParams.STREAM_URL ); + if( strs != null ) { + if( !enableRemoteStreams ) { + throw new SolrException( 400, "Remote Streaming is disabled." ); + } + for( final String url : strs ) { + final URLConnection conn = new URL(url).openConnection(); + streams.add( new ContentStream() { + public String getContentType() { return conn.getContentType(); } + public String getName() { return url; } + public Long getSize() { return new Long( conn.getContentLength() ); } + public String getSourceInfo() { + return SolrParams.STREAM_URL; + } + public InputStream getStream() throws IOException { + return conn.getInputStream(); + } + }); + } + } + + // Check for streams in the request parameters + strs = params.getParams( SolrParams.STREAM_BODY ); + if( strs != null ) { + for( final String body : strs ) { + streams.add( new ContentStream() { + public String getContentType() { return null; } // Is there anything meaningful? + public String getName() { return null; } + public Long getSize() { return null; } + public String getSourceInfo() { + return SolrParams.STREAM_BODY; + } + public InputStream getStream() throws IOException { + return new ByteArrayInputStream( body.getBytes() ); + } + }); + } + } + + SolrQueryRequestBase q = new SolrQueryRequestBase( core, params ) { }; + if( streams != null && streams.size() > 0 ) { + q.setContentStreams( streams ); + } + + return q; + } + + + /** + * Given a standard query string map it into solr params + */ + public static MultiMapSolrParams parseQueryString(String queryString) + { + Map map = new HashMap(); + if( queryString != null && queryString.length() > 0 ) { + for( String kv : queryString.split( "&" ) ) { + int idx = kv.indexOf( '=' ); + if( idx > 0 ) { + String name = URLDecoder.decode( kv.substring( 0, idx )); + String value = URLDecoder.decode( kv.substring( idx+1 )); + MultiMapSolrParams.addParam( name, value, map ); + } + else { + String name = URLDecoder.decode( kv ); + MultiMapSolrParams.addParam( name, "", map ); + } + } + } + return new MultiMapSolrParams( map ); + } +} + +//----------------------------------------------------------------- +//----------------------------------------------------------------- + +// I guess we don't really even need the interface, but i'll keep it here just for kicks +interface SolrRequestParser +{ + public SolrParams parseParamsAndFillStreams( + final HttpServletRequest req, ArrayList streams ) throws Exception; +} + + +//----------------------------------------------------------------- +//----------------------------------------------------------------- + +/** + * The simple parser just uses the params directly + */ +class SimpleRequestParser implements SolrRequestParser +{ + public SolrParams parseParamsAndFillStreams( + final HttpServletRequest req, ArrayList streams ) throws Exception + { + return new ServletSolrParams(req); + } +} + + +/** + * The simple parser just uses the params directly + */ +class RawRequestParser implements SolrRequestParser +{ + public SolrParams parseParamsAndFillStreams( + final HttpServletRequest req, ArrayList streams ) throws Exception + { + streams.add( new ContentStream() { + public String getContentType() { + return req.getContentType(); + } + public String getName() { + return null; // Is there any meaningfull name? + } + public String getSourceInfo() { + return null; // Is there any meaningfull name? + } + public Long getSize() { + String v = req.getHeader( "Content-Length" ); + if( v != null ) { + return Long.valueOf( v ); + } + return null; + } + public InputStream getStream() throws IOException { + return req.getInputStream(); + } + }); + return SolrRequestParsers.parseQueryString( req.getQueryString() ); + } +} + + + +/** + * Extract Multipart streams + */ +class MultipartRequestParser implements SolrRequestParser +{ + private long uploadLimitKB; + + public MultipartRequestParser( long limit ) + { + uploadLimitKB = limit; + } + + public SolrParams parseParamsAndFillStreams( + final HttpServletRequest req, ArrayList streams ) throws Exception + { + if( !ServletFileUpload.isMultipartContent(req) ) { + throw new SolrException( 400, "Not multipart content! "+req.getContentType() ); + } + + MultiMapSolrParams params = SolrRequestParsers.parseQueryString( req.getQueryString() ); + + // Create a factory for disk-based file items + DiskFileItemFactory factory = new DiskFileItemFactory(); + + // Set factory constraints + // TODO - configure factory.setSizeThreshold(yourMaxMemorySize); + // TODO - configure factory.setRepository(yourTempDirectory); + + // Create a new file upload handler + ServletFileUpload upload = new ServletFileUpload(factory); + upload.setSizeMax( uploadLimitKB*1024 ); + + // Parse the request + List items = upload.parseRequest(req); + Iterator iter = items.iterator(); + while (iter.hasNext()) { + FileItem item = (FileItem) iter.next(); + + // If its a form field, put it in our parameter map + if (item.isFormField()) { + MultiMapSolrParams.addParam( + item.getFieldName(), + item.getString(), params.getMap() ); + } + // Only add it if it actually has something... + else if( item.getSize() > 0 ) { + streams.add( new FileItemContentStream( item ) ); + } + } + return params; + } + + /** + * Wrap a FileItem as a ContentStream + */ + private static class FileItemContentStream implements ContentStream + { + FileItem item; + + public FileItemContentStream( FileItem f ) + { + item = f; + } + + public String getContentType() { + return item.getContentType(); + } + + public String getName() { + return item.getName(); + } + + public InputStream getStream() throws IOException { + return item.getInputStream(); + } + + public String getSourceInfo() { + return item.getFieldName(); + } + + public Long getSize() + { + return item.getSize(); + } + } +} + + +/** + * The default Logic + */ +class StandardRequestParser implements SolrRequestParser +{ + MultipartRequestParser multipart; + RawRequestParser raw; + + StandardRequestParser( MultipartRequestParser multi, RawRequestParser raw ) + { + this.multipart = multi; + this.raw = raw; + } + + public SolrParams parseParamsAndFillStreams( + final HttpServletRequest req, ArrayList streams ) throws Exception + { + String method = req.getMethod().toUpperCase(); + if( "GET".equals( method ) ) { + return new ServletSolrParams(req); + } + if( "POST".equals( method ) ) { + String contentType = req.getContentType(); + if( contentType != null ) { + if( "application/x-www-form-urlencoded".equals( contentType.toLowerCase() ) ) { + return new ServletSolrParams(req); // just get the params from parameterMap + } + if( ServletFileUpload.isMultipartContent(req) ) { + return multipart.parseParamsAndFillStreams(req, streams); + } + } + return raw.parseParamsAndFillStreams(req, streams); + } + throw new SolrException( 400, "Unsuported method: "+method ); + } +} + + + diff --git a/src/webapp/src/org/apache/solr/servlet/SolrServlet.java b/src/webapp/src/org/apache/solr/servlet/SolrServlet.java index 1ef1752164f..e66da9b7b8f 100644 --- a/src/webapp/src/org/apache/solr/servlet/SolrServlet.java +++ b/src/webapp/src/org/apache/solr/servlet/SolrServlet.java @@ -17,29 +17,27 @@ package org.apache.solr.servlet; -import org.apache.solr.core.Config; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrException; -import org.apache.solr.request.SolrQueryResponse; -import org.apache.solr.request.QueryResponseWriter; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.logging.Logger; -import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; -import javax.naming.NoInitialContextException; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.logging.Logger; + +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrException; +import org.apache.solr.request.QueryResponseWriter; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.request.SolrRequestHandler; /** * @author yonik * @author Mike Baranczak */ +@Deprecated public class SolrServlet extends HttpServlet { final Logger log = Logger.getLogger(SolrServlet.class.getName()); @@ -47,38 +45,10 @@ public class SolrServlet extends HttpServlet { public void init() throws ServletException { log.info("SolrServlet.init()"); - try { - Context c = new InitialContext(); - - /*** - System.out.println("Enumerating JNDI Context=" + c); - NamingEnumeration en = c.list("java:comp/env"); - while (en.hasMore()) { - NameClassPair ncp = en.next(); - System.out.println(" ENTRY:" + ncp); - } - System.out.println("JNDI lookup=" + c.lookup("java:comp/env/solr/home")); - ***/ - - String home = (String)c.lookup("java:comp/env/solr/home"); - if (home!=null) Config.setInstanceDir(home); - } catch (NoInitialContextException e) { - log.info("JNDI not configured for Solr (NoInitialContextEx)"); - } catch (NamingException e) { - log.info("No /solr/home in JNDI"); - } - - log.info("user.dir=" + System.getProperty("user.dir")); core = SolrCore.getSolrCore(); - log.info("SolrServlet.init() done"); } - public void destroy() { - core.close(); - super.destroy(); - } - public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { doGet(request,response); } @@ -87,7 +57,13 @@ public class SolrServlet extends HttpServlet { SolrServletRequest solrReq = new SolrServletRequest(core, request);; SolrQueryResponse solrRsp = new SolrQueryResponse(); try { - core.execute(solrReq, solrRsp); + + SolrRequestHandler handler = core.getRequestHandler(solrReq.getQueryType()); + if (handler==null) { + log.warning("Unknown Request Handler '" + solrReq.getQueryType() +"' :" + solrReq); + throw new SolrException(400,"Unknown Request Handler '" + solrReq.getQueryType() + "'", true); + } + core.execute(handler, solrReq, solrRsp ); if (solrRsp.getException() == null) { QueryResponseWriter responseWriter = core.getQueryResponseWriter(solrReq); response.setContentType(responseWriter.getContentType(solrReq, solrRsp)); @@ -127,14 +103,4 @@ public class SolrServlet extends HttpServlet { SolrException.log(log,e); } } - - final int getParam(HttpServletRequest request, String param, int defval) { - final String pval = request.getParameter(param); - return (pval==null) ? defval : Integer.parseInt(pval); - } - - final boolean paramExists(HttpServletRequest request, String param) { - return request.getParameter(param)!=null ? true : false; - } - } diff --git a/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java b/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java index b1cb5c56130..a7509604be5 100644 --- a/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java +++ b/src/webapp/src/org/apache/solr/servlet/SolrUpdateServlet.java @@ -15,41 +15,46 @@ package org.apache.solr.servlet;/** * limitations under the License. */ -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrException; -import org.apache.solr.request.XMLResponseWriter; -import org.apache.solr.request.SolrQueryResponse; -import org.apache.solr.request.QueryResponseWriter; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.logging.Logger; +import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.servlet.ServletException; -import java.util.logging.Logger; -import java.io.IOException; -import java.io.BufferedReader; -import java.io.PrintWriter; + +import org.apache.solr.core.SolrException; +import org.apache.solr.handler.XmlUpdateRequestHandler; +import org.apache.solr.request.QueryResponseWriter; +import org.apache.solr.request.XMLResponseWriter; +import org.apache.solr.util.XML; /** * @author yonik * @version $Id$ */ +@Deprecated public class SolrUpdateServlet extends HttpServlet { final Logger log = Logger.getLogger(SolrUpdateServlet.class.getName()); - private SolrCore core; + XmlUpdateRequestHandler legacyUpdateHandler; XMLResponseWriter xmlResponseWriter; public void init() throws ServletException { - core = SolrCore.getSolrCore(); + legacyUpdateHandler = new XmlUpdateRequestHandler(); + legacyUpdateHandler.init( null ); + log.info("SolrUpdateServlet.init() done"); } public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { BufferedReader requestReader = request.getReader(); response.setContentType(QueryResponseWriter.CONTENT_TYPE_XML_UTF8); - PrintWriter responseWriter = response.getWriter(); - core.update(requestReader, responseWriter); + + PrintWriter writer = response.getWriter(); + legacyUpdateHandler.doLegacyUpdate(requestReader, writer); } }