SOLR-3439: Make SolrCell easier to use out of the box

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1369433 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -217,6 +217,10 @@ Other Changes
   using a file (this previously worked in debug mode only).  When configuration errors are 
   encountered, the error message is returned in XML format.  (James Dyer)
   
+* SOLR-3439: Make SolrCell easier to use out of the box. Also improves "/browse" to display
+  rich-text documents correctly, along with facets for author and content_type.  
+  With the new "content" field, highlighting of body is supported. See also SOLR-3672 for
+  easier posting of a whole directory structure. (Jack Krupansky, janhoy)
 
 ==================  4.0.0-ALPHA ==================
 More information about this release, including any errata related to the 

solr/contrib/velocity/src/java/org/apache/solr/response/VelocityResponseWriter.java

@@ -30,6 +30,8 @@ import org.apache.velocity.runtime.RuntimeConstants;
 import org.apache.velocity.tools.generic.*;
 
 import java.io.*;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 
 public class VelocityResponseWriter implements QueryResponseWriter {
@@ -81,6 +83,64 @@ public class VelocityResponseWriter implements QueryResponseWriter {
 
     context.put("engine", engine);  // for $engine.resourceExists(...)
 
+    // Mimetype to extension map for detecting file type and show icon
+    // List of types match the icons in /solr/img/filetypes
+    Map<String, String> mimeToExt = new HashMap<String, String>() {{
+      put("application/x-7z-compressed", "7z");
+      put("application/postscript", "ai");
+      put("application/pgp-signature", "asc");
+      put("application/octet-stream", "bin");
+      put("application/x-bzip2", "bz2");
+      put("text/x-c", "c");
+      put("application/vnd.ms-htmlhelp", "chm");
+      put("application/java-vm", "class");
+      put("text/css", "css");
+      put("text/csv", "csv");
+      put("application/x-debian-package", "deb");
+      put("application/msword", "doc");
+      put("message/rfc822", "eml");
+      put("image/gif", "gif");
+      put("application/winhlp", "hlp");
+      put("text/html", "html");
+      put("application/java-archive", "jar");
+      put("text/x-java-source", "java");
+      put("image/jpeg", "jpeg");
+      put("application/javascript", "js");
+      put("application/vnd.oasis.opendocument.chart", "odc");
+      put("application/vnd.oasis.opendocument.formula", "odf");
+      put("application/vnd.oasis.opendocument.graphics", "odg");
+      put("application/vnd.oasis.opendocument.image", "odi");
+      put("application/vnd.oasis.opendocument.presentation", "odp");
+      put("application/vnd.oasis.opendocument.spreadsheet", "ods");
+      put("application/vnd.oasis.opendocument.text", "odt");
+      put("application/pdf", "pdf");
+      put("application/pgp-encrypted", "pgp");
+      put("image/png", "png");
+      put("application/vnd.ms-powerpoint", "ppt");
+      put("audio/x-pn-realaudio", "ram");
+      put("application/x-rar-compressed", "rar");
+      put("application/vnd.rn-realmedia", "rm");
+      put("application/rtf", "rtf");
+      put("application/x-shockwave-flash", "swf");
+      put("application/vnd.sun.xml.calc", "sxc");
+      put("application/vnd.sun.xml.draw", "sxd");
+      put("application/vnd.sun.xml.impress", "sxi");
+      put("application/vnd.sun.xml.writer", "sxw");
+      put("application/x-tar", "tar");
+      put("application/x-tex", "tex");
+      put("text/plain", "txt");
+      put("text/x-vcard", "vcf");
+      put("application/vnd.visio", "vsd");
+      put("audio/x-wav", "wav");
+      put("audio/x-ms-wma", "wma");
+      put("video/x-ms-wmv", "wmv");
+      put("application/vnd.ms-excel", "xls");
+      put("application/xml", "xml");
+      put("application/x-xpinstall", "xpi");
+      put("application/zip", "zip");
+    }};
+    context.put("mimeToExt", mimeToExt);
+    
     String layout_template = request.getParams().get("v.layout");
     String json_wrapper = request.getParams().get("v.json");
     boolean wrap_response = (layout_template != null) || (json_wrapper != null);

solr/example/solr/collection1/conf/schema.xml

@@ -109,7 +109,10 @@
    <!-- Common metadata fields, named specifically to match up with
      SolrCell metadata when parsing rich documents such as Word, PDF.
      Some fields are multiValued only because Tika currently may return
-     multiple values for them.
+     multiple values for them. Some metadata is parsed from the documents,
+     but there are some which come from the client context:
+       "content_type": From the HTTP headers of incoming stream
+       "resourcename": From SolrCell request param resource.name
    -->
    <field name="title" type="text_general" indexed="true" stored="true" multiValued="true"/>
    <field name="subject" type="text_general" indexed="true" stored="true"/>
@@ -118,10 +121,18 @@
    <field name="author" type="text_general" indexed="true" stored="true"/>
    <field name="keywords" type="text_general" indexed="true" stored="true"/>
    <field name="category" type="text_general" indexed="true" stored="true"/>
+   <field name="resourcename" type="text_general" indexed="true" stored="true"/>
+   <field name="url" type="text_general" indexed="true" stored="true"/>
    <field name="content_type" type="string" indexed="true" stored="true" multiValued="true"/>
    <field name="last_modified" type="date" indexed="true" stored="true"/>
    <field name="links" type="string" indexed="true" stored="true" multiValued="true"/>
 
+   <!-- Main body of document extracted by SolrCell.
+        NOTE: This field is not indexed by default, since it is also copied to "text"
+        using copyField below. This is to save space. Use this field for returning and
+        highlighting document content. Use the "text" field to search the content. -->
+   <field name="content" type="text_general" indexed="false" stored="true" multiValued="true"/>
+   
 
    <!-- catchall field, containing all other searchable text fields (implemented
         via copyField further on in this schema  -->
@@ -233,6 +244,19 @@
    <!-- Copy the price into a currency enabled field (default USD) -->
    <copyField source="price" dest="price_c"/>
 
+   <!-- Text fields from SolrCell to search by default in our catch-all field -->
+   <copyField source="title" dest="text"/>
+   <copyField source="author" dest="text"/>
+   <copyField source="description" dest="text"/>
+   <copyField source="keywords" dest="text"/>
+   <copyField source="content" dest="text"/>
+   <copyField source="content_type" dest="text"/>
+   <copyField source="resourcename" dest="text"/>
+   <copyField source="url" dest="text"/>
+
+   <!-- Create a string version of author for faceting -->
+   <copyField source="author" dest="author_s"/>
+	
    <!-- Above, multiple source fields are copied to the [text] field. 
 	  Another way to map multiple source fields to the same 
 	  destination field is to use the dynamic field syntax. 

solr/example/solr/collection1/conf/solrconfig.xml

@@ -850,6 +850,7 @@
        <str name="defType">edismax</str>
        <str name="qf">
           text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+          title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
        </str>
        <str name="df">text</str>
        <str name="mm">100%</str>
@@ -859,14 +860,17 @@
 
        <str name="mlt.qf">
          text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+         title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
        </str>
-       <str name="mlt.fl">text,features,name,sku,id,manu,cat</str>
+       <str name="mlt.fl">text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename</str>
        <int name="mlt.count">3</int>
 
        <!-- Faceting defaults -->
        <str name="facet">on</str>
        <str name="facet.field">cat</str>
        <str name="facet.field">manu_exact</str>
+       <str name="facet.field">content_type</str>
+       <str name="facet.field">author_s</str>
        <str name="facet.query">ipod</str>
        <str name="facet.query">GB</str>
        <str name="facet.mincount">1</str>
@@ -889,9 +893,18 @@
 
        <!-- Highlighting defaults -->
        <str name="hl">on</str>
-       <str name="hl.fl">text features name</str>
+       <str name="hl.fl">content features title name</str>
+       <str name="hl.encoder">html</str>
+       <str name="hl.simple.pre">&lt;b&gt;</str>
+       <str name="hl.simple.post">&lt;/b&gt;</str>
+       <str name="f.title.hl.fragsize">0</str>
+       <str name="f.title.hl.alternateField">title</str>
        <str name="f.name.hl.fragsize">0</str>
        <str name="f.name.hl.alternateField">name</str>
+       <str name="f.content.hl.snippets">3</str>
+       <str name="f.content.hl.fragsize">200</str>
+       <str name="f.content.hl.alternateField">content</str>
+       <str name="f.content.hl.maxAlternateFieldLength">750</str>
 
        <!-- Spell checking defaults -->
        <str name="spellcheck">on</str>
@@ -952,9 +965,6 @@
                   startup="lazy"
                   class="solr.extraction.ExtractingRequestHandler" >
     <lst name="defaults">
-      <!-- All the main content goes into "text"... if you need to return
-           the extracted text or do highlighting, use a stored field. -->
-      <str name="fmap.content">text</str>
       <str name="lowernames">true</str>
       <str name="uprefix">ignored_</str>
 

solr/example/solr/collection1/conf/velocity/VM_global_library.vm

@@ -133,10 +133,14 @@
 
 #macro(field $f)
   #if($response.response.highlighting.get($docId).get($f).get(0))
-    $!response.response.highlighting.get($docId).get($f).get(0)
+    #set($pad = "")
+    #foreach($v in $response.response.highlighting.get($docId).get($f))
+$pad$v##
+      #set($pad = " ... ")
+    #end
   #else
     #foreach($v in $doc.getFieldValues($f))
-      $v
+$v##
     #end
   #end
 #end  

solr/example/solr/collection1/conf/velocity/debug.vm

@@ -0,0 +1,17 @@
+#if($params.getBool("debugQuery",false))
+  <a href="#" onclick='jQuery(this).siblings("pre").toggle(); return false;'>toggle explain</a>
+  <pre style="display:none">$response.getExplainMap().get($doc.getFirstValue('id'))</pre>
+  <a href="#" onclick='jQuery(this).siblings("pre2").toggle(); return false;'>toggle all fields</a>
+  <pre2 style="display:none">
+  #foreach($fieldname in $doc.fieldNames)
+     <br>
+       <span class="field-name">$fieldname :</span>
+       <span>
+       #foreach($value in $doc.getFieldValues($fieldname))
+         $esc.html($value)
+       #end
+       </span>
+  #end
+   </br>
+  </pre2>
+#end

solr/example/solr/collection1/conf/velocity/facet_fields.vm

@@ -1,6 +1,8 @@
 #if($response.facetFields)
     <h2 #annTitle("Facets generated by adding &facet.field= to the request")>Field Facets</h2>
     #foreach($field in $response.facetFields)
+      ## Hide facets without value
+      #if($field.values.size() > 0)
       <span class="facet-field">$field.name</span>
 
       <ul>
@@ -10,3 +12,4 @@
       </ul>
       #end
     #end
+  #end

solr/example/solr/collection1/conf/velocity/facet_ranges.vm

@@ -1,5 +1,7 @@
 <h2 #annTitle("Facets generated by adding &facet.range= to the request")>Range Facets</h2>
 #foreach ($field in $response.response.facet_counts.facet_ranges)
+  ## Hide facets without value
+  #if($field.value.counts.size() > 0)
 	#set($name = $field.key)
 	#set($display = $name)
 	#set($f = $field.value.counts)
@@ -9,4 +11,5 @@
 	#set($before = $field.value.before)
 	#set($after = $field.value.after)
 	#display_facet_range($f, $display, $name, $start, $end, $gap, $before, $after)
+  #end
 #end

solr/example/solr/collection1/conf/velocity/hit.vm

@@ -1,5 +1,11 @@
 #set($docId = $doc.getFieldValue('id'))
 
 <div class="result-document">
-  #parse("doc.vm")
+#if($doc.getFieldValue('name'))
+  #parse("product-doc.vm")
+#elseif($doc.getFieldValue('compName_s'))
+  #parse("join-doc.vm")
+#else
+  #parse("richtext-doc.vm")
+#end
 </div>

solr/example/solr/collection1/conf/velocity/join-doc.vm

@@ -0,0 +1,4 @@
+<div class="result-title"><b>#field('compName_s')</b></div>
+<div>Id: #field('id') (company-details document for <a href="http://wiki.apache.org/solr/Join" target="_new">join</a>)</div>
+<div>Address: #field('address_s')</div>
+#parse('debug.vm')

solr/example/solr/collection1/conf/velocity/main.css

@@ -167,6 +167,10 @@ a {
   width:60%;
 }
 
+.result-body{
+  background: #ddd;
+}
+
 .mlt{
   
 }

solr/example/solr/collection1/conf/velocity/product-doc.vm

@@ -1,7 +1,8 @@
-<div class="result-title"><b>#field('name')</b><span class="mlt">#if($params.getBool('mlt', false) == false)<a href="#lensNoQ&q=id:$docId&mlt=true">More Like This</a>#end</span></div>
+<div class="result-title"><b>#field('name')</b><span class="mlt">   #if($params.getBool('mlt', false) == false)<a href="#lensNoQ&q=id:$docId&mlt=true">More Like This</a>#end</span></div>
 ##do we have a physical store for this product
 #set($store = $doc.getFieldValue('store'))
 #if($store)<div class="map"><img src="http://maps.google.com/maps/api/staticmap?&zoom=12&size=150x80&maptype=roadmap&markers=$doc.getFieldValue('store')&sensor=false" /><div><small><a target="_map" href="http://maps.google.com/?q=$store&amp;source=embed">Larger Map</a></small></div></div>#end
+<div>Id: #field('id')</div>
 <div>Price: #field('price_c')</div>
 <div>Features: #field('features')</div>
 <div>In Stock: #field('inStock')</div>
@@ -23,20 +24,4 @@
     <div>No Similar Items Found</div>
   #end
 </div>
-#if($params.getBool("debugQuery",false))
-  <a href="#" onclick='jQuery(this).siblings("pre").toggle(); return false;'>toggle explain</a>
-  <pre style="display:none">$response.getExplainMap().get($doc.getFirstValue('id'))</pre>
-  <a href="#" onclick='jQuery(this).siblings("pre2").toggle(); return false;'>toggle all fields</a>
-  <pre2 style="display:none">
-  #foreach($fieldname in $doc.fieldNames)
-     <br>
-       <span class="field-name">$fieldname :</span>
-       <span>
-       #foreach($value in $doc.getFieldValues($fieldname))
-         $value
-       #end
-       </span>
-  #end
-   </br>
-  </pre2>
-#end
+#parse('debug.vm')

solr/example/solr/collection1/conf/velocity/richtext-doc.vm

@@ -0,0 +1,52 @@
+#if($doc.getFieldValue('title'))
+  #set($title = $esc.html($doc.getFirstValue('title')))
+#else
+  #set($title = "["+$doc.getFieldValue('id')+"]")
+#end
+#if($doc.getFieldValue('url'))
+  #set($url = $doc.getFieldValue('url'))
+#elseif($doc.getFieldValue('resourcename'))
+  #set($url = "file:///$doc.getFieldValue('resourcename')")
+#else
+  #set($url = "$doc.getFieldValue('id')")
+#end
+#set($supportedtypes = "7z;ai;aiff;asc;audio;bin;bz2;c;cfc;cfm;chm;class;conf;cpp;cs;css;csv;deb;divx;doc;dot;eml;enc;file;gif;gz;hlp;htm;html;image;iso;jar;java;jpeg;jpg;js;lua;m;mm;mov;mp3;mpg;odc;odf;odg;odi;odp;ods;odt;ogg;pdf;pgp;php;pl;png;ppt;ps;py;ram;rar;rb;rm;rpm;rtf;sig;sql;swf;sxc;sxd;sxi;sxw;tar;tex;tgz;txt;vcf;video;vsd;wav;wma;wmv;xls;xml;xpi;xvid;zip")
+#set($ct = $list.get($doc.getFirstValue('content_type').split(";"),0))
+#set($filename = $doc.getFieldValue('resourcename'))
+#set($filetype = $mimeToExt.get($ct))
+#if(!$filetype)#set($filetype = $filename.substring($filename.lastIndexOf(".")).substring(1))#end
+#if(!$filetype)#set($filetype = "file")#end
+#if(!$supportedtypes.contains($filetype))#set($filetype = "file")#end
+<div class="result-title">
+## Small file type icons from http://www.splitbrain.org/projects/file_icons (public domain)
+<img src="#{url_root}/img/filetypes/${filetype}.png" align="center">
+<a href="${url}" target="_blank"><b>$title</b></a><span class="mlt">   #if($params.getBool('mlt', false) == false)<a href="#lensNoQ&q=id:%22$docId%22&mlt=true">More Like This</a>#end</span></div>
+<div>Id: #field('id')</div>
+<div>
+#if($doc.getFieldValue('resourcename'))Resource name: $filename 
+#elseif($url)URL: $url
+#end
+#if($ct) ($ct)#end
+</div>
+#if($doc.getFieldValue('author'))<div>Author: #field('author')</div>#end
+#if($doc.getFieldValue('last_modified'))<div>Last-modified: #field('last_modified')</div>#end
+<div class="result-body">#field('content')</div>
+<div class="mlt">
+  #set($mlt = $mltResults.get($docId))
+  #set($mltOn = $params.getBool('mlt'))
+  #if($mltOn == true)<div class="field-name">Similar Items</div>#end
+  #if ($mltOn && $mlt && $mlt.size() > 0)
+  <ul>
+    #foreach($mltHit in $mlt)
+      #set($mltId = $mltHit.getFieldValue('id'))
+      <li><div><a href="#url_for_home?q=id:$mltId">$mltId</a></div><div><span class="field-name">Title:</span> $mltHit.getFieldValue('title')</div>
+        <div><span class="field-name">Author:</span> $mltHit.getFieldValue('author') <span class="field-name">Description:</span> $mltHit.getFieldValue('description')</div>
+
+      </li>
+    #end
+  </ul>
+  #elseif($mltOn && $mlt.size() == 0)
+    <div>No Similar Items Found</div>
+  #end
+</div>
+#parse('debug.vm')

solr/webapp/web/img/filetypes/README

@@ -0,0 +1,27 @@
+http://www.splitbrain.org/projects/file_icons
+
+Released to the Public Domain
+Free to use. Provided as is. No warranties.
+
+Note: The big majority of icons where created by the creators listed
+      below. Only a few ones where found on the net. They were too
+      widespread to determine the original author and thus were
+      considered public domain.
+      If you are the author of one of those icons just send a short
+      mail to either be included in the list below or have the icon
+      removed from the package.
+
+Creators:
+
+  Andreas Gohr <andi@splitbrain.org>
+  Michael Klier <chi@chimeric.de>
+  Andreas Barton <andreas.barton@web.de>
+  Hubert Chathi <hubert@uhoreg.ca>
+  Johan Koehne <johankohne@gmail.com>
+  Rudi von Staden <rudivs@iafrica.com>
+  Daniel Darvish <ddarvish@hibm.org>
+  Andy Pascall <apascall@engineering.ucsb.edu>
+  Seth <seth.holcomb@gmail.com>
+  David Carella <david.carella@gmail.com>
+  Tom N. Harris <telliamed@fastmail.us>
+  Brandon Carmon Colvin <b.carmon.colvin@gmail.com>