SOLR-2444, SOLR-705 -- Updating Transformers to use LocalParam syntax (surrounded by [ ])

adding SHARD_URL as a parameter

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1133505 13f79535-47bb-0310-9956-ffa450edef68

solr/src/common/org/apache/solr/common/params/ShardParams.java

@@ -34,6 +34,9 @@ public interface ShardParams {
   /** whether the request goes to a shard */
   public static final String IS_SHARD = "isShard";
   
+  /** The requested URL for this shard */
+  public static final String SHARD_URL = "shard.url";
+  
   /** query type for shard requests */
   public static final String SHARDS_QT = "shards.qt";
 }

solr/src/java/org/apache/solr/handler/component/SearchHandler.java

@@ -290,6 +290,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware
               params.remove("indent");
               params.remove(CommonParams.HEADER_ECHO_PARAMS);
               params.set(ShardParams.IS_SHARD, true);  // a sub (shard) request
+              params.set(ShardParams.SHARD_URL, shard); // so the shard knows what was asked
               if (rb.requestInfo != null) {
                 // we could try and detect when this is needed, but it could be tricky
                 params.set("NOW", Long.toString(rb.requestInfo.getNOW().getTime()));

solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java

@@ -16,6 +16,8 @@
  */
 package org.apache.solr.response.transform;
 
+import java.util.Map;
+
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
@@ -28,8 +30,8 @@ import org.apache.solr.request.SolrQueryRequest;
 public class DocIdAugmenterFactory extends TransformerFactory
 {
   @Override
-  public DocTransformer create(String field, String arg, SolrQueryRequest req) {
+  public DocTransformer create(String field, Map<String,String> args, SolrQueryRequest req) {
-    if( arg != null ) {
+    if( !args.isEmpty() ) {
       throw new SolrException( ErrorCode.BAD_REQUEST,
           "DocIdAugmenter does not take any arguments" );
     }

solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java

@@ -17,6 +17,7 @@
 package org.apache.solr.response.transform;
 
 import java.io.IOException;
+import java.util.Map;
 
 import org.apache.lucene.search.Explanation;
 import org.apache.solr.common.SolrDocument;
@@ -63,12 +64,12 @@ public class ExplainAugmenterFactory extends TransformerFactory
   }
 
   @Override
-  public DocTransformer create(String field, String arg, SolrQueryRequest req) {
+  public DocTransformer create(String field, Map<String,String> args, SolrQueryRequest req) {
-    Style style = (arg==null)?defaultStyle:getStyle(arg);
+    String s = args.get("style");
+    Style style = (s==null)?defaultStyle:getStyle(s);
     return new ExplainAugmenter( field, style );
   }
 
-
   static class ExplainAugmenter extends TransformerWithContext
   {
     final String name;

solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java

@@ -16,6 +16,9 @@
  */
 package org.apache.solr.response.transform;
 
+import java.util.Map;
+
+import org.apache.solr.common.params.ShardParams;
 import org.apache.solr.request.SolrQueryRequest;
 
 
@@ -26,11 +29,17 @@ import org.apache.solr.request.SolrQueryRequest;
 public class ShardAugmenterFactory extends TransformerFactory
 {
   @Override
-  public DocTransformer create(String field, String arg, SolrQueryRequest req) {
+  public DocTransformer create(String field, Map<String,String> args, SolrQueryRequest req) {
-    String id = "TODO... find ID";
+    String v = req.getParams().get(ShardParams.SHARD_URL);
-    // Maybe it is stored in the context?
+    if( v == null ) {
-    // is it a request variable?
+      if( req.getParams().getBool(ShardParams.IS_SHARD, false) ) {
-    return new ValueAugmenter( field, id );
+        v = "[unknown]";
+      }
+      else {
+        v = "[not a shard request]";
+      }
+    }
+    return new ValueAugmenter( field, v );
   }
 }
 

solr/src/java/org/apache/solr/response/transform/TransformerFactory.java

@@ -20,7 +20,6 @@ package org.apache.solr.response.transform;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -38,7 +37,7 @@ public abstract class TransformerFactory implements NamedListInitializedPlugin
     defaultUserArgs = (String)args.get( "args" );
   }
 
-  public abstract DocTransformer create(String field, String args, SolrQueryRequest req);
+  public abstract DocTransformer create(String field, Map<String,String> args, SolrQueryRequest req);
 
   public static final Map<String,TransformerFactory> defaultFactories = new HashMap<String,TransformerFactory>();
   static {

solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java

@@ -16,6 +16,8 @@
  */
 package org.apache.solr.response.transform;
 
+import java.util.Map;
+
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
@@ -40,12 +42,9 @@ public class ValueAugmenterFactory extends TransformerFactory
     }
   }
 
-  public static Object getObjectFrom( String str )
+  public static Object getObjectFrom( String val, String type )
   {
-    int idx = str.indexOf( ':' );
+    if( type != null ) {
-    if( idx > 0 ) {
-      String type = str.substring(0,idx);
-      String val = str.substring(idx+1);
       try {
         if( "int".equals( type ) ) return Integer.valueOf( val );
         if( "double".equals( type ) ) return Double.valueOf( val );
@@ -57,14 +56,20 @@ public class ValueAugmenterFactory extends TransformerFactory
             "Unable to parse "+type+"="+val, ex );
       }
     }
-    return str;
+    return val;
   }
 
   @Override
-  public DocTransformer create(String field, String arg, SolrQueryRequest req) {
+  public DocTransformer create(String field, Map<String,String> args, SolrQueryRequest req) {
     Object val = value;
     if( val == null ) {
-      val = (arg==null)?defaultValue:getObjectFrom(arg);
+      String v = args.get("v");
+      if( v == null ) {
+        val = defaultValue;
+      }
+      else {
+        val = getObjectFrom(v, args.get("t"));
+      }
       if( val == null ) {
         throw new SolrException( ErrorCode.BAD_REQUEST,
             "ValueAugmenter is missing a value -- should be defined in solrconfig or inline" );

solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java

@@ -16,19 +16,18 @@
  */
 package org.apache.solr.response.transform;
 
+import java.io.IOException;
+import java.util.Map;
+
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.util.ReaderUtil;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.core.SolrCore;
 import org.apache.solr.search.QParser;
 import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.search.function.DocValues;
 import org.apache.solr.search.function.ValueSource;
 
-import java.io.IOException;
-import java.util.Map;
-
 /**
  * Add values from a ValueSource (function query etc)
  *
@@ -65,7 +64,7 @@ public class ValueSourceAugmenter extends DocTransformer
     docValuesArr = new DocValues[readerContexts.length];
 
     searcher = qparser.getReq().getSearcher();
-    this.fcontext = valueSource.newContext(searcher);
+    this.fcontext = ValueSource.newContext(searcher);
   }
 
 

solr/src/java/org/apache/solr/search/ReturnFields.java

@@ -26,7 +26,6 @@ import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.MapSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.SolrCore;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.transform.DocTransformer;
 import org.apache.solr.response.transform.DocTransformers;
@@ -217,10 +216,22 @@ public class ReturnFields
           Map<String,String> augmenterArgs = new HashMap<String,String>();
           int end = QueryParsing.parseLocalParams(funcStr, 0, augmenterArgs, req.getParams(), "[", ']');
           sp.pos += end;
-          String augmenterName = augmenterArgs.get("type");    // [foo] is short for [type=foo] in localParams syntax
+          
-          // TODO: look up and add the augmenter.  If the form was myalias:[myaugmenter], then "key" will be myalias
+          // [foo] is short for [type=foo] in localParams syntax
-          SolrParams augmenterParams = new MapSolrParams(augmenterArgs);
+          String augmenterName = augmenterArgs.remove("type"); 
-          log.info("Parsed augmenter " + augmenterParams + " with alias " + key);  // TODO: remove log statement after augmenter works
+          String disp = key;
+          if( disp == null ) {
+            disp = '['+augmenterName+']';
+          }
+
+          TransformerFactory factory = req.getCore().getTransformerFactory( augmenterName );
+          if( factory != null ) {
+            augmenters.addTransformer( factory.create(disp, augmenterArgs, req) );
+          }
+          else {
+            // unknown transformer?
+          }
+          addField(field, disp, augmenters, req);
           continue;
         }
 
@@ -323,26 +334,6 @@ public class ReturnFields
       _wantsScore = true;
       augmenters.addTransformer( new ScoreAugmenter( disp ) );
     }
-    else if( field.charAt(0)=='_'&& field.charAt(field.length()-1)=='_' ) {
-      String name = field;
-      String args = null;
-      int idx = field.indexOf( ':' );
-      if( idx > 0 ) {
-        name = field.substring(1,idx);
-        args = field.substring(idx+1,field.length()-1);
-      }
-      else {
-        name = field.substring(1,field.length()-1 );
-      }
-
-      TransformerFactory factory = req.getCore().getTransformerFactory( name );
-      if( factory != null ) {
-        augmenters.addTransformer( factory.create(disp, args, req) );
-      }
-      else {
-        // unknown field?
-      }
-    }
   }
 
   public Set<String> getLuceneFieldNames()

solr/src/test/org/apache/solr/client/solrj/SolrExampleTests.java

@@ -51,7 +51,6 @@ import org.apache.solr.common.util.XML;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.FacetParams;
-import org.junit.Ignore;
 import org.junit.Test;
 
 /**
@@ -380,7 +379,7 @@ abstract public class SolrExampleTests extends SolrJettyTestBase
   }
 
 
-  @Test @Ignore   // TODO: re-enable when new transformer syntax is implemented
+  @Test
   public void testAugmentFields() throws Exception
   {    
     SolrServer server = getSolrServer();
@@ -405,7 +404,7 @@ abstract public class SolrExampleTests extends SolrJettyTestBase
     
     SolrQuery query = new SolrQuery();
     query.setQuery( "*:*" );
-    query.set( CommonParams.FL, "id,price,_docid_,_explain:nl_,score,aaa:_value:aaa_,ten:_value:int:10_" );
+    query.set( CommonParams.FL, "id,price,[docid],[explain style=nl],score,aaa:[value v=aaa],ten:[value v=10 t=int]" );
     query.addSortField( "price", SolrQuery.ORDER.asc );
     QueryResponse rsp = server.query( query );
     
@@ -419,12 +418,12 @@ abstract public class SolrExampleTests extends SolrJettyTestBase
     assertEquals( 1.0f, out2.getFieldValue( "score" ) );
     
     // check that the docid is one bigger
-    int id1 = (Integer)out1.getFieldValue( "_docid_" );
+    int id1 = (Integer)out1.getFieldValue( "[docid]" );
-    int id2 = (Integer)out2.getFieldValue( "_docid_" );
+    int id2 = (Integer)out2.getFieldValue( "[docid]" );
     assertTrue( "should be bigger ["+id1+","+id2+"]", id2 > id1 );
     
     // The score from explain should be the same as the score
-    NamedList explain = (NamedList)out1.getFieldValue( "_explain:nl_" );
+    NamedList explain = (NamedList)out1.getFieldValue( "[explain]" );
     assertEquals( out1.get( "score"), explain.get( "value" ) );
     
     // Augmented _value_ with alias

solr/src/test/org/apache/solr/client/solrj/embedded/MultiCoreExampleJettyTest.java

@@ -18,8 +18,19 @@
 package org.apache.solr.client.solrj.embedded;
 
 import org.apache.solr.client.solrj.MultiCoreExampleTestBase;
+import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServer;
 import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest.ACTION;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.search.SolrIndexSearcher.QueryResult;
+import org.junit.Test;
 
 /**
  * TODO? perhaps use:
@@ -95,4 +106,51 @@ public class MultiCoreExampleJettyTest extends MultiCoreExampleTestBase {
       throw new RuntimeException( ex );
     }
   }
+  
+  
+
+  @Test
+  public void testDistributed() throws Exception
+  {
+    UpdateRequest up = new UpdateRequest();
+    up.setAction( ACTION.COMMIT, true, true );
+    up.deleteByQuery( "*:*" );
+    up.process( getSolrCore0() );
+    up.process( getSolrCore1() );
+    up.clear();
+    
+    // Add something to each core
+    SolrInputDocument doc = new SolrInputDocument();
+    // Add to core0
+    doc.setField( "id", "core0" );
+    up.add( doc );
+    up.process( getSolrCore0() );
+    up.clear();
+
+    // Add to core1
+    doc.setField( "id", "core1" );
+    up.add( doc );
+    up.process( getSolrCore1() );
+    up.clear();
+    
+    SolrQuery q = new SolrQuery();
+    QueryRequest r = new QueryRequest( q );
+    q.setQuery( "*:*" );
+    assertEquals( 1, r.process( getSolrCore0() ).getResults().size() );
+    assertEquals( 1, r.process( getSolrCore1() ).getResults().size() );
+    
+    // Distributed
+    String baseURL = "localhost:"+port+context+"/";
+    q = new SolrQuery( "*:*" );
+    q.set( ShardParams.SHARDS, baseURL+"core0,"+baseURL+"core1" );
+    q.set( "fl", "id,s:[shard]" );
+    r = new QueryRequest( q );
+    SolrDocumentList docs = r.process( getSolrCore0() ).getResults();
+    assertEquals( 2, docs.size() );
+    for( SolrDocument d : docs ) {
+      String id = (String)d.get("id");
+      String shard = (String)d.get("s");
+      assertEquals(baseURL+id, shard); // The shard ends with the core name
+    }
+  }
 }

solr/src/test/org/apache/solr/search/TestSolrQueryParser.java

@@ -65,10 +65,10 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
     assertTrue( rf.wantsAllFields() );
     assertNull( rf.getTransformer() );
     
-    rf = new ReturnFields( req("fl", "_explain_") );
+    rf = new ReturnFields( req("fl", "[explain]") );
     assertFalse( rf.wantsScore() );
     assertFalse( rf.wantsField( "id" ) );
-    assertEquals( "_explain_", rf.getTransformer().getName() );
+    assertEquals( "[explain]", rf.getTransformer().getName() );
 
     // Check that we want wildcards
     rf = new ReturnFields( req("fl", "id,aaa*,*bbb") );