LUCENE-840: benchmarking code correctness tests were added.

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@521526 13f79535-47bb-0310-9956-ffa450edef68

common-build.xml

@@ -194,6 +194,9 @@
       <sysproperty key="docs.dir" file="src/test"/>
       <sysproperty key="index.dir" file="${build.dir}/test/index"/>
 
+      <!-- contrib/benchmark uses this system property to locate defined tasks -->
+      <sysproperty key="tasks.dir" file="${build.dir}/classes/java/org/apache/lucene/benchmark/byTask/tasks"/>
+
       <formatter type="xml"/>
       <formatter type="brief" usefile="false"/>
       <batchtest fork="yes" todir="${junit.output.dir}" unless="testcase">

contrib/benchmark/CHANGES.txt

@@ -4,6 +4,11 @@ The Benchmark contrib package contains code for benchmarking Lucene in a variety
 
 $Id:$
 
+3/21/07
+
+Tests (for benchmarking code correctness) were added - LUCENE-840.
+To be invoked by "ant test" from contrib/benchmark. (Doron Cohen)
+
 3/19/07
 
 1. Introduced an AbstractQueryMaker to hold common QueryMaker code. (GSI)

contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/Benchmark.java

@@ -18,6 +18,8 @@ package org.apache.lucene.benchmark.byTask;
  */
 
 import java.io.File;
+import java.io.FileReader;
+import java.io.Reader;
 
 import org.apache.lucene.benchmark.byTask.utils.Algorithm;
 import org.apache.lucene.benchmark.byTask.utils.Config;
@@ -37,11 +39,38 @@ import org.apache.lucene.benchmark.byTask.utils.Config;
  * <li>TODO - perf report adequate to include in Lucene nightly build site? (so we can easily track performance changes.)</li>
  * <li>TODO - add overall time control for repeated execution (vs. current by-count only).</li>
  * <li>TODO - query maker that is based on index statistics.</li>
- * <li>TODO - prpoerties documentation - each task should document the properties it relies on.</li> 
  * </ol>
  */
 public class Benchmark {
 
+  private PerfRunData runData;
+  private Algorithm algorithm;
+  private boolean executed;
+  
+  public Benchmark (Reader algReader) throws Exception {
+    // prepare run data
+    try {
+      runData = new PerfRunData(new Config(algReader));
+    } catch (Exception e) {
+      throw new Exception("Error: cannot init PerfRunData!",e);
+    }
+    
+    // parse algorithm
+    try {
+      algorithm = new Algorithm(runData);
+    } catch (Exception e) {
+      throw new Exception("Error: cannot understand algorithm!",e);
+    }
+  }
+  
+  public synchronized void  execute() throws Exception {
+    if (executed) {
+      throw new Exception("Benchmark was already executed");
+    }
+    executed = true;
+    algorithm.execute();
+  }
+  
   /**
    * Run the benchmark algorithm.
    * @param args benchmark config and algorithm files
@@ -60,32 +89,22 @@ public class Benchmark {
       System.exit(1);
     }
     
-    // last preparations
-    PerfRunData runData = null;
-    try {
-      runData = new PerfRunData(new Config(algFile));
-    } catch (Exception e) {
-      System.err.println("Error: cannot init PerfRunData: "+e.getMessage());
-      e.printStackTrace();
-      System.exit(1);
-    }
+    System.out.println("Running algorithm from: "+algFile.getAbsolutePath());
     
-    // parse algorithm
-    Algorithm algorithm = null;
+    Benchmark benchmark = null;
     try {
-      algorithm = new Algorithm(runData);
+      benchmark = new Benchmark(new FileReader(algFile));
     } catch (Exception e) {
-      System.err.println("Error: cannot understand algorithm from file: "+algFile.getAbsolutePath());
       e.printStackTrace();
       System.exit(1);
     }
 
     System.out.println("------------> algorithm:");
-    System.out.println(algorithm.toString());
+    System.out.println(benchmark.getAlgorithm().toString());
 
     // execute
     try {
-      algorithm.execute();
+      benchmark.execute();
     } catch (Exception e) {
       System.err.println("Error: cannot execute the algorithm! "+e.getMessage());
       e.printStackTrace();
@@ -97,4 +116,18 @@ public class Benchmark {
 
   }
 
+  /**
+   * @return Returns the algorithm.
+   */
+  public Algorithm getAlgorithm() {
+    return algorithm;
+  }
+
+  /**
+   * @return Returns the runData.
+   */
+  public PerfRunData getRunData() {
+    return runData;
+  }
+
 }

contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Algorithm.java

@@ -223,7 +223,32 @@ public class Algorithm {
   public void execute() throws Exception {
     sequence.doLogic();
   }
-  
+
+  /**
+   * Expert: for test purposes, return all tasks participating in this algorithm.
+   * @return all tasks participating in this algorithm.
+   */
+  public ArrayList extractTasks() {
+    ArrayList res = new ArrayList();
+    extractTasks(res, sequence);
+    return res;
+  }
+  private void extractTasks (ArrayList extrct, TaskSequence seq) {
+    if (seq==null) 
+      return;
+    extrct.add(seq);
+    ArrayList t = sequence.getTasks();
+    if (t==null) 
+      return;
+    for (int i = 0; i < t.size(); i++) {
+      PerfTask p = (PerfTask) t.get(0);
+      if (p instanceof TaskSequence) {
+        extractTasks(extrct, (TaskSequence)p);
+      } else {
+        extrct.add(p);
+      }
+    }
+  }
   
 }
 

contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java

@@ -19,9 +19,8 @@ package org.apache.lucene.benchmark.byTask.utils;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileReader;
 import java.io.IOException;
+import java.io.Reader;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -49,10 +48,10 @@ public class Config {
    * @param algFile file containing both algorithm and config properties.
    * @throws IOException
    */
-  public Config (File algFile) throws IOException {
+  public Config (Reader algReader) throws IOException {
     // read alg file to array of lines
     ArrayList lines = new ArrayList();
-    BufferedReader r = new BufferedReader(new FileReader(algFile));
+    BufferedReader r = new BufferedReader(algReader);
     int lastConfigLine=0;
     for (String line = r.readLine(); line!=null; line=r.readLine()) {
       lines.add(line);

contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java

@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.benchmark.byTask;
+
+import java.io.StringReader;
+
+import org.apache.lucene.benchmark.byTask.Benchmark;
+import org.apache.lucene.benchmark.byTask.tasks.CountingSearchTestTask;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+
+import junit.framework.TestCase;
+
+/**
+ * Test very simply that perf tasks - simple algorithms - are doing what they should.
+ */
+public class TestPerfTasksLogic extends TestCase {
+
+  private static final boolean DEBUG = false;
+  static final String NEW_LINE = System.getProperty("line.separator");
+  
+  // properties in effect in all tests here
+  static final String propLines [] = {
+    "directory=RAMDirectory",
+    "print.props=false",
+  };
+  
+  /**
+   * @param name test name
+   */
+  public TestPerfTasksLogic(String name) {
+    super(name);
+  }
+
+  /**
+   * Test index creation logic
+   */
+  public void testIndexAndSearchTasks() throws Exception {
+    // 1. alg definition (required in every "logic" test)
+    String algLines[] = {
+        "ResetSystemErase",
+        "CreateIndex",
+        "{ AddDoc } : 1000",
+        "Optimize",
+        "CloseIndex",
+        "OpenReader",
+        "{ CountingSearchTest } : 200",
+        "CloseReader",
+        "[ CountingSearchTest > : 70",
+        "[ CountingSearchTest > : 9",
+    };
+    
+    // 2. we test this value later
+    CountingSearchTestTask.numSearches = 0;
+    
+    // 3. execute the algorithm  (required in every "logic" test)
+    Benchmark benchmark = execBenchmark(algLines);
+
+    // 4. test specific checks after the benchmark run completed.
+    assertEquals("TestSearchTask was supposed to be called!",279,CountingSearchTestTask.numSearches);
+    assertTrue("Index does not exist?...!", IndexReader.indexExists(benchmark.getRunData().getDirectory()));
+    // now we should be able to open the index for write. 
+    IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),null,false);
+    iw.close();
+    IndexReader ir = IndexReader.open(benchmark.getRunData().getDirectory());
+    assertEquals("1000 docs were added to the index, this is what we expect to find!",1000,ir.numDocs());
+  }
+
+  // create the benchmark and execute it. 
+  private Benchmark execBenchmark(String[] algLines) throws Exception {
+    String algText = algLinesToText(algLines);
+    logTstLogic(algText);
+    Benchmark benchmark = new Benchmark(new StringReader(algText));
+    benchmark.execute();
+    return benchmark;
+  }
+  
+  // catenate alg lines to make the alg text
+  private String algLinesToText(String[] algLines) {
+    String indent = "  ";
+    StringBuffer sb = new StringBuffer();
+    for (int i = 0; i < propLines.length; i++) {
+      sb.append(indent).append(propLines[i]).append(NEW_LINE);
+    }
+    for (int i = 0; i < algLines.length; i++) {
+      sb.append(indent).append(algLines[i]).append(NEW_LINE);
+    }
+    return sb.toString();
+  }
+
+  private void logTstLogic (String txt) {
+    if (!DEBUG) 
+      return;
+    System.out.println("Test logic of:");
+    System.out.println(txt);
+  }
+
+}

contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksParse.java

@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.benchmark.byTask;
+
+import java.io.File;
+import java.io.StringReader;
+import java.lang.reflect.Modifier;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.lucene.benchmark.byTask.tasks.PerfTask;
+import org.apache.lucene.benchmark.byTask.tasks.TaskSequence;
+import org.apache.lucene.benchmark.byTask.utils.Algorithm;
+
+import junit.framework.TestCase;
+
+/**
+ * Test very simply that perf tasks are parses as expected.
+ */
+public class TestPerfTasksParse extends TestCase {
+
+  private static final boolean DEBUG = false;
+  static final String NEW_LINE = System.getProperty("line.separator");
+  static final String INDENT = "  ";
+
+  // properties in effect in all tests here
+  static final String propPart = 
+    INDENT+"directory=RAMDirectory" + NEW_LINE +
+    INDENT+"print.props=false" + NEW_LINE
+  ;
+
+  /*
+   * All known tasks. 
+   * As new tasks are added, add them here.
+   * It would be nice to do that automatically, unfortunately
+   * Java does not provide a "get all classes in package" or
+   * "get all sub-classes" functionality.  
+   */
+  static String singleTaskAlgs [];
+  
+  /* (non-Javadoc)
+   * @see junit.framework.TestCase#setUp()
+   */
+  protected void setUp() throws Exception {
+    super.setUp();
+    if (singleTaskAlgs==null) {
+      singleTaskAlgs = findTasks();
+    }
+  }
+
+  // one time initialization
+  static String [] findTasks () throws Exception {
+    ArrayList tsks = new ArrayList();
+    // init with tasks we know about
+    tsks.add(  " AddDoc                   "  );
+    tsks.add(  " AddDoc(1000.0)           "  );
+    tsks.add(  " ClearStats               "  );
+    tsks.add(  " CloseIndex               "  );
+    tsks.add(  " CloseReader              "  );
+    tsks.add(  " CreateIndex              "  );
+    tsks.add(  " DeleteDoc                "  );
+    tsks.add(  " DeleteDoc(500.0)         "  );
+    tsks.add(  " NewRound                 "  );
+    tsks.add(  " OpenIndex                "  );
+    tsks.add(  " OpenReader               "  );
+    tsks.add(  " Optimize                 "  );
+    tsks.add(  " RepAll                   "  );
+    tsks.add(  " RepSelectByPref prefix   "  );
+    tsks.add(  " RepSumByNameRound        "  );
+    tsks.add(  " RepSumByName             "  );
+    tsks.add(  " RepSumByPrefRound prefix "  );
+    tsks.add(  " RepSumByPref   prefix    "  );
+    tsks.add(  " ResetInputs              "  );
+    tsks.add(  " ResetSystemErase         "  );
+    tsks.add(  " ResetSystemSoft          "  );
+    tsks.add(  " Search                   "  );
+    tsks.add(  " SearchTravRet            "  );
+    tsks.add(  " SearchTravRet(100.0)     "  );
+    tsks.add(  " SearchTrav               "  );
+    tsks.add(  " SearchTrav(50.0)         "  );
+    tsks.add(  " SetProp                  "  );
+    tsks.add(  " SetProp(name,value)      "  );
+    tsks.add(  " Warm                     "  );
+    
+    // if tasks.dir property is defined, look for additional tasks.
+    // this somewhat covers tasks that would be added in the future, in case
+    // the list above is not updated to cover them.
+    // some tasks would be tested more than once this way, but that's ok.
+    String tasksDir = System.getProperty("tasks.dir");
+    if (tasksDir !=null) {
+      String pkgPrefix = PerfTask.class.getPackage().getName()+".";
+      String taskNames[] = new File(tasksDir).list();
+      for (int i = 0; i < taskNames.length; i++) {
+        String name = taskNames[i].trim();
+        if (!name.endsWith("Task.class"))
+          continue; // Task class file only
+        name = name.substring(0,name.length()-6);
+        Class cls = Class.forName(pkgPrefix+name);
+        if (Modifier.isAbstract(cls.getModifiers()) || Modifier.isInterface(cls.getModifiers())) 
+          continue; // skip sbstract classes
+        if (!PerfTask.class.isAssignableFrom(cls))
+          continue; // not a task
+        name = name.substring(0,name.length()-4);
+        if (name.startsWith("Rep") && name.indexOf("Pref")>=0)
+          name += " prefix";
+        tsks.add(" "+name+" ");
+      }
+    }
+    return (String[]) tsks.toArray(new String[0]);
+  }
+  
+  
+  /**
+   * @param name test name
+   */
+  public TestPerfTasksParse(String name) {
+    super(name);
+  }
+
+  /**
+   * Test the parsing of very simple tasks, for all tasks
+   */
+  public void testAllTasksSimpleParse() {
+    doTestAllTasksSimpleParse(false,false);
+  }
+  
+  /**
+   * Test the parsing of simple sequential sequences, for all tasks
+   */
+  public void testAllTasksSimpleParseSequntial() {
+    doTestAllTasksSimpleParse(true,false);
+  }
+
+  /**
+   * Test the parsing of simple parallel sequences, for all tasks
+   */
+  public void testAllTasksSimpleParseParallel() {
+    doTestAllTasksSimpleParse(true,true);
+  }
+  
+  // utility for simple parsing testing of all tasks.
+  private void doTestAllTasksSimpleParse(boolean parOrSeq, boolean par) {
+    for (int i = 0; i < singleTaskAlgs.length; i++) {
+      String testedTask = singleTaskAlgs[i];
+      if (parOrSeq) {
+        if (par) {
+          testedTask = "[ " + testedTask + " ] : 2";
+        } else {
+          testedTask = "{ " + testedTask + " } : 3";
+        }
+      }
+      try {
+        String algText = propPart+INDENT+testedTask;
+        logTstParsing(algText);
+        Benchmark benchmark = new Benchmark(new StringReader(algText));
+        Algorithm alg = benchmark.getAlgorithm();
+        ArrayList algTasks = alg.extractTasks();
+        // must find a task with this name in the algorithm
+        boolean foundName = false;
+        boolean foundPar = false;
+        String theTask = singleTaskAlgs[i].replaceAll(" +"," ").trim();
+        for (Iterator iter = algTasks.iterator(); iter.hasNext();) {
+          PerfTask task = (PerfTask) iter.next();
+          foundName |= (task.toString().indexOf(theTask)>=0);
+          foundPar |= (task instanceof TaskSequence && ((TaskSequence)task).isParallel());
+        }
+        assertTrue("Task "+testedTask+" was not found in "+alg.toString(),foundName);
+        if (parOrSeq) {
+          if (par) {
+            assertTrue("Task "+testedTask+" was supposed to be parallel in "+alg.toString(),foundPar);
+          } else {
+            assertFalse("Task "+testedTask+" was not supposed to be parallel in "+alg.toString(),foundPar);
+          }
+        }
+      } catch (Exception e) {
+        System.out.flush();
+        e.printStackTrace();
+        fail(e.getMessage());
+      }
+    }
+  }
+
+  /**
+   * Test the repetiotion parsing for parallel tasks
+   */
+  public void testParseParallelTaskSequenceRepetition() throws Exception {
+    String taskStr = "AddDoc";
+    String parsedTasks = "[ "+taskStr+" ] : 1000";
+    Benchmark benchmark = new Benchmark(new StringReader(propPart+parsedTasks));
+    Algorithm alg = benchmark.getAlgorithm();
+    ArrayList algTasks = alg.extractTasks();
+    boolean foundAdd = false;
+    for (Iterator iter = algTasks.iterator(); iter.hasNext();) {
+       PerfTask task = (PerfTask) iter.next();
+       if (task.toString().indexOf(taskStr)>=0) {
+          foundAdd = true;
+       }
+       if (task instanceof TaskSequence) {
+         assertEquals("repetions should be 1000 for "+parsedTasks, 1000, ((TaskSequence) task).getRepetitions());
+         assertTrue("sequence for "+parsedTasks+" should be parallel!", ((TaskSequence) task).isParallel());
+       }
+       assertTrue("Task "+taskStr+" was not found in "+alg.toString(),foundAdd);
+    }
+  }
+
+  /**
+   * Test the repetiotion parsing for sequential  tasks
+   */
+  public void testParseTaskSequenceRepetition() throws Exception {
+    String taskStr = "AddDoc";
+    String parsedTasks = "{ "+taskStr+" } : 1000";
+    Benchmark benchmark = new Benchmark(new StringReader(propPart+parsedTasks));
+    Algorithm alg = benchmark.getAlgorithm();
+    ArrayList algTasks = alg.extractTasks();
+    boolean foundAdd = false;
+    for (Iterator iter = algTasks.iterator(); iter.hasNext();) {
+       PerfTask task = (PerfTask) iter.next();
+       if (task.toString().indexOf(taskStr)>=0) {
+          foundAdd = true;
+       }
+       if (task instanceof TaskSequence) {
+         assertEquals("repetions should be 1000 for "+parsedTasks, 1000, ((TaskSequence) task).getRepetitions());
+         assertFalse("sequence for "+parsedTasks+" should be sequential!", ((TaskSequence) task).isParallel());
+       }
+       assertTrue("Task "+taskStr+" was not found in "+alg.toString(),foundAdd);
+    }
+  }
+
+  private void logTstParsing (String txt) {
+    if (!DEBUG) 
+      return;
+    System.out.println("Test parsing of");
+    System.out.println(txt);
+  }
+
+}

contrib/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CountingSearchTestTask.java

@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.benchmark.byTask.tasks;
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+
+/**
+ * Test Search task which counts number of searches.
+ */
+public class CountingSearchTestTask extends SearchTask {
+
+  public static int numSearches = 0; 
+
+  public CountingSearchTestTask(PerfRunData runData) {
+    super(runData);
+  }
+
+  public int doLogic() throws Exception {
+    int res = super.doLogic();
+    incrNumSearches();
+    return res;
+  }
+
+  private static synchronized void incrNumSearches() {
+    numSearches++;
+  }
+
+}