From 9e65cd554f2e36e72b08c6a6d2625f9457f19143 Mon Sep 17 00:00:00 2001 From: Doron Cohen Date: Thu, 3 Jan 2008 07:44:40 +0000 Subject: [PATCH] LUCENE-1116: contrib/benchmark quality package improvements (MRR, Trec1MQ) git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@608370 13f79535-47bb-0310-9956-ffa450edef68 --- contrib/benchmark/CHANGES.txt | 7 ++ .../benchmark/quality/QualityBenchmark.java | 45 +++++++++- .../benchmark/quality/QualityStats.java | 30 ++++++- .../benchmark/quality/trec/QueryDriver.java | 3 +- .../benchmark/quality/trec/Trec1MQReader.java | 87 +++++++++++++++++++ .../quality/utils/SubmissionReport.java | 16 +++- .../benchmark/quality/TestQualityRun.java | 5 +- 7 files changed, 182 insertions(+), 11 deletions(-) create mode 100755 contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java diff --git a/contrib/benchmark/CHANGES.txt b/contrib/benchmark/CHANGES.txt index e46edfdb71d..081cb3f38e4 100644 --- a/contrib/benchmark/CHANGES.txt +++ b/contrib/benchmark/CHANGES.txt @@ -4,6 +4,13 @@ The Benchmark contrib package contains code for benchmarking Lucene in a variety $Id:$ +01/03/08 + LUCENE-1116: quality package improvements: + - add MRR computation; + - allow control of max #queries to run; + - verify log & report are flushed. + - add TREC query reader for the 1MQ track. + 12/31/07 LUCENE-1102: EnwikiDocMaker now indexes the docid field, so results might not be comparable with results prior to this change, although it is doubted that this one small field makes much difference. diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java index 92a22828082..3ed30d0068f 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java @@ -51,6 +51,12 @@ public class QualityBenchmark { /** index field to extract doc name for each search result; used for judging the results. */ protected String docNameField; + + /** maximal number of queries that this quality benchmark runs. Default: maxint. Useful for debugging. */ + private int maxQueries = Integer.MAX_VALUE; + + /** maximal number of results to collect for each query. Default: 1000. */ + private int maxResults = 1000; /** * Create a QualityBenchmark. @@ -71,7 +77,6 @@ public class QualityBenchmark { /** * Run the quality benchmark. - * @param maxResults how many results to collect for each quality query. * @param judge the judge that can tell if a certain result doc is relevant for a certain quality query. * If null, no judgements would be made. Usually null for a submission run. * @param submitRep submission report is created if non null. @@ -79,10 +84,11 @@ public class QualityBenchmark { * @return QualityStats of each quality query that was executed. * @throws Exception if quality benchmark failed to run. */ - public QualityStats [] execute(int maxResults, Judge judge, SubmissionReport submitRep, + public QualityStats [] execute(Judge judge, SubmissionReport submitRep, PrintWriter qualityLog) throws Exception { - QualityStats stats[] = new QualityStats[qualityQueries.length]; - for (int i=0; i + * Reciprocal rank is defined as 1/r where r is the + * rank of the first correct result, or 0 if there are no correct + * results within the top 5 results. + *

+ * This follows the definition in + * + * Question Answering - CNLP at the TREC-10 Question Answering Track. + */ + public double getMRR() { + return mrr; + } + + /** * Returns the search time in milliseconds for the measured query. */ diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java index 9fc28ff6624..d29df9ec73b 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java @@ -46,8 +46,9 @@ public class QueryDriver { // run the benchmark QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField); + qrun.setMaxResults(maxResults); SubmissionReport submitLog = null; - QualityStats stats[] = qrun.execute(maxResults, judge, submitLog, logger); + QualityStats stats[] = qrun.execute(judge, submitLog, logger); // print an avarage sum of the results QualityStats avg = QualityStats.average(stats); diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java new file mode 100755 index 00000000000..c2598994103 --- /dev/null +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.quality.trec; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.lucene.benchmark.quality.QualityQuery; + +/** + * Read topics of TREC 1MQ track. + *

+ * Expects this topic format - + * 

+ *   qnum:qtext
+ *
+ * Comment lines starting with '#' are ignored. + *

+ * All topics will have a single name value pair. + */ +public class Trec1MQReader { + + private String name; + + /** + * Constructor for Trec's 1MQ TopicsReader + * @param name name of name-value pair to set for all queries. + */ + public Trec1MQReader(String name) { + super(); + this.name = name; + } + + /** + * Read quality queries from trec 1MQ format topics file. + * @param reader where queries are read from. + * @return the result quality queries. + * @throws IOException if cannot read the queries. + */ + public QualityQuery[] readQueries(BufferedReader reader) throws IOException { + ArrayList res = new ArrayList(); + String line; + try { + while (null!=(line=reader.readLine())) { + line = line.trim(); + if (line.startsWith("#")) { + continue; + } + // id + int k = line.indexOf(":"); + String id = line.substring(0,k).trim(); + // qtext + String qtext = line.substring(k+1).trim(); + // we got a topic! + HashMap fields = new HashMap(); + fields.put(name,qtext); + //System.out.println("id: "+id+" qtext: "+qtext+" line: "+line); + QualityQuery topic = new QualityQuery(id,fields); + res.add(topic); + } + } finally { + reader.close(); + } + // sort result array (by ID) + QualityQuery qq[] = (QualityQuery[]) res.toArray(new QualityQuery[0]); + Arrays.sort(qq); + return qq; + } + +} diff --git a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java index 4f592a15f05..f9ea2d0d29b 100644 --- a/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java +++ b/contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java @@ -35,13 +35,16 @@ public class SubmissionReport { private NumberFormat nf; private PrintWriter logger; + private String name; /** * Constructor for SubmissionReport. * @param logger if null, no submission data is created. + * @param name name of this run. */ - public SubmissionReport (PrintWriter logger) { + public SubmissionReport (PrintWriter logger, String name) { this.logger = logger; + this.name = name; nf = NumberFormat.getInstance(); nf.setMaximumFractionDigits(4); nf.setMinimumFractionDigits(4); @@ -66,14 +69,21 @@ public class SubmissionReport { String docName = xt.docName(searcher,sd[i].doc); logger.println( qq.getQueryID() + sep + - '0' + sep + + "Q0" + sep + format(docName,20) + sep + format(""+i,7) + sep + - nf.format(sd[i].score) + nf.format(sd[i].score) + sep + + name ); } } + public void flush() { + if (logger!=null) { + logger.flush(); + } + } + private static String padd = " "; private String format(String s, int minLen) { s = (s==null ? "" : s); diff --git a/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java b/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java index b37ca3bf2e9..22686edf317 100644 --- a/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java +++ b/contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java @@ -87,8 +87,9 @@ public class TestQualityRun extends TestCase { QualityQueryParser qqParser = new SimpleQQParser("title","body"); QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField); - SubmissionReport submitLog = DEBUG ? new SubmissionReport(logger) : null; - QualityStats stats[] = qrun.execute(maxResults, judge, submitLog, logger); + SubmissionReport submitLog = DEBUG ? new SubmissionReport(logger, "TestRun") : null; + qrun.setMaxResults(maxResults); + QualityStats stats[] = qrun.execute(judge, submitLog, logger); // --------- verify by the way judgments were altered for this test: // for some queries, depending on m = qnum % 8