From d80355fd2175029ecdfb65402ec6286baec7b33b Mon Sep 17 00:00:00 2001
From: Robert Muir <rmuir@apache.org>
Date: Thu, 2 Feb 2012 15:46:43 +0000
Subject: [PATCH] LUCENE-3566: parameterize H1 and H2

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1239680 13f79535-47bb-0310-9956-ffa450edef68
---
 .../search/similarities/NormalizationH1.java  | 20 +++++++++++++
 .../search/similarities/NormalizationH2.java  | 21 +++++++++++++-
 .../similarities/DFRSimilarityFactory.java    | 29 +++++++++++++------
 .../similarities/IBSimilarityFactory.java     |  2 +-
 .../src/test-files/solr/conf/schema-dfr.xml   | 12 ++++++++
 .../TestDFRSimilarityFactory.java             | 14 +++++++++
 6 files changed, 87 insertions(+), 11 deletions(-)

diff --git a/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java b/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java
index 77b18055903..c667d783614 100644
--- a/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java
+++ b/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java
@@ -19,9 +19,25 @@ package org.apache.lucene.search.similarities;
 
 /**
  * Normalization model that assumes a uniform distribution of the term frequency.
+ * <p>While this model is parameterless in the
+ * <a href="http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.101.742">
+ * original article</a>, <a href="http://dl.acm.org/citation.cfm?id=1835490">
+ * information-based models</a> (see {@link IBSimilarity}) introduced a
+ * multiplying factor.
+ * The default value for the {@code c} parameter is {@code 1}.</p>
  * @lucene.experimental
  */
 public class NormalizationH1 extends Normalization {
+  private final float c;
+  
+  public NormalizationH1(float c) {
+    this.c = c;
+  }
+  
+  public NormalizationH1() {
+    this(1);
+  }
+  
   @Override
   public final float tfn(BasicStats stats, float tf, float len) {
     return tf * stats.getAvgFieldLength() / len;
@@ -31,4 +47,8 @@ public class NormalizationH1 extends Normalization {
   public String toString() {
     return "1";
   }
+  
+  public float getC() {
+    return c;
+  }
 }
diff --git a/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java b/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java
index 9055e6f7f73..12890dfcb51 100644
--- a/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java
+++ b/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java
@@ -22,16 +22,35 @@ import static org.apache.lucene.search.similarities.SimilarityBase.log2;
 /**
  * Normalization model in which the term frequency is inversely related to the
  * length.
+ * <p>While this model is parameterless in the
+ * <a href="http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.101.742">
+ * original article</a>, the <a href="http://theses.gla.ac.uk/1570/">thesis</a>
+ * introduces the parameterized variant.
+ * The default value for the {@code c} parameter is {@code 1}.</p>
  * @lucene.experimental
  */
 public class NormalizationH2 extends Normalization {
+  private final float c;
+  
+  public NormalizationH2(float c) {
+    this.c = c;
+  }
+
+  public NormalizationH2() {
+    this(1);
+  }
+  
   @Override
   public final float tfn(BasicStats stats, float tf, float len) {
-    return (float)(tf * log2(1 + stats.getAvgFieldLength() / len));
+    return (float)(tf * log2(1 + c * stats.getAvgFieldLength() / len));
   }
 
   @Override
   public String toString() {
     return "2";
   }
+  
+  public float getC() {
+    return c;
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java
index 23ef1a724c1..59daf076125 100644
--- a/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java
+++ b/solr/core/src/java/org/apache/solr/search/similarities/DFRSimilarityFactory.java
@@ -70,8 +70,18 @@ import org.apache.solr.schema.SimilarityFactory;
  *        <ul>
  *           <li>{@link NormalizationH1 H1}: Uniform distribution of term
  *               frequency
+ *               <ul>
+ *                  <li>parameter c (float): hyper-parameter that controls
+ *                      the term frequency normalization with respect to the
+ *                      document length. The default is <code>1</code>
+ *               </ul>
  *           <li>{@link NormalizationH2 H2}: term frequency density inversely
  *               related to length
+ *               <ul>
+ *                  <li>parameter c (float): hyper-parameter that controls
+ *                      the term frequency normalization with respect to the
+ *                      document length. The default is <code>1</code>
+ *                </ul>
  *           <li>{@link NormalizationH3 H3}: term frequency normalization
  *               provided by Dirichlet prior
  *               <ul>
@@ -110,7 +120,7 @@ public class DFRSimilarityFactory extends SimilarityFactory {
     basicModel = parseBasicModel(params.get("basicModel"));
     afterEffect = parseAfterEffect(params.get("afterEffect"));
     normalization = parseNormalization(
-        params.get("normalization"), params.get("mu"), params.get("z"));
+        params.get("normalization"), params.get("c"), params.get("mu"), params.get("z"));
   }
   
   private BasicModel parseBasicModel(String expr) {
@@ -146,11 +156,7 @@ public class DFRSimilarityFactory extends SimilarityFactory {
   }
   
   // also used by IBSimilarityFactory
-  static Normalization parseNormalization(String expr, String mu, String z) {
-    if (mu != null && z != null) {
-      throw new RuntimeException(
-          "specifying mu and z make no sense for: " + expr);
-    }
+  static Normalization parseNormalization(String expr, String c, String mu, String z) {
     if (mu != null && !"H3".equals(expr)) {
       throw new RuntimeException(
           "parameter mu only makes sense for normalization H3");
@@ -159,11 +165,16 @@ public class DFRSimilarityFactory extends SimilarityFactory {
       throw new RuntimeException(
           "parameter z only makes sense for normalization Z");
     }
-    
+    if (c != null && !("H1".equals(expr) || "H2".equals(expr))) {
+      throw new RuntimeException(
+          "parameter c only makese sense for normalizations H1 and H2");
+    }
     if ("H1".equals(expr)) {
-      return new NormalizationH1();
+      return (c != null) ? new NormalizationH1(Float.parseFloat(c))
+                         : new NormalizationH1();
     } else if ("H2".equals(expr)) {
-      return new NormalizationH2();
+      return (c != null) ? new NormalizationH2(Float.parseFloat(c))
+                         : new NormalizationH2();
     } else if ("H3".equals(expr)) {
       return (mu != null) ? new NormalizationH3(Float.parseFloat(mu))
                           : new NormalizationH3();
diff --git a/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java b/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java
index ffb7872d960..012272cf125 100644
--- a/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java
+++ b/solr/core/src/java/org/apache/solr/search/similarities/IBSimilarityFactory.java
@@ -79,7 +79,7 @@ public class IBSimilarityFactory extends SimilarityFactory {
     distribution = parseDistribution(params.get("distribution"));
     lambda = parseLambda(params.get("lambda"));
     normalization = DFRSimilarityFactory.parseNormalization(
-        params.get("normalization"), params.get("mu"), params.get("z"));
+        params.get("normalization"), params.get("c"), params.get("mu"), params.get("z"));
   }
   
   private Distribution parseDistribution(String expr) {
diff --git a/solr/core/src/test-files/solr/conf/schema-dfr.xml b/solr/core/src/test-files/solr/conf/schema-dfr.xml
index fdc4330f124..7ddf5a63399 100644
--- a/solr/core/src/test-files/solr/conf/schema-dfr.xml
+++ b/solr/core/src/test-files/solr/conf/schema-dfr.xml
@@ -42,6 +42,17 @@
         <float name="mu">900</float>
       </similarity>
     </fieldType>
+    
+    <!-- with parameter C -->
+    <fieldType name="text_paramc" class="solr.TextField">
+      <analyzer class="org.apache.lucene.analysis.standard.StandardAnalyzer"/>
+      <similarity class="solr.DFRSimilarityFactory">
+        <str name="basicModel">P</str>
+        <str name="afterEffect">L</str>
+        <str name="normalization">H2</str>
+        <float name="c">7</float>
+      </similarity>
+    </fieldType>
    
   </types>
 
@@ -49,6 +60,7 @@
     <field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
     <field name="text" type="text" indexed="true" stored="false"/>
     <field name="text_params" type="text_params" indexed="true" stored="false"/>
+    <field name="text_paramc" type="text_paramc" indexed="true" stored="false"/>
   </fields>
 
   <defaultSearchField>text</defaultSearchField>
diff --git a/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java b/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java
index 1ba40b67c3f..42b39f5137c 100644
--- a/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java
+++ b/solr/core/src/test/org/apache/solr/search/similarities/TestDFRSimilarityFactory.java
@@ -18,7 +18,9 @@ package org.apache.solr.search.similarities;
  */
 
 import org.apache.lucene.search.similarities.AfterEffectB;
+import org.apache.lucene.search.similarities.AfterEffectL;
 import org.apache.lucene.search.similarities.BasicModelIF;
+import org.apache.lucene.search.similarities.BasicModelP;
 import org.apache.lucene.search.similarities.DFRSimilarity;
 import org.apache.lucene.search.similarities.NormalizationH2;
 import org.apache.lucene.search.similarities.NormalizationH3;
@@ -55,4 +57,16 @@ public class TestDFRSimilarityFactory extends BaseSimilarityTestCase {
     NormalizationH3 norm = (NormalizationH3) dfr.getNormalization();
     assertEquals(900f, norm.getMu(), 0.01f);
   }
+  
+  /** LUCENE-3566 */
+  public void testParameterC() throws Exception {
+    Similarity sim = getSimilarity("text_paramc");
+    assertEquals(DFRSimilarity.class, sim.getClass());
+    DFRSimilarity dfr = (DFRSimilarity) sim;
+    assertEquals(BasicModelP.class, dfr.getBasicModel().getClass());
+    assertEquals(AfterEffectL.class, dfr.getAfterEffect().getClass());
+    assertEquals(NormalizationH2.class, dfr.getNormalization().getClass());
+    NormalizationH2 norm = (NormalizationH2) dfr.getNormalization();
+    assertEquals(7f, norm.getC(), 0.01f);
+  }
 }