From 9c8ffabfe3ed75768e770f9f3a0171f77fac0e1e Mon Sep 17 00:00:00 2001
From: Christine Poerschke <cpoerschke@apache.org>
Date: Fri, 12 Oct 2018 17:08:35 +0100
Subject: [PATCH] SOLR-12780: Add support for Leaky ReLU and TanH activations
 in contrib/ltr NeuralNetworkModel class. (Kamuela Lau, Christine Poerschke)

---
 solr/CHANGES.txt                              |  3 ++
 .../solr/ltr/model/NeuralNetworkModel.java    | 34 +++++++++++++++++--
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 628abb8bc09..3a58a68f844 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -144,6 +144,9 @@ New Features
 
 * SOLR-12843: Implement a MultiContentWriter in SolrJ to post multiple files/payload at once (noble)
 
+* SOLR-12780: Add support for Leaky ReLU and TanH activations in contrib/ltr NeuralNetworkModel class.
+  (Kamuela Lau, Christine Poerschke)
+
 Other Changes
 ----------------------
 
diff --git a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/NeuralNetworkModel.java b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/NeuralNetworkModel.java
index 798b81c2916..fa92374169c 100644
--- a/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/NeuralNetworkModel.java
+++ b/solr/contrib/ltr/src/java/org/apache/solr/ltr/model/NeuralNetworkModel.java
@@ -31,7 +31,7 @@ import org.apache.solr.util.SolrPluginUtils;
  * A scoring model that computes document scores using a neural network.
  * <p>
  * Supported <a href="https://en.wikipedia.org/wiki/Activation_function">activation functions</a> are:
- * <code>identity</code>, <code>relu</code>, <code>sigmoid</code> and
+ * <code>identity</code>, <code>relu</code>, <code>sigmoid</code>, <code>tanh</code>, <code>leakyrelu</code> and
  * contributions to support additional activation functions are welcome.
  * <p>
  * Example configuration:
@@ -60,8 +60,20 @@ import org.apache.solr.util.SolrPluginUtils;
                 "activation" : "relu"
             },
             {
-                "matrix" : [ [ 27.0, 28.0 ] ],
-                "bias" : [ 29.0 ],
+                "matrix" : [ [ 27.0, 28.0 ],
+                             [ 29.0, 30.0 ] ],
+                "bias" : [ 31.0, 32.0 ],
+                "activation" : "leakyrelu"
+            },
+            {
+                "matrix" : [ [ 33.0, 34.0 ],
+                             [ 35.0, 36.0 ] ],
+                "bias" : [ 37.0, 38.0 ],
+                "activation" : "tanh"
+            },
+            {
+                "matrix" : [ [ 39.0, 40.0 ] ],
+                "bias" : [ 41.0 ],
                 "activation" : "identity"
             }
         ]
@@ -144,6 +156,22 @@ public class NeuralNetworkModel extends LTRScoringModel {
             }
           };
           break;
+        case "leakyrelu":
+          this.activation = new Activation() {
+            @Override
+            public float apply(float in) {
+              return in < 0 ? 0.01f * in : in;
+            }
+          };
+          break;
+        case "tanh":
+          this.activation = new Activation() {
+            @Override
+            public float apply(float in) {
+              return (float)Math.tanh(in);
+            }
+          };
+          break;
         case "sigmoid":
           this.activation = new Activation() {
             @Override