SOLR-10278: suggesters implemented

solr/solrj/src/java/org/apache/solr/recipe/AddReplicaSuggester.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+import java.util.Map;
+
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.recipe.RuleSorter.BaseSuggester;
+import org.apache.solr.recipe.RuleSorter.Session;
+
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
+
+class AddReplicaSuggester extends BaseSuggester {
+
+  AddReplicaSuggester(String coll, String shard, Session session) {
+    super(coll, shard, session);
+  }
+
+  Map get() {
+    Map operation = tryEachNode(true);
+    if (operation == null) operation = tryEachNode(false);
+    return operation;
+  }
+
+  Map tryEachNode(boolean strict) {
+    //iterate through elements and identify the least loaded
+    for (int i = matrix.size() - 1; i >= 0; i--) {
+      Row row = matrix.get(i);
+      row = row.addReplica(coll, shard);
+      row.violations.clear();
+      for (Clause clause : session.getRuleSorter().clauses) {
+        if (strict || clause.strict) clause.test(row);
+      }
+      if (row.violations.isEmpty()) {
+        return Utils.makeMap("operation", ADDREPLICA,
+            COLLECTION_PROP, coll,
+            SHARD_ID_PROP, shard,
+            NODE, row.node);
+      }
+    }
+    return null;
+  }
+
+}

solr/solrj/src/java/org/apache/solr/recipe/Cell.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+import java.io.IOException;
+
+import org.apache.solr.common.MapWriter;
+
+class Cell implements MapWriter {
+  final int index;
+  final String name;
+  Object val, val_;
+
+  Cell(int index, String name, Object val) {
+    this.index = index;
+    this.name = name;
+    this.val = val;
+  }
+
+  Cell(int index, String name, Object val, Object val_) {
+    this.index = index;
+    this.name = name;
+    this.val = val;
+    this.val_ = val_;
+  }
+
+  @Override
+  public void writeMap(EntryWriter ew) throws IOException {
+    ew.put(name, val);
+  }
+
+  public Cell copy() {
+    return new Cell(index, name, val, val_);
+  }
+}

solr/solrj/src/java/org/apache/solr/recipe/Clause.java

@@ -26,10 +26,8 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.recipe.RuleSorter.ReplicaStat;
-import org.apache.solr.recipe.RuleSorter.Row;
 
 import static java.util.Collections.singletonMap;
 import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
@@ -46,7 +44,7 @@ import static org.apache.solr.recipe.Operand.WILDCARD;
 import static org.apache.solr.recipe.RuleSorter.ANY;
 import static org.apache.solr.recipe.RuleSorter.EACH;
 
-
+// a set of conditions in a policy
 public class Clause implements MapWriter {
   Map<String, Object> original;
   Condition collection, shard, replica, tag;

solr/solrj/src/java/org/apache/solr/recipe/MoveReplicaSuggester.java

@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+import java.util.Map;
+
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.recipe.RuleSorter.BaseSuggester;
+import org.apache.solr.recipe.RuleSorter.Session;
+
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
+import static org.apache.solr.common.params.CoreAdminParams.REPLICA;
+
+public class MoveReplicaSuggester  extends BaseSuggester{
+
+  MoveReplicaSuggester(String coll, String shard, Session session) {
+    super(coll, shard, session);
+  }
+
+  Map get() {
+    Map operation = tryEachNode(true);
+    if (operation == null) operation = tryEachNode(false);
+    return operation;
+  }
+
+  Map tryEachNode(boolean strict) {
+    //iterate through elements and identify the least loaded
+    for (int i = 0; i < matrix.size(); i++) {
+      Row fromRow = matrix.get(i);
+      Pair<Row, RuleSorter.ReplicaStat> pair = fromRow.removeReplica(coll, shard);
+      fromRow = pair.first();
+      if(fromRow == null){
+        //no such replica available
+        continue;
+      }
+
+      for (Clause clause : session.getRuleSorter().clauses) {
+        if (strict || clause.strict) clause.test(fromRow);
+      }
+      if (fromRow.violations.isEmpty()) {
+        for (int j = matrix.size() - 1; j > i; i--) {
+          Row targetRow = matrix.get(i);
+          targetRow = targetRow.addReplica(coll, shard);
+          for (Clause clause : session.getRuleSorter().clauses) {
+            if (strict || clause.strict) clause.test(targetRow);
+          }
+          if (targetRow.violations.isEmpty()) {
+                return Utils.makeMap("operation", MOVEREPLICA.toLower(),
+                    COLLECTION_PROP, coll,
+                    SHARD_ID_PROP, shard,
+                    NODE, fromRow.node,
+                    REPLICA, pair.second().name,
+                    "target", targetRow.node);
+          }
+        }
+      }
+    }
+    return null;
+  }
+
+}

solr/solrj/src/java/org/apache/solr/recipe/Preference.java

@@ -38,7 +38,7 @@ class Preference {
   // there are 2 modes of compare.
   // recursive, it uses the precision to tie & when there is a tie use the next preference to compare
   // in non-recursive mode, precision is not taken into consideration and sort is done on actual value
-  int compare(RuleSorter.Row r1, RuleSorter.Row r2, boolean recursive) {
+  int compare(Row r1, Row r2, boolean recursive) {
     Object o1 = recursive ? r1.cells[idx].val_ : r1.cells[idx].val;
     Object o2 = recursive ? r2.cells[idx].val_ : r2.cells[idx].val;
     int result = 0;
@@ -50,9 +50,9 @@ class Preference {
   }
 
   //sets the new value according to precision in val_
-  void setApproxVal(List<RuleSorter.Row> tmpMatrix) {
+  void setApproxVal(List<Row> tmpMatrix) {
     Object prevVal = null;
-    for (RuleSorter.Row row : tmpMatrix) {
+    for (Row row : tmpMatrix) {
       prevVal = row.cells[idx].val_ =
           prevVal == null || Math.abs(((Number) prevVal).longValue() - ((Number) row.cells[idx].val).longValue()) > precision ?
               row.cells[idx].val :

solr/solrj/src/java/org/apache/solr/recipe/Row.java

@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.recipe;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Pair;
+import org.apache.solr.common.util.Utils;
+
+import static org.apache.solr.common.params.CoreAdminParams.NODE;
+
+
+class Row implements MapWriter {
+  public final String node;
+  final Cell[] cells;
+  Map<String, Map<String, List<RuleSorter.ReplicaStat>>> replicaInfo;
+  List<Clause> violations = new ArrayList<>();
+  boolean anyValueMissing = false;
+
+  Row(String node, List<String> params, RuleSorter.NodeValueProvider snitch) {
+    replicaInfo = snitch.getReplicaCounts(node, params);
+    if (replicaInfo == null) replicaInfo = Collections.emptyMap();
+    this.node = node;
+    cells = new Cell[params.size()];
+    Map<String, Object> vals = snitch.getValues(node, params);
+    for (int i = 0; i < params.size(); i++) {
+      String s = params.get(i);
+      cells[i] = new Cell(i, s, vals.get(s));
+      if (NODE.equals(s)) cells[i].val = node;
+      if (cells[i].val == null) anyValueMissing = true;
+    }
+  }
+
+  Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<RuleSorter.ReplicaStat>>> replicaInfo, List<Clause> violations) {
+    this.node = node;
+    this.cells = new Cell[cells.length];
+    for (int i = 0; i < this.cells.length; i++) {
+      this.cells[i] = cells[i].copy();
+
+    }
+    this.anyValueMissing = anyValueMissing;
+    this.replicaInfo = replicaInfo;
+    this.violations = violations;
+  }
+
+  @Override
+  public void writeMap(EntryWriter ew) throws IOException {
+    ew.put(node, (IteratorWriter) iw -> {
+      iw.add((MapWriter) e -> e.put("replicas", replicaInfo));
+      for (Cell cell : cells) iw.add(cell);
+    });
+  }
+
+  public Row copy() {
+    return new Row(node, cells, anyValueMissing, Utils.getDeepCopy(replicaInfo, 2), new ArrayList<>(violations));
+  }
+
+  public Object getVal(String name) {
+    for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
+    return null;
+  }
+
+  @Override
+  public String toString() {
+    return node;
+  }
+
+  Row addReplica(String coll, String shard) {
+    Row row = copy();
+    Map<String, List<RuleSorter.ReplicaStat>> c = row.replicaInfo.get(coll);
+    if (c == null) row.replicaInfo.put(coll, c = new HashMap<>());
+    List<RuleSorter.ReplicaStat> s = c.get(shard);
+    if (s == null) c.put(shard, s = new ArrayList<>());
+    return row;
+
+
+  }
+
+  Pair<Row, RuleSorter.ReplicaStat> removeReplica(String coll, String shard) {
+    Row row = copy();
+    Map<String, List<RuleSorter.ReplicaStat>> c = row.replicaInfo.get(coll);
+    if(c == null) return null;
+    List<RuleSorter.ReplicaStat> s = c.get(shard);
+    if (s == null || s.isEmpty()) return null;
+    return new Pair(row,s.remove(0));
+
+  }
+}

solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java

@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -29,18 +30,12 @@ import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
-import org.apache.solr.common.IteratorWriter;
 import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.params.CollectionParams.CollectionAction;
 import org.apache.solr.common.util.Utils;
 
 import static java.util.Collections.singletonList;
 import static java.util.stream.Collectors.toList;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICA;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
-import static org.apache.solr.common.params.CoreAdminParams.NODE;
 
 public class RuleSorter {
   public static final String EACH = "#EACH";
@@ -91,6 +86,10 @@ public class RuleSorter {
           .collect(Collectors.toList());
     }
 
+    RuleSorter getRuleSorter() {
+      return RuleSorter.this;
+
+    }
 
     /**Apply the preferences and conditions
      */
@@ -122,10 +121,10 @@ public class RuleSorter {
           .collect(Collectors.toMap(r -> r.node, r -> r.violations));
     }
 
-    public Operation suggest(CollectionAction action) {
-      if (!supportedActions.contains(action))
-        throw new UnsupportedOperationException(action.toString() + "is not supported");
-      return null;
+    public Map suggest(CollectionAction action, String collection, String shard) {
+      Suggester op = ops.get(action);
+      if (op == null) throw new UnsupportedOperationException(action.toString() + "is not supported");
+      return op.suggest(collection, shard, this);
     }
 
     @Override
@@ -151,7 +150,7 @@ public class RuleSorter {
   }
 
 
-  private static List<Map<String, Object>> getListOfMap(String key, Map<String, Object> jsonMap) {
+  static List<Map<String, Object>> getListOfMap(String key, Map<String, Object> jsonMap) {
     Object o = jsonMap.get(key);
     if (o != null) {
       if (!(o instanceof List)) o = singletonList(o);
@@ -190,96 +189,6 @@ public class RuleSorter {
     }
   }
 
-  static class Row implements MapWriter {
-    public final String node;
-    final Cell[] cells;
-    Map<String, Map<String, List<ReplicaStat>>> replicaInfo;
-    List<Clause> violations = new ArrayList<>();
-    boolean anyValueMissing = false;
-
-    Row(String node, List<String> params, NodeValueProvider snitch) {
-      replicaInfo = snitch.getReplicaCounts(node, params);
-      if (replicaInfo == null) replicaInfo = Collections.emptyMap();
-      this.node = node;
-      cells = new Cell[params.size()];
-      Map<String, Object> vals = snitch.getValues(node, params);
-      for (int i = 0; i < params.size(); i++) {
-        String s = params.get(i);
-        cells[i] = new Cell(i, s, vals.get(s));
-        if (NODE.equals(s)) cells[i].val = node;
-        if (cells[i].val == null) anyValueMissing = true;
-      }
-    }
-
-    Row(String node, Cell[] cells, boolean anyValueMissing, Map<String, Map<String, List<ReplicaStat>>> replicaInfo) {
-      this.node = node;
-      this.cells = new Cell[cells.length];
-      for (int i = 0; i < this.cells.length; i++) {
-        this.cells[i] = cells[i].copy();
-
-      }
-      this.anyValueMissing = anyValueMissing;
-      this.replicaInfo = replicaInfo;
-    }
-
-    @Override
-    public void writeMap(EntryWriter ew) throws IOException {
-      ew.put(node, (IteratorWriter) iw -> {
-        iw.add((MapWriter) e -> e.put("replicas", replicaInfo));
-        for (Cell cell : cells) iw.add(cell);
-      });
-    }
-
-    public Row copy() {
-      return new Row(node, cells, anyValueMissing, replicaInfo);
-    }
-
-    public Object getVal(String name) {
-      for (Cell cell : cells) if (cell.name.equals(name)) return cell.val;
-      return null;
-    }
-
-    @Override
-    public String toString() {
-      return node;
-    }
-  }
-
-  static class Cell implements MapWriter {
-    final int index;
-    final String name;
-    Object val, val_;
-
-    Cell(int index, String name, Object val) {
-      this.index = index;
-      this.name = name;
-      this.val = val;
-    }
-
-    Cell(int index, String name, Object val, Object val_) {
-      this.index = index;
-      this.name = name;
-      this.val = val;
-      this.val_ = val_;
-    }
-
-    @Override
-    public void writeMap(EntryWriter ew) throws IOException {
-      ew.put(name, val);
-    }
-
-    public Cell copy() {
-      return new Cell(index, name, val, val_);
-    }
-  }
-
-  static class Operation {
-    CollectionAction action;
-    String node, collection, shard, replica;
-    String targetNode;
-
-  }
-
 
   static class ReplicaStat implements MapWriter {
     final String name;
@@ -304,12 +213,39 @@ public class RuleSorter {
      * Get the details of each replica in a node. It attempts to fetch as much details about
      * the replica as mentioned in the keys list
      * <p>
-     * the format is {collection:shard :[{replicaetails}]}
+     * the format is {collection:shard :[{replicadetails}]}
      */
     Map<String, Map<String, List<ReplicaStat>>> getReplicaCounts(String node, Collection<String> keys);
   }
 
-  private static final Set<CollectionAction> supportedActions = new HashSet<>(Arrays.asList(ADDREPLICA, DELETEREPLICA, MOVEREPLICA, SPLITSHARD));
+  interface Suggester {
+    Map<String, Object> suggest(String coll, String shard, Session session);
+
+  }
+
+  static class BaseSuggester {
+    final String coll;
+    final String shard;
+    final RuleSorter.Session session;
+    List<Row> matrix;
+
+    BaseSuggester(String coll, String shard, RuleSorter.Session session) {
+      this.coll = coll;
+      this.shard = shard;
+      this.session = session;
+      matrix = session.getMatrixCopy();
+    }
+
+
+  }
+
+  private static final Map<CollectionAction, Suggester> ops = new HashMap<>();
+
+  static {
+    ops.put(CollectionAction.ADDREPLICA, (coll, shard, session) -> new AddReplicaSuggester(coll, shard, session).get());
+    ops.put(CollectionAction.MOVEREPLICA, (coll, shard, session) -> new MoveReplicaSuggester(coll, shard, session).get());
+  }
+
 
 
 }

solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java

@@ -153,7 +153,7 @@ public class TestRuleSorter extends SolrTestCaseJ4 {
     session = ruleSorter.createSession(Arrays.asList("node1", "node2", "node3", "node4"), snitch);
 
     session.applyRules();
-    List<RuleSorter.Row> l = session.getSorted();
+    List<Row> l = session.getSorted();
     assertEquals("node1",l.get(0).node);
     assertEquals("node3",l.get(1).node);
     assertEquals("node4",l.get(2).node);