diff --git a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java index 0049a5bda99..15733fbbd43 100644 --- a/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/IteratorWriter.java @@ -19,6 +19,9 @@ package org.apache.solr.common; import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; /** * Interface to help do push writing to an array @@ -62,4 +65,16 @@ public interface IteratorWriter { return this; } } + default List toList( List l) throws IOException { + writeIter(new IteratorWriter.ItemWriter() { + @Override + public IteratorWriter.ItemWriter add(Object o) throws IOException { + if (o instanceof MapWriter) o = ((MapWriter) o).toMap(new LinkedHashMap<>()); + if (o instanceof IteratorWriter) o = ((IteratorWriter) o).toList(new ArrayList<>()); + l.add(o); + return this; + } + }); + return l; + } } diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java index 8fb9d031b27..24307d6887e 100644 --- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java +++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java @@ -19,6 +19,8 @@ package org.apache.solr.common; import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -34,6 +36,8 @@ public interface MapWriter extends MapSerializable { writeMap(new EntryWriter() { @Override public EntryWriter put(String k, Object v) throws IOException { + if (v instanceof MapWriter) v = ((MapWriter) v).toMap(new LinkedHashMap<>()); + if (v instanceof IteratorWriter) v = ((IteratorWriter) v).toList(new ArrayList<>()); map.put(k, v); return this; } diff --git a/solr/solrj/src/java/org/apache/solr/recipe/Operand.java b/solr/solrj/src/java/org/apache/solr/recipe/Operand.java new file mode 100644 index 00000000000..f755dad915a --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/recipe/Operand.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.recipe; + +import java.util.Objects; + +/** + * Created by noble on 3/6/17. + */ +public enum Operand { + EQUAL(""), + NOT_EQUAL("!") { + @Override + public boolean canMatch(Object ruleVal, Object testVal) { + return !super.canMatch(ruleVal, testVal); + } + }, + GREATER_THAN(">") { + @Override + public Object match(String val) { + return checkNumeric(super.match(val)); + } + + + @Override + public boolean canMatch(Object ruleVal, Object testVal) { + return testVal != null && compareNum(ruleVal, testVal) == 1; + } + + }, + LESS_THAN("<") { + @Override + public int compare(Object n1Val, Object n2Val) { + return GREATER_THAN.compare(n1Val, n2Val) * -1; + } + + @Override + public boolean canMatch(Object ruleVal, Object testVal) { + return testVal != null && compareNum(ruleVal, testVal) == -1; + } + + @Override + public Object match(String val) { + return checkNumeric(super.match(val)); + } + }; + public final String operand; + + Operand(String val) { + this.operand = val; + } + + public String toStr(Object expectedVal) { + return operand + expectedVal.toString(); + } + + Object checkNumeric(Object val) { + if (val == null) return null; + try { + return Integer.parseInt(val.toString()); + } catch (NumberFormatException e) { + throw new RuntimeException("for operand " + operand + " the value must be numeric"); + } + } + + public Object match(String val) { + if (operand.isEmpty()) return val; + return val.startsWith(operand) ? val.substring(1) : null; + } + + public boolean canMatch(Object ruleVal, Object testVal) { + return Objects.equals(String.valueOf(ruleVal), String.valueOf(testVal)); + } + + + public int compare(Object n1Val, Object n2Val) { + return 0; + } + + public int compareNum(Object n1Val, Object n2Val) { + Integer n1 = (Integer) parseObj(n1Val, Integer.class); + Integer n2 = (Integer) parseObj(n2Val, Integer.class); + return n1 > n2 ? -1 : Objects.equals(n1, n2) ? 0 : 1; + } + + Object parseObj(Object o, Class typ) { + if (o == null) return o; + if (typ == String.class) return String.valueOf(o); + if (typ == Integer.class) { + return Integer.parseInt(String.valueOf(o)); + } + return o; + } +} diff --git a/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java b/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java new file mode 100644 index 00000000000..1ed6800250d --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/recipe/RuleSorter.java @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.recipe; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.solr.common.IteratorWriter; +import org.apache.solr.common.MapWriter; +import org.apache.solr.common.util.Utils; + +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP; +import static org.apache.solr.recipe.Operand.GREATER_THAN; +import static org.apache.solr.recipe.Operand.LESS_THAN; +import static org.apache.solr.recipe.Operand.NOT_EQUAL; + +public class RuleSorter { + public static final String ALL = "#ALL"; + public static final String EACH = "#EACH"; + List conditionClauses = new ArrayList<>(); + List preferences = new ArrayList<>(); + List params= new ArrayList<>(); + + + public RuleSorter(Map jsonMap) { + List> l = getListOfMap("conditions", jsonMap); + conditionClauses = l.stream().map(Clause::new).collect(toList()); + l = getListOfMap("preferences", jsonMap); + preferences = l.stream().map(Preference::new).collect(toList()); + for (int i = 0; i < preferences.size() - 1; i++) { + Preference preference = preferences.get(i); + preference.next = preferences.get(i + 1); + } + + for (Clause c : conditionClauses) { + for (Condition condition : c.conditions) params.add(condition.name); + } + + for (Preference preference : preferences) { + if (params.contains(preference.name.name())) { + throw new RuntimeException(preference.name + " is repeated"); + } + params.add(preference.name.toString()); + preference.idx = params.size() - 1; + } + + } + + public class Session implements MapWriter { + private final List nodes; + private final NodeValueProvider snitch; + private List matrix; + List paramsList = new ArrayList<>(params); + + private Session(List nodes, NodeValueProvider snitch) { + this.nodes = nodes; + this.snitch = snitch; + matrix = new ArrayList<>(nodes.size()); + for (String node : nodes) matrix.add(new Row(node, paramsList, snitch)); + + } + + public void sort() { + if (preferences.size() > 1) { + //this is to set the approximate value according to the precision + ArrayList tmpMatrix = new ArrayList<>(matrix); + for (Preference p : preferences) { + Collections.sort(tmpMatrix, (r1, r2) -> p.compare(r1, r2, false)); + p.setNewVal(tmpMatrix); + } + //approximate values are set now. Let's do recursive sorting + Collections.sort(matrix, (r1, r2) -> preferences.get(0).compare(r1, r2, true)); + } + } + + @Override + public void writeMap(EntryWriter ew) throws IOException { + for (int i = 0; i < matrix.size(); i++) { + Row row = matrix.get(i); + ew.put(row.node, row); + } + } + + @Override + public String toString() { + return Utils.toJSONString(toMap(new LinkedHashMap<>())); + } + public List getSorted(){ + return Collections.unmodifiableList(matrix); + } + } + + + public Session createSession(List nodes, NodeValueProvider snitch) { + return new Session(nodes, snitch); + } + + + private static List> getListOfMap(String key, Map jsonMap) { + Object o = jsonMap.get(key); + if (o != null) { + if (!(o instanceof List)) o = singletonList(o); + return (List) o; + } else { + return Collections.emptyList(); + } + } + + + static class Clause { + List conditions; + boolean strict = true; + + Clause(Map m) { + conditions = m.entrySet().stream() + .filter(e -> !"strict".equals(e.getKey().trim())) + .map(Condition::new) + .collect(toList()); + Object o = m.get("strict"); + if (o == null) return; + strict = o instanceof Boolean ? (Boolean) o : Boolean.parseBoolean(o.toString()); + } + + } + + static class Condition { + String name; + Object val; + Operand operand; + + Condition(Map.Entry m) { + Object expectedVal; + try { + this.name = m.getKey().trim(); + String value = m.getValue().toString().trim(); + if ((expectedVal = NOT_EQUAL.match(value)) != null) { + operand = NOT_EQUAL; + } else if ((expectedVal = GREATER_THAN.match(value)) != null) { + operand = GREATER_THAN; + } else if ((expectedVal = LESS_THAN.match(value)) != null) { + operand = LESS_THAN; + } else { + operand = Operand.EQUAL; + expectedVal = value; + } + + if (name.equals(REPLICA_PROP)) { + if (!ALL.equals(expectedVal)) { + try { + expectedVal = Integer.parseInt(expectedVal.toString()); + } catch (NumberFormatException e) { + throw new RuntimeException("The replica tag value can only be '*' or an integer"); + } + } + } + + } catch (Exception e) { + throw new IllegalArgumentException("Invalid condition : " + name + ":" + val, e); + } + this.val = expectedVal; + + + } + } + + static class Preference { + final SortParam name; + Integer precision; + final Sort sort; + Preference next; + public int idx; + + Preference(Map m) { + sort = Sort.get(m); + name = SortParam.get(m.get(sort.name()).toString()); + Object p = m.getOrDefault("precision", 0); + precision = p instanceof Number ? ((Number) p).intValue() : Integer.parseInt(p.toString()); + + } + + // there are 2 modes of compare. + // recursive, it uses the precision to tie & when there is a tie use the next preference to compare + // in non-recursive mode, precision is not taken into consideration and sort is done on actual value + int compare(Row r1, Row r2, boolean recursive) { + Object o1 = recursive ? r1.cells[idx].val_ : r1.cells[idx].val; + Object o2 = recursive ? r2.cells[idx].val_ : r2.cells[idx].val; + int result = 0; + if (o1 instanceof Integer && o2 instanceof Integer) result = ((Integer) o1).compareTo((Integer) o2); + if (o1 instanceof Long && o2 instanceof Long) result = ((Long) o1).compareTo((Long) o2); + if (o1 instanceof Float && o2 instanceof Float) result = ((Float) o1).compareTo((Float) o2); + if (o1 instanceof Double && o2 instanceof Double) result = ((Double) o1).compareTo((Double) o2); + return result == 0 ? next == null ? 0 : next.compare(r1, r2, recursive) : sort.sortval * result; + } + + //sets the new value according to precision in val_ + void setNewVal(List tmpMatrix) { + Object prevVal = null; + for (Row row : tmpMatrix) { + prevVal = row.cells[idx].val_ = + prevVal == null || Math.abs(((Number) prevVal).longValue() - ((Number) row.cells[idx].val).longValue()) > precision ? + row.cells[idx].val : + prevVal; + } + } + } + + enum SortParam { + freedisk, cores, heap, cpu; + + static SortParam get(String m) { + for (SortParam p : values()) if (p.name().equals(m)) return p; + throw new RuntimeException("Sort must be on one of these " + Arrays.asList(values())); + } + + } + + enum Sort { + maximize(1), minimize(-1); + final int sortval; + + Sort(int i) { + sortval = i; + } + + public static Sort get(Map m) { + if (m.containsKey(maximize.name()) && m.containsKey(minimize.name())) { + throw new RuntimeException("Cannot have both 'maximize' and 'minimize'"); + } + if (m.containsKey(maximize.name())) return maximize; + if (m.containsKey(minimize.name())) return minimize; + throw new RuntimeException("must have either 'maximize' or 'minimize'"); + } + } + + public static class Row implements MapWriter { + public final String node; + final Cell[] cells; + boolean anyValueMissing = false; + + Row(String node, List params, NodeValueProvider snitch) { + this.node = node; + cells = new Cell[params.size()]; + Map vals = new HashMap<>(); + for (String param : params) vals.put(param, null); + snitch.getValues(node, vals); + for (int i = 0; i < params.size(); i++) { + String s = params.get(i); + cells[i] = new Cell(i, s, vals.get(s)); + if (cells[i].val == null) anyValueMissing = true; + } + } + + @Override + public void writeMap(EntryWriter ew) throws IOException { + ew.put(node, (IteratorWriter) iw -> { + for (Cell cell : cells) iw.add(cell); + }); + } + } + + static class Cell implements MapWriter { + final int index; + final String name; + Object val, val_; + + Cell(int index, String name, Object val) { + this.index = index; + this.name = name; + this.val = val; + } + + @Override + public void writeMap(EntryWriter ew) throws IOException { + ew.put(name, val); + } + } + + interface NodeValueProvider { + + void getValues(String node, Map valuesMap); + + } +} diff --git a/solr/solrj/src/java/org/apache/solr/recipe/package-info.java b/solr/solrj/src/java/org/apache/solr/recipe/package-info.java new file mode 100644 index 00000000000..d9730150856 --- /dev/null +++ b/solr/solrj/src/java/org/apache/solr/recipe/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Common classes for recipe parsing filtering nodes & sorting + */ + +package org.apache.solr.recipe; \ No newline at end of file diff --git a/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java b/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java new file mode 100644 index 00000000000..e72aafb987f --- /dev/null +++ b/solr/solrj/src/test/org/apache/solr/recipe/TestRuleSorter.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.recipe; + + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.util.Utils; + +public class TestRuleSorter extends SolrTestCaseJ4 { + public void testRuleParsing() { + + String rules = "{" + + "conditions:[{node:'!overseer', strict:false}, " + + "{replica:'<2',node:'*', shard:'#EACH'}]," + + " preferences:[" + + "{minimize:cores , precision:2}," + + "{maximize:freedisk, precision:50}, " + + "{minimize:heap, precision:1000}]}"; + + + Map nodeValues = (Map) Utils.fromJSONString( "{" + + "node1:{cores:12, freedisk: 334, heap:10480}," + + "node2:{cores:4, freedisk: 749, heap:6873}," + + "node3:{cores:7, freedisk: 262, heap:7834}," + + "node4:{cores:8, freedisk: 375, heap:16900}" + + "}"); + + RuleSorter ruleSorter = new RuleSorter((Map) Utils.fromJSONString(rules)); + RuleSorter.Session session = ruleSorter.createSession(Arrays.asList("node1", "node2","node3","node4"), (node, valuesMap) -> { + Map n = (Map) nodeValues.get(node); + valuesMap.entrySet().stream().forEach(e -> e.setValue(n.get(e.getKey()))); + }); + session.sort(); + List l = session.getSorted(); + assertEquals("node1",l.get(0).node); + assertEquals("node3",l.get(1).node); + assertEquals("node4",l.get(2).node); + assertEquals("node2",l.get(3).node); + + +// System.out.println(session); + + } + + + + +}