mirror of https://github.com/apache/lucene.git
SOLR-6349 + SOLR-6682: Added support for stats.field localparams to enable/disable individual stats; Fix response when using EnumField with StatsComponent
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1665579 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Chris M. Hostetter
parent
ce47eee2f4
commit
10dd59701c
|
|
@ -87,6 +87,7 @@ com.sun.jersey.version = 1.9
|
|||
/org.apache.ant/ant = 1.8.2
|
||||
/org.apache.avro/avro = 1.7.5
|
||||
/org.apache.commons/commons-compress = 1.8.1
|
||||
/org.apache.commons/commons-math3 = 3.4.1
|
||||
/org.apache.derby/derby = 10.9.1.0
|
||||
/org.apache.directory.server/apacheds-all = 2.0.0-M15
|
||||
|
||||
|
|
|
|||
|
|
@ -158,6 +158,12 @@ New Features
|
|||
|
||||
* SOLR-7126: Secure loading of runtime external jars (Noble Paul)
|
||||
|
||||
* SOLR-6349: Added support for stats.field localparams to enable/disable individual stats to
|
||||
limit the amount of computation done and the amount of data returned.
|
||||
eg: stats.field={!min=true max=true}field_name
|
||||
(Tomas Fernandez-Lobbe, Xu Zhang, hossman)
|
||||
|
||||
|
||||
Bug Fixes
|
||||
----------------------
|
||||
|
||||
|
|
@ -221,6 +227,9 @@ Bug Fixes
|
|||
looking for start.jar.
|
||||
(Xu Zhang via Shawn Heisey)
|
||||
|
||||
* SOLR-6682: Fix response when using EnumField with StatsComponent
|
||||
(Xu Zhang via hossman)
|
||||
|
||||
Optimizations
|
||||
----------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
<dependency org="com.google.guava" name="guava" rev="${/com.google.guava/guava}" conf="compile"/>
|
||||
<dependency org="com.spatial4j" name="spatial4j" rev="${/com.spatial4j/spatial4j}" conf="compile"/>
|
||||
<dependency org="org.antlr" name="antlr-runtime" rev="${/org.antlr/antlr-runtime}" conf="compile"/>
|
||||
<dependency org="org.apache.commons" name="commons-math3" rev="${/org.apache.commons/commons-math3}" conf="test"/>
|
||||
<dependency org="org.ow2.asm" name="asm" rev="${/org.ow2.asm/asm}" conf="compile"/>
|
||||
<dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" conf="compile"/>
|
||||
<dependency org="org.restlet.jee" name="org.restlet" rev="${/org.restlet.jee/org.restlet}" conf="compile"/>
|
||||
|
|
|
|||
|
|
@ -278,8 +278,7 @@ public class PivotFacetProcessor extends SimpleFacets
|
|||
for (StatsField statsField : statsFields) {
|
||||
stv.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(subset));
|
||||
}
|
||||
// for pivots, we *always* include requested stats - even if 'empty'
|
||||
pivot.add("stats", StatsComponent.convertToResponse(true, stv));
|
||||
pivot.add("stats", StatsComponent.convertToResponse(stv));
|
||||
}
|
||||
values.add( pivot );
|
||||
}
|
||||
|
|
|
|||
|
|
@ -183,8 +183,7 @@ public class PivotFacetValue {
|
|||
}
|
||||
if (null != statsValues) {
|
||||
newList.add(PivotListEntry.STATS.getName(),
|
||||
// for pivots, we *always* include requested stats - even if 'empty'
|
||||
StatsComponent.convertToResponse(true, statsValues));
|
||||
StatsComponent.convertToResponse(statsValues));
|
||||
}
|
||||
return newList;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ public class StatsComponent extends SearchComponent {
|
|||
@Override
|
||||
public void process(ResponseBuilder rb) throws IOException {
|
||||
if (!rb.doStats) return;
|
||||
|
||||
|
||||
boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
|
||||
Map<String, StatsValues> statsValues = new LinkedHashMap<>();
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ public class StatsComponent extends SearchComponent {
|
|||
statsValues.put(statsField.getOutputKey(), statsField.computeLocalStatsValues(docs));
|
||||
}
|
||||
|
||||
rb.rsp.add( "stats", convertToResponse(isShard, statsValues) );
|
||||
rb.rsp.add( "stats", convertToResponse(statsValues) );
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -122,7 +122,7 @@ public class StatsComponent extends SearchComponent {
|
|||
// so that "result" is already stored in the response (for aesthetics)
|
||||
|
||||
Map<String, StatsValues> allStatsValues = rb._statsInfo.getAggregateStatsValues();
|
||||
rb.rsp.add("stats", convertToResponse(false, allStatsValues));
|
||||
rb.rsp.add("stats", convertToResponse(allStatsValues));
|
||||
|
||||
rb._statsInfo = null; // free some objects
|
||||
}
|
||||
|
|
@ -142,7 +142,7 @@ public class StatsComponent extends SearchComponent {
|
|||
* including the esoteric "stats_fields" wrapper.
|
||||
*/
|
||||
public static NamedList<NamedList<NamedList<?>>> convertToResponse
|
||||
(boolean force, Map<String,StatsValues> statsValues) {
|
||||
(Map<String,StatsValues> statsValues) {
|
||||
|
||||
NamedList<NamedList<NamedList<?>>> stats = new SimpleOrderedMap<>();
|
||||
NamedList<NamedList<?>> stats_fields = new SimpleOrderedMap<>();
|
||||
|
|
@ -151,11 +151,7 @@ public class StatsComponent extends SearchComponent {
|
|||
for (Map.Entry<String,StatsValues> entry : statsValues.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
NamedList stv = entry.getValue().getStatsValues();
|
||||
if (force || ((Long) stv.get("count") != 0)) {
|
||||
stats_fields.add(key, stv);
|
||||
} else {
|
||||
stats_fields.add(key, null);
|
||||
}
|
||||
stats_fields.add(key, stv);
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,30 +19,32 @@ package org.apache.solr.handler.component;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
import java.util.IdentityHashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.search.*;
|
||||
import org.apache.lucene.queries.function.FunctionQuery;
|
||||
import org.apache.lucene.queries.function.ValueSource;
|
||||
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
|
||||
import org.apache.lucene.queries.function.valuesource.FieldCacheSource;
|
||||
|
||||
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.SolrException.ErrorCode;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.StatsParams;
|
||||
import org.apache.solr.common.util.StrUtils;
|
||||
import org.apache.solr.request.SolrQueryRequest; // jdocs
|
||||
import org.apache.solr.request.DocValuesStats;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.schema.IndexSchema;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
|
|
@ -60,6 +62,74 @@ import org.apache.solr.search.SyntaxError;
|
|||
* @see StatsComponent
|
||||
*/
|
||||
public class StatsField {
|
||||
|
||||
/**
|
||||
* An enumeration representing the sumer set of all possible stat values that can be computed.
|
||||
* Each of these enum values can be specified as a local param in a <code>stats.field</code>
|
||||
* (eg: <code>stats.field={!min=true mean=true}my_field_name</code>) but not all enum values
|
||||
* are valid for all field types (eg: <code>mean</code> is meaningless for String fields)
|
||||
*
|
||||
* @lucene.internal
|
||||
* @lucene.experimental
|
||||
*/
|
||||
public static enum Stat {
|
||||
min(true),
|
||||
max(true),
|
||||
missing(true),
|
||||
sum(true),
|
||||
count(true),
|
||||
mean(false, sum, count),
|
||||
sumOfSquares(true),
|
||||
stddev(false, sum, count, sumOfSquares),
|
||||
calcdistinct(true);
|
||||
|
||||
private final List<Stat> distribDeps;
|
||||
|
||||
/**
|
||||
* Sole constructor for Stat enum values
|
||||
* @param deps the set of stat values, other then this one, which are a distributed
|
||||
* dependency and must be computed and returned by each individual shards in
|
||||
* order to compute <i>this</i> stat over the entire distributed result set.
|
||||
* @param selfDep indicates that when computing this stat across a distributed result
|
||||
* set, each shard must compute this stat <i>in addition to</i> any other
|
||||
* distributed dependences.
|
||||
* @see #getDistribDeps
|
||||
*/
|
||||
Stat(boolean selfDep, Stat... deps) {
|
||||
distribDeps = new ArrayList<Stat>(deps.length+1);
|
||||
distribDeps.addAll(Arrays.asList(deps));
|
||||
if (selfDep) {
|
||||
distribDeps.add(this);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a String, returns the corrisponding Stat enum value if any, otherwise returns null.
|
||||
*/
|
||||
public static Stat forName(String paramKey) {
|
||||
try {
|
||||
return Stat.valueOf(paramKey);
|
||||
} catch (IllegalArgumentException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The stats that must be computed and returned by each shard involved in a distributed
|
||||
* request in order to compute the overall value for this stat across the entire distributed
|
||||
* result set. A Stat instance may include itself in the <code>getDistribDeps()</code> result,
|
||||
* but that is not always the case.
|
||||
*/
|
||||
public EnumSet<Stat> getDistribDeps() {
|
||||
return EnumSet.copyOf(this.distribDeps);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The set of stats computed by default when no localparams are used to specify explicit stats
|
||||
*/
|
||||
public final static Set<Stat> DEFAULT_STATS = Collections.<Stat>unmodifiableSet
|
||||
(EnumSet.of(Stat.min, Stat.max, Stat.missing, Stat.sum, Stat.count, Stat.mean, Stat.sumOfSquares, Stat.stddev));
|
||||
|
||||
private final SolrIndexSearcher searcher;
|
||||
private final ResponseBuilder rb;
|
||||
|
|
@ -68,10 +138,13 @@ public class StatsField {
|
|||
private final ValueSource valueSource; // may be null if simple field stats
|
||||
private final SchemaField schemaField; // may be null if function/query stats
|
||||
private final String key;
|
||||
private final boolean calcDistinct; // TODO: put this inside localParams ? SOLR-6349 ?
|
||||
private final boolean topLevelCalcDistinct;
|
||||
private final String[] facets;
|
||||
private final List<String> tagList;
|
||||
private final List<String> excludeTagList;
|
||||
private final EnumSet<Stat> statsToCalculate = EnumSet.noneOf(Stat.class);
|
||||
private final EnumSet<Stat> statsInResponse = EnumSet.noneOf(Stat.class);
|
||||
private final boolean isShard;
|
||||
|
||||
/**
|
||||
* @param rb the current request/response
|
||||
|
|
@ -84,6 +157,7 @@ public class StatsField {
|
|||
|
||||
SolrParams params = rb.req.getParams();
|
||||
try {
|
||||
isShard = params.getBool("isShard", false);
|
||||
SolrParams localParams = QueryParsing.getLocalParams(originalParam, params);
|
||||
if (null == localParams) {
|
||||
// simplest possible input: bare string (field name)
|
||||
|
|
@ -91,7 +165,9 @@ public class StatsField {
|
|||
customParams.add(QueryParsing.V, originalParam);
|
||||
localParams = customParams;
|
||||
}
|
||||
|
||||
this.localParams = localParams;
|
||||
|
||||
|
||||
String parserName = localParams.get(QueryParsing.TYPE);
|
||||
SchemaField sf = null;
|
||||
|
|
@ -141,11 +217,12 @@ public class StatsField {
|
|||
// default to entire original param str.
|
||||
originalParam));
|
||||
|
||||
|
||||
this.calcDistinct = null == schemaField
|
||||
? params.getBool(StatsParams.STATS_CALC_DISTINCT, false)
|
||||
: params.getFieldBool(schemaField.getName(), StatsParams.STATS_CALC_DISTINCT, false);
|
||||
|
||||
this.topLevelCalcDistinct = null == schemaField
|
||||
? params.getBool(StatsParams.STATS_CALC_DISTINCT, false)
|
||||
: params.getFieldBool(schemaField.getName(), StatsParams.STATS_CALC_DISTINCT, false);
|
||||
|
||||
populateStatsSets();
|
||||
|
||||
String[] facets = params.getFieldParams(key, StatsParams.STATS_FACET);
|
||||
this.facets = (null == facets) ? new String[0] : facets;
|
||||
String tagStr = localParams.get(CommonParams.TAG);
|
||||
|
|
@ -360,15 +437,6 @@ public class StatsField {
|
|||
return valueSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether or not the effective value of the {@link StatsParams#STATS_CALC_DISTINCT} param
|
||||
* is true or false for this StatsField
|
||||
*/
|
||||
public boolean getCalcDistinct() {
|
||||
return calcDistinct;
|
||||
}
|
||||
|
||||
|
||||
public List<String> getTagList() {
|
||||
return tagList;
|
||||
}
|
||||
|
|
@ -377,4 +445,59 @@ public class StatsField {
|
|||
return "StatsField<" + originalParam + ">";
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A helper method which inspects the {@link #localParams} associated with this StatsField,
|
||||
* and uses them to populate the {@link #statsInResponse} and {@link #statsToCalculate} data
|
||||
* structures
|
||||
*/
|
||||
private void populateStatsSets() {
|
||||
|
||||
boolean statSpecifiedByLocalParam = false;
|
||||
// local individual stat
|
||||
Iterator<String> itParams = localParams.getParameterNamesIterator();
|
||||
while (itParams.hasNext()) {
|
||||
String paramKey = itParams.next();
|
||||
Stat stat = Stat.forName(paramKey);
|
||||
if (stat != null) {
|
||||
statSpecifiedByLocalParam = true;
|
||||
// TODO: this isn't going to work for planned "non-boolean' stats - eg: SOLR-6350, SOLR-6968
|
||||
if (localParams.getBool(paramKey, false)) {
|
||||
statsInResponse.add(stat);
|
||||
statsToCalculate.addAll(stat.getDistribDeps());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if no individual stat setting.
|
||||
if ( ! statSpecifiedByLocalParam ) {
|
||||
statsInResponse.addAll(DEFAULT_STATS);
|
||||
for (Stat stat : statsInResponse) {
|
||||
statsToCalculate.addAll(stat.getDistribDeps());
|
||||
}
|
||||
}
|
||||
|
||||
// calcDistinct has special "default" behavior using top level CalcDistinct param
|
||||
if (topLevelCalcDistinct && localParams.getBool(Stat.calcdistinct.toString(), true)) {
|
||||
statsInResponse.add(Stat.calcdistinct);
|
||||
statsToCalculate.addAll(Stat.calcdistinct.getDistribDeps());
|
||||
}
|
||||
}
|
||||
|
||||
public boolean calculateStats(Stat stat) {
|
||||
return statsToCalculate.contains(stat);
|
||||
}
|
||||
|
||||
public boolean includeInResponse(Stat stat) {
|
||||
if (isShard) {
|
||||
return statsToCalculate.contains(stat);
|
||||
}
|
||||
|
||||
if (statsInResponse.contains(stat)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import org.apache.solr.common.EnumFieldValue;
|
|||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.common.util.SimpleOrderedMap;
|
||||
import org.apache.solr.handler.component.StatsField.Stat;
|
||||
import org.apache.solr.schema.*;
|
||||
|
||||
/**
|
||||
|
|
@ -79,17 +80,19 @@ abstract class AbstractStatsValues<T> implements StatsValues {
|
|||
/** Tracks all data about tthe stats we need to collect */
|
||||
final protected StatsField statsField;
|
||||
|
||||
/**
|
||||
* local copy to save method dispatch in tight loops
|
||||
* @see StatsField#getCalcDistinct
|
||||
*/
|
||||
final protected boolean calcDistinct;
|
||||
|
||||
/** may be null if we are collecting stats directly from a function ValueSource */
|
||||
final protected SchemaField sf;
|
||||
/** may be null if we are collecting stats directly from a function ValueSource */
|
||||
final protected FieldType ft;
|
||||
|
||||
// final booleans from StatsField to allow better inlining & JIT optimizing
|
||||
final protected boolean computeCount;
|
||||
final protected boolean computeMissing;
|
||||
final protected boolean computeCalcDistinct;
|
||||
final protected boolean computeMin;
|
||||
final protected boolean computeMax;
|
||||
final protected boolean computeMinOrMax;
|
||||
|
||||
/**
|
||||
* Either a function value source to collect from, or the ValueSource associated
|
||||
* with a single valued field we are collecting from. Will be null until/unless
|
||||
|
|
@ -112,15 +115,21 @@ abstract class AbstractStatsValues<T> implements StatsValues {
|
|||
protected long missing;
|
||||
protected long count;
|
||||
protected long countDistinct;
|
||||
protected Set<T> distinctValues;
|
||||
protected final Set<T> distinctValues;
|
||||
|
||||
// facetField facetValue
|
||||
protected Map<String, Map<String, StatsValues>> facets = new HashMap<>();
|
||||
|
||||
protected AbstractStatsValues(StatsField statsField) {
|
||||
this.statsField = statsField;
|
||||
this.calcDistinct = statsField.getCalcDistinct();
|
||||
this.distinctValues = new TreeSet<>();
|
||||
this.computeCount = statsField.calculateStats(Stat.count);
|
||||
this.computeMissing = statsField.calculateStats(Stat.missing);
|
||||
this.computeCalcDistinct = statsField.calculateStats(Stat.calcdistinct);
|
||||
this.computeMin = statsField.calculateStats(Stat.min);
|
||||
this.computeMax = statsField.calculateStats(Stat.max);
|
||||
this.computeMinOrMax = computeMin || computeMax;
|
||||
|
||||
this.distinctValues = computeCalcDistinct ? new TreeSet<>() : null;
|
||||
|
||||
// alternatively, we could refactor a common base class that doesn't know/care
|
||||
// about either SchemaField or ValueSource - but then there would be a lot of
|
||||
|
|
@ -149,14 +158,20 @@ abstract class AbstractStatsValues<T> implements StatsValues {
|
|||
*/
|
||||
@Override
|
||||
public void accumulate(NamedList stv) {
|
||||
count += (Long) stv.get("count");
|
||||
missing += (Long) stv.get("missing");
|
||||
if (calcDistinct) {
|
||||
if (computeCount) {
|
||||
count += (Long) stv.get("count");
|
||||
}
|
||||
if (computeMissing) {
|
||||
missing += (Long) stv.get("missing");
|
||||
}
|
||||
if (computeCalcDistinct) {
|
||||
distinctValues.addAll((Collection<T>) stv.get("distinctValues"));
|
||||
countDistinct = distinctValues.size();
|
||||
}
|
||||
|
||||
updateMinMax((T) stv.get("min"), (T) stv.get("max"));
|
||||
|
||||
if (computeMinOrMax) {
|
||||
updateMinMax((T) stv.get("min"), (T) stv.get("max"));
|
||||
}
|
||||
updateTypeSpecificStats(stv);
|
||||
|
||||
NamedList f = (NamedList) stv.get(FACETS);
|
||||
|
|
@ -197,12 +212,16 @@ abstract class AbstractStatsValues<T> implements StatsValues {
|
|||
}
|
||||
|
||||
public void accumulate(T value, int count) {
|
||||
this.count += count;
|
||||
if (calcDistinct) {
|
||||
if (computeCount) {
|
||||
this.count += count;
|
||||
}
|
||||
if (computeCalcDistinct) {
|
||||
distinctValues.add(value);
|
||||
countDistinct = distinctValues.size();
|
||||
}
|
||||
updateMinMax(value, value);
|
||||
if (computeMinOrMax) {
|
||||
updateMinMax(value, value);
|
||||
}
|
||||
updateTypeSpecificStats(value, count);
|
||||
}
|
||||
|
||||
|
|
@ -211,7 +230,9 @@ abstract class AbstractStatsValues<T> implements StatsValues {
|
|||
*/
|
||||
@Override
|
||||
public void missing() {
|
||||
missing++;
|
||||
if (computeMissing) {
|
||||
missing++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -237,27 +258,39 @@ abstract class AbstractStatsValues<T> implements StatsValues {
|
|||
public NamedList<?> getStatsValues() {
|
||||
NamedList<Object> res = new SimpleOrderedMap<>();
|
||||
|
||||
res.add("min", min);
|
||||
res.add("max", max);
|
||||
res.add("count", count);
|
||||
res.add("missing", missing);
|
||||
if (calcDistinct) {
|
||||
if (statsField.includeInResponse(Stat.min)) {
|
||||
res.add("min", min);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.max)) {
|
||||
res.add("max", max);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.count)) {
|
||||
res.add("count", count);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.missing)) {
|
||||
res.add("missing", missing);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.calcdistinct)) {
|
||||
res.add("distinctValues", distinctValues);
|
||||
res.add("countDistinct", countDistinct);
|
||||
}
|
||||
|
||||
addTypeSpecificStats(res);
|
||||
|
||||
if (!facets.isEmpty()) {
|
||||
|
||||
// add the facet stats
|
||||
NamedList<NamedList<?>> nl = new SimpleOrderedMap<>();
|
||||
for (Map.Entry<String, Map<String, StatsValues>> entry : facets.entrySet()) {
|
||||
NamedList<NamedList<?>> nl2 = new SimpleOrderedMap<>();
|
||||
nl.add(entry.getKey(), nl2);
|
||||
for (Map.Entry<String, StatsValues> e2 : entry.getValue().entrySet()) {
|
||||
nl2.add(e2.getKey(), e2.getValue().getStatsValues());
|
||||
}
|
||||
// add the facet stats
|
||||
NamedList<NamedList<?>> nl = new SimpleOrderedMap<>();
|
||||
for (Map.Entry<String, Map<String, StatsValues>> entry : facets.entrySet()) {
|
||||
NamedList<NamedList<?>> nl2 = new SimpleOrderedMap<>();
|
||||
nl.add(entry.getKey(), nl2);
|
||||
for (Map.Entry<String, StatsValues> e2 : entry.getValue().entrySet()) {
|
||||
nl2.add(e2.getKey(), e2.getValue().getStatsValues());
|
||||
}
|
||||
}
|
||||
res.add(FACETS, nl);
|
||||
}
|
||||
res.add(FACETS, nl);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
|
|
@ -314,8 +347,13 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
|
|||
double sum;
|
||||
double sumOfSquares;
|
||||
|
||||
final protected boolean computeSum;
|
||||
final protected boolean computeSumOfSquares;
|
||||
|
||||
public NumericStatsValues(StatsField statsField) {
|
||||
super(statsField);
|
||||
this.computeSum = statsField.calculateStats(Stat.sum);
|
||||
this.computeSumOfSquares = statsField.calculateStats(Stat.sumOfSquares);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -332,8 +370,12 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
|
|||
*/
|
||||
@Override
|
||||
public void updateTypeSpecificStats(NamedList stv) {
|
||||
sum += ((Number)stv.get("sum")).doubleValue();
|
||||
sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
|
||||
if (computeSum) {
|
||||
sum += ((Number)stv.get("sum")).doubleValue();
|
||||
}
|
||||
if (computeSumOfSquares) {
|
||||
sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -342,8 +384,12 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
|
|||
@Override
|
||||
public void updateTypeSpecificStats(Number v, int count) {
|
||||
double value = v.doubleValue();
|
||||
sumOfSquares += (value * value * count); // for std deviation
|
||||
sum += value * count;
|
||||
if (computeSumOfSquares) {
|
||||
sumOfSquares += (value * value * count); // for std deviation
|
||||
}
|
||||
if (computeSum) {
|
||||
sum += value * count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -351,22 +397,23 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
|
|||
*/
|
||||
@Override
|
||||
protected void updateMinMax(Number min, Number max) {
|
||||
if (null == min) {
|
||||
assert null == max : "min is null but max isn't ? ==> " + max;
|
||||
return; // No-Op
|
||||
}
|
||||
|
||||
assert null != max : "max is null but min isn't ? ==> " + min;
|
||||
|
||||
// we always use the double value, because that way the response Object class is
|
||||
// we always use the double values, because that way the response Object class is
|
||||
// consistent regardless of whether we only have 1 value or many that we min/max
|
||||
//
|
||||
// TODO: would be nice to have subclasses for each type of Number ... breaks backcompat
|
||||
double minD = min.doubleValue();
|
||||
double maxD = max.doubleValue();
|
||||
|
||||
this.min = (null == this.min) ? minD : Math.min(this.min.doubleValue(), minD);
|
||||
this.max = (null == this.max) ? maxD : Math.max(this.max.doubleValue(), maxD);
|
||||
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
|
||||
if (null != min) {
|
||||
double minD = min.doubleValue();
|
||||
this.min = (null == this.min) ? minD : Math.min(this.min.doubleValue(), minD);
|
||||
}
|
||||
}
|
||||
if (computeMax) { // nested if to encourage JIT to optimize aware final var?
|
||||
if (null != max) {
|
||||
double maxD = max.doubleValue();
|
||||
this.max = (null == this.max) ? maxD : Math.max(this.max.doubleValue(), maxD);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -376,10 +423,18 @@ class NumericStatsValues extends AbstractStatsValues<Number> {
|
|||
*/
|
||||
@Override
|
||||
protected void addTypeSpecificStats(NamedList<Object> res) {
|
||||
res.add("sum", sum);
|
||||
res.add("sumOfSquares", sumOfSquares);
|
||||
res.add("mean", sum / count);
|
||||
res.add("stddev", getStandardDeviation());
|
||||
if (statsField.includeInResponse(Stat.sum)) {
|
||||
res.add("sum", sum);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.sumOfSquares)) {
|
||||
res.add("sumOfSquares", sumOfSquares);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.mean)) {
|
||||
res.add("mean", sum / count);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.stddev)) {
|
||||
res.add("stddev", getStandardDeviation());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -424,14 +479,20 @@ class EnumStatsValues extends AbstractStatsValues<EnumFieldValue> {
|
|||
* {@inheritDoc}
|
||||
*/
|
||||
protected void updateMinMax(EnumFieldValue min, EnumFieldValue max) {
|
||||
if (max != null) {
|
||||
if (max.compareTo(this.max) > 0)
|
||||
this.max = max;
|
||||
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
|
||||
if (null != min) {
|
||||
if (null == this.min || (min.compareTo(this.min) < 0)) {
|
||||
this.min = min;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (computeMax) { // nested if to encourage JIT to optimize aware final var?
|
||||
if (null != max) {
|
||||
if (null == this.max || (max.compareTo(this.max) > 0)) {
|
||||
this.max = max;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.min == null)
|
||||
this.min = min;
|
||||
else if (this.min.compareTo(min) > 0)
|
||||
this.min = min;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -470,8 +531,13 @@ class DateStatsValues extends AbstractStatsValues<Date> {
|
|||
private long sum = 0;
|
||||
double sumOfSquares = 0;
|
||||
|
||||
final protected boolean computeSum;
|
||||
final protected boolean computeSumOfSquares;
|
||||
|
||||
public DateStatsValues(StatsField statsField) {
|
||||
super(statsField);
|
||||
this.computeSum = statsField.calculateStats(Stat.sum);
|
||||
this.computeSumOfSquares = statsField.calculateStats(Stat.sumOfSquares);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -488,9 +554,10 @@ class DateStatsValues extends AbstractStatsValues<Date> {
|
|||
*/
|
||||
@Override
|
||||
protected void updateTypeSpecificStats(NamedList stv) {
|
||||
Date date = (Date) stv.get("sum");
|
||||
if (date != null) {
|
||||
sum += date.getTime();
|
||||
if (computeSum) {
|
||||
sum += ((Date) stv.get("sum")).getTime();
|
||||
}
|
||||
if (computeSumOfSquares) {
|
||||
sumOfSquares += ((Number)stv.get("sumOfSquares")).doubleValue();
|
||||
}
|
||||
}
|
||||
|
|
@ -501,8 +568,12 @@ class DateStatsValues extends AbstractStatsValues<Date> {
|
|||
@Override
|
||||
public void updateTypeSpecificStats(Date v, int count) {
|
||||
long value = v.getTime();
|
||||
sumOfSquares += (value * value * count); // for std deviation
|
||||
sum += value * count;
|
||||
if (computeSumOfSquares) {
|
||||
sumOfSquares += (value * value * count); // for std deviation
|
||||
}
|
||||
if (computeSum) {
|
||||
sum += value * count;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -510,11 +581,15 @@ class DateStatsValues extends AbstractStatsValues<Date> {
|
|||
*/
|
||||
@Override
|
||||
protected void updateMinMax(Date min, Date max) {
|
||||
if(null != min && (this.min==null || this.min.after(min))) {
|
||||
this.min = min;
|
||||
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
|
||||
if (null != min && (this.min==null || this.min.after(min))) {
|
||||
this.min = min;
|
||||
}
|
||||
}
|
||||
if(null != max && (this.max==null || this.max.before(max))) {
|
||||
this.max = max;
|
||||
if (computeMax) { // nested if to encourage JIT to optimize aware final var?
|
||||
if (null != max && (this.max==null || this.max.before(max))) {
|
||||
this.max = max;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -525,15 +600,18 @@ class DateStatsValues extends AbstractStatsValues<Date> {
|
|||
*/
|
||||
@Override
|
||||
protected void addTypeSpecificStats(NamedList<Object> res) {
|
||||
if(sum<=0) {
|
||||
return; // date==0 is meaningless
|
||||
if (statsField.includeInResponse(Stat.sum)) {
|
||||
res.add("sum", new Date(sum));
|
||||
}
|
||||
res.add("sum", new Date(sum));
|
||||
if (count > 0) {
|
||||
res.add("mean", new Date(sum / count));
|
||||
if (statsField.includeInResponse(Stat.mean)) {
|
||||
res.add("mean", (count > 0) ? new Date(sum / count) : null);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.sumOfSquares)) {
|
||||
res.add("sumOfSquares", sumOfSquares);
|
||||
}
|
||||
if (statsField.includeInResponse(Stat.stddev)) {
|
||||
res.add("stddev", getStandardDeviation());
|
||||
}
|
||||
res.add("sumOfSquares", sumOfSquares);
|
||||
res.add("stddev", getStandardDeviation());
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -594,8 +672,12 @@ class StringStatsValues extends AbstractStatsValues<String> {
|
|||
*/
|
||||
@Override
|
||||
protected void updateMinMax(String min, String max) {
|
||||
this.max = max(this.max, max);
|
||||
this.min = min(this.min, min);
|
||||
if (computeMin) { // nested if to encourage JIT to optimize aware final var?
|
||||
this.min = min(this.min, min);
|
||||
}
|
||||
if (computeMax) { // nested if to encourage JIT to optimize aware final var?
|
||||
this.max = max(this.max, max);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -56,7 +56,6 @@ public class DocValuesStats {
|
|||
public static StatsValues getCounts(SolrIndexSearcher searcher, StatsField statsField, DocSet docs, String[] facet) throws IOException {
|
||||
|
||||
final SchemaField schemaField = statsField.getSchemaField();
|
||||
final boolean calcDistinct = statsField.getCalcDistinct();
|
||||
|
||||
assert null != statsField.getSchemaField()
|
||||
: "DocValuesStats requires a StatsField using a SchemaField";
|
||||
|
|
|
|||
|
|
@ -23,35 +23,17 @@ import java.util.Map;
|
|||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.SortedDocValues;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.index.TermsEnum;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.search.TermRangeQuery;
|
||||
import org.apache.lucene.uninverting.DocTermOrds;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.BytesRefBuilder;
|
||||
import org.apache.lucene.util.CharsRefBuilder;
|
||||
import org.apache.lucene.util.FixedBitSet;
|
||||
import org.apache.lucene.util.UnicodeUtil;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.FacetParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.handler.component.FieldFacetStats;
|
||||
import org.apache.solr.handler.component.StatsField;
|
||||
import org.apache.solr.handler.component.StatsValues;
|
||||
import org.apache.solr.handler.component.StatsValuesFactory;
|
||||
import org.apache.solr.schema.FieldType;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.schema.TrieField;
|
||||
import org.apache.solr.search.BitDocSet;
|
||||
import org.apache.solr.search.DocIterator;
|
||||
import org.apache.solr.search.DocSet;
|
||||
import org.apache.solr.search.SolrCache;
|
||||
import org.apache.solr.search.SolrIndexSearcher;
|
||||
import org.apache.solr.util.LongPriorityQueue;
|
||||
import org.apache.solr.util.PrimUtils;
|
||||
|
||||
/**
|
||||
*
|
||||
|
|
@ -84,7 +66,6 @@ import org.apache.solr.util.PrimUtils;
|
|||
*
|
||||
*/
|
||||
public class UnInvertedField extends DocTermOrds {
|
||||
private static int TNUM_OFFSET=2;
|
||||
|
||||
static class TopTerm {
|
||||
BytesRef term;
|
||||
|
|
@ -214,420 +195,6 @@ public class UnInvertedField extends DocTermOrds {
|
|||
return numTermsInField;
|
||||
}
|
||||
|
||||
public NamedList<Integer> getCounts(SolrIndexSearcher searcher, DocSet baseDocs, int offset, int limit, Integer mincount, boolean missing, String sort, String prefix) throws IOException {
|
||||
use.incrementAndGet();
|
||||
|
||||
FieldType ft = searcher.getSchema().getFieldType(field);
|
||||
|
||||
NamedList<Integer> res = new NamedList<>(); // order is important
|
||||
|
||||
DocSet docs = baseDocs;
|
||||
int baseSize = docs.size();
|
||||
int maxDoc = searcher.maxDoc();
|
||||
|
||||
//System.out.println("GET COUNTS field=" + field + " baseSize=" + baseSize + " minCount=" + mincount + " maxDoc=" + maxDoc + " numTermsInField=" + numTermsInField);
|
||||
if (baseSize >= mincount) {
|
||||
|
||||
final int[] index = this.index;
|
||||
// tricky: we add more more element than we need because we will reuse this array later
|
||||
// for ordering term ords before converting to term labels.
|
||||
final int[] counts = new int[numTermsInField + 1];
|
||||
|
||||
//
|
||||
// If there is prefix, find its start and end term numbers
|
||||
//
|
||||
int startTerm = 0;
|
||||
int endTerm = numTermsInField; // one past the end
|
||||
|
||||
TermsEnum te = getOrdTermsEnum(searcher.getLeafReader());
|
||||
if (te != null && prefix != null && prefix.length() > 0) {
|
||||
final BytesRefBuilder prefixBr = new BytesRefBuilder();
|
||||
prefixBr.copyChars(prefix);
|
||||
if (te.seekCeil(prefixBr.get()) == TermsEnum.SeekStatus.END) {
|
||||
startTerm = numTermsInField;
|
||||
} else {
|
||||
startTerm = (int) te.ord();
|
||||
}
|
||||
prefixBr.append(UnicodeUtil.BIG_TERM);
|
||||
if (te.seekCeil(prefixBr.get()) == TermsEnum.SeekStatus.END) {
|
||||
endTerm = numTermsInField;
|
||||
} else {
|
||||
endTerm = (int) te.ord();
|
||||
}
|
||||
}
|
||||
|
||||
/***********
|
||||
// Alternative 2: get the docSet of the prefix (could take a while) and
|
||||
// then do the intersection with the baseDocSet first.
|
||||
if (prefix != null && prefix.length() > 0) {
|
||||
docs = searcher.getDocSet(new ConstantScorePrefixQuery(new Term(field, ft.toInternal(prefix))), docs);
|
||||
// The issue with this method are problems of returning 0 counts for terms w/o
|
||||
// the prefix. We can't just filter out those terms later because it may
|
||||
// mean that we didn't collect enough terms in the queue (in the sorted case).
|
||||
}
|
||||
***********/
|
||||
|
||||
boolean doNegative = baseSize > maxDoc >> 1 && termInstances > 0
|
||||
&& startTerm==0 && endTerm==numTermsInField
|
||||
&& docs instanceof BitDocSet;
|
||||
|
||||
if (doNegative) {
|
||||
FixedBitSet bs = ((BitDocSet)docs).getBits().clone();
|
||||
bs.flip(0, maxDoc);
|
||||
// TODO: when iterator across negative elements is available, use that
|
||||
// instead of creating a new bitset and inverting.
|
||||
docs = new BitDocSet(bs, maxDoc - baseSize);
|
||||
// simply negating will mean that we have deleted docs in the set.
|
||||
// that should be OK, as their entries in our table should be empty.
|
||||
//System.out.println(" NEG");
|
||||
}
|
||||
|
||||
// For the biggest terms, do straight set intersections
|
||||
for (TopTerm tt : bigTerms.values()) {
|
||||
//System.out.println(" do big termNum=" + tt.termNum + " term=" + tt.term.utf8ToString());
|
||||
// TODO: counts could be deferred if sorted==false
|
||||
if (tt.termNum >= startTerm && tt.termNum < endTerm) {
|
||||
counts[tt.termNum] = searcher.numDocs(new TermQuery(new Term(field, tt.term)), docs);
|
||||
//System.out.println(" count=" + counts[tt.termNum]);
|
||||
} else {
|
||||
//System.out.println("SKIP term=" + tt.termNum);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: we could short-circuit counting altogether for sorted faceting
|
||||
// where we already have enough terms from the bigTerms
|
||||
|
||||
// TODO: we could shrink the size of the collection array, and
|
||||
// additionally break when the termNumber got above endTerm, but
|
||||
// it would require two extra conditionals in the inner loop (although
|
||||
// they would be predictable for the non-prefix case).
|
||||
// Perhaps a different copy of the code would be warranted.
|
||||
|
||||
if (termInstances > 0) {
|
||||
DocIterator iter = docs.iterator();
|
||||
while (iter.hasNext()) {
|
||||
int doc = iter.nextDoc();
|
||||
//System.out.println("iter doc=" + doc);
|
||||
int code = index[doc];
|
||||
|
||||
if ((code & 0xff)==1) {
|
||||
//System.out.println(" ptr");
|
||||
int pos = code>>>8;
|
||||
int whichArray = (doc >>> 16) & 0xff;
|
||||
byte[] arr = tnums[whichArray];
|
||||
int tnum = 0;
|
||||
for(;;) {
|
||||
int delta = 0;
|
||||
for(;;) {
|
||||
byte b = arr[pos++];
|
||||
delta = (delta << 7) | (b & 0x7f);
|
||||
if ((b & 0x80) == 0) break;
|
||||
}
|
||||
if (delta == 0) break;
|
||||
tnum += delta - TNUM_OFFSET;
|
||||
//System.out.println(" tnum=" + tnum);
|
||||
counts[tnum]++;
|
||||
}
|
||||
} else {
|
||||
//System.out.println(" inlined");
|
||||
int tnum = 0;
|
||||
int delta = 0;
|
||||
for (;;) {
|
||||
delta = (delta << 7) | (code & 0x7f);
|
||||
if ((code & 0x80)==0) {
|
||||
if (delta==0) break;
|
||||
tnum += delta - TNUM_OFFSET;
|
||||
//System.out.println(" tnum=" + tnum);
|
||||
counts[tnum]++;
|
||||
delta = 0;
|
||||
}
|
||||
code >>>= 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
final CharsRefBuilder charsRef = new CharsRefBuilder();
|
||||
|
||||
int off=offset;
|
||||
int lim=limit>=0 ? limit : Integer.MAX_VALUE;
|
||||
|
||||
if (sort.equals(FacetParams.FACET_SORT_COUNT) || sort.equals(FacetParams.FACET_SORT_COUNT_LEGACY)) {
|
||||
int maxsize = limit>0 ? offset+limit : Integer.MAX_VALUE-1;
|
||||
maxsize = Math.min(maxsize, numTermsInField);
|
||||
LongPriorityQueue queue = new LongPriorityQueue(Math.min(maxsize,1000), maxsize, Long.MIN_VALUE);
|
||||
|
||||
int min=mincount-1; // the smallest value in the top 'N' values
|
||||
//System.out.println("START=" + startTerm + " END=" + endTerm);
|
||||
for (int i=startTerm; i<endTerm; i++) {
|
||||
int c = doNegative ? maxTermCounts[i] - counts[i] : counts[i];
|
||||
if (c>min) {
|
||||
// NOTE: we use c>min rather than c>=min as an optimization because we are going in
|
||||
// index order, so we already know that the keys are ordered. This can be very
|
||||
// important if a lot of the counts are repeated (like zero counts would be).
|
||||
|
||||
// smaller term numbers sort higher, so subtract the term number instead
|
||||
long pair = (((long)c)<<32) + (Integer.MAX_VALUE - i);
|
||||
boolean displaced = queue.insert(pair);
|
||||
if (displaced) min=(int)(queue.top() >>> 32);
|
||||
}
|
||||
}
|
||||
|
||||
// now select the right page from the results
|
||||
|
||||
// if we are deep paging, we don't have to order the highest "offset" counts.
|
||||
int collectCount = Math.max(0, queue.size() - off);
|
||||
assert collectCount <= lim;
|
||||
|
||||
// the start and end indexes of our list "sorted" (starting with the highest value)
|
||||
int sortedIdxStart = queue.size() - (collectCount - 1);
|
||||
int sortedIdxEnd = queue.size() + 1;
|
||||
final long[] sorted = queue.sort(collectCount);
|
||||
|
||||
final int[] indirect = counts; // reuse the counts array for the index into the tnums array
|
||||
assert indirect.length >= sortedIdxEnd;
|
||||
|
||||
for (int i=sortedIdxStart; i<sortedIdxEnd; i++) {
|
||||
long pair = sorted[i];
|
||||
int c = (int)(pair >>> 32);
|
||||
int tnum = Integer.MAX_VALUE - (int)pair;
|
||||
|
||||
indirect[i] = i; // store the index for indirect sorting
|
||||
sorted[i] = tnum; // reuse the "sorted" array to store the term numbers for indirect sorting
|
||||
|
||||
// add a null label for now... we'll fill it in later.
|
||||
res.add(null, c);
|
||||
}
|
||||
|
||||
// now sort the indexes by the term numbers
|
||||
PrimUtils.sort(sortedIdxStart, sortedIdxEnd, indirect, new PrimUtils.IntComparator() {
|
||||
@Override
|
||||
public int compare(int a, int b) {
|
||||
return (int)sorted[a] - (int)sorted[b];
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean lessThan(int a, int b) {
|
||||
return sorted[a] < sorted[b];
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(int a, int b) {
|
||||
return sorted[a] == sorted[b];
|
||||
}
|
||||
});
|
||||
|
||||
// convert the term numbers to term values and set
|
||||
// as the label
|
||||
//System.out.println("sortStart=" + sortedIdxStart + " end=" + sortedIdxEnd);
|
||||
for (int i=sortedIdxStart; i<sortedIdxEnd; i++) {
|
||||
int idx = indirect[i];
|
||||
int tnum = (int)sorted[idx];
|
||||
final String label = getReadableValue(getTermValue(te, tnum), ft, charsRef);
|
||||
//System.out.println(" label=" + label);
|
||||
res.setName(idx - sortedIdxStart, label);
|
||||
}
|
||||
|
||||
} else {
|
||||
// add results in index order
|
||||
int i=startTerm;
|
||||
if (mincount<=0) {
|
||||
// if mincount<=0, then we won't discard any terms and we know exactly
|
||||
// where to start.
|
||||
i=startTerm+off;
|
||||
off=0;
|
||||
}
|
||||
|
||||
for (; i<endTerm; i++) {
|
||||
int c = doNegative ? maxTermCounts[i] - counts[i] : counts[i];
|
||||
if (c<mincount || --off>=0) continue;
|
||||
if (--lim<0) break;
|
||||
|
||||
final String label = getReadableValue(getTermValue(te, i), ft, charsRef);
|
||||
res.add(label, c);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (missing) {
|
||||
// TODO: a faster solution for this?
|
||||
res.add(null, SimpleFacets.getFieldMissingCount(searcher, baseDocs, field));
|
||||
}
|
||||
|
||||
//System.out.println(" res=" + res);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect statistics about the UninvertedField. Code is very similar to {@link #getCounts(org.apache.solr.search.SolrIndexSearcher, org.apache.solr.search.DocSet, int, int, Integer, boolean, String, String)}
|
||||
* It can be used to calculate stats on multivalued fields.
|
||||
* <p>
|
||||
* This method is mainly used by the {@link org.apache.solr.handler.component.StatsComponent}.
|
||||
*
|
||||
* @param searcher The Searcher to use to gather the statistics
|
||||
* @param baseDocs The {@link org.apache.solr.search.DocSet} to gather the stats on
|
||||
* @param statsField the {@link StatsField} param corrisponding to a real {@link SchemaField} to compute stats over
|
||||
* @param facet One or more fields to facet on.
|
||||
* @return The {@link org.apache.solr.handler.component.StatsValues} collected
|
||||
* @throws IOException If there is a low-level I/O error.
|
||||
*/
|
||||
public StatsValues getStats(SolrIndexSearcher searcher, DocSet baseDocs, StatsField statsField, String[] facet) throws IOException {
|
||||
//this function is ripped off nearly wholesale from the getCounts function to use
|
||||
//for multiValued fields within the StatsComponent. may be useful to find common
|
||||
//functionality between the two and refactor code somewhat
|
||||
use.incrementAndGet();
|
||||
|
||||
assert null != statsField.getSchemaField()
|
||||
: "DocValuesStats requires a StatsField using a SchemaField";
|
||||
|
||||
SchemaField sf = statsField.getSchemaField();
|
||||
// FieldType ft = sf.getType();
|
||||
|
||||
StatsValues allstats = StatsValuesFactory.createStatsValues(statsField);
|
||||
|
||||
DocSet docs = baseDocs;
|
||||
int baseSize = docs.size();
|
||||
int maxDoc = searcher.maxDoc();
|
||||
|
||||
if (baseSize <= 0) return allstats;
|
||||
|
||||
DocSet missing = docs.andNot( searcher.getDocSet(new TermRangeQuery(field, null, null, false, false)) );
|
||||
|
||||
int i = 0;
|
||||
final FieldFacetStats[] finfo = new FieldFacetStats[facet.length];
|
||||
//Initialize facetstats, if facets have been passed in
|
||||
SortedDocValues si;
|
||||
for (String f : facet) {
|
||||
SchemaField facet_sf = searcher.getSchema().getField(f);
|
||||
finfo[i] = new FieldFacetStats(searcher, facet_sf, statsField);
|
||||
i++;
|
||||
}
|
||||
|
||||
final int[] index = this.index;
|
||||
final int[] counts = new int[numTermsInField];//keep track of the number of times we see each word in the field for all the documents in the docset
|
||||
|
||||
TermsEnum te = getOrdTermsEnum(searcher.getLeafReader());
|
||||
|
||||
boolean doNegative = false;
|
||||
if (finfo.length == 0) {
|
||||
//if we're collecting statistics with a facet field, can't do inverted counting
|
||||
doNegative = baseSize > maxDoc >> 1 && termInstances > 0
|
||||
&& docs instanceof BitDocSet;
|
||||
}
|
||||
|
||||
if (doNegative) {
|
||||
FixedBitSet bs = ((BitDocSet) docs).getBits().clone();
|
||||
bs.flip(0, maxDoc);
|
||||
// TODO: when iterator across negative elements is available, use that
|
||||
// instead of creating a new bitset and inverting.
|
||||
docs = new BitDocSet(bs, maxDoc - baseSize);
|
||||
// simply negating will mean that we have deleted docs in the set.
|
||||
// that should be OK, as their entries in our table should be empty.
|
||||
}
|
||||
|
||||
// For the biggest terms, do straight set intersections
|
||||
for (TopTerm tt : bigTerms.values()) {
|
||||
// TODO: counts could be deferred if sorted==false
|
||||
if (tt.termNum >= 0 && tt.termNum < numTermsInField) {
|
||||
final Term t = new Term(field, tt.term);
|
||||
if (finfo.length == 0) {
|
||||
counts[tt.termNum] = searcher.numDocs(new TermQuery(t), docs);
|
||||
} else {
|
||||
//COULD BE VERY SLOW
|
||||
//if we're collecting stats for facet fields, we need to iterate on all matching documents
|
||||
DocSet bigTermDocSet = searcher.getDocSet(new TermQuery(t)).intersection(docs);
|
||||
DocIterator iter = bigTermDocSet.iterator();
|
||||
while (iter.hasNext()) {
|
||||
int doc = iter.nextDoc();
|
||||
counts[tt.termNum]++;
|
||||
for (FieldFacetStats f : finfo) {
|
||||
f.facetTermNum(doc, tt.termNum);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (termInstances > 0) {
|
||||
DocIterator iter = docs.iterator();
|
||||
while (iter.hasNext()) {
|
||||
int doc = iter.nextDoc();
|
||||
int code = index[doc];
|
||||
|
||||
if ((code & 0xff) == 1) {
|
||||
int pos = code >>> 8;
|
||||
int whichArray = (doc >>> 16) & 0xff;
|
||||
byte[] arr = tnums[whichArray];
|
||||
int tnum = 0;
|
||||
for (; ;) {
|
||||
int delta = 0;
|
||||
for (; ;) {
|
||||
byte b = arr[pos++];
|
||||
delta = (delta << 7) | (b & 0x7f);
|
||||
if ((b & 0x80) == 0) break;
|
||||
}
|
||||
if (delta == 0) break;
|
||||
tnum += delta - TNUM_OFFSET;
|
||||
counts[tnum]++;
|
||||
for (FieldFacetStats f : finfo) {
|
||||
f.facetTermNum(doc, tnum);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
int tnum = 0;
|
||||
int delta = 0;
|
||||
for (; ;) {
|
||||
delta = (delta << 7) | (code & 0x7f);
|
||||
if ((code & 0x80) == 0) {
|
||||
if (delta == 0) break;
|
||||
tnum += delta - TNUM_OFFSET;
|
||||
counts[tnum]++;
|
||||
for (FieldFacetStats f : finfo) {
|
||||
f.facetTermNum(doc, tnum);
|
||||
}
|
||||
delta = 0;
|
||||
}
|
||||
code >>>= 8;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add results in index order
|
||||
for (i = 0; i < numTermsInField; i++) {
|
||||
int c = doNegative ? maxTermCounts[i] - counts[i] : counts[i];
|
||||
if (c == 0) continue;
|
||||
BytesRef value = getTermValue(te, i);
|
||||
|
||||
allstats.accumulate(value, c);
|
||||
//as we've parsed the termnum into a value, lets also accumulate fieldfacet statistics
|
||||
for (FieldFacetStats f : finfo) {
|
||||
f.accumulateTermNum(i, value);
|
||||
}
|
||||
}
|
||||
|
||||
int c = missing.size();
|
||||
allstats.addMissing(c);
|
||||
|
||||
if (finfo.length > 0) {
|
||||
for (FieldFacetStats f : finfo) {
|
||||
Map<String, StatsValues> facetStatsValues = f.facetStatsValues;
|
||||
FieldType facetType = searcher.getSchema().getFieldType(f.name);
|
||||
for (Map.Entry<String,StatsValues> entry : facetStatsValues.entrySet()) {
|
||||
String termLabel = entry.getKey();
|
||||
int missingCount = searcher.numDocs(new TermQuery(new Term(f.name, facetType.toInternal(termLabel))), missing);
|
||||
entry.getValue().addMissing(missingCount);
|
||||
}
|
||||
allstats.addFacet(f.name, facetStatsValues);
|
||||
}
|
||||
}
|
||||
|
||||
return allstats;
|
||||
|
||||
}
|
||||
|
||||
String getReadableValue(BytesRef termval, FieldType ft, CharsRefBuilder charsRef) {
|
||||
return ft.indexedToReadable(termval, charsRef).toString();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,13 +21,12 @@ import java.io.IOException;
|
|||
import java.io.Writer;
|
||||
import java.util.*;
|
||||
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.IndexableField;
|
||||
import org.apache.solr.client.solrj.io.TupleStream;
|
||||
import org.apache.solr.client.solrj.io.Tuple;
|
||||
import org.apache.lucene.index.StorableField;
|
||||
import org.apache.lucene.index.StoredDocument;
|
||||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.solr.common.EnumFieldValue;
|
||||
import org.apache.solr.common.SolrDocument;
|
||||
import org.apache.solr.common.SolrDocumentList;
|
||||
import org.apache.solr.common.util.Base64;
|
||||
|
|
@ -202,6 +201,8 @@ public abstract class TextResponseWriter {
|
|||
} else if (val instanceof BytesRef) {
|
||||
BytesRef arr = (BytesRef)val;
|
||||
writeByteArr(name, arr.bytes, arr.offset, arr.length);
|
||||
} else if (val instanceof EnumFieldValue) {
|
||||
writeStr(name, val.toString(), true);
|
||||
} else {
|
||||
// default... for debugging only
|
||||
writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
|
||||
|
|
|
|||
|
|
@ -410,6 +410,7 @@
|
|||
<tokenizer class="solr.WhitespaceTokenizerFactory" />
|
||||
</analyzer>
|
||||
</fieldType>
|
||||
<fieldType name="severityType" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"/>
|
||||
|
||||
</types>
|
||||
|
||||
|
|
@ -548,6 +549,10 @@
|
|||
|
||||
<field name="payloadDelimited" type="payloadDelimited" />
|
||||
|
||||
<!-- EnumType -->
|
||||
<field name="severity" type="severityType" indexed="true" stored="true" multiValued="false"/>
|
||||
|
||||
|
||||
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
|
||||
will be used if the name matches any of the patterns.
|
||||
RESTRICTION: the glob-like pattern in the name attribute must have
|
||||
|
|
@ -620,6 +625,7 @@
|
|||
<dynamicField name="*_path" type="path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
|
||||
<dynamicField name="*_ancestor" type="ancestor_path" indexed="true" stored="true" omitNorms="true" multiValued="true" />
|
||||
|
||||
<dynamicField name="*_sev_enum" type="severityType" indexed="true" stored="true" docValues="true" multiValued="true" />
|
||||
</fields>
|
||||
|
||||
<defaultSearchField>text</defaultSearchField>
|
||||
|
|
|
|||
|
|
@ -283,6 +283,9 @@ valued. -->
|
|||
</fieldType>
|
||||
|
||||
|
||||
<!-- EnumType -->
|
||||
<fieldType name="severityType" class="solr.EnumField" enumsConfig="enumsConfig.xml" enumName="severity"/>
|
||||
|
||||
<!-- Valid attributes for fields:
|
||||
name: mandatory - the name for the field
|
||||
type: mandatory - the name of a previously defined type from the <types> section
|
||||
|
|
@ -322,6 +325,10 @@ valued. -->
|
|||
<field name="cat_floatDocValues" type="float" indexed="true" stored="true" docValues="true" multiValued="true" />
|
||||
<field name="cat_length" type="text_length" indexed="true" stored="true" multiValued="true"/>
|
||||
|
||||
|
||||
<!-- EnumType -->
|
||||
<field name="severity" type="severityType" indexed="true" stored="true" multiValued="false"/>
|
||||
|
||||
<!-- Dynamic field definitions. If a field name is not found, dynamicFields
|
||||
will be used if the name matches any of the patterns.
|
||||
RESTRICTION: the glob-like pattern in the name attribute must have
|
||||
|
|
|
|||
|
|
@ -19,26 +19,41 @@ package org.apache.solr;
|
|||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.lucene.util.LuceneTestCase.Slow;
|
||||
|
||||
import org.apache.solr.client.solrj.SolrClient;
|
||||
import org.apache.solr.client.solrj.SolrResponse;
|
||||
import org.apache.solr.client.solrj.SolrServerException;
|
||||
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
|
||||
import org.apache.solr.client.solrj.impl.HttpSolrClient;
|
||||
import org.apache.solr.client.solrj.response.FacetField;
|
||||
import org.apache.solr.client.solrj.response.QueryResponse;
|
||||
import org.apache.solr.client.solrj.response.RangeFacet;
|
||||
import org.apache.solr.client.solrj.response.FieldStatsInfo;
|
||||
import org.apache.solr.cloud.ChaosMonkey;
|
||||
import org.apache.solr.common.EnumFieldValue;
|
||||
import org.apache.solr.common.SolrException;
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.ModifiableSolrParams;
|
||||
import org.apache.solr.common.params.ShardParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.StatsParams;
|
||||
import org.apache.solr.common.util.NamedList;
|
||||
import org.apache.solr.handler.component.ShardResponse;
|
||||
import org.apache.solr.handler.component.ShardRequest;
|
||||
import org.apache.solr.handler.component.StatsField.Stat;
|
||||
import org.apache.solr.handler.component.TrackingShardHandlerFactory;
|
||||
import org.apache.solr.handler.component.TrackingShardHandlerFactory.ShardRequestAndParams;
|
||||
import org.apache.solr.handler.component.TrackingShardHandlerFactory.RequestTrackingQueue;
|
||||
import org.apache.solr.handler.component.StatsComponentTest.StatSetCombinations;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.EnumSet;
|
||||
|
||||
/**
|
||||
* TODO? perhaps use:
|
||||
|
|
@ -63,18 +78,25 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
String missingField="ignore_exception__missing_but_valid_field_t";
|
||||
String invalidField="ignore_exception__invalid_field_not_in_schema";
|
||||
|
||||
@Override
|
||||
protected String getSolrXml() {
|
||||
return "solr-trackingshardhandler.xml";
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test() throws Exception {
|
||||
QueryResponse rsp = null;
|
||||
int backupStress = stress; // make a copy so we can restore
|
||||
|
||||
|
||||
del("*:*");
|
||||
indexr(id,1, i1, 100, tlong, 100,t1,"now is the time for all good men",
|
||||
"foo_sev_enum", "Medium",
|
||||
tdate_a, "2010-04-20T11:00:00Z",
|
||||
tdate_b, "2009-08-20T11:00:00Z",
|
||||
"foo_f", 1.414f, "foo_b", "true", "foo_d", 1.414d);
|
||||
indexr(id,2, i1, 50 , tlong, 50,t1,"to come to the aid of their country.",
|
||||
"foo_sev_enum", "Medium",
|
||||
"foo_sev_enum", "High",
|
||||
tdate_a, "2010-05-02T11:00:00Z",
|
||||
tdate_b, "2009-11-02T11:00:00Z");
|
||||
indexr(id,3, i1, 2, tlong, 2,t1,"how now brown cow",
|
||||
|
|
@ -90,6 +112,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
indexr(id,7, i1, 123, tlong, 123 ,t1,"humpty dumpy had a great fall");
|
||||
indexr(id,8, i1, 876, tlong, 876,
|
||||
tdate_b, "2010-01-05T11:00:00Z",
|
||||
"foo_sev_enum", "High",
|
||||
t1,"all the kings horses and all the kings men");
|
||||
indexr(id,9, i1, 7, tlong, 7,t1,"couldn't put humpty together again");
|
||||
|
||||
|
|
@ -97,6 +120,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
|
||||
indexr(id,10, i1, 4321, tlong, 4321,t1,"this too shall pass");
|
||||
indexr(id,11, i1, -987, tlong, 987,
|
||||
"foo_sev_enum", "Medium",
|
||||
t1,"An eye for eye only ends up making the whole world blind.");
|
||||
indexr(id,12, i1, 379, tlong, 379,
|
||||
t1,"Great works are performed, not by strength, but by perseverance.");
|
||||
|
|
@ -363,7 +387,7 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
query("q","*:*", "rows",100, "facet","true", "facet.field",missingField, "facet.mincount",2);
|
||||
// test field that is valid in schema and missing in some shards
|
||||
query("q","*:*", "rows",100, "facet","true", "facet.field",oddField, "facet.mincount",2);
|
||||
|
||||
|
||||
query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", "stats_dt");
|
||||
query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", i1);
|
||||
query("q","*:*", "sort",i1+" desc", "stats", "true", "stats.field", tdate_a);
|
||||
|
|
@ -392,6 +416,283 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
"stats.field", i1,
|
||||
"stats.field", tdate_a,
|
||||
"stats.field", tdate_b);
|
||||
|
||||
// only ask for "min" and "mean", explicitly exclude deps of mean, whitebox check shard responses
|
||||
try {
|
||||
RequestTrackingQueue trackingQueue = new RequestTrackingQueue();
|
||||
TrackingShardHandlerFactory.setTrackingQueue(jettys, trackingQueue);
|
||||
|
||||
rsp = query("q","*:*", "sort",i1+" desc", "stats", "true",
|
||||
"stats.field", "{!min=true sum=false mean=true count=false}" + i1);
|
||||
FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1);
|
||||
assertNotNull("no stats for " + i1, s);
|
||||
//
|
||||
assertEquals("wrong min", -987.0D, (Double)s.getMin(), 0.0001D );
|
||||
assertEquals("wrong mean", 377.153846D, (Double)s.getMean(), 0.0001D );
|
||||
//
|
||||
assertNull("expected null for count", s.getCount());
|
||||
assertNull("expected null for calcDistinct", s.getCountDistinct());
|
||||
assertNull("expected null for distinct vals", s.getDistinctValues());
|
||||
assertNull("expected null for max", s.getMax());
|
||||
assertNull("expected null for missing", s.getMissing());
|
||||
assertNull("expected null for stddev", s.getStddev());
|
||||
assertNull("expected null for sum", s.getSum());
|
||||
|
||||
// sanity check deps relationship
|
||||
for (Stat dep : EnumSet.of(Stat.sum, Stat.count)) {
|
||||
assertTrue("Purpose of this test is to ensure that asking for some stats works even when the deps " +
|
||||
"of those stats are explicitly excluded -- but the expected dep relationshp is no longer valid. " +
|
||||
"ie: who changed the code and didn't change this test?, expected: " + dep,
|
||||
Stat.mean.getDistribDeps().contains(dep));
|
||||
}
|
||||
|
||||
// check our shard requests & responses - ensure we didn't get unneccessary stats from every shard
|
||||
int numStatsShardRequests = 0;
|
||||
EnumSet<Stat> shardStatsExpected = EnumSet.of(Stat.min, Stat.sum, Stat.count);
|
||||
for (List<ShardRequestAndParams> shard : trackingQueue.getAllRequests().values()) {
|
||||
for (ShardRequestAndParams shardReq : shard) {
|
||||
if (shardReq.params.getBool(StatsParams.STATS, false)) {
|
||||
numStatsShardRequests++;
|
||||
for (ShardResponse shardRsp : shardReq.sreq.responses) {
|
||||
NamedList<Object> shardStats =
|
||||
((NamedList<NamedList<NamedList<Object>>>)
|
||||
shardRsp.getSolrResponse().getResponse().get("stats")).get("stats_fields").get(i1);
|
||||
|
||||
assertNotNull("no stard stats for " + i1, shardStats);
|
||||
//
|
||||
for (Map.Entry<String,Object> entry : shardStats) {
|
||||
Stat found = Stat.forName(entry.getKey());
|
||||
assertNotNull("found shardRsp stat key we were not expecting: " + entry, found);
|
||||
assertTrue("found stat we were not expecting: " + entry, shardStatsExpected.contains(found));
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assertTrue("did't see any stats=true shard requests", 0 < numStatsShardRequests);
|
||||
} finally {
|
||||
TrackingShardHandlerFactory.setTrackingQueue(jettys, null);
|
||||
}
|
||||
|
||||
// only ask for "min", "mean" and "stddev",
|
||||
rsp = query("q","*:*", "sort",i1+" desc", "stats", "true",
|
||||
"stats.field", "{!min=true mean=true stddev=true}" + i1);
|
||||
{ // don't leak variables
|
||||
FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1);
|
||||
assertNotNull("no stats for " + i1, s);
|
||||
//
|
||||
assertEquals("wrong min", -987.0D, (Double)s.getMin(), 0.0001D );
|
||||
assertEquals("wrong mean", 377.153846D, (Double)s.getMean(), 0.0001D );
|
||||
assertEquals("wrong stddev", 1271.76215D, (Double)s.getStddev(), 0.0001D );
|
||||
//
|
||||
assertNull("expected null for count", s.getCount());
|
||||
assertNull("expected null for calcDistinct", s.getCountDistinct());
|
||||
assertNull("expected null for distinct vals", s.getDistinctValues());
|
||||
assertNull("expected null for max", s.getMax());
|
||||
assertNull("expected null for missing", s.getMissing());
|
||||
assertNull("expected null for sum", s.getSum());
|
||||
}
|
||||
|
||||
final String[] stats = new String[] {
|
||||
"min", "max", "sum", "sumOfSquares", "stddev", "mean", "missing", "count"
|
||||
};
|
||||
|
||||
// ask for arbitrary pairs of stats
|
||||
for (String stat1 : stats) {
|
||||
for (String stat2 : stats) {
|
||||
// NOTE: stat1 might equal stat2 - good edge case to test for
|
||||
|
||||
rsp = query("q","*:*", "sort",i1+" desc", "stats", "true",
|
||||
"stats.field", "{!" + stat1 + "=true " + stat2 + "=true}" + i1);
|
||||
|
||||
final List<String> statsExpected = new ArrayList<String>(2);
|
||||
statsExpected.add(stat1);
|
||||
if ( ! stat1.equals(stat2) ) {
|
||||
statsExpected.add(stat2);
|
||||
}
|
||||
|
||||
// ignore the FieldStatsInfo convinience class, and look directly at the NamedList
|
||||
// so we don't need any sort of crazy reflection
|
||||
NamedList<Object> svals =
|
||||
((NamedList<NamedList<NamedList<Object>>>)
|
||||
rsp.getResponse().get("stats")).get("stats_fields").get(i1);
|
||||
|
||||
assertNotNull("no stats for field " + i1, svals);
|
||||
assertEquals("wrong quantity of stats", statsExpected.size(), svals.size());
|
||||
|
||||
|
||||
for (String s : statsExpected) {
|
||||
assertNotNull("stat shouldn't be null: " + s, svals.get(s));
|
||||
assertTrue("stat should be a Number: " + s + " -> " + svals.get(s).getClass(),
|
||||
svals.get(s) instanceof Number);
|
||||
// some loose assertions since we're iterating over various stats
|
||||
if (svals.get(s) instanceof Double) {
|
||||
Double val = (Double) svals.get(s);
|
||||
assertFalse("stat shouldn't be NaN: " + s, val.isNaN());
|
||||
assertFalse("stat shouldn't be Inf: " + s, val.isInfinite());
|
||||
assertFalse("stat shouldn't be 0: " + s, val.equals(0.0D));
|
||||
} else {
|
||||
// count or missing
|
||||
assertTrue("stat should be count of missing: " + s,
|
||||
("count".equals(s) || "missing".equals(s)));
|
||||
assertTrue("stat should be a Long: " + s + " -> " + svals.get(s).getClass(),
|
||||
svals.get(s) instanceof Long);
|
||||
Long val = (Long) svals.get(s);
|
||||
assertFalse("stat shouldn't be 0: " + s, val.equals(0L));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// all of these diff ways of asking for min & calcdistinct should have the same result
|
||||
for (SolrParams p : new SolrParams[] {
|
||||
params("stats.field", "{!min=true calcdistinct=true}" + i1),
|
||||
params("stats.calcdistinct", "true",
|
||||
"stats.field", "{!min=true}" + i1),
|
||||
params("f."+i1+".stats.calcdistinct", "true",
|
||||
"stats.field", "{!min=true}" + i1),
|
||||
params("stats.calcdistinct", "false",
|
||||
"f."+i1+".stats.calcdistinct", "true",
|
||||
"stats.field", "{!min=true}" + i1),
|
||||
params("stats.calcdistinct", "false",
|
||||
"f."+i1+".stats.calcdistinct", "false",
|
||||
"stats.field", "{!min=true calcdistinct=true}" + i1),
|
||||
}) {
|
||||
|
||||
rsp = query(SolrParams.wrapDefaults
|
||||
(p, params("q","*:*", "sort",i1+" desc", "stats", "true")));
|
||||
FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1);
|
||||
assertNotNull(p+" no stats for " + i1, s);
|
||||
//
|
||||
assertEquals(p+" wrong min", -987.0D, (Double)s.getMin(), 0.0001D );
|
||||
assertEquals(p+" wrong calcDistinct", new Long(13), s.getCountDistinct());
|
||||
assertNotNull(p+" expected non-null list for distinct vals", s.getDistinctValues());
|
||||
assertEquals(p+" expected list for distinct vals", 13, s.getDistinctValues().size());
|
||||
//
|
||||
assertNull(p+" expected null for mean", s.getMean() );
|
||||
assertNull(p+" expected null for count", s.getCount());
|
||||
assertNull(p+" expected null for max", s.getMax());
|
||||
assertNull(p+" expected null for missing", s.getMissing());
|
||||
assertNull(p+" expected null for stddev", s.getStddev());
|
||||
assertNull(p+" expected null for sum", s.getSum());
|
||||
|
||||
}
|
||||
|
||||
// all of these diff ways of excluding calcdistinct should have the same result
|
||||
for (SolrParams p : new SolrParams[] {
|
||||
params("stats.field", "{!min=true calcdistinct=false}" + i1),
|
||||
params("stats.calcdistinct", "false",
|
||||
"stats.field", "{!min=true}" + i1),
|
||||
params("f."+i1+".stats.calcdistinct", "false",
|
||||
"stats.field", "{!min=true}" + i1),
|
||||
params("stats.calcdistinct", "true",
|
||||
"f."+i1+".stats.calcdistinct", "false",
|
||||
"stats.field", "{!min=true}" + i1),
|
||||
params("stats.calcdistinct", "true",
|
||||
"f."+i1+".stats.calcdistinct", "true",
|
||||
"stats.field", "{!min=true calcdistinct=false}" + i1),
|
||||
}) {
|
||||
|
||||
rsp = query(SolrParams.wrapDefaults
|
||||
(p, params("q","*:*", "sort",i1+" desc", "stats", "true")));
|
||||
FieldStatsInfo s = rsp.getFieldStatsInfo().get(i1);
|
||||
assertNotNull(p+" no stats for " + i1, s);
|
||||
//
|
||||
assertEquals(p+" wrong min", -987.0D, (Double)s.getMin(), 0.0001D );
|
||||
//
|
||||
assertNull(p+" expected null for calcDistinct", s.getCountDistinct());
|
||||
assertNull(p+" expected null for distinct vals", s.getDistinctValues());
|
||||
//
|
||||
assertNull(p+" expected null for mean", s.getMean() );
|
||||
assertNull(p+" expected null for count", s.getCount());
|
||||
assertNull(p+" expected null for max", s.getMax());
|
||||
assertNull(p+" expected null for missing", s.getMissing());
|
||||
assertNull(p+" expected null for stddev", s.getStddev());
|
||||
assertNull(p+" expected null for sum", s.getSum());
|
||||
|
||||
}
|
||||
|
||||
// this field doesn't exist in any doc in the result set.
|
||||
// ensure we get expected values for the stats we ask for, but null for the stats
|
||||
rsp = query("q","*:*", "sort",i1+" desc", "stats", "true",
|
||||
"stats.field", "{!min=true mean=true stddev=true}does_not_exist_i");
|
||||
{ // don't leak variables
|
||||
FieldStatsInfo s = rsp.getFieldStatsInfo().get("does_not_exist_i");
|
||||
assertNotNull("no stats for bogus field", s);
|
||||
|
||||
// things we explicit expect because we asked for them
|
||||
// NOTE: min is expected to be null even though requested because of no values
|
||||
assertEquals("wrong min", null, s.getMin());
|
||||
assertTrue("mean should be NaN", ((Double)s.getMean()).isNaN());
|
||||
assertEquals("wrong stddev", 0.0D, (Double)s.getStddev(), 0.0D );
|
||||
|
||||
// things that we didn't ask for, so they better be null
|
||||
assertNull("expected null for count", s.getCount());
|
||||
assertNull("expected null for calcDistinct", s.getCountDistinct());
|
||||
assertNull("expected null for distinct vals", s.getDistinctValues());
|
||||
assertNull("expected null for max", s.getMax());
|
||||
assertNull("expected null for missing", s.getMissing());
|
||||
assertNull("expected null for sum", s.getSum());
|
||||
}
|
||||
|
||||
// look at stats on non numeric fields
|
||||
//
|
||||
// not all stats are supported on every field type, so some of these permutations will
|
||||
// result in no stats being computed but this at least lets us sanity check that for each
|
||||
// of these field+stats(s) combinations we get consistent results between the distribted
|
||||
// request and the single node situation.
|
||||
EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);
|
||||
int numTotalStatQueries = 0;
|
||||
// don't go overboard, just do all permutations of 1 or 2 stat params, for each field & query
|
||||
final int numStatParamsAtOnce = 2;
|
||||
for (int numParams = 1; numParams <= numStatParamsAtOnce; numParams++) {
|
||||
for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
|
||||
|
||||
for (String field : new String[] {
|
||||
"foo_f", i1, tlong, tdate_a, oddField, "foo_sev_enum",
|
||||
// fields that no doc has any value in
|
||||
"bogus___s", "bogus___f", "bogus___i", "bogus___tdt", "bogus___sev_enum"
|
||||
}) {
|
||||
|
||||
for ( String q : new String[] {
|
||||
"*:*", // all docs
|
||||
"bogus___s:bogus", // no docs
|
||||
"id:" + random().nextInt(50 ), // 0 or 1 doc...
|
||||
"id:" + random().nextInt(50 ),
|
||||
"id:" + random().nextInt(100),
|
||||
"id:" + random().nextInt(100),
|
||||
"id:" + random().nextInt(200)
|
||||
}) {
|
||||
|
||||
// EnumSets use natural ordering, we want to randomize the order of the params
|
||||
List<Stat> combo = new ArrayList<Stat>(set);
|
||||
Collections.shuffle(combo, random());
|
||||
|
||||
StringBuilder paras = new StringBuilder("{!key=k ");
|
||||
|
||||
for (Stat stat : combo) {
|
||||
paras.append(stat + "=true ");
|
||||
}
|
||||
|
||||
paras.append("}").append(field);
|
||||
numTotalStatQueries++;
|
||||
rsp = query("q","*:*", "rows", "0", "stats", "true",
|
||||
"stats.field", paras.toString());
|
||||
// simple assert, mostly relying on comparison with single shard
|
||||
FieldStatsInfo s = rsp.getFieldStatsInfo().get("k");
|
||||
assertNotNull(s);
|
||||
|
||||
// TODO: if we had a programatic way to determine what stats are supported
|
||||
// by what field types, we could make more confident asserts here.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assertEquals("Sanity check failed: either test broke, or test changed, or you adjusted Stat enum" +
|
||||
" (adjust constant accordingly if intentional)",
|
||||
3465, numTotalStatQueries);
|
||||
|
||||
|
||||
/*** TODO: the failure may come back in "exception"
|
||||
try {
|
||||
|
|
@ -567,6 +868,25 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
|
|||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
String fieldName = "severity";
|
||||
indexr("id", "1", fieldName, "Not Available");
|
||||
indexr("id", "2", fieldName, "Low");
|
||||
indexr("id", "3", fieldName, "Medium");
|
||||
indexr("id", "4", fieldName, "High");
|
||||
indexr("id", "5", fieldName, "Critical");
|
||||
|
||||
commit();
|
||||
|
||||
rsp = query("q", "*:*", "stats", "true", "stats.field", fieldName);
|
||||
assertEquals(new EnumFieldValue(0, "Not Available"),
|
||||
rsp.getFieldStatsInfo().get(fieldName).getMin());
|
||||
query("q", "*:*", "stats", "true", "stats.field", fieldName,
|
||||
StatsParams.STATS_CALC_DISTINCT, "true");
|
||||
assertEquals(new EnumFieldValue(4, "Critical"),
|
||||
rsp.getFieldStatsInfo().get(fieldName).getMax());
|
||||
query("q", "*:*", "stats", "true", "stats.field", fieldName,
|
||||
"stats.facet", fieldName);
|
||||
}
|
||||
|
||||
protected void checkMinCountsField(List<FacetField.Count> counts, Object[] pairs) {
|
||||
|
|
|
|||
|
|
@ -139,7 +139,8 @@ public class CollectionsAPIDistributedZkTest extends AbstractFullDistribZkTestBa
|
|||
// for now, always upload the config and schema to the canonical names
|
||||
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, getCloudSolrConfig(), "solrconfig.xml");
|
||||
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "schema.xml", "schema.xml");
|
||||
|
||||
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "enumsConfig.xml", "enumsConfig.xml");
|
||||
|
||||
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "solrconfig.snippet.randomindexconfig.xml");
|
||||
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "stopwords.txt");
|
||||
AbstractZkTestCase.putConfig("conf2", zkClient, solrhome, "protwords.txt");
|
||||
|
|
|
|||
|
|
@ -89,19 +89,23 @@ public class DistributedFacetPivotSmallAdvancedTest extends BaseDistributedSearc
|
|||
handle.put("maxScore", SKIPVAL);
|
||||
|
||||
doTestDeepPivotStatsOnString();
|
||||
doTestTopStatsWithRefinement();
|
||||
|
||||
doTestTopStatsWithRefinement(true);
|
||||
doTestTopStatsWithRefinement(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* we need to ensure that stats never "overcount" the values from a single shard
|
||||
* even if we hit that shard with a refinement request
|
||||
*/
|
||||
private void doTestTopStatsWithRefinement() throws Exception {
|
||||
|
||||
|
||||
private void doTestTopStatsWithRefinement(final boolean allStats) throws Exception {
|
||||
|
||||
String stat_param = allStats ?
|
||||
"{!tag=s1}foo_i" : "{!tag=s1 min=true max=true count=true missing=true}foo_i";
|
||||
|
||||
ModifiableSolrParams coreParams = params("q", "*:*", "rows", "0",
|
||||
"stats", "true",
|
||||
"stats.field", "{!tag=s1}foo_i" );
|
||||
"stats.field", stat_param );
|
||||
ModifiableSolrParams facetParams = new ModifiableSolrParams(coreParams);
|
||||
facetParams.add(params("facet", "true",
|
||||
"facet.limit", "1",
|
||||
|
|
@ -128,10 +132,18 @@ public class DistributedFacetPivotSmallAdvancedTest extends BaseDistributedSearc
|
|||
assertEquals(msg, 91.0, fieldStatsInfo.getMax());
|
||||
assertEquals(msg, 10, (long) fieldStatsInfo.getCount());
|
||||
assertEquals(msg, 0, (long) fieldStatsInfo.getMissing());
|
||||
assertEquals(msg, 248.0, fieldStatsInfo.getSum());
|
||||
assertEquals(msg, 15294.0, fieldStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(msg, 24.8, (double) fieldStatsInfo.getMean(), 0.1E-7);
|
||||
assertEquals(msg, 31.87405772027709, fieldStatsInfo.getStddev(), 0.1E-7);
|
||||
|
||||
if (allStats) {
|
||||
assertEquals(msg, 248.0, fieldStatsInfo.getSum());
|
||||
assertEquals(msg, 15294.0, fieldStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(msg, 24.8, (double) fieldStatsInfo.getMean(), 0.1E-7);
|
||||
assertEquals(msg, 31.87405772027709, fieldStatsInfo.getStddev(), 0.1E-7);
|
||||
} else {
|
||||
assertNull(msg, fieldStatsInfo.getSum());
|
||||
assertNull(msg, fieldStatsInfo.getSumOfSquares());
|
||||
assertNull(msg, fieldStatsInfo.getMean());
|
||||
assertNull(msg, fieldStatsInfo.getStddev());
|
||||
}
|
||||
|
||||
if (params.getBool("facet", false)) {
|
||||
// if this was a facet request, then the top pivot constraint and pivot
|
||||
|
|
@ -156,6 +168,12 @@ public class DistributedFacetPivotSmallAdvancedTest extends BaseDistributedSearc
|
|||
assertEquals(4, (long) dublinMicrosoftStatsInfo.getCount());
|
||||
assertEquals(0, (long) dublinMicrosoftStatsInfo.getMissing());
|
||||
|
||||
if (! allStats) {
|
||||
assertNull(msg, dublinMicrosoftStatsInfo.getSum());
|
||||
assertNull(msg, dublinMicrosoftStatsInfo.getSumOfSquares());
|
||||
assertNull(msg, dublinMicrosoftStatsInfo.getMean());
|
||||
assertNull(msg, dublinMicrosoftStatsInfo.getStddev());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -334,16 +334,21 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas
|
|||
}
|
||||
}
|
||||
|
||||
doTestDeepPivotStats();
|
||||
doTestDeepPivotStats(false); // all price stats
|
||||
doTestDeepPivotStats(true); // just the mean price stat
|
||||
|
||||
doTestPivotStatsFromOneShard();
|
||||
}
|
||||
|
||||
private void doTestDeepPivotStats() throws Exception {
|
||||
/**
|
||||
* @param justMean - only the mean stat is requested/computed
|
||||
*/
|
||||
private void doTestDeepPivotStats(boolean justMean) throws Exception {
|
||||
SolrParams params = params("q", "*:*", "rows", "0",
|
||||
"facet", "true", "stats", "true",
|
||||
"facet.pivot", "{!stats=s1}place_t,company_t",
|
||||
"stats.field", "{!key=avg_price tag=s1}price_ti");
|
||||
"stats.field", ("{!key=avg_price tag=s1 "+
|
||||
(justMean ? "mean=true" : "") +"}price_ti"));
|
||||
QueryResponse rsp = query(params);
|
||||
|
||||
List<PivotField> placePivots = rsp.getFacetPivot().get("place_t,company_t");
|
||||
|
|
@ -357,15 +362,24 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas
|
|||
assertEquals(4, microsoftPivotField.getCount());
|
||||
|
||||
FieldStatsInfo dublinMicrosoftStatsInfo = microsoftPivotField.getFieldStatsInfo().get("avg_price");
|
||||
assertEquals(15.0, dublinMicrosoftStatsInfo.getMin());
|
||||
assertEquals(29.0, dublinMicrosoftStatsInfo.getMax());
|
||||
assertEquals(3, (long) dublinMicrosoftStatsInfo.getCount());
|
||||
assertEquals(1, (long) dublinMicrosoftStatsInfo.getMissing());
|
||||
assertEquals(63.0, dublinMicrosoftStatsInfo.getSum());
|
||||
assertEquals(1427.0, dublinMicrosoftStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(21.0, (double) dublinMicrosoftStatsInfo.getMean(), 0.1E-7);
|
||||
assertEquals(7.211102550927978, dublinMicrosoftStatsInfo.getStddev(), 0.1E-7);
|
||||
|
||||
if (justMean) {
|
||||
assertNull(dublinMicrosoftStatsInfo.getMin());
|
||||
assertNull(dublinMicrosoftStatsInfo.getMax());
|
||||
assertNull(dublinMicrosoftStatsInfo.getCount());
|
||||
assertNull(dublinMicrosoftStatsInfo.getMissing());
|
||||
assertNull(dublinMicrosoftStatsInfo.getSum());
|
||||
assertNull(dublinMicrosoftStatsInfo.getSumOfSquares());
|
||||
assertNull(dublinMicrosoftStatsInfo.getStddev());
|
||||
} else {
|
||||
assertEquals(15.0, dublinMicrosoftStatsInfo.getMin());
|
||||
assertEquals(29.0, dublinMicrosoftStatsInfo.getMax());
|
||||
assertEquals(3, (long) dublinMicrosoftStatsInfo.getCount());
|
||||
assertEquals(1, (long) dublinMicrosoftStatsInfo.getMissing());
|
||||
assertEquals(63.0, dublinMicrosoftStatsInfo.getSum());
|
||||
assertEquals(1427.0, dublinMicrosoftStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(7.211102550927978, dublinMicrosoftStatsInfo.getStddev(), 0.1E-7);
|
||||
}
|
||||
|
||||
PivotField cardiffPivotField = placePivots.get(2);
|
||||
assertEquals("cardiff", cardiffPivotField.getValue());
|
||||
|
|
@ -376,15 +390,24 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas
|
|||
assertEquals(3, polecatPivotField.getCount());
|
||||
|
||||
FieldStatsInfo cardiffPolecatStatsInfo = polecatPivotField.getFieldStatsInfo().get("avg_price");
|
||||
assertEquals(15.0, cardiffPolecatStatsInfo.getMin());
|
||||
assertEquals(39.0, cardiffPolecatStatsInfo.getMax());
|
||||
assertEquals(2, (long) cardiffPolecatStatsInfo.getCount());
|
||||
assertEquals(1, (long) cardiffPolecatStatsInfo.getMissing());
|
||||
assertEquals(54.0, cardiffPolecatStatsInfo.getSum());
|
||||
assertEquals(1746.0, cardiffPolecatStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(27.0, (double) cardiffPolecatStatsInfo.getMean(), 0.1E-7);
|
||||
assertEquals(16.97056274847714, cardiffPolecatStatsInfo.getStddev(), 0.1E-7);
|
||||
|
||||
if (justMean) {
|
||||
assertNull(cardiffPolecatStatsInfo.getMin());
|
||||
assertNull(cardiffPolecatStatsInfo.getMax());
|
||||
assertNull(cardiffPolecatStatsInfo.getCount());
|
||||
assertNull(cardiffPolecatStatsInfo.getMissing());
|
||||
assertNull(cardiffPolecatStatsInfo.getSum());
|
||||
assertNull(cardiffPolecatStatsInfo.getSumOfSquares());
|
||||
assertNull(cardiffPolecatStatsInfo.getStddev());
|
||||
} else {
|
||||
assertEquals(15.0, cardiffPolecatStatsInfo.getMin());
|
||||
assertEquals(39.0, cardiffPolecatStatsInfo.getMax());
|
||||
assertEquals(2, (long) cardiffPolecatStatsInfo.getCount());
|
||||
assertEquals(1, (long) cardiffPolecatStatsInfo.getMissing());
|
||||
assertEquals(54.0, cardiffPolecatStatsInfo.getSum());
|
||||
assertEquals(1746.0, cardiffPolecatStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(16.97056274847714, cardiffPolecatStatsInfo.getStddev(), 0.1E-7);
|
||||
}
|
||||
|
||||
PivotField krakowPivotField = placePivots.get(3);
|
||||
assertEquals("krakow", krakowPivotField.getValue());
|
||||
|
|
@ -395,14 +418,25 @@ public class DistributedFacetPivotSmallTest extends BaseDistributedSearchTestCas
|
|||
assertEquals(1, fujitsuPivotField.getCount());
|
||||
|
||||
FieldStatsInfo krakowFujitsuStatsInfo = fujitsuPivotField.getFieldStatsInfo().get("avg_price");
|
||||
assertEquals(null, krakowFujitsuStatsInfo.getMin());
|
||||
assertEquals(null, krakowFujitsuStatsInfo.getMax());
|
||||
assertEquals(0, (long) krakowFujitsuStatsInfo.getCount());
|
||||
assertEquals(1, (long) krakowFujitsuStatsInfo.getMissing());
|
||||
assertEquals(0.0, krakowFujitsuStatsInfo.getSum());
|
||||
assertEquals(0.0, krakowFujitsuStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(Double.NaN, (double) krakowFujitsuStatsInfo.getMean(), 0.1E-7);
|
||||
assertEquals(0.0, krakowFujitsuStatsInfo.getStddev(), 0.1E-7);
|
||||
if (justMean) {
|
||||
assertNull(krakowFujitsuStatsInfo.getMin());
|
||||
assertNull(krakowFujitsuStatsInfo.getMax());
|
||||
assertNull(krakowFujitsuStatsInfo.getCount());
|
||||
assertNull(krakowFujitsuStatsInfo.getMissing());
|
||||
assertNull(krakowFujitsuStatsInfo.getSum());
|
||||
assertNull(krakowFujitsuStatsInfo.getSumOfSquares());
|
||||
assertNull(krakowFujitsuStatsInfo.getStddev());
|
||||
} else {
|
||||
assertEquals(null, krakowFujitsuStatsInfo.getMin());
|
||||
assertEquals(null, krakowFujitsuStatsInfo.getMax());
|
||||
assertEquals(0, (long) krakowFujitsuStatsInfo.getCount());
|
||||
assertEquals(1, (long) krakowFujitsuStatsInfo.getMissing());
|
||||
assertEquals(0.0, krakowFujitsuStatsInfo.getSum());
|
||||
assertEquals(0.0, krakowFujitsuStatsInfo.getSumOfSquares(), 0.1E-7);
|
||||
assertEquals(Double.NaN, (double) krakowFujitsuStatsInfo.getMean(), 0.1E-7);
|
||||
assertEquals(0.0, krakowFujitsuStatsInfo.getStddev(), 0.1E-7);
|
||||
}
|
||||
}
|
||||
|
||||
// Useful to check for errors, orders lists and does toString() equality check
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ public class FacetPivotSmallTest extends SolrTestCaseJ4 {
|
|||
params.add("facet", "true");
|
||||
params.add("facet.pivot", "{!stats=s1}place_t,company_t");
|
||||
params.add("stats", "true");
|
||||
params.add("stats.field", "{!key=avg_price tag=s1 mean=true}price_ti");
|
||||
params.add("stats.field", "{!key=avg_price tag=s1}price_ti");
|
||||
|
||||
SolrQueryRequest req = req(params);
|
||||
final String statsPrefix = "//lst[@name='facet_counts']/lst[@name='facet_pivot']/arr[@name='place_t,company_t']/lst";
|
||||
|
|
@ -174,6 +174,8 @@ public class FacetPivotSmallTest extends SolrTestCaseJ4 {
|
|||
dublinMicrosoftStats + "/double[@name='sumOfSquares'][.=1427.0]",
|
||||
dublinMicrosoftStats + "/double[@name='mean'][.=21.0]",
|
||||
dublinMicrosoftStats + "/double[@name='stddev'][.=7.211102550927978]",
|
||||
// if new stats are supported, this will break - update test to assert values for each
|
||||
"count(" + dublinMicrosoftStats + "/*)=8",
|
||||
|
||||
cardiffPolecatStats + "/double[@name='min'][.=15.0]",
|
||||
cardiffPolecatStats + "/double[@name='max'][.=39.0]",
|
||||
|
|
@ -183,6 +185,8 @@ public class FacetPivotSmallTest extends SolrTestCaseJ4 {
|
|||
cardiffPolecatStats + "/double[@name='sumOfSquares'][.=1746.0]",
|
||||
cardiffPolecatStats + "/double[@name='mean'][.=27.0]",
|
||||
cardiffPolecatStats + "/double[@name='stddev'][.=16.97056274847714]",
|
||||
// if new stats are supported, this will break - update test to assert values for each
|
||||
"count(" + cardiffPolecatStats + "/*)=8",
|
||||
|
||||
krakowFujitsuStats + "/null[@name='min']",
|
||||
krakowFujitsuStats + "/null[@name='max']",
|
||||
|
|
@ -191,7 +195,10 @@ public class FacetPivotSmallTest extends SolrTestCaseJ4 {
|
|||
krakowFujitsuStats + "/double[@name='sum'][.=0.0]",
|
||||
krakowFujitsuStats + "/double[@name='sumOfSquares'][.=0.0]",
|
||||
krakowFujitsuStats + "/double[@name='mean'][.='NaN']",
|
||||
krakowFujitsuStats + "/double[@name='stddev'][.=0.0]"
|
||||
krakowFujitsuStats + "/double[@name='stddev'][.=0.0]",
|
||||
// if new stats are supported, this will break - update test to assert values for each
|
||||
"count(" + krakowFujitsuStats + "/*)=8"
|
||||
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ import java.text.SimpleDateFormat;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.Date;
|
||||
import java.util.Iterator;
|
||||
import java.util.EnumSet;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
|
|
@ -29,19 +31,21 @@ import java.util.TimeZone;
|
|||
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
|
||||
|
||||
import org.apache.solr.common.params.CommonParams;
|
||||
import org.apache.solr.common.params.MapSolrParams;
|
||||
import org.apache.solr.common.params.SolrParams;
|
||||
import org.apache.solr.common.params.StatsParams;
|
||||
import org.apache.solr.core.SolrCore;
|
||||
import org.apache.solr.handler.component.StatsField.Stat;
|
||||
import org.apache.solr.request.LocalSolrQueryRequest;
|
||||
import org.apache.solr.request.SolrQueryRequest;
|
||||
import org.apache.solr.response.SolrQueryResponse;
|
||||
import org.apache.solr.schema.SchemaField;
|
||||
import org.apache.solr.util.AbstractSolrTestCase;
|
||||
|
||||
import org.apache.commons.math3.util.Combinations;
|
||||
|
||||
import org.junit.BeforeClass;
|
||||
|
||||
|
||||
|
|
@ -647,8 +651,20 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
args.put("indent", "true");
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
|
||||
|
||||
assertQ("test string statistics values", req,
|
||||
"//null[@name='active_i'][.='']");
|
||||
assertQ("test string statistics values", req
|
||||
,"//lst[@name='active_i']/long[@name='count'][.='0']"
|
||||
,"//lst[@name='active_i']/long[@name='missing'][.='4']"
|
||||
|
||||
,"//lst[@name='active_i']/null[@name='min']"
|
||||
,"//lst[@name='active_i']/null[@name='max']"
|
||||
,"//lst[@name='active_i']/double[@name='sum'][.='0.0']"
|
||||
,"//lst[@name='active_i']/double[@name='sumOfSquares'][.='0.0']"
|
||||
,"//lst[@name='active_i']/double[@name='stddev'][.='0.0']"
|
||||
,"//lst[@name='active_i']/double[@name='mean'][.='NaN']"
|
||||
// if new stats are supported, this will break - update test to assert values for each
|
||||
,"count(//lst[@name='active_i']/*)=8"
|
||||
|
||||
);
|
||||
}
|
||||
|
||||
public void testFieldStatisticsResultsStringFieldAlwaysMissing() throws Exception {
|
||||
|
|
@ -667,8 +683,15 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
args.put("indent", "true");
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
|
||||
|
||||
assertQ("test string statistics values", req,
|
||||
"//null[@name='active_s'][.='']");
|
||||
assertQ("test string statistics values", req
|
||||
,"//lst[@name='active_s']/long[@name='count'][.='0']"
|
||||
,"//lst[@name='active_s']/long[@name='missing'][.='4']"
|
||||
|
||||
,"//lst[@name='active_s']/null[@name='min']"
|
||||
,"//lst[@name='active_s']/null[@name='max']"
|
||||
// if new stats are supported, this will break - update test to assert values for each
|
||||
,"count(//lst[@name='active_s']/*)=4"
|
||||
);
|
||||
}
|
||||
|
||||
//SOLR-3160
|
||||
|
|
@ -688,8 +711,20 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
args.put("indent", "true");
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
|
||||
|
||||
assertQ("test string statistics values", req,
|
||||
"//null[@name='active_dt'][.='']");
|
||||
assertQ("test string statistics values", req
|
||||
,"//lst[@name='active_dt']/long[@name='count'][.='0']"
|
||||
,"//lst[@name='active_dt']/long[@name='missing'][.='3']"
|
||||
|
||||
,"//lst[@name='active_dt']/null[@name='min']"
|
||||
,"//lst[@name='active_dt']/null[@name='max']"
|
||||
,"//lst[@name='active_dt']/null[@name='mean']"
|
||||
,"//lst[@name='active_dt']/date[@name='sum'][.='1970-01-01T00:00:00Z']"
|
||||
,"//lst[@name='active_dt']/double[@name='sumOfSquares'][.='0.0']"
|
||||
,"//lst[@name='active_dt']/double[@name='stddev'][.='0.0']"
|
||||
|
||||
// if new stats are supported, this will break - update test to assert values for each
|
||||
,"count(//lst[@name='active_dt']/*)=8"
|
||||
);
|
||||
}
|
||||
|
||||
public void testStatsFacetMultivaluedErrorHandling() throws Exception {
|
||||
|
|
@ -813,8 +848,8 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
args.put(CommonParams.Q, "*:*");
|
||||
args.put(StatsParams.STATS, "true");
|
||||
args.put(StatsParams.STATS_FIELD, fieldName);
|
||||
args.put("indent", "true");
|
||||
args.put(StatsParams.STATS_CALC_DISTINCT, "true");
|
||||
args.put("indent", "true");
|
||||
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
|
||||
|
||||
|
|
@ -860,8 +895,8 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
args.put(StatsParams.STATS, "true");
|
||||
args.put(StatsParams.STATS_FIELD, fieldName);
|
||||
args.put(StatsParams.STATS_FACET, fieldName);
|
||||
args.put("indent", "true");
|
||||
args.put(StatsParams.STATS_CALC_DISTINCT, "true");
|
||||
args.put("indent", "true");
|
||||
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
|
||||
|
||||
|
|
@ -987,27 +1022,107 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
|
||||
assertU(commit());
|
||||
|
||||
Map<String, String> args = new HashMap<>();
|
||||
args.put(CommonParams.Q, "*:*");
|
||||
args.put(StatsParams.STATS, "true");
|
||||
args.put(StatsParams.STATS_FIELD, fieldName);
|
||||
args.put(StatsParams.STATS_CALC_DISTINCT, "true");
|
||||
args.put("indent", "true");
|
||||
SolrQueryRequest req = new LocalSolrQueryRequest(core, new MapSolrParams(args));
|
||||
final SolrParams baseParams = params(CommonParams.Q, "*:*",
|
||||
"indent", "true",
|
||||
StatsParams.STATS, "true");
|
||||
|
||||
assertQ("test min/max on docValues and multiValued", req
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='min'][.='-3.0']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='max'][.='16.0']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='count'][.='12']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='sum'][.='38.0']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='9']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='mean'][.='3.1666666666666665']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='stddev'][.='5.638074031784151']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='sumOfSquares'][.='470.0']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='missing'][.='0']");
|
||||
SolrQueryRequest req1 = req(baseParams,
|
||||
StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
StatsParams.STATS_FIELD, fieldName);
|
||||
SolrQueryRequest req2 = req(baseParams,
|
||||
StatsParams.STATS_FIELD,
|
||||
"{!min=true, max=true, count=true, sum=true, mean=true, stddev=true, sumOfSquares=true, missing=true, calcdistinct=true}" + fieldName);
|
||||
|
||||
for (SolrQueryRequest req : new SolrQueryRequest[] { req1, req2 }) {
|
||||
assertQ("test status on docValues and multiValued: " + req.toString(), req
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='min'][.='-3.0']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='max'][.='16.0']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='count'][.='12']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='sum'][.='38.0']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='mean'][.='3.1666666666666665']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='stddev'][.='5.638074031784151']"
|
||||
, "//lst[@name='" + fieldName + "']/double[@name='sumOfSquares'][.='470.0']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='missing'][.='0']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='9']"
|
||||
// always comes along with countDistinct
|
||||
, "count(//lst[@name='" + fieldName + "']/arr[@name='distinctValues']/float)=9"
|
||||
// if new default stats are added, this will break - update test to assert values for each
|
||||
,"count(//lst[@name='" + fieldName + "']/*)=10"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
public void testEnumFieldTypeStatus() throws Exception {
|
||||
clearIndex();
|
||||
|
||||
String fieldName = "severity";
|
||||
assertU(adoc("id", "0", fieldName, "Not Available"));
|
||||
assertU(adoc("id", "1", fieldName, "Not Available"));
|
||||
assertU(adoc("id", "2", fieldName, "Not Available"));
|
||||
assertU(adoc("id", "3", fieldName, "Not Available"));
|
||||
assertU(adoc("id", "4", fieldName, "Not Available"));
|
||||
assertU(adoc("id", "5", fieldName, "Low"));
|
||||
assertU(adoc("id", "6", fieldName, "Low"));
|
||||
assertU(adoc("id", "7", fieldName, "Low"));
|
||||
assertU(adoc("id", "8", fieldName, "Low"));
|
||||
assertU(adoc("id", "9", fieldName, "Medium"));
|
||||
assertU(adoc("id", "10", fieldName, "Medium"));
|
||||
assertU(adoc("id", "11", fieldName, "Medium"));
|
||||
assertU(adoc("id", "12", fieldName, "High"));
|
||||
assertU(adoc("id", "13", fieldName, "High"));
|
||||
assertU(adoc("id", "14", fieldName, "Critical"));
|
||||
|
||||
|
||||
for (int i = 20; i <= 30; i++) {
|
||||
assertU(adoc("id", "" + i));
|
||||
}
|
||||
|
||||
assertU(commit());
|
||||
|
||||
assertQ("enum", req("q","*:*", "stats", "true", "stats.field", fieldName)
|
||||
, "//lst[@name='" + fieldName + "']/str[@name='min'][.='Not Available']"
|
||||
, "//lst[@name='" + fieldName + "']/str[@name='max'][.='Critical']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='count'][.='15']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='missing'][.='11']");
|
||||
|
||||
|
||||
assertQ("enum calcdistinct", req("q","*:*", "stats", "true", "stats.field", fieldName,
|
||||
StatsParams.STATS_CALC_DISTINCT, "true")
|
||||
, "//lst[@name='" + fieldName + "']/str[@name='min'][.='Not Available']"
|
||||
, "//lst[@name='" + fieldName + "']/str[@name='max'][.='Critical']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='count'][.='15']"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='countDistinct'][.='5']"
|
||||
, "count(//lst[@name='" + fieldName + "']/arr[@name='distinctValues']/*)=5"
|
||||
, "//lst[@name='" + fieldName + "']/long[@name='missing'][.='11']");
|
||||
|
||||
|
||||
final String pre = "//lst[@name='stats_fields']/lst[@name='"+fieldName+"']/lst[@name='facets']/lst[@name='severity']";
|
||||
|
||||
assertQ("enum + stats.facet", req("q","*:*", "stats", "true", "stats.field", fieldName,
|
||||
"stats.facet", fieldName)
|
||||
, pre + "/lst[@name='High']/str[@name='min'][.='High']"
|
||||
, pre + "/lst[@name='High']/str[@name='max'][.='High']"
|
||||
, pre + "/lst[@name='High']/long[@name='count'][.='2']"
|
||||
, pre + "/lst[@name='High']/long[@name='missing'][.='0']"
|
||||
, pre + "/lst[@name='Low']/str[@name='min'][.='Low']"
|
||||
, pre + "/lst[@name='Low']/str[@name='max'][.='Low']"
|
||||
, pre + "/lst[@name='Low']/long[@name='count'][.='4']"
|
||||
, pre + "/lst[@name='Low']/long[@name='missing'][.='0']"
|
||||
, pre + "/lst[@name='Medium']/str[@name='min'][.='Medium']"
|
||||
, pre + "/lst[@name='Medium']/str[@name='max'][.='Medium']"
|
||||
, pre + "/lst[@name='Medium']/long[@name='count'][.='3']"
|
||||
, pre + "/lst[@name='Medium']/long[@name='missing'][.='0']"
|
||||
, pre + "/lst[@name='Not Available']/str[@name='min'][.='Not Available']"
|
||||
, pre + "/lst[@name='Not Available']/str[@name='max'][.='Not Available']"
|
||||
, pre + "/lst[@name='Not Available']/long[@name='count'][.='5']"
|
||||
, pre + "/lst[@name='Not Available']/long[@name='missing'][.='0']"
|
||||
, pre + "/lst[@name='Critical']/str[@name='min'][.='Critical']"
|
||||
, pre + "/lst[@name='Critical']/str[@name='max'][.='Critical']"
|
||||
, pre + "/lst[@name='Critical']/long[@name='count'][.='1']"
|
||||
, pre + "/lst[@name='Critical']/long[@name='missing'][.='0']"
|
||||
);
|
||||
}
|
||||
|
||||
private Doc createDocValuesDocument(List<FldType> types, String fieldName, String id, Comparable... values) throws Exception {
|
||||
Doc doc = createDoc(types);
|
||||
doc.getValues("id").set(0, id);
|
||||
|
|
@ -1020,30 +1135,302 @@ public class StatsComponentTest extends AbstractSolrTestCase {
|
|||
return cat_docValues;
|
||||
}
|
||||
|
||||
public void testIndividualStatLocalParams() throws Exception {
|
||||
final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";
|
||||
|
||||
assertU(adoc("id", "1", "a_f", "2.3", "b_f", "9.7", "a_i", "9", "foo_t", "how now brown cow"));
|
||||
assertU(commit());
|
||||
|
||||
// some quick sanity check assertions...
|
||||
|
||||
// trivial check that we only get the exact 2 we ask for
|
||||
assertQ("ask for and get only 2 stats",
|
||||
req("q","*:*", "stats", "true",
|
||||
"stats.field", "{!key=k mean=true min=true}a_i")
|
||||
, kpre + "double[@name='mean'][.='9.0']"
|
||||
, kpre + "double[@name='min'][.='9.0']"
|
||||
, "count(" + kpre + "*)=2"
|
||||
);
|
||||
|
||||
// for stats that are true/false, sanity check false does it's job
|
||||
assertQ("min=true & max=false: only min should come back",
|
||||
req("q","*:*", "stats", "true",
|
||||
"stats.field", "{!key=k max=false min=true}a_i")
|
||||
, kpre + "double[@name='min'][.='9.0']"
|
||||
, "count(" + kpre + "*)=1"
|
||||
);
|
||||
assertQ("min=false: localparam stat means ignore default set, "+
|
||||
"but since only local param is false no stats should be returned",
|
||||
req("q","*:*", "stats", "true",
|
||||
"stats.field", "{!key=k min=false}a_i")
|
||||
, "count(" + kpre + "*)=0"
|
||||
);
|
||||
|
||||
double sum = 0;
|
||||
double sumOfSquares = 0;
|
||||
final int count = 20;
|
||||
for (int i = 0; i < count; i++) {
|
||||
assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i", String.valueOf(i%10), "foo_t", "how now brown cow"));
|
||||
sum+=i%10;
|
||||
sumOfSquares+=(i%10)*(i%10);
|
||||
}
|
||||
|
||||
assertU(commit());
|
||||
|
||||
EnumSet<Stat> allStats = EnumSet.allOf(Stat.class);
|
||||
|
||||
Map<Stat, String> expectedStats = new HashMap<>();
|
||||
expectedStats.put(Stat.min, "0.0");
|
||||
expectedStats.put(Stat.max, "9.0");
|
||||
expectedStats.put(Stat.missing, "0");
|
||||
expectedStats.put(Stat.sum, String.valueOf(sum));
|
||||
expectedStats.put(Stat.count, String.valueOf(count));
|
||||
expectedStats.put(Stat.mean, String.valueOf(sum/count));
|
||||
expectedStats.put(Stat.sumOfSquares, String.valueOf(sumOfSquares));
|
||||
expectedStats.put(Stat.stddev, String.valueOf(Math.sqrt(((count * sumOfSquares) - (sum * sum)) / (20 * (count - 1.0D)))));
|
||||
expectedStats.put(Stat.calcdistinct, "10");
|
||||
|
||||
Map<Stat, String> expectedType = new HashMap<>();
|
||||
expectedType.put(Stat.min, "double");
|
||||
expectedType.put(Stat.max, "double");
|
||||
expectedType.put(Stat.missing, "long");
|
||||
expectedType.put(Stat.sum, "double");
|
||||
expectedType.put(Stat.count, "long");
|
||||
expectedType.put(Stat.mean, "double");
|
||||
expectedType.put(Stat.sumOfSquares, "double");
|
||||
expectedType.put(Stat.stddev, "double");
|
||||
expectedType.put(Stat.calcdistinct, "long");
|
||||
|
||||
// canary in the coal mine
|
||||
assertEquals("size of expectedStats doesn't match all known stats; " +
|
||||
"enum was updated w/o updating test?",
|
||||
expectedStats.size(), allStats.size());
|
||||
assertEquals("size of expectedType doesn't match all known stats; " +
|
||||
"enum was updated w/o updating test?",
|
||||
expectedType.size(), allStats.size());
|
||||
|
||||
// whitebox test: explicitly ask for isShard=true with an individual stat
|
||||
for (Stat stat : expectedStats.keySet()) {
|
||||
EnumSet<Stat> distribDeps = stat.getDistribDeps();
|
||||
|
||||
StringBuilder exclude = new StringBuilder();
|
||||
List<String> testParas = new ArrayList<String>(distribDeps.size() + 2);
|
||||
int calcdistinctFudge = 0;
|
||||
|
||||
for (Stat perShardStat : distribDeps ){
|
||||
String key = perShardStat.toString();
|
||||
if (perShardStat.equals(Stat.calcdistinct)) {
|
||||
// this abomination breaks all the rules - uses a diff response key and triggers
|
||||
// the additional "distinctValues" stat
|
||||
key = "countDistinct";
|
||||
calcdistinctFudge++;
|
||||
testParas.add("count(" + kpre + "arr[@name='distinctValues']/*)=10");
|
||||
}
|
||||
testParas.add(kpre + expectedType.get(perShardStat) +
|
||||
"[@name='" + key + "'][.='" + expectedStats.get(perShardStat) + "']");
|
||||
// even if we go out of our way to exclude the dependent stats,
|
||||
// the shard should return them since they are a dependency for the requested stat
|
||||
exclude.append(perShardStat + "=false ");
|
||||
}
|
||||
testParas.add("count(" + kpre + "*)=" + (distribDeps.size() + calcdistinctFudge));
|
||||
|
||||
assertQ("ask for only "+stat+", with isShard=true, and expect only deps: " + distribDeps,
|
||||
req("q", "*:*", "isShard", "true", "stats", "true",
|
||||
"stats.field", "{!key=k " + exclude + stat + "=true}a_i")
|
||||
, testParas.toArray(new String[testParas.size()])
|
||||
);
|
||||
}
|
||||
|
||||
// test all the possible combinations (of all possible sizes) of stats params
|
||||
for (int numParams = 1; numParams <= allStats.size(); numParams++) {
|
||||
for (EnumSet<Stat> set : new StatSetCombinations(numParams, allStats)) {
|
||||
|
||||
// EnumSets use natural ordering, we want to randomize the order of the params
|
||||
List<Stat> combo = new ArrayList<Stat>(set);
|
||||
Collections.shuffle(combo, random());
|
||||
|
||||
StringBuilder paras = new StringBuilder("{!key=k ");
|
||||
List<String> testParas = new ArrayList<String>(numParams + 2);
|
||||
|
||||
int calcdistinctFudge = 0;
|
||||
for (Stat stat : combo) {
|
||||
String key = stat.toString();
|
||||
if (stat.equals(Stat.calcdistinct)) {
|
||||
// this abomination breaks all the rules - uses a diff response key and triggers
|
||||
// the additional "distinctValues" stat
|
||||
key = "countDistinct";
|
||||
calcdistinctFudge++;
|
||||
testParas.add("count(" + kpre + "arr[@name='distinctValues']/*)=10");
|
||||
}
|
||||
paras.append(stat + "=true ");
|
||||
testParas.add(kpre + expectedType.get(stat) + "[@name='" + key + "'][.='" + expectedStats.get(stat) + "']");
|
||||
}
|
||||
|
||||
paras.append("}a_i");
|
||||
testParas.add("count(" + kpre + "*)=" + (combo.size() + calcdistinctFudge));
|
||||
|
||||
assertQ("ask for an get only: "+ combo,
|
||||
req("q","*:*", "stats", "true",
|
||||
"stats.field", paras.toString())
|
||||
, testParas.toArray(new String[testParas.size()])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// public void testOtherFacetStatsResult() throws Exception {
|
||||
//
|
||||
// assertU(adoc("id", "1", "stats_tls_dv", "10", "active_i", "1"));
|
||||
// assertU(adoc("id", "2", "stats_tls_dv", "20", "active_i", "1"));
|
||||
// assertU(commit());
|
||||
// assertU(adoc("id", "3", "stats_tls_dv", "30", "active_i", "2"));
|
||||
// assertU(adoc("id", "4", "stats_tls_dv", "40", "active_i", "2"));
|
||||
// assertU(commit());
|
||||
//
|
||||
// final String pre = "//lst[@name='stats_fields']/lst[@name='stats_tls_dv']/lst[@name='facets']/lst[@name='active_i']";
|
||||
//
|
||||
// assertQ("test value for active_s=true", req("q", "*:*", "stats", "true", "stats.field", "stats_tls_dv", "stats.facet", "active_i","indent", "true")
|
||||
// , "*[count("+pre+")=1]"
|
||||
// , pre+"/lst[@name='1']/double[@name='min'][.='10.0']"
|
||||
// , pre+"/lst[@name='1']/double[@name='max'][.='20.0']"
|
||||
// , pre+"/lst[@name='1']/double[@name='sum'][.='30.0']"
|
||||
// , pre+"/lst[@name='1']/long[@name='count'][.='2']"
|
||||
// , pre+"/lst[@name='1']/long[@name='missing'][.='0']"
|
||||
// , pre + "/lst[@name='true']/long[@name='countDistinct'][.='2']"
|
||||
// , "count(" + pre + "/lst[@name='true']/arr[@name='distinctValues']/*)=2"
|
||||
// , pre+"/lst[@name='1']/double[@name='sumOfSquares'][.='500.0']"
|
||||
// , pre+"/lst[@name='1']/double[@name='mean'][.='15.0']"
|
||||
// , pre+"/lst[@name='1']/double[@name='stddev'][.='7.0710678118654755']"
|
||||
// );
|
||||
// }
|
||||
// Test for Solr-6349
|
||||
public void testCalcDistinctStats() throws Exception {
|
||||
final String kpre = XPRE + "lst[@name='stats_fields']/lst[@name='k']/";
|
||||
final String min = "count(" + kpre +"/double[@name='min'])";
|
||||
final String countDistinct = "count(" + kpre +"/long[@name='countDistinct'])";
|
||||
final String distinctValues = "count(" + kpre +"/arr[@name='distinctValues'])";
|
||||
|
||||
final int count = 20;
|
||||
for (int i = 0; i < count; i++) {
|
||||
assertU(adoc("id", String.valueOf(i), "a_f", "2.3", "b_f", "9.7", "a_i",
|
||||
String.valueOf(i % 10), "foo_t", "how now brown cow"));
|
||||
}
|
||||
|
||||
assertU(commit());
|
||||
|
||||
String[] baseParams = new String[] { "q", "*:*", "stats", "true","indent", "true" };
|
||||
|
||||
for (SolrParams p : new SolrParams[] {
|
||||
params("stats.field", "{!key=k}a_i"),
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k}a_i"),
|
||||
params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k}a_i"),
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
"f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k}a_i"),
|
||||
params("stats.field", "{!key=k min='true'}a_i"),
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
"f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
"stats.field", "{!key=k min='true' calcdistinct='false'}a_i"),
|
||||
}) {
|
||||
|
||||
assertQ("min is either default or explicitly requested; "+
|
||||
"countDistinct & distinctValues either default or explicitly prevented"
|
||||
, req(p, baseParams)
|
||||
, min + "=1"
|
||||
, countDistinct + "=0"
|
||||
, distinctValues + "=0");
|
||||
}
|
||||
|
||||
for (SolrParams p : new SolrParams[] {
|
||||
params("stats.calcdistinct", "true",
|
||||
"stats.field", "{!key=k}a_i"),
|
||||
params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
"stats.field", "{!key=k}a_i"),
|
||||
params("stats.calcdistinct", "false",
|
||||
"f.a_i." + StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
"stats.field", "{!key=k}a_i"),
|
||||
params("stats.calcdistinct", "false ",
|
||||
"stats.field", "{!key=k min=true calcdistinct=true}a_i"),
|
||||
params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k min=true calcdistinct=true}a_i"),
|
||||
params("stats.calcdistinct", "false ",
|
||||
"f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k min=true calcdistinct=true}a_i"),
|
||||
}) {
|
||||
|
||||
assertQ("min is either default or explicitly requested; " +
|
||||
"countDistinct & distinctValues explicitly requested"
|
||||
, req(p, baseParams)
|
||||
, min + "=1"
|
||||
, countDistinct + "=1"
|
||||
, distinctValues + "=1");
|
||||
}
|
||||
|
||||
for (SolrParams p : new SolrParams[] {
|
||||
params("stats.field", "{!key=k calcdistinct=true}a_i"),
|
||||
|
||||
params("stats.calcdistinct", "true",
|
||||
"stats.field", "{!key=k min='false'}a_i"),
|
||||
|
||||
params("stats.calcdistinct", "true",
|
||||
"stats.field", "{!key=k max='true' min='false'}a_i"),
|
||||
|
||||
params("stats.calcdistinct", "false",
|
||||
"stats.field", "{!key=k calcdistinct=true}a_i"),
|
||||
params("f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k calcdistinct=true}a_i"),
|
||||
params("stats.calcdistinct", "false",
|
||||
"f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k calcdistinct=true}a_i"),
|
||||
params("stats.calcdistinct", "false",
|
||||
"f.a_i." + StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k min='false' calcdistinct=true}a_i"),
|
||||
}) {
|
||||
|
||||
assertQ("min is explicitly excluded; " +
|
||||
"countDistinct & distinctValues explicitly requested"
|
||||
, req(p, baseParams)
|
||||
, min + "=0"
|
||||
, countDistinct + "=1"
|
||||
, distinctValues + "=1");
|
||||
}
|
||||
|
||||
for (SolrParams p : new SolrParams[] {
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "true",
|
||||
"stats.field", "{!key=k min=true}a_i"),
|
||||
params("f.a_i.stats.calcdistinct", "true",
|
||||
"stats.field", "{!key=k min=true}a_i"),
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"f.a_i.stats.calcdistinct", "true",
|
||||
"stats.field", "{!key=k min=true}a_i"),
|
||||
params("f.a_i.stats.calcdistinct", "false",
|
||||
"stats.field", "{!key=k min=true calcdistinct=true}a_i"),
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"stats.field", "{!key=k min=true calcdistinct=true}a_i"),
|
||||
params(StatsParams.STATS_CALC_DISTINCT, "false",
|
||||
"f.a_i.stats.calcdistinct", "false",
|
||||
"stats.field", "{!key=k min=true calcdistinct=true}a_i"),
|
||||
}) {
|
||||
|
||||
assertQ("min is explicitly requested; " +
|
||||
"countDistinct & distinctValues explicitly requested"
|
||||
, req(p, baseParams)
|
||||
, min + "=1"
|
||||
, countDistinct + "=1"
|
||||
, distinctValues + "=1");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* given a comboSize and an EnumSet of Stats, generates iterators that produce every possible
|
||||
* enum combination of that size
|
||||
*/
|
||||
public static final class StatSetCombinations implements Iterable<EnumSet<Stat>> {
|
||||
// we need an array so we can do fixed index offset lookups
|
||||
private final Stat[] all;
|
||||
private final Combinations intCombos;
|
||||
public StatSetCombinations(int comboSize, EnumSet<Stat> universe) {
|
||||
// NOTE: should not need to sort, EnumSet uses natural ordering
|
||||
all = universe.toArray(new Stat[universe.size()]);
|
||||
intCombos = new Combinations(all.length, comboSize);
|
||||
}
|
||||
public Iterator<EnumSet<Stat>> iterator() {
|
||||
return new Iterator<EnumSet<Stat>>() {
|
||||
final Iterator<int[]> wrapped = intCombos.iterator();
|
||||
public void remove() {
|
||||
wrapped.remove();
|
||||
}
|
||||
public boolean hasNext() {
|
||||
return wrapped.hasNext();
|
||||
}
|
||||
public EnumSet<Stat> next() {
|
||||
EnumSet<Stat> result = EnumSet.noneOf(Stat.class);
|
||||
int[] indexes = wrapped.next();
|
||||
for (int i = 0; i < indexes.length; i++) {
|
||||
result.add(all[indexes[i]]);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
3ac44a8664228384bc68437264cf7c4cf112f579
|
||||
|
|
@ -0,0 +1,457 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
|
||||
Apache Commons Math includes the following code provided to the ASF under the
|
||||
Apache License 2.0:
|
||||
|
||||
- The inverse error function implementation in the Erf class is based on CUDA
|
||||
code developed by Mike Giles, Oxford-Man Institute of Quantitative Finance,
|
||||
and published in GPU Computing Gems, volume 2, 2010 (grant received on
|
||||
March 23th 2013)
|
||||
- The LinearConstraint, LinearObjectiveFunction, LinearOptimizer,
|
||||
RelationShip, SimplexSolver and SimplexTableau classes in package
|
||||
org.apache.commons.math3.optimization.linear include software developed by
|
||||
Benjamin McCann (http://www.benmccann.com) and distributed with
|
||||
the following copyright: Copyright 2009 Google Inc. (grant received on
|
||||
March 16th 2009)
|
||||
- The class "org.apache.commons.math3.exception.util.LocalizedFormatsTest" which
|
||||
is an adapted version of "OrekitMessagesTest" test class for the Orekit library
|
||||
- The "org.apache.commons.math3.analysis.interpolation.HermiteInterpolator"
|
||||
has been imported from the Orekit space flight dynamics library.
|
||||
|
||||
===============================================================================
|
||||
|
||||
|
||||
|
||||
APACHE COMMONS MATH DERIVATIVE WORKS:
|
||||
|
||||
The Apache commons-math library includes a number of subcomponents
|
||||
whose implementation is derived from original sources written
|
||||
in C or Fortran. License terms of the original sources
|
||||
are reproduced below.
|
||||
|
||||
===============================================================================
|
||||
For the lmder, lmpar and qrsolv Fortran routine from minpack and translated in
|
||||
the LevenbergMarquardtOptimizer class in package
|
||||
org.apache.commons.math3.optimization.general
|
||||
Original source copyright and license statement:
|
||||
|
||||
Minpack Copyright Notice (1999) University of Chicago. All rights reserved
|
||||
|
||||
Redistribution and use in source and binary forms, with or
|
||||
without modification, are permitted provided that the
|
||||
following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above
|
||||
copyright notice, this list of conditions and the following
|
||||
disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above
|
||||
copyright notice, this list of conditions and the following
|
||||
disclaimer in the documentation and/or other materials
|
||||
provided with the distribution.
|
||||
|
||||
3. The end-user documentation included with the
|
||||
redistribution, if any, must include the following
|
||||
acknowledgment:
|
||||
|
||||
"This product includes software developed by the
|
||||
University of Chicago, as Operator of Argonne National
|
||||
Laboratory.
|
||||
|
||||
Alternately, this acknowledgment may appear in the software
|
||||
itself, if and wherever such third-party acknowledgments
|
||||
normally appear.
|
||||
|
||||
4. WARRANTY DISCLAIMER. THE SOFTWARE IS SUPPLIED "AS IS"
|
||||
WITHOUT WARRANTY OF ANY KIND. THE COPYRIGHT HOLDER, THE
|
||||
UNITED STATES, THE UNITED STATES DEPARTMENT OF ENERGY, AND
|
||||
THEIR EMPLOYEES: (1) DISCLAIM ANY WARRANTIES, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES
|
||||
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE
|
||||
OR NON-INFRINGEMENT, (2) DO NOT ASSUME ANY LEGAL LIABILITY
|
||||
OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR
|
||||
USEFULNESS OF THE SOFTWARE, (3) DO NOT REPRESENT THAT USE OF
|
||||
THE SOFTWARE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS, (4)
|
||||
DO NOT WARRANT THAT THE SOFTWARE WILL FUNCTION
|
||||
UNINTERRUPTED, THAT IT IS ERROR-FREE OR THAT ANY ERRORS WILL
|
||||
BE CORRECTED.
|
||||
|
||||
5. LIMITATION OF LIABILITY. IN NO EVENT WILL THE COPYRIGHT
|
||||
HOLDER, THE UNITED STATES, THE UNITED STATES DEPARTMENT OF
|
||||
ENERGY, OR THEIR EMPLOYEES: BE LIABLE FOR ANY INDIRECT,
|
||||
INCIDENTAL, CONSEQUENTIAL, SPECIAL OR PUNITIVE DAMAGES OF
|
||||
ANY KIND OR NATURE, INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
PROFITS OR LOSS OF DATA, FOR ANY REASON WHATSOEVER, WHETHER
|
||||
SUCH LIABILITY IS ASSERTED ON THE BASIS OF CONTRACT, TORT
|
||||
(INCLUDING NEGLIGENCE OR STRICT LIABILITY), OR OTHERWISE,
|
||||
EVEN IF ANY OF SAID PARTIES HAS BEEN WARNED OF THE
|
||||
POSSIBILITY OF SUCH LOSS OR DAMAGES.
|
||||
===============================================================================
|
||||
|
||||
Copyright and license statement for the odex Fortran routine developed by
|
||||
E. Hairer and G. Wanner and translated in GraggBulirschStoerIntegrator class
|
||||
in package org.apache.commons.math3.ode.nonstiff:
|
||||
|
||||
|
||||
Copyright (c) 2004, Ernst Hairer
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are
|
||||
met:
|
||||
|
||||
- Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
- Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
|
||||
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
||||
TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
|
||||
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
|
||||
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
===============================================================================
|
||||
|
||||
Copyright and license statement for the original Mersenne twister C
|
||||
routines translated in MersenneTwister class in package
|
||||
org.apache.commons.math3.random:
|
||||
|
||||
Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions
|
||||
are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
3. The names of its contributors may not be used to endorse or promote
|
||||
products derived from this software without specific prior written
|
||||
permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
|
||||
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
|
||||
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
||||
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
||||
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
||||
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
===============================================================================
|
||||
|
||||
The initial code for shuffling an array (originally in class
|
||||
"org.apache.commons.math3.random.RandomDataGenerator", now replaced by
|
||||
a method in class "org.apache.commons.math3.util.MathArrays") was
|
||||
inspired from the algorithm description provided in
|
||||
"Algorithms", by Ian Craw and John Pulham (University of Aberdeen 1999).
|
||||
The textbook (containing a proof that the shuffle is uniformly random) is
|
||||
available here:
|
||||
http://citeseerx.ist.psu.edu/viewdoc/download;?doi=10.1.1.173.1898&rep=rep1&type=pdf
|
||||
|
||||
===============================================================================
|
||||
License statement for the direction numbers in the resource files for Sobol sequences.
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
Licence pertaining to sobol.cc and the accompanying sets of direction numbers
|
||||
|
||||
-----------------------------------------------------------------------------
|
||||
Copyright (c) 2008, Frances Y. Kuo and Stephen Joe
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the names of the copyright holders nor the names of the
|
||||
University of New South Wales and the University of Waikato
|
||||
and its contributors may be used to endorse or promote products derived
|
||||
from this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
|
||||
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
===============================================================================
|
||||
|
||||
The initial commit of package "org.apache.commons.math3.ml.neuralnet" is
|
||||
an adapted version of code developed in the context of the Data Processing
|
||||
and Analysis Consortium (DPAC) of the "Gaia" project of the European Space
|
||||
Agency (ESA).
|
||||
===============================================================================
|
||||
|
||||
The initial commit of the class "org.apache.commons.math3.special.BesselJ" is
|
||||
an adapted version of code translated from the netlib Fortran program, rjbesl
|
||||
http://www.netlib.org/specfun/rjbesl by R.J. Cody at Argonne National
|
||||
Laboratory (USA). There is no license or copyright statement included with the
|
||||
original Fortran sources.
|
||||
===============================================================================
|
||||
|
||||
|
||||
The BracketFinder (package org.apache.commons.math3.optimization.univariate)
|
||||
and PowellOptimizer (package org.apache.commons.math3.optimization.general)
|
||||
classes are based on the Python code in module "optimize.py" (version 0.5)
|
||||
developed by Travis E. Oliphant for the SciPy library (http://www.scipy.org/)
|
||||
Copyright © 2003-2009 SciPy Developers.
|
||||
|
||||
SciPy license
|
||||
Copyright © 2001, 2002 Enthought, Inc.
|
||||
All rights reserved.
|
||||
|
||||
Copyright © 2003-2013 SciPy Developers.
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright
|
||||
notice, this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of Enthought nor the names of the SciPy Developers may
|
||||
be used to endorse or promote products derived from this software without
|
||||
specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY
|
||||
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY
|
||||
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
===============================================================================
|
||||
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
Apache Commons Math
|
||||
Copyright 2001-2015 The Apache Software Foundation
|
||||
|
||||
This product includes software developed at
|
||||
The Apache Software Foundation (http://www.apache.org/).
|
||||
|
||||
This product includes software developed for Orekit by
|
||||
CS Systèmes d'Information (http://www.c-s.fr/)
|
||||
Copyright 2010-2012 CS Systèmes d'Information
|
||||
|
|
@ -750,7 +750,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
try {
|
||||
String m = (null == message) ? "" : message + " ";
|
||||
String response = h.query(req);
|
||||
|
||||
|
||||
if (req.getParams().getBool("facet", false)) {
|
||||
// add a test to ensure that faceting did not throw an exception
|
||||
// internally, where it would be added to facet_counts/exception
|
||||
|
|
@ -1888,6 +1888,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
|
|||
FileUtils.copyFile(new File(top, "open-exchange-rates.json"), new File(subHome, "open-exchange-rates.json"));
|
||||
FileUtils.copyFile(new File(top, "protwords.txt"), new File(subHome, "protwords.txt"));
|
||||
FileUtils.copyFile(new File(top, "schema.xml"), new File(subHome, "schema.xml"));
|
||||
FileUtils.copyFile(new File(top, "enumsConfig.xml"), new File(subHome, "enumsConfig.xml"));
|
||||
FileUtils.copyFile(new File(top, "solrconfig.snippet.randomindexconfig.xml"), new File(subHome, "solrconfig.snippet.randomindexconfig.xml"));
|
||||
FileUtils.copyFile(new File(top, "solrconfig.xml"), new File(subHome, "solrconfig.xml"));
|
||||
FileUtils.copyFile(new File(top, "stopwords.txt"), new File(subHome, "stopwords.txt"));
|
||||
|
|
|
|||
Loading…
Reference in New Issue