honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Histogram Facet: Improve value field case performance, fix wrong total computation with multi valued fields by introducing total_count, closes #829.

This commit is contained in:
kimchy 2011-04-04 21:30:46 +03:00
parent 58c606d6d9
commit 46088b9f8a
18 changed files with 490 additions and 436 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
modules/elasticsearch/src/main/java/org/elasticsearch/common/CacheRecycler.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
import org.elasticsearch.common.trove.map.hash.*;
import java.lang.ref.SoftReference;
@ -29,6 +30,9 @@ import java.util.Deque;
public class CacheRecycler {
public static void clear() {
longObjectHashMap.remove();
longObjectHashMap2.remove();
longLongHashMap.remove();
intIntHashMap.remove();
floatIntHashMap.remove();
doubleIntHashMap.remove();
@ -38,11 +42,69 @@ public class CacheRecycler {
intArray.remove();
}
// ----- ExtTLongObjectHashMap ----
private static ThreadLocal<SoftReference<Deque<ExtTLongObjectHashMap>>> longObjectHashMap2 = new ThreadLocal<SoftReference<Deque<ExtTLongObjectHashMap>>>();
public static <T> ExtTLongObjectHashMap<T> popLongObjectMap2() {
SoftReference<Deque<ExtTLongObjectHashMap>> ref = longObjectHashMap2.get();
Deque<ExtTLongObjectHashMap> deque = ref == null ? null : ref.get();
if (deque == null) {
deque = new ArrayDeque<ExtTLongObjectHashMap>();
longObjectHashMap2.set(new SoftReference<Deque<ExtTLongObjectHashMap>>(deque));
}
if (deque.isEmpty()) {
return new ExtTLongObjectHashMap();
}
ExtTLongObjectHashMap map = deque.pollFirst();
map.clear();
return map;
}
public static void pushLongObjectMap2(ExtTLongObjectHashMap map) {
SoftReference<Deque<ExtTLongObjectHashMap>> ref = longObjectHashMap2.get();
Deque<ExtTLongObjectHashMap> deque = ref == null ? null : ref.get();
if (deque == null) {
deque = new ArrayDeque<ExtTLongObjectHashMap>();
longObjectHashMap2.set(new SoftReference<Deque<ExtTLongObjectHashMap>>(deque));
}
deque.add(map);
}
// ----- ExtTLongObjectHashMap ----
private static ThreadLocal<SoftReference<Deque<ExtTLongObjectHashMap>>> longObjectHashMap = new ThreadLocal<SoftReference<Deque<ExtTLongObjectHashMap>>>();
public static <T> ExtTLongObjectHashMap<T> popLongObjectMap() {
SoftReference<Deque<ExtTLongObjectHashMap>> ref = longObjectHashMap.get();
Deque<ExtTLongObjectHashMap> deque = ref == null ? null : ref.get();
if (deque == null) {
deque = new ArrayDeque<ExtTLongObjectHashMap>();
longObjectHashMap.set(new SoftReference<Deque<ExtTLongObjectHashMap>>(deque));
}
if (deque.isEmpty()) {
return new ExtTLongObjectHashMap();
}
ExtTLongObjectHashMap map = deque.pollFirst();
map.clear();
return map;
}
public static void pushLongObjectMap(ExtTLongObjectHashMap map) {
SoftReference<Deque<ExtTLongObjectHashMap>> ref = longObjectHashMap.get();
Deque<ExtTLongObjectHashMap> deque = ref == null ? null : ref.get();
if (deque == null) {
deque = new ArrayDeque<ExtTLongObjectHashMap>();
longObjectHashMap.set(new SoftReference<Deque<ExtTLongObjectHashMap>>(deque));
}
deque.add(map);
}
// ----- TLongLongHashMap ----
private static ThreadLocal<SoftReference<Deque<TLongLongHashMap>>> longLongHashMap = new ThreadLocal<SoftReference<Deque<TLongLongHashMap>>>();
public static TLongLongHashMap popLongLongMap() {
SoftReference<Deque<TLongLongHashMap>> ref = longLongHashMap.get();
Deque<TLongLongHashMap> deque = ref == null ? null : ref.get();

36
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/CountAndTotalHistogramFacetCollector.java → modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/FullHistogramFacetCollector.java
View File

@ -20,8 +20,8 @@
package org.elasticsearch.search.facet.histogram;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.common.trove.map.hash.TLongDoubleHashMap;
import org.elasticsearch.common.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.field.data.NumericFieldData;
@ -40,7 +40,7 @@ import java.io.IOException;
*
* @author kimchy (shay.banon)
*/
public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector {
public class FullHistogramFacetCollector extends AbstractFacetCollector {
private final String indexFieldName;
@ -54,7 +54,7 @@ public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector
private final HistogramProc histoProc;
public CountAndTotalHistogramFacetCollector(String facetName, String fieldName, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
public FullHistogramFacetCollector(String facetName, String fieldName, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
super(facetName);
this.comparatorType = comparatorType;
this.fieldDataCache = context.fieldDataCache();
@ -86,7 +86,8 @@ public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector
}
@Override public Facet facet() {
return new InternalCountAndTotalHistogramFacet(facetName, comparatorType, histoProc.counts(), histoProc.totals());
CacheRecycler.pushLongObjectMap(histoProc.entries);
return new InternalFullHistogramFacet(facetName, comparatorType, histoProc.entries.valueCollection());
}
public static long bucket(double value, long interval) {
@ -95,11 +96,9 @@ public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector
public static class HistogramProc implements NumericFieldData.DoubleValueInDocProc {
private final long interval;
final long interval;
private final TLongLongHashMap counts = new TLongLongHashMap();
private final TLongDoubleHashMap totals = new TLongDoubleHashMap();
final ExtTLongObjectHashMap<InternalFullHistogramFacet.FullEntry> entries = CacheRecycler.popLongObjectMap();
public HistogramProc(long interval) {
this.interval = interval;
@ -107,16 +106,15 @@ public class CountAndTotalHistogramFacetCollector extends AbstractFacetCollector
@Override public void onValue(int docId, double value) {
long bucket = bucket(value, interval);
counts.adjustOrPutValue(bucket, 1, 1);
totals.adjustOrPutValue(bucket, value, value);
}
public TLongLongHashMap counts() {
return counts;
}
public TLongDoubleHashMap totals() {
return totals;
InternalFullHistogramFacet.FullEntry entry = entries.get(bucket);
if (entry == null) {
entry = new InternalFullHistogramFacet.FullEntry(bucket, 1, 1, value);
entries.put(bucket, entry);
} else {
entry.count++;
entry.totalCount++;
entry.total += value;
}
}
}
}

31
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/HistogramFacet.java
View File

@ -51,18 +51,39 @@ public interface HistogramFacet extends Facet, Iterable<HistogramFacet.Entry> {
KEY((byte) 0, "key", new Comparator<Entry>() {
@Override public int compare(Entry o1, Entry o2) {
// push nulls to the end
if (o1 == null) {
return 1;
}
if (o2 == null) {
return -1;
}
return (o1.key() < o2.key() ? -1 : (o1.key() == o2.key() ? 0 : 1));
}
}),
COUNT((byte) 1, "count", new Comparator<Entry>() {
@Override public int compare(Entry o1, Entry o2) {
// push nulls to the end
if (o1 == null) {
return 1;
}
if (o2 == null) {
return -1;
}
return (o1.count() < o2.count() ? -1 : (o1.count() == o2.count() ? 0 : 1));
}
}),
TOTAL((byte) 2, "total", new Comparator<Entry>() {
@Override public int compare(Entry o1, Entry o2) {
// push nulls to the end
if (o1 == null) {
return 1;
}
if (o2 == null) {
return -1;
}
return (o1.total() < o2.total() ? -1 : (o1.total() == o2.total() ? 0 : 1));
}
});
@ -136,6 +157,16 @@ public interface HistogramFacet extends Facet, Iterable<HistogramFacet.Entry> {
*/
long getCount();
/**
* The total count of values aggregated to compute the total.
*/
long totalCount();
/**
* The total count of values aggregated to compute the total.
*/
long getTotalCount();
/**
* The sum / total of the value field that fall within this key "interval".
*/

2
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/HistogramFacetProcessor.java
View File

@ -106,7 +106,7 @@ public class HistogramFacetProcessor extends AbstractComponent implements FacetP
} else if (valueField == null) {
return new CountHistogramFacetCollector(facetName, keyField, interval, comparatorType, context);
} else if (keyField.equals(valueField)) {
return new CountAndTotalHistogramFacetCollector(facetName, keyField, interval, comparatorType, context);
return new FullHistogramFacetCollector(facetName, keyField, interval, comparatorType, context);
} else {
// we have a value field, and its different than the key
return new KeyValueHistogramFacetCollector(facetName, keyField, valueField, interval, comparatorType, context);

302
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountAndTotalHistogramFacet.java
View File

@ -1,302 +0,0 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.facet.histogram;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.trove.iterator.TLongDoubleIterator;
import org.elasticsearch.common.trove.iterator.TLongLongIterator;
import org.elasticsearch.common.trove.map.hash.TLongDoubleHashMap;
import org.elasticsearch.common.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.search.facet.Facet;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
/**
* @author kimchy (shay.banon)
*/
public class InternalCountAndTotalHistogramFacet extends InternalHistogramFacet {
private static final String STREAM_TYPE = "ctHistogram";
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);
}
static Stream STREAM = new Stream() {
@Override public Facet readFacet(String type, StreamInput in) throws IOException {
return readHistogramFacet(in);
}
};
@Override public String streamType() {
return STREAM_TYPE;
}
/**
* A histogram entry representing a single entry within the result of a histogram facet.
*/
public class CountAndTotalEntry implements Entry {
private final long key;
private final long count;
private final double total;
public CountAndTotalEntry(long key, long count, double total) {
this.key = key;
this.count = count;
this.total = total;
}
/**
* The key value of the histogram.
*/
public long key() {
return key;
}
/**
* The key value of the histogram.
*/
public long getKey() {
return key();
}
/**
* The number of hits that fall within that key "range" or "interval".
*/
public long count() {
return count;
}
/**
* The number of hits that fall within that key "range" or "interval".
*/
public long getCount() {
return count();
}
/**
* The sum / total of the value field that fall within this key "interval".
*/
public double total() {
return total;
}
/**
* The sum / total of the value field that fall within this key "interval".
*/
public double getTotal() {
return total();
}
/**
* The mean of this facet interval.
*/
public double mean() {
return total / count;
}
/**
* The mean of this facet interval.
*/
public double getMean() {
return mean();
}
}
private String name;
private ComparatorType comparatorType;
TLongLongHashMap counts;
TLongDoubleHashMap totals;
CountAndTotalEntry[] entries = null;
private InternalCountAndTotalHistogramFacet() {
}
public InternalCountAndTotalHistogramFacet(String name, ComparatorType comparatorType, TLongLongHashMap counts, TLongDoubleHashMap totals) {
this.name = name;
this.comparatorType = comparatorType;
this.counts = counts;
this.totals = totals;
}
@Override public String name() {
return this.name;
}
@Override public String getName() {
return name();
}
@Override public String type() {
return TYPE;
}
@Override public String getType() {
return type();
}
@Override public List<CountAndTotalEntry> entries() {
return Arrays.asList(computeEntries());
}
@Override public List<CountAndTotalEntry> getEntries() {
return entries();
}
@Override public Iterator<Entry> iterator() {
return (Iterator) entries().iterator();
}
private CountAndTotalEntry[] computeEntries() {
if (entries != null) {
return entries;
}
entries = new CountAndTotalEntry[counts.size()];
int i = 0;
for (TLongLongIterator it = counts.iterator(); it.hasNext();) {
it.advance();
entries[i++] = new CountAndTotalEntry(it.key(), it.value(), totals.get(it.key()));
}
Arrays.sort(entries, comparatorType.comparator());
return entries;
}
@Override public Facet reduce(String name, List<Facet> facets) {
if (facets.size() == 1) {
return facets.get(0);
}
TLongLongHashMap counts = null;
TLongDoubleHashMap totals = null;
InternalCountAndTotalHistogramFacet firstHistoFacet = (InternalCountAndTotalHistogramFacet) facets.get(0);
for (Facet facet : facets) {
InternalCountAndTotalHistogramFacet histoFacet = (InternalCountAndTotalHistogramFacet) facet;
if (!histoFacet.counts.isEmpty()) {
if (counts == null) {
counts = histoFacet.counts;
} else {
for (TLongLongIterator it = histoFacet.counts.iterator(); it.hasNext();) {
it.advance();
counts.adjustOrPutValue(it.key(), it.value(), it.value());
}
}
}
if (!histoFacet.totals.isEmpty()) {
if (totals == null) {
totals = histoFacet.totals;
} else {
for (TLongDoubleIterator it = histoFacet.totals.iterator(); it.hasNext();) {
it.advance();
totals.adjustOrPutValue(it.key(), it.value(), it.value());
}
}
}
}
if (counts == null) {
counts = InternalCountAndTotalHistogramFacet.EMPTY_LONG_LONG_MAP;
}
if (totals == null) {
totals = InternalCountAndTotalHistogramFacet.EMPTY_LONG_DOUBLE_MAP;
}
firstHistoFacet.counts = counts;
firstHistoFacet.totals = totals;
return firstHistoFacet;
}
static final class Fields {
static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
static final XContentBuilderString ENTRIES = new XContentBuilderString("entries");
static final XContentBuilderString KEY = new XContentBuilderString("key");
static final XContentBuilderString COUNT = new XContentBuilderString("count");
static final XContentBuilderString TOTAL = new XContentBuilderString("total");
static final XContentBuilderString MEAN = new XContentBuilderString("mean");
}
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name);
builder.field(Fields._TYPE, HistogramFacet.TYPE);
builder.startArray(Fields.ENTRIES);
for (Entry entry : computeEntries()) {
builder.startObject();
builder.field(Fields.KEY, entry.key());
builder.field(Fields.COUNT, entry.count());
builder.field(Fields.TOTAL, entry.total());
builder.field(Fields.MEAN, entry.mean());
builder.endObject();
}
builder.endArray();
builder.endObject();
return builder;
}
public static InternalCountAndTotalHistogramFacet readHistogramFacet(StreamInput in) throws IOException {
InternalCountAndTotalHistogramFacet facet = new InternalCountAndTotalHistogramFacet();
facet.readFrom(in);
return facet;
}
@Override public void readFrom(StreamInput in) throws IOException {
name = in.readUTF();
comparatorType = ComparatorType.fromId(in.readByte());
int size = in.readVInt();
if (size == 0) {
counts = EMPTY_LONG_LONG_MAP;
totals = EMPTY_LONG_DOUBLE_MAP;
} else {
counts = new TLongLongHashMap(size);
totals = new TLongDoubleHashMap(size);
for (int i = 0; i < size; i++) {
long key = in.readLong();
counts.put(key, in.readVLong());
totals.put(key, in.readDouble());
}
}
}
@Override public void writeTo(StreamOutput out) throws IOException {
out.writeUTF(name);
out.writeByte(comparatorType.id());
// optimize the write, since we know we have the same buckets as keys
out.writeVInt(counts.size());
for (TLongLongIterator it = counts.iterator(); it.hasNext();) {
it.advance();
out.writeLong(it.key());
out.writeVLong(it.value());
out.writeDouble(totals.get(it.key()));
}
}
static final TLongLongHashMap EMPTY_LONG_LONG_MAP = new TLongLongHashMap();
static final TLongDoubleHashMap EMPTY_LONG_DOUBLE_MAP = new TLongDoubleHashMap();
}

52
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java
View File

@ -66,59 +66,43 @@ public class InternalCountHistogramFacet extends InternalHistogramFacet {
this.count = count;
}
/**
* The key value of the histogram.
*/
public long key() {
@Override public long key() {
return key;
}
/**
* The key value of the histogram.
*/
public long getKey() {
@Override public long getKey() {
return key();
}
/**
* The number of hits that fall within that key "range" or "interval".
*/
public long count() {
@Override public long count() {
return count;
}
/**
* The number of hits that fall within that key "range" or "interval".
*/
public long getCount() {
@Override public long getCount() {
return count();
}
/**
* The sum / total of the value field that fall within this key "interval".
*/
public double total() {
return -1;
@Override public double total() {
return Double.NaN;
}
/**
* The sum / total of the value field that fall within this key "interval".
*/
public double getTotal() {
@Override public double getTotal() {
return total();
}
/**
* The mean of this facet interval.
*/
public double mean() {
return -1;
@Override public long totalCount() {
return 0;
}
/**
* The mean of this facet interval.
*/
public double getMean() {
@Override public long getTotalCount() {
return 0;
}
@Override public double mean() {
return Double.NaN;
}
@Override public double getMean() {
return mean();
}
}

261
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalFullHistogramFacet.java Normal file
View File

@ -0,0 +1,261 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.facet.histogram;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.collect.ImmutableList;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.search.facet.Facet;
import java.io.IOException;
import java.util.*;
/**
* @author kimchy (shay.banon)
*/
public class InternalFullHistogramFacet extends InternalHistogramFacet {
private static final String STREAM_TYPE = "fHistogram";
public static void registerStreams() {
Streams.registerStream(STREAM, STREAM_TYPE);
}
static Stream STREAM = new Stream() {
@Override public Facet readFacet(String type, StreamInput in) throws IOException {
return readHistogramFacet(in);
}
};
@Override public String streamType() {
return STREAM_TYPE;
}
/**
* A histogram entry representing a single entry within the result of a histogram facet.
*/
public static class FullEntry implements Entry {
long key;
long count;
long totalCount;
double total;
public FullEntry(long key, long count, long totalCount, double total) {
this.key = key;
this.count = count;
this.totalCount = totalCount;
this.total = total;
}
@Override public long key() {
return key;
}
@Override public long getKey() {
return key();
}
@Override public long count() {
return count;
}
@Override public long getCount() {
return count();
}
@Override public double total() {
return total;
}
@Override public double getTotal() {
return total();
}
@Override public long totalCount() {
return totalCount;
}
@Override public long getTotalCount() {
return this.totalCount;
}
@Override public double mean() {
return total / totalCount;
}
@Override public double getMean() {
return total / totalCount;
}
}
private String name;
private ComparatorType comparatorType;
Collection<FullEntry> entries = ImmutableList.of();
private InternalFullHistogramFacet() {
}
public InternalFullHistogramFacet(String name, ComparatorType comparatorType, Collection<FullEntry> entries) {
this.name = name;
this.comparatorType = comparatorType;
this.entries = entries;
}
@Override public String name() {
return this.name;
}
@Override public String getName() {
return name();
}
@Override public String type() {
return TYPE;
}
@Override public String getType() {
return type();
}
@Override public List<FullEntry> entries() {
if (!(entries instanceof List)) {
entries = new ArrayList<FullEntry>(entries);
}
return (List<FullEntry>) entries;
}
@Override public List<FullEntry> getEntries() {
return entries();
}
@Override public Iterator<Entry> iterator() {
return (Iterator) entries().iterator();
}
@Override public Facet reduce(String name, List<Facet> facets) {
if (facets.size() == 1) {
// we need to sort it
InternalFullHistogramFacet internalFacet = (InternalFullHistogramFacet) facets.get(0);
if (!internalFacet.entries.isEmpty()) {
List<FullEntry> entries = internalFacet.entries();
Collections.sort(entries, comparatorType.comparator());
}
return internalFacet;
}
ExtTLongObjectHashMap<FullEntry> map = CacheRecycler.popLongObjectMap2();
for (Facet facet : facets) {
InternalFullHistogramFacet histoFacet = (InternalFullHistogramFacet) facet;
for (Entry entry : histoFacet) {
FullEntry fullEntry = (FullEntry) entry;
FullEntry current = map.get(fullEntry.key);
if (current != null) {
current.count += fullEntry.count;
current.total += fullEntry.total;
current.totalCount += fullEntry.totalCount;
} else {
map.put(fullEntry.key, fullEntry);
}
}
}
// sort
Object[] values = map.internalValues();
Arrays.sort(values, (Comparator) comparatorType.comparator());
List<FullEntry> ordered = new ArrayList<FullEntry>(map.size());
for (int i = 0; i < map.size(); i++) {
FullEntry value = (FullEntry) values[i];
if (value == null) {
break;
}
ordered.add(value);
}
CacheRecycler.pushLongObjectMap2(map);
return new InternalFullHistogramFacet(name, comparatorType, ordered);
}
static final class Fields {
static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
static final XContentBuilderString ENTRIES = new XContentBuilderString("entries");
static final XContentBuilderString KEY = new XContentBuilderString("key");
static final XContentBuilderString COUNT = new XContentBuilderString("count");
static final XContentBuilderString TOTAL = new XContentBuilderString("total");
static final XContentBuilderString TOTAL_COUNT = new XContentBuilderString("total_count");
static final XContentBuilderString MEAN = new XContentBuilderString("mean");
}
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(name);
builder.field(Fields._TYPE, HistogramFacet.TYPE);
builder.startArray(Fields.ENTRIES);
for (Entry entry : entries) {
builder.startObject();
builder.field(Fields.KEY, entry.key());
builder.field(Fields.COUNT, entry.count());
builder.field(Fields.TOTAL, entry.total());
builder.field(Fields.TOTAL_COUNT, entry.totalCount());
builder.field(Fields.MEAN, entry.mean());
builder.endObject();
}
builder.endArray();
builder.endObject();
return builder;
}
public static InternalFullHistogramFacet readHistogramFacet(StreamInput in) throws IOException {
InternalFullHistogramFacet facet = new InternalFullHistogramFacet();
facet.readFrom(in);
return facet;
}
@Override public void readFrom(StreamInput in) throws IOException {
name = in.readUTF();
comparatorType = ComparatorType.fromId(in.readByte());
int size = in.readVInt();
entries = new ArrayList<FullEntry>(size);
for (int i = 0; i < size; i++) {
entries.add(new FullEntry(in.readLong(), in.readVLong(), in.readVLong(), in.readDouble()));
}
}
@Override public void writeTo(StreamOutput out) throws IOException {
out.writeUTF(name);
out.writeByte(comparatorType.id());
// optimize the write, since we know we have the same buckets as keys
out.writeVInt(entries.size());
for (FullEntry entry : entries) {
out.writeLong(entry.key);
out.writeVLong(entry.count);
out.writeVLong(entry.totalCount);
out.writeDouble(entry.total);
}
}
}

2
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalHistogramFacet.java
View File

@ -30,7 +30,7 @@ import java.util.List;
public abstract class InternalHistogramFacet implements HistogramFacet, InternalFacet {
public static void registerStreams() {
InternalCountAndTotalHistogramFacet.registerStreams();
InternalFullHistogramFacet.registerStreams();
InternalCountHistogramFacet.registerStreams();
}

96
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/KeyValueHistogramFacetCollector.java
View File

@ -20,8 +20,8 @@
package org.elasticsearch.search.facet.histogram;
import org.apache.lucene.index.IndexReader;
import org.elasticsearch.common.trove.map.hash.TLongDoubleHashMap;
import org.elasticsearch.common.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.field.data.NumericFieldData;
@ -55,10 +55,8 @@ public class KeyValueHistogramFacetCollector extends AbstractFacetCollector {
private NumericFieldData keyFieldData;
private final FieldDataType valueFieldDataType;
private NumericFieldData valueFieldData;
private final TLongLongHashMap counts = new TLongLongHashMap();
private final TLongDoubleHashMap totals = new TLongDoubleHashMap();
private final HistogramProc histoProc;
public KeyValueHistogramFacetCollector(String facetName, String keyFieldName, String valueFieldName, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
super(facetName);
@ -85,52 +83,66 @@ public class KeyValueHistogramFacetCollector extends AbstractFacetCollector {
}
valueIndexFieldName = mapper.names().indexName();
valueFieldDataType = mapper.fieldDataType();
histoProc = new HistogramProc(interval);
}
@Override protected void doCollect(int doc) throws IOException {
if (keyFieldData.multiValued()) {
if (valueFieldData.multiValued()) {
// both multi valued, intersect based on the minimum size
double[] keys = keyFieldData.doubleValues(doc);
double[] values = valueFieldData.doubleValues(doc);
int size = Math.min(keys.length, values.length);
for (int i = 0; i < size; i++) {
long bucket = CountAndTotalHistogramFacetCollector.bucket(keys[i], interval);
counts.adjustOrPutValue(bucket, 1, 1);
totals.adjustOrPutValue(bucket, values[i], values[i]);
}
} else {
// key multi valued, value is a single value
double value = valueFieldData.doubleValue(doc);
for (double key : keyFieldData.doubleValues(doc)) {
long bucket = CountAndTotalHistogramFacetCollector.bucket(key, interval);
counts.adjustOrPutValue(bucket, 1, 1);
totals.adjustOrPutValue(bucket, value, value);
}
}
} else {
// single key value, compute the bucket once
long bucket = CountAndTotalHistogramFacetCollector.bucket(keyFieldData.doubleValue(doc), interval);
if (valueFieldData.multiValued()) {
for (double value : valueFieldData.doubleValues(doc)) {
counts.adjustOrPutValue(bucket, 1, 1);
totals.adjustOrPutValue(bucket, value, value);
}
} else {
// both key and value are not multi valued
double value = valueFieldData.doubleValue(doc);
counts.adjustOrPutValue(bucket, 1, 1);
totals.adjustOrPutValue(bucket, value, value);
}
}
keyFieldData.forEachValueInDoc(doc, histoProc);
}
@Override protected void doSetNextReader(IndexReader reader, int docBase) throws IOException {
keyFieldData = (NumericFieldData) fieldDataCache.cache(keyFieldDataType, reader, keyIndexFieldName);
valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, reader, valueIndexFieldName);
histoProc.valueFieldData = (NumericFieldData) fieldDataCache.cache(valueFieldDataType, reader, valueIndexFieldName);
}
@Override public Facet facet() {
return new InternalCountAndTotalHistogramFacet(facetName, comparatorType, counts, totals);
CacheRecycler.pushLongObjectMap(histoProc.entries);
return new InternalFullHistogramFacet(facetName, comparatorType, histoProc.entries.valueCollection());
}
public static class HistogramProc implements NumericFieldData.DoubleValueInDocProc {
final long interval;
final ExtTLongObjectHashMap<InternalFullHistogramFacet.FullEntry> entries = CacheRecycler.popLongObjectMap();
NumericFieldData valueFieldData;
public HistogramProc(long interval) {
this.interval = interval;
}
@Override public void onValue(int docId, double value) {
long bucket = FullHistogramFacetCollector.bucket(value, interval);
InternalFullHistogramFacet.FullEntry entry = entries.get(bucket);
if (entry == null) {
if (valueFieldData.multiValued()) {
double[] valuesValues = valueFieldData.doubleValues(docId);
entry = new InternalFullHistogramFacet.FullEntry(bucket, 1, valuesValues.length, 0);
for (double valueValue : valuesValues) {
entry.total += valueValue;
}
entries.put(bucket, entry);
} else {
double valueValue = valueFieldData.doubleValue(docId);
entry = new InternalFullHistogramFacet.FullEntry(bucket, 1, 1, valueValue);
entries.put(bucket, entry);
}
} else {
entry.count++;
if (valueFieldData.multiValued()) {
double[] valuesValues = valueFieldData.doubleValues(docId);
entry.totalCount += valuesValues.length;
for (double valueValue : valuesValues) {
entry.total += valueValue;
}
} else {
entry.totalCount++;
double valueValue = valueFieldData.doubleValue(docId);
entry.total += valueValue;
}
}
}
}
}

30
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/KeyValueScriptHistogramFacetCollector.java
View File

@ -21,8 +21,8 @@ package org.elasticsearch.search.facet.histogram;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.common.trove.map.hash.TLongDoubleHashMap;
import org.elasticsearch.common.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
import org.elasticsearch.index.cache.field.data.FieldDataCache;
import org.elasticsearch.index.field.data.FieldDataType;
import org.elasticsearch.index.field.data.NumericFieldData;
@ -98,7 +98,8 @@ public class KeyValueScriptHistogramFacetCollector extends AbstractFacetCollecto
}
@Override public Facet facet() {
return new InternalCountAndTotalHistogramFacet(facetName, comparatorType, histoProc.counts(), histoProc.totals());
CacheRecycler.pushLongObjectMap(histoProc.entries);
return new InternalFullHistogramFacet(facetName, comparatorType, histoProc.entries.valueCollection());
}
public static long bucket(double value, long interval) {
@ -111,9 +112,7 @@ public class KeyValueScriptHistogramFacetCollector extends AbstractFacetCollecto
private final SearchScript valueScript;
private final TLongLongHashMap counts = new TLongLongHashMap();
private final TLongDoubleHashMap totals = new TLongDoubleHashMap();
final ExtTLongObjectHashMap<InternalFullHistogramFacet.FullEntry> entries = CacheRecycler.popLongObjectMap();
public HistogramProc(long interval, SearchScript valueScript) {
this.interval = interval;
@ -122,19 +121,18 @@ public class KeyValueScriptHistogramFacetCollector extends AbstractFacetCollecto
@Override public void onValue(int docId, double value) {
valueScript.setNextDocId(docId);
long bucket = bucket(value, interval);
counts.adjustOrPutValue(bucket, 1, 1);
double scriptValue = valueScript.runAsDouble();
totals.adjustOrPutValue(bucket, scriptValue, scriptValue);
}
public TLongLongHashMap counts() {
return counts;
}
public TLongDoubleHashMap totals() {
return totals;
InternalFullHistogramFacet.FullEntry entry = entries.get(bucket);
if (entry == null) {
entry = new InternalFullHistogramFacet.FullEntry(bucket, 1, 1, scriptValue);
entries.put(bucket, entry);
} else {
entry.count++;
entry.totalCount++;
entry.total += scriptValue;
}
}
}
}

23
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/ScriptHistogramFacetCollector.java
View File

@ -21,8 +21,8 @@ package org.elasticsearch.search.facet.histogram;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.common.trove.map.hash.TLongDoubleHashMap;
import org.elasticsearch.common.trove.map.hash.TLongLongHashMap;
import org.elasticsearch.common.CacheRecycler;
import org.elasticsearch.common.trove.ExtTLongObjectHashMap;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.AbstractFacetCollector;
import org.elasticsearch.search.facet.Facet;
@ -44,9 +44,7 @@ public class ScriptHistogramFacetCollector extends AbstractFacetCollector {
private final HistogramFacet.ComparatorType comparatorType;
private final TLongLongHashMap counts = new TLongLongHashMap();
private final TLongDoubleHashMap totals = new TLongDoubleHashMap();
final ExtTLongObjectHashMap<InternalFullHistogramFacet.FullEntry> entries = CacheRecycler.popLongObjectMap();
public ScriptHistogramFacetCollector(String facetName, String scriptLang, String keyScript, String valueScript, Map<String, Object> params, long interval, HistogramFacet.ComparatorType comparatorType, SearchContext context) {
super(facetName);
@ -66,8 +64,16 @@ public class ScriptHistogramFacetCollector extends AbstractFacetCollector {
bucket = bucket(keyScript.runAsDouble(), interval);
}
double value = valueScript.runAsDouble();
counts.adjustOrPutValue(bucket, 1, 1);
totals.adjustOrPutValue(bucket, value, value);
InternalFullHistogramFacet.FullEntry entry = entries.get(bucket);
if (entry == null) {
entry = new InternalFullHistogramFacet.FullEntry(bucket, 1, 1, value);
entries.put(bucket, entry);
} else {
entry.count++;
entry.totalCount++;
entry.total += value;
}
}
@Override public void setScorer(Scorer scorer) throws IOException {
@ -81,7 +87,8 @@ public class ScriptHistogramFacetCollector extends AbstractFacetCollector {
}
@Override public Facet facet() {
return new InternalCountAndTotalHistogramFacet(facetName, comparatorType, counts, totals);
CacheRecycler.pushLongObjectMap(entries);
return new InternalFullHistogramFacet(facetName, comparatorType, entries.valueCollection());
}
public static long bucket(double value, long interval) {

1
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/bytes/InternalByteTermsFacet.java
View File

@ -173,7 +173,6 @@ public class InternalByteTermsFacet extends InternalTermsFacet {
}
InternalByteTermsFacet first = (InternalByteTermsFacet) facets.get(0);
TByteIntHashMap aggregated = CacheRecycler.popByteIntMap();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {

1
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/doubles/InternalDoubleTermsFacet.java
View File

@ -175,7 +175,6 @@ public class InternalDoubleTermsFacet extends InternalTermsFacet {
}
InternalDoubleTermsFacet first = (InternalDoubleTermsFacet) facets.get(0);
TDoubleIntHashMap aggregated = CacheRecycler.popDoubleIntMap();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {
InternalDoubleTermsFacet mFacet = (InternalDoubleTermsFacet) facet;

1
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/floats/InternalFloatTermsFacet.java
View File

@ -175,7 +175,6 @@ public class InternalFloatTermsFacet extends InternalTermsFacet {
}
InternalFloatTermsFacet first = (InternalFloatTermsFacet) facets.get(0);
TFloatIntHashMap aggregated = CacheRecycler.popFloatIntMap();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {

1
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ints/InternalIntTermsFacet.java
View File

@ -172,7 +172,6 @@ public class InternalIntTermsFacet extends InternalTermsFacet {
}
InternalIntTermsFacet first = (InternalIntTermsFacet) facets.get(0);
TIntIntHashMap aggregated = CacheRecycler.popIntIntMap();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {

1
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/ip/InternalIpTermsFacet.java
View File

@ -176,7 +176,6 @@ public class InternalIpTermsFacet extends InternalTermsFacet {
}
InternalIpTermsFacet first = (InternalIpTermsFacet) facets.get(0);
TLongIntHashMap aggregated = CacheRecycler.popLongIntMap();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {

1
modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/terms/shorts/InternalShortTermsFacet.java
View File

@ -172,7 +172,6 @@ public class InternalShortTermsFacet extends InternalTermsFacet {
}
InternalShortTermsFacet first = (InternalShortTermsFacet) facets.get(0);
TShortIntHashMap aggregated = CacheRecycler.popShortIntMap();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {
InternalShortTermsFacet mFacet = (InternalShortTermsFacet) facet;

21
modules/test/integration/src/test/java/org/elasticsearch/test/integration/search/facet/SimpleFacetsTests.java
View File

@ -948,10 +948,12 @@ public class SimpleFacetsTests extends AbstractNodesTests {
assertThat(facet.entries().size(), equalTo(2));
assertThat(facet.entries().get(0).key(), equalTo(1000l));
assertThat(facet.entries().get(0).count(), equalTo(2l));
assertThat(facet.entries().get(0).totalCount(), equalTo(2l));
assertThat(facet.entries().get(0).total(), equalTo(2120d));
assertThat(facet.entries().get(0).mean(), equalTo(1060d));
assertThat(facet.entries().get(1).key(), equalTo(1100l));
assertThat(facet.entries().get(1).count(), equalTo(1l));
assertThat(facet.entries().get(1).totalCount(), equalTo(1l));
assertThat(facet.entries().get(1).total(), equalTo(1175d));
assertThat(facet.entries().get(1).mean(), equalTo(1175d));
@ -960,14 +962,17 @@ public class SimpleFacetsTests extends AbstractNodesTests {
assertThat(facet.entries().size(), equalTo(3));
assertThat(facet.entries().get(0).key(), equalTo(10l));
assertThat(facet.entries().get(0).count(), equalTo(3l));
assertThat(facet.entries().get(0).totalCount(), equalTo(3l));
assertThat(facet.entries().get(0).total(), equalTo(45d));
assertThat(facet.entries().get(0).mean(), equalTo(15d));
assertThat(facet.entries().get(1).key(), equalTo(20l));
assertThat(facet.entries().get(1).count(), equalTo(2l));
assertThat(facet.entries().get(1).totalCount(), equalTo(2l));
assertThat(facet.entries().get(1).total(), equalTo(48d));
assertThat(facet.entries().get(1).mean(), equalTo(24d));
assertThat(facet.entries().get(2).key(), equalTo(30l));
assertThat(facet.entries().get(2).count(), equalTo(1l));
assertThat(facet.entries().get(2).totalCount(), equalTo(1l));
assertThat(facet.entries().get(2).total(), equalTo(31d));
assertThat(facet.entries().get(2).mean(), equalTo(31d));
@ -975,11 +980,13 @@ public class SimpleFacetsTests extends AbstractNodesTests {
assertThat(facet.name(), equalTo("stats3"));
assertThat(facet.entries().size(), equalTo(2));
assertThat(facet.entries().get(0).key(), equalTo(1000l));
assertThat(facet.entries().get(0).count(), equalTo(4l));
assertThat(facet.entries().get(0).count(), equalTo(2l));
assertThat(facet.entries().get(0).totalCount(), equalTo(4l));
assertThat(facet.entries().get(0).total(), equalTo(82d));
assertThat(facet.entries().get(0).mean(), equalTo(20.5d));
assertThat(facet.entries().get(1).key(), equalTo(1100l));
assertThat(facet.entries().get(1).count(), equalTo(2l));
assertThat(facet.entries().get(1).count(), equalTo(1l));
assertThat(facet.entries().get(1).totalCount(), equalTo(2l));
assertThat(facet.entries().get(1).total(), equalTo(42d));
assertThat(facet.entries().get(1).mean(), equalTo(21d));
@ -988,10 +995,12 @@ public class SimpleFacetsTests extends AbstractNodesTests {
assertThat(facet.entries().size(), equalTo(2));
assertThat(facet.entries().get(0).key(), equalTo(0l));
assertThat(facet.entries().get(0).count(), equalTo(2l));
assertThat(facet.entries().get(0).totalCount(), equalTo(2l));
assertThat(facet.entries().get(0).total(), equalTo(2120d));
assertThat(facet.entries().get(0).mean(), equalTo(1060d));
assertThat(facet.entries().get(1).key(), equalTo(2l));
assertThat(facet.entries().get(1).count(), equalTo(1l));
assertThat(facet.entries().get(1).totalCount(), equalTo(1l));
assertThat(facet.entries().get(1).total(), equalTo(1175d));
assertThat(facet.entries().get(1).mean(), equalTo(1175d));
@ -1008,10 +1017,12 @@ public class SimpleFacetsTests extends AbstractNodesTests {
assertThat(facet.entries().size(), equalTo(2));
assertThat(facet.entries().get(0).key(), equalTo(1000l));
assertThat(facet.entries().get(0).count(), equalTo(2l));
assertThat(facet.entries().get(0).totalCount(), equalTo(2l));
assertThat(facet.entries().get(0).total(), equalTo(2120d));
assertThat(facet.entries().get(0).mean(), equalTo(1060d));
assertThat(facet.entries().get(1).key(), equalTo(1100l));
assertThat(facet.entries().get(1).count(), equalTo(1l));
assertThat(facet.entries().get(1).totalCount(), equalTo(1l));
assertThat(facet.entries().get(1).total(), equalTo(1175d));
assertThat(facet.entries().get(1).mean(), equalTo(1175d));
@ -1020,22 +1031,20 @@ public class SimpleFacetsTests extends AbstractNodesTests {
assertThat(facet.entries().size(), equalTo(2));
assertThat(facet.entries().get(0).key(), equalTo(1000l));
assertThat(facet.entries().get(0).count(), equalTo(2l));
assertThat(facet.entries().get(0).total(), equalTo(-1d));
assertThat(facet.entries().get(0).mean(), equalTo(-1d));
assertThat(facet.entries().get(1).key(), equalTo(1100l));
assertThat(facet.entries().get(1).count(), equalTo(1l));
assertThat(facet.entries().get(1).total(), equalTo(-1d));
assertThat(facet.entries().get(1).mean(), equalTo(-1d));
facet = searchResponse.facets().facet("stats8");
assertThat(facet.name(), equalTo("stats8"));
assertThat(facet.entries().size(), equalTo(2));
assertThat(facet.entries().get(0).key(), equalTo(1000l));
assertThat(facet.entries().get(0).count(), equalTo(2l));
assertThat(facet.entries().get(0).totalCount(), equalTo(2l));
assertThat(facet.entries().get(0).total(), equalTo(2d));
assertThat(facet.entries().get(0).mean(), equalTo(1d));
assertThat(facet.entries().get(1).key(), equalTo(1100l));
assertThat(facet.entries().get(1).count(), equalTo(1l));
assertThat(facet.entries().get(1).totalCount(), equalTo(1l));
assertThat(facet.entries().get(1).total(), equalTo(1d));
assertThat(facet.entries().get(1).mean(), equalTo(1d));
}