From 8617e3b0e36d76bb38ad0840127032c8ba304806 Mon Sep 17 00:00:00 2001 From: kimchy Date: Wed, 23 Feb 2011 05:55:23 +0200 Subject: [PATCH] enhance memory usage when doing reduce on histogram facets --- ...ternalCountAndTotalDateHistogramFacet.java | 23 ++++++++----------- .../InternalCountDateHistogramFacet.java | 23 ++++++++----------- ...alueScriptDateHistogramFacetCollector.java | 8 ------- .../InternalCountAndTotalHistogramFacet.java | 23 ++++++++----------- .../InternalCountHistogramFacet.java | 23 ++++++++----------- 5 files changed, 36 insertions(+), 64 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountAndTotalDateHistogramFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountAndTotalDateHistogramFacet.java index 78ed582a7ff..7bc24875b78 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountAndTotalDateHistogramFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountAndTotalDateHistogramFacet.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.facet.datehistogram; -import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.trove.iterator.TLongDoubleIterator; @@ -32,10 +31,9 @@ import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.histogram.HistogramFacet; import java.io.IOException; -import java.util.Collection; +import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.TreeSet; /** * @author kimchy (shay.banon) @@ -114,7 +112,7 @@ public class InternalCountAndTotalDateHistogramFacet extends InternalDateHistogr TLongDoubleHashMap totals; - Collection entries = null; + CountAndTotalEntry[] entries = null; private InternalCountAndTotalDateHistogramFacet() { } @@ -143,11 +141,7 @@ public class InternalCountAndTotalDateHistogramFacet extends InternalDateHistogr } @Override public List entries() { - computeEntries(); - if (!(entries instanceof List)) { - entries = ImmutableList.copyOf(entries); - } - return (List) entries; + return Arrays.asList(computeEntries()); } @Override public List getEntries() { @@ -155,19 +149,20 @@ public class InternalCountAndTotalDateHistogramFacet extends InternalDateHistogr } @Override public Iterator iterator() { - return (Iterator) computeEntries().iterator(); + return (Iterator) entries().iterator(); } - private Collection computeEntries() { + private CountAndTotalEntry[] computeEntries() { if (entries != null) { return entries; } - TreeSet set = new TreeSet(comparatorType.comparator()); + entries = new CountAndTotalEntry[counts.size()]; + int i = 0; for (TLongLongIterator it = counts.iterator(); it.hasNext();) { it.advance(); - set.add(new CountAndTotalEntry(it.key(), it.value(), totals.get(it.key()))); + entries[i++] = new CountAndTotalEntry(it.key(), it.value(), totals.get(it.key())); } - entries = set; + Arrays.sort(entries, comparatorType.comparator()); return entries; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java index 8be5dfe5a4b..507e239c65e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/InternalCountDateHistogramFacet.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.facet.datehistogram; -import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.trove.iterator.TLongLongIterator; @@ -30,10 +29,9 @@ import org.elasticsearch.search.facet.Facet; import org.elasticsearch.search.facet.histogram.HistogramFacet; import java.io.IOException; -import java.util.Collection; +import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.TreeSet; /** * @author kimchy (shay.banon) @@ -108,7 +106,7 @@ public class InternalCountDateHistogramFacet extends InternalDateHistogramFacet TLongLongHashMap counts; - Collection entries = null; + CountEntry[] entries = null; private InternalCountDateHistogramFacet() { } @@ -136,11 +134,7 @@ public class InternalCountDateHistogramFacet extends InternalDateHistogramFacet } @Override public List entries() { - computeEntries(); - if (!(entries instanceof List)) { - entries = ImmutableList.copyOf(entries); - } - return (List) entries; + return Arrays.asList(computeEntries()); } @Override public List getEntries() { @@ -148,19 +142,20 @@ public class InternalCountDateHistogramFacet extends InternalDateHistogramFacet } @Override public Iterator iterator() { - return (Iterator) computeEntries().iterator(); + return (Iterator) entries().iterator(); } - private Collection computeEntries() { + private CountEntry[] computeEntries() { if (entries != null) { return entries; } - TreeSet set = new TreeSet(comparatorType.comparator()); + entries = new CountEntry[counts.size()]; + int i = 0; for (TLongLongIterator it = counts.iterator(); it.hasNext();) { it.advance(); - set.add(new CountEntry(it.key(), it.value())); + entries[i++] = new CountEntry(it.key(), it.value()); } - entries = set; + Arrays.sort(entries, comparatorType.comparator()); return entries; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java index 5c6bc373893..002c2740123 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetCollector.java @@ -158,13 +158,5 @@ public class ValueScriptDateHistogramFacetCollector extends AbstractFacetCollect double scriptValue = valueScript.runAsDouble(); totals.adjustOrPutValue(bucket, scriptValue, scriptValue); } - - public TLongLongHashMap counts() { - return counts; - } - - public TLongDoubleHashMap totals() { - return totals; - } } } \ No newline at end of file diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountAndTotalHistogramFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountAndTotalHistogramFacet.java index bcc74b8050d..64cecb59cae 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountAndTotalHistogramFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountAndTotalHistogramFacet.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.facet.histogram; -import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.trove.iterator.TLongDoubleIterator; @@ -31,10 +30,9 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; import java.io.IOException; -import java.util.Collection; +import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.TreeSet; /** * @author kimchy (shay.banon) @@ -137,7 +135,7 @@ public class InternalCountAndTotalHistogramFacet extends InternalHistogramFacet TLongDoubleHashMap totals; - Collection entries = null; + CountAndTotalEntry[] entries = null; private InternalCountAndTotalHistogramFacet() { } @@ -166,11 +164,7 @@ public class InternalCountAndTotalHistogramFacet extends InternalHistogramFacet } @Override public List entries() { - computeEntries(); - if (!(entries instanceof List)) { - entries = ImmutableList.copyOf(entries); - } - return (List) entries; + return Arrays.asList(computeEntries()); } @Override public List getEntries() { @@ -178,19 +172,20 @@ public class InternalCountAndTotalHistogramFacet extends InternalHistogramFacet } @Override public Iterator iterator() { - return (Iterator) computeEntries().iterator(); + return (Iterator) entries().iterator(); } - private Collection computeEntries() { + private CountAndTotalEntry[] computeEntries() { if (entries != null) { return entries; } - TreeSet set = new TreeSet(comparatorType.comparator()); + entries = new CountAndTotalEntry[counts.size()]; + int i = 0; for (TLongLongIterator it = counts.iterator(); it.hasNext();) { it.advance(); - set.add(new CountAndTotalEntry(it.key(), it.value(), totals.get(it.key()))); + entries[i++] = new CountAndTotalEntry(it.key(), it.value(), totals.get(it.key())); } - entries = set; + Arrays.sort(entries, comparatorType.comparator()); return entries; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java index 9fb5cce6f92..d2d985a443f 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/search/facet/histogram/InternalCountHistogramFacet.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.facet.histogram; -import org.elasticsearch.common.collect.ImmutableList; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.trove.iterator.TLongLongIterator; @@ -29,10 +28,9 @@ import org.elasticsearch.common.xcontent.XContentBuilderString; import org.elasticsearch.search.facet.Facet; import java.io.IOException; -import java.util.Collection; +import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.TreeSet; /** * @author kimchy (shay.banon) @@ -131,7 +129,7 @@ public class InternalCountHistogramFacet extends InternalHistogramFacet { TLongLongHashMap counts; - Collection entries = null; + CountEntry[] entries = null; private InternalCountHistogramFacet() { } @@ -159,11 +157,7 @@ public class InternalCountHistogramFacet extends InternalHistogramFacet { } @Override public List entries() { - computeEntries(); - if (!(entries instanceof List)) { - entries = ImmutableList.copyOf(entries); - } - return (List) entries; + return Arrays.asList(computeEntries()); } @Override public List getEntries() { @@ -171,19 +165,20 @@ public class InternalCountHistogramFacet extends InternalHistogramFacet { } @Override public Iterator iterator() { - return (Iterator) computeEntries().iterator(); + return (Iterator) entries().iterator(); } - private Collection computeEntries() { + private CountEntry[] computeEntries() { if (entries != null) { return entries; } - TreeSet set = new TreeSet(comparatorType.comparator()); + entries = new CountEntry[counts.size()]; + int i = 0; for (TLongLongIterator it = counts.iterator(); it.hasNext();) { it.advance(); - set.add(new CountEntry(it.key(), it.value())); + entries[i++] = new CountEntry(it.key(), it.value()); } - entries = set; + Arrays.sort(entries, comparatorType.comparator()); return entries; }