honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Aggregations: Added pre and post offset to histogram aggregation 

Added preOffset and postOffset parameters to the API for the histogram aggregation which work in the same way as in the date histogram

Closes #6605
This commit is contained in:
Colin Goodheart-Smithe 2014-07-23 13:49:38 +01:00
parent f5d1e0a37d
commit 127649d174
11 changed files with 207 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
src/main/java/org/elasticsearch/common/rounding/Rounding.java
View File

@ -128,6 +128,108 @@ public abstract class Rounding implements Streamable {
}
}
public static class FactorRounding extends Rounding {
final static byte ID = 7;
private Rounding rounding;
private float factor;
FactorRounding() { // for serialization
}
FactorRounding(Rounding rounding, float factor) {
this.rounding = rounding;
this.factor = factor;
}
@Override
public byte id() {
return ID;
}
@Override
public long roundKey(long utcMillis) {
return rounding.roundKey((long) (factor * utcMillis));
}
@Override
public long valueForKey(long key) {
return rounding.valueForKey(key);
}
@Override
public long nextRoundingValue(long value) {
return rounding.nextRoundingValue(value);
}
@Override
public void readFrom(StreamInput in) throws IOException {
rounding = (TimeZoneRounding) Rounding.Streams.read(in);
factor = in.readFloat();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
Rounding.Streams.write(rounding, out);
out.writeFloat(factor);
}
}
public static class PrePostRounding extends Rounding {
final static byte ID = 8;
private Rounding rounding;
private long preOffset;
private long postOffset;
PrePostRounding() { // for serialization
}
public PrePostRounding(Rounding intervalRounding, long preOffset, long postOffset) {
this.rounding = intervalRounding;
this.preOffset = preOffset;
this.postOffset = postOffset;
}
@Override
public byte id() {
return ID;
}
@Override
public long roundKey(long value) {
return rounding.roundKey(value + preOffset);
}
@Override
public long valueForKey(long key) {
return postOffset + rounding.valueForKey(key);
}
@Override
public long nextRoundingValue(long value) {
return postOffset + rounding.nextRoundingValue(value - postOffset);
}
@Override
public void readFrom(StreamInput in) throws IOException {
rounding = Rounding.Streams.read(in);
preOffset = in.readVLong();
postOffset = in.readVLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
Rounding.Streams.write(rounding, out);
out.writeVLong(preOffset);
out.writeVLong(postOffset);
}
}
public static class Streams {
public static void write(Rounding rounding, StreamOutput out) throws IOException {
@ -146,8 +248,8 @@ public abstract class Rounding implements Streamable {
case TimeZoneRounding.UTCIntervalTimeZoneRounding.ID: rounding = new TimeZoneRounding.UTCIntervalTimeZoneRounding(); break;
case TimeZoneRounding.TimeIntervalTimeZoneRounding.ID: rounding = new TimeZoneRounding.TimeIntervalTimeZoneRounding(); break;
case TimeZoneRounding.DayIntervalTimeZoneRounding.ID: rounding = new TimeZoneRounding.DayIntervalTimeZoneRounding(); break;
case TimeZoneRounding.FactorTimeZoneRounding.ID: rounding = new TimeZoneRounding.FactorTimeZoneRounding(); break;
case TimeZoneRounding.PrePostTimeZoneRounding.ID: rounding = new TimeZoneRounding.PrePostTimeZoneRounding(); break;
case TimeZoneRounding.FactorRounding.ID: rounding = new FactorRounding(); break;
case PrePostRounding.ID: rounding = new PrePostRounding(); break;
default: throw new ElasticsearchException("unknown rounding id [" + id + "]");
}
rounding.readFrom(in);

110
src/main/java/org/elasticsearch/common/rounding/TimeZoneRounding.java
View File

@ -94,8 +94,8 @@ public abstract class TimeZoneRounding extends Rounding {
return this;
}
public TimeZoneRounding build() {
TimeZoneRounding timeZoneRounding;
public Rounding build() {
Rounding timeZoneRounding;
if (unit != null) {
if (preTz.equals(DateTimeZone.UTC) && postTz.equals(DateTimeZone.UTC)) {
timeZoneRounding = new UTCTimeZoneRoundingFloor(unit);
@ -114,10 +114,10 @@ public abstract class TimeZoneRounding extends Rounding {
}
}
if (preOffset != 0 || postOffset != 0) {
timeZoneRounding = new PrePostTimeZoneRounding(timeZoneRounding, preOffset, postOffset);
timeZoneRounding = new PrePostRounding(timeZoneRounding, preOffset, postOffset);
}
if (factor != 1.0f) {
timeZoneRounding = new FactorTimeZoneRounding(timeZoneRounding, factor);
timeZoneRounding = new FactorRounding(timeZoneRounding, factor);
}
return timeZoneRounding;
}
@ -439,106 +439,4 @@ public abstract class TimeZoneRounding extends Rounding {
out.writeSharedString(postTz.getID());
}
}
static class FactorTimeZoneRounding extends TimeZoneRounding {
final static byte ID = 7;
private TimeZoneRounding timeZoneRounding;
private float factor;
FactorTimeZoneRounding() { // for serialization
}
FactorTimeZoneRounding(TimeZoneRounding timeZoneRounding, float factor) {
this.timeZoneRounding = timeZoneRounding;
this.factor = factor;
}
@Override
public byte id() {
return ID;
}
@Override
public long roundKey(long utcMillis) {
return timeZoneRounding.roundKey((long) (factor * utcMillis));
}
@Override
public long valueForKey(long key) {
return timeZoneRounding.valueForKey(key);
}
@Override
public long nextRoundingValue(long value) {
return timeZoneRounding.nextRoundingValue(value);
}
@Override
public void readFrom(StreamInput in) throws IOException {
timeZoneRounding = (TimeZoneRounding) Rounding.Streams.read(in);
factor = in.readFloat();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
Rounding.Streams.write(timeZoneRounding, out);
out.writeFloat(factor);
}
}
static class PrePostTimeZoneRounding extends TimeZoneRounding {
final static byte ID = 8;
private TimeZoneRounding timeZoneRounding;
private long preOffset;
private long postOffset;
PrePostTimeZoneRounding() { // for serialization
}
PrePostTimeZoneRounding(TimeZoneRounding timeZoneRounding, long preOffset, long postOffset) {
this.timeZoneRounding = timeZoneRounding;
this.preOffset = preOffset;
this.postOffset = postOffset;
}
@Override
public byte id() {
return ID;
}
@Override
public long roundKey(long utcMillis) {
return timeZoneRounding.roundKey(utcMillis + preOffset);
}
@Override
public long valueForKey(long key) {
return postOffset + timeZoneRounding.valueForKey(key);
}
@Override
public long nextRoundingValue(long value) {
return postOffset + timeZoneRounding.nextRoundingValue(value - postOffset);
}
@Override
public void readFrom(StreamInput in) throws IOException {
timeZoneRounding = (TimeZoneRounding) Rounding.Streams.read(in);
preOffset = in.readVLong();
postOffset = in.readVLong();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
Rounding.Streams.write(timeZoneRounding, out);
out.writeVLong(preOffset);
out.writeVLong(postOffset);
}
}
}

3
src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/DateHistogramParser.java
View File

@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.rounding.DateTimeUnit;
import org.elasticsearch.common.rounding.Rounding;
import org.elasticsearch.common.rounding.TimeZoneRounding;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentParser;
@ -192,7 +193,7 @@ public class DateHistogramParser implements Aggregator.Parser {
tzRoundingBuilder = TimeZoneRounding.builder(TimeValue.parseTimeValue(interval, null));
}
TimeZoneRounding rounding = tzRoundingBuilder
Rounding rounding = tzRoundingBuilder
.preZone(preZone).postZone(postZone)
.preZoneAdjustLargeInterval(preZoneAdjustLargeInterval)
.preOffset(preOffset).postOffset(postOffset)

21
src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramBuilder.java
View File

@ -35,6 +35,8 @@ public class HistogramBuilder extends ValuesSourceAggregationBuilder<HistogramBu
private Long minDocCount;
private Long extendedBoundsMin;
private Long extendedBoundsMax;
private Long preOffset;
private Long postOffset;
/**
* Constructs a new histogram aggregation builder.
@ -84,6 +86,16 @@ public class HistogramBuilder extends ValuesSourceAggregationBuilder<HistogramBu
return this;
}
public HistogramBuilder preOffset(long preOffset) {
this.preOffset = preOffset;
return this;
}
public HistogramBuilder postOffset(long postOffset) {
this.postOffset = postOffset;
return this;
}
@Override
protected XContentBuilder doInternalXContent(XContentBuilder builder, Params params) throws IOException {
if (interval == null) {
@ -96,6 +108,15 @@ public class HistogramBuilder extends ValuesSourceAggregationBuilder<HistogramBu
order.toXContent(builder, params);
}
if (preOffset != null) {
builder.field("pre_offset", preOffset);
}
if (postOffset != null) {
builder.field("post_offset", postOffset);
}
if (minDocCount != null) {
builder.field("min_doc_count", minDocCount);
}

10
src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/HistogramParser.java
View File

@ -57,6 +57,8 @@ public class HistogramParser implements Aggregator.Parser {
InternalOrder order = (InternalOrder) InternalOrder.KEY_ASC;
long interval = -1;
ExtendedBounds extendedBounds = null;
long preOffset = 0;
long postOffset = 0;
XContentParser.Token token;
String currentFieldName = null;
@ -72,6 +74,10 @@ public class HistogramParser implements Aggregator.Parser {
minDocCount = parser.longValue();
} else if ("keyed".equals(currentFieldName)) {
keyed = parser.booleanValue();
} else if ("pre_offset".equals(currentFieldName) || "preOffset".equals(currentFieldName)) {
preOffset = parser.longValue();
} else if ("post_offset".equals(currentFieldName) || "postOffset".equals(currentFieldName)) {
postOffset = parser.longValue();
} else {
throw new SearchParseException(context, "Unknown key for a " + token + " in aggregation [" + aggregationName + "]: [" + currentFieldName + "].");
}
@ -116,7 +122,11 @@ public class HistogramParser implements Aggregator.Parser {
if (interval < 0) {
throw new SearchParseException(context, "Missing required field [interval] for histogram aggregation [" + aggregationName + "]");
}
Rounding rounding = new Rounding.Interval(interval);
if (preOffset != 0 || postOffset != 0) {
rounding = new Rounding.PrePostRounding((Rounding.Interval) rounding, preOffset, postOffset);
}
if (extendedBounds != null) {
// with numeric histogram, we can process here and fail fast if the bounds are invalid

10
src/main/java/org/elasticsearch/search/facet/datehistogram/CountDateHistogramFacetExecutor.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedNumericDocValues;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.common.recycler.Recycler;
import org.elasticsearch.common.rounding.TimeZoneRounding;
import org.elasticsearch.common.rounding.Rounding;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.search.facet.FacetExecutor;
import org.elasticsearch.search.facet.InternalFacet;
@ -38,13 +38,13 @@ import java.io.IOException;
*/
public class CountDateHistogramFacetExecutor extends FacetExecutor {
private final TimeZoneRounding tzRounding;
private final Rounding tzRounding;
private final IndexNumericFieldData indexFieldData;
final DateHistogramFacet.ComparatorType comparatorType;
final Recycler.V<LongLongOpenHashMap> counts;
public CountDateHistogramFacetExecutor(IndexNumericFieldData indexFieldData, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, CacheRecycler cacheRecycler) {
public CountDateHistogramFacetExecutor(IndexNumericFieldData indexFieldData, Rounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, CacheRecycler cacheRecycler) {
this.comparatorType = comparatorType;
this.indexFieldData = indexFieldData;
this.tzRounding = tzRounding;
@ -101,9 +101,9 @@ public class CountDateHistogramFacetExecutor extends FacetExecutor {
public static class DateHistogramProc extends LongFacetAggregatorBase {
private final LongLongOpenHashMap counts;
private final TimeZoneRounding tzRounding;
private final Rounding tzRounding;
public DateHistogramProc(LongLongOpenHashMap counts, TimeZoneRounding tzRounding) {
public DateHistogramProc(LongLongOpenHashMap counts, Rounding tzRounding) {
this.counts = counts;
this.tzRounding = tzRounding;
}

3
src/main/java/org/elasticsearch/search/facet/datehistogram/DateHistogramFacetParser.java
View File

@ -24,6 +24,7 @@ import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.rounding.DateTimeUnit;
import org.elasticsearch.common.rounding.Rounding;
import org.elasticsearch.common.rounding.TimeZoneRounding;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
@ -179,7 +180,7 @@ public class DateHistogramFacetParser extends AbstractComponent implements Facet
tzRoundingBuilder = TimeZoneRounding.builder(TimeValue.parseTimeValue(interval, null));
}
TimeZoneRounding tzRounding = tzRoundingBuilder
Rounding tzRounding = tzRoundingBuilder
.preZone(preZone).postZone(postZone)
.preZoneAdjustLargeInterval(preZoneAdjustLargeInterval)
.preOffset(preOffset).postOffset(postOffset)

10
src/main/java/org/elasticsearch/search/facet/datehistogram/ValueDateHistogramFacetExecutor.java
View File

@ -24,7 +24,7 @@ import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.SortedNumericDocValues;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.common.recycler.Recycler;
import org.elasticsearch.common.rounding.TimeZoneRounding;
import org.elasticsearch.common.rounding.Rounding;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.search.facet.DoubleFacetAggregatorBase;
@ -43,11 +43,11 @@ public class ValueDateHistogramFacetExecutor extends FacetExecutor {
private final IndexNumericFieldData keyIndexFieldData;
private final IndexNumericFieldData valueIndexFieldData;
private final DateHistogramFacet.ComparatorType comparatorType;
final TimeZoneRounding tzRounding;
final Rounding tzRounding;
final Recycler.V<LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry>> entries;
public ValueDateHistogramFacetExecutor(IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, CacheRecycler cacheRecycler) {
public ValueDateHistogramFacetExecutor(IndexNumericFieldData keyIndexFieldData, IndexNumericFieldData valueIndexFieldData, Rounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, CacheRecycler cacheRecycler) {
this.comparatorType = comparatorType;
this.keyIndexFieldData = keyIndexFieldData;
this.valueIndexFieldData = valueIndexFieldData;
@ -105,13 +105,13 @@ public class ValueDateHistogramFacetExecutor extends FacetExecutor {
public static class DateHistogramProc extends LongFacetAggregatorBase {
final LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry> entries;
private final TimeZoneRounding tzRounding;
private final Rounding tzRounding;
SortedNumericDoubleValues valueValues;
final ValueAggregator valueAggregator = new ValueAggregator();
public DateHistogramProc(TimeZoneRounding tzRounding, LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry> entries) {
public DateHistogramProc(Rounding tzRounding, LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry> entries) {
this.tzRounding = tzRounding;
this.entries = entries;
}

10
src/main/java/org/elasticsearch/search/facet/datehistogram/ValueScriptDateHistogramFacetExecutor.java
View File

@ -25,7 +25,7 @@ import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.search.Scorer;
import org.elasticsearch.cache.recycler.CacheRecycler;
import org.elasticsearch.common.recycler.Recycler;
import org.elasticsearch.common.rounding.TimeZoneRounding;
import org.elasticsearch.common.rounding.Rounding;
import org.elasticsearch.index.fielddata.IndexNumericFieldData;
import org.elasticsearch.script.SearchScript;
import org.elasticsearch.search.facet.FacetExecutor;
@ -44,11 +44,11 @@ public class ValueScriptDateHistogramFacetExecutor extends FacetExecutor {
private final IndexNumericFieldData keyIndexFieldData;
private final DateHistogramFacet.ComparatorType comparatorType;
final SearchScript valueScript;
final TimeZoneRounding tzRounding;
final Rounding tzRounding;
final Recycler.V<LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry>> entries;
public ValueScriptDateHistogramFacetExecutor(IndexNumericFieldData keyIndexFieldData, SearchScript valueScript, TimeZoneRounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, CacheRecycler cacheRecycler) {
public ValueScriptDateHistogramFacetExecutor(IndexNumericFieldData keyIndexFieldData, SearchScript valueScript, Rounding tzRounding, DateHistogramFacet.ComparatorType comparatorType, CacheRecycler cacheRecycler) {
this.comparatorType = comparatorType;
this.keyIndexFieldData = keyIndexFieldData;
this.valueScript = valueScript;
@ -110,12 +110,12 @@ public class ValueScriptDateHistogramFacetExecutor extends FacetExecutor {
public static class DateHistogramProc extends LongFacetAggregatorBase {
private final TimeZoneRounding tzRounding;
private final Rounding tzRounding;
protected final SearchScript valueScript;
final LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry> entries;
public DateHistogramProc(TimeZoneRounding tzRounding, SearchScript valueScript, final LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry> entries) {
public DateHistogramProc(Rounding tzRounding, SearchScript valueScript, final LongObjectOpenHashMap<InternalFullDateHistogramFacet.FullEntry> entries) {
this.tzRounding = tzRounding;
this.valueScript = valueScript;
this.entries = entries;

6
src/test/java/org/elasticsearch/common/rounding/TimeZoneRoundingTests.java
View File

@ -33,7 +33,7 @@ public class TimeZoneRoundingTests extends ElasticsearchTestCase {
@Test
public void testUTCMonthRounding() {
TimeZoneRounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).build();
Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.MONTH_OF_YEAR).build();
assertThat(tzRounding.round(utc("2009-02-03T01:01:01")), equalTo(utc("2009-02-01T00:00:00.000Z")));
assertThat(tzRounding.nextRoundingValue(utc("2009-02-01T00:00:00.000Z")), equalTo(utc("2009-03-01T00:00:00.000Z")));
@ -48,7 +48,7 @@ public class TimeZoneRoundingTests extends ElasticsearchTestCase {
@Test
public void testDayTimeZoneRounding() {
TimeZoneRounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).preZone(DateTimeZone.forOffsetHours(-2)).build();
Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.DAY_OF_MONTH).preZone(DateTimeZone.forOffsetHours(-2)).build();
assertThat(tzRounding.round(0), equalTo(0l - TimeValue.timeValueHours(24).millis()));
assertThat(tzRounding.nextRoundingValue(0l - TimeValue.timeValueHours(24).millis()), equalTo(0l));
@ -67,7 +67,7 @@ public class TimeZoneRoundingTests extends ElasticsearchTestCase {
@Test
public void testTimeTimeZoneRounding() {
TimeZoneRounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).preZone(DateTimeZone.forOffsetHours(-2)).build();
Rounding tzRounding = TimeZoneRounding.builder(DateTimeUnit.HOUR_OF_DAY).preZone(DateTimeZone.forOffsetHours(-2)).build();
assertThat(tzRounding.round(0), equalTo(0l));
assertThat(tzRounding.nextRoundingValue(0l), equalTo(TimeValue.timeValueHours(1l).getMillis()));

46
src/test/java/org/elasticsearch/search/aggregations/bucket/HistogramTests.java
View File

@ -129,6 +129,52 @@ public class HistogramTests extends ElasticsearchIntegrationTest {
}
}
@Test
public void singleValuedField_withPreOffset() throws Exception {
long preOffsetMultiplier = randomIntBetween(2, 10);
SearchResponse response = client().prepareSearch("idx")
.addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).preOffset(preOffsetMultiplier * interval))
.execute().actionGet();
assertSearchResponse(response);
Histogram histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
assertThat(histo.getBuckets().size(), equalTo(numValueBuckets));
for (int i = 0; i < numValueBuckets; ++i) {
Histogram.Bucket bucket = histo.getBucketByKey((i + preOffsetMultiplier) * interval);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsNumber().longValue(), equalTo((long) (i + preOffsetMultiplier) * interval));
assertThat(bucket.getDocCount(), equalTo(valueCounts[i]));
}
}
@Test
public void singleValuedField_withPostOffset() throws Exception {
long postOffsetMultiplier = randomIntBetween(2, 10);
SearchResponse response = client().prepareSearch("idx")
.addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(interval).postOffset(postOffsetMultiplier * interval))
.execute().actionGet();
assertSearchResponse(response);
Histogram histo = response.getAggregations().get("histo");
assertThat(histo, notNullValue());
assertThat(histo.getName(), equalTo("histo"));
assertThat(histo.getBuckets().size(), equalTo(numValueBuckets));
for (int i = 0; i < numValueBuckets; ++i) {
Histogram.Bucket bucket = histo.getBucketByKey((i + postOffsetMultiplier) * interval);
assertThat(bucket, notNullValue());
assertThat(bucket.getKeyAsNumber().longValue(), equalTo((long) (i + postOffsetMultiplier) * interval));
assertThat(bucket.getDocCount(), equalTo(valueCounts[i]));
}
}
@Test
public void singleValuedField_OrderedByKeyAsc() throws Exception {
SearchResponse response = client().prepareSearch("idx")