[7.x] Support geotile_grid aggregation in composite agg sources () ()

* Support geotile_grid aggregation in composite agg sources ()

Adds support for `geotile_grid` as a source in composite aggs. 

Part of this change includes adding a new docFormat of `GEOTILE` that formats a hashed `long` value into a geotile formatting string `zoom/x/y`.
This commit is contained in:
Benjamin Trent 2019-09-05 13:22:57 -05:00 committed by GitHub
parent 828ff01515
commit d912a49c6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 514 additions and 30 deletions

View File

@ -12,6 +12,8 @@ setup:
type: keyword
long:
type: long
geo_point:
type: geo_point
nested:
type: nested
properties:
@ -38,25 +40,25 @@ setup:
index:
index: test
id: 1
body: { "keyword": "foo", "long": [10, 20], "nested": [{"nested_long": 10}, {"nested_long": 20}] }
body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] }
- do:
index:
index: test
id: 2
body: { "keyword": ["foo", "bar"] }
body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" }
- do:
index:
index: test
id: 3
body: { "keyword": "bar", "long": [100, 0], "nested": [{"nested_long": 10}, {"nested_long": 0}] }
body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] }
- do:
index:
index: test
id: 4
body: { "keyword": "bar", "long": [1000, 0], "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
- do:
index:
@ -650,3 +652,87 @@ setup:
}
]
---
"Simple Composite aggregation with GeoTile grid":
- skip:
version: " - 7.4.99"
reason: geotile_grid is not supported until 7.5.0
- do:
search:
rest_total_hits_as_int: true
index: test
body:
aggregations:
test:
composite:
sources: [
"geo": {
"geotile_grid": {
"field": "geo_point",
"precision": 12
}
},
{
"kw": {
"terms": {
"field": "keyword"
}
}
}
]
- match: {hits.total: 6}
- length: { aggregations.test.buckets: 4 }
- match: { aggregations.test.buckets.0.key.geo: "12/730/1590" }
- match: { aggregations.test.buckets.0.key.kw: "foo" }
- match: { aggregations.test.buckets.0.doc_count: 1 }
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
- match: { aggregations.test.buckets.1.key.kw: "bar" }
- match: { aggregations.test.buckets.1.doc_count: 2 }
- match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" }
- match: { aggregations.test.buckets.2.key.kw: "foo" }
- match: { aggregations.test.buckets.2.doc_count: 1 }
- match: { aggregations.test.buckets.3.key.geo: "12/2048/0" }
- match: { aggregations.test.buckets.3.key.kw: "bar" }
- match: { aggregations.test.buckets.3.doc_count: 1 }
---
"Simple Composite aggregation with geotile grid add aggregate after":
- skip:
version: " - 7.4.99"
reason: geotile_grid is not supported until 7.5.0
- do:
search:
rest_total_hits_as_int: true
index: test
body:
aggregations:
test:
composite:
sources: [
"geo": {
"geotile_grid": {
"field": "geo_point",
"precision": 12
}
},
{
"kw": {
"terms": {
"field": "keyword"
}
}
}
]
after: { "geo": "12/730/1590", "kw": "foo" }
- match: {hits.total: 6}
- length: { aggregations.test.buckets: 3 }
- match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" }
- match: { aggregations.test.buckets.0.key.kw: "bar" }
- match: { aggregations.test.buckets.0.doc_count: 2 }
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
- match: { aggregations.test.buckets.1.key.kw: "foo" }
- match: { aggregations.test.buckets.1.doc_count: 1 }
- match: { aggregations.test.buckets.2.key.geo: "12/2048/0" }
- match: { aggregations.test.buckets.2.key.kw: "bar" }
- match: { aggregations.test.buckets.2.doc_count: 1 }

View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.time.DateMathParser;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.geometry.utils.Geohash;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import java.io.IOException;
import java.net.InetAddress;
@ -257,6 +258,28 @@ public interface DocValueFormat extends NamedWriteable {
}
};
DocValueFormat GEOTILE = new DocValueFormat() {
@Override
public String getWriteableName() {
return "geo_tile";
}
@Override
public void writeTo(StreamOutput out) {
}
@Override
public String format(long value) {
return GeoTileUtils.stringEncode(value);
}
@Override
public String format(double value) {
return format((long) value);
}
};
DocValueFormat BOOLEAN = new DocValueFormat() {
@Override

View File

@ -704,6 +704,7 @@ public class SearchModule {
registerValueFormat(DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new);
registerValueFormat(DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new);
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
registerValueFormat(DocValueFormat.GEOTILE.getWriteableName(), in -> DocValueFormat.GEOTILE);
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);

View File

@ -42,6 +42,7 @@ import org.elasticsearch.search.aggregations.LeafBucketCollector;
import org.elasticsearch.search.aggregations.MultiBucketCollector;
import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.internal.SearchContext;
@ -299,6 +300,17 @@ final class CompositeAggregator extends BucketsAggregator {
reverseMul
);
} else if (config.valuesSource() instanceof CellIdSource) {
final CellIdSource cis = (CellIdSource) config.valuesSource();
return new GeoTileValuesSource(
bigArrays,
config.fieldType(),
cis::longValues,
LongUnaryOperator.identity(),
config.format(),
config.missingBucket(),
size,
reverseMul);
} else if (config.valuesSource() instanceof ValuesSource.Numeric) {
final ValuesSource.Numeric vs = (ValuesSource.Numeric) config.valuesSource();
if (vs.isFloatingPoint()) {

View File

@ -19,6 +19,7 @@
package org.elasticsearch.search.aggregations.bucket.composite;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
@ -67,6 +68,12 @@ public class CompositeValuesSourceParserHelper {
code = 1;
} else if (builder.getClass() == HistogramValuesSourceBuilder.class) {
code = 2;
} else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) {
if (out.getVersion().before(Version.V_7_5_0)) {
throw new IOException("Attempting to serialize [" + builder.getClass().getSimpleName()
+ "] to a node with unsupported version [" + out.getVersion() + "]");
}
code = 3;
} else {
throw new IOException("invalid builder type: " + builder.getClass().getSimpleName());
}
@ -83,6 +90,8 @@ public class CompositeValuesSourceParserHelper {
return new DateHistogramValuesSourceBuilder(in);
case 2:
return new HistogramValuesSourceBuilder(in);
case 3:
return new GeoTileGridValuesSourceBuilder(in);
default:
throw new IOException("Invalid code " + code);
}
@ -112,6 +121,9 @@ public class CompositeValuesSourceParserHelper {
case HistogramValuesSourceBuilder.TYPE:
builder = HistogramValuesSourceBuilder.parse(name, parser);
break;
case GeoTileGridValuesSourceBuilder.TYPE:
builder = GeoTileGridValuesSourceBuilder.parse(name, parser);
break;
default:
throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type);
}

View File

@ -0,0 +1,122 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.bucket.composite;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import org.elasticsearch.search.aggregations.support.ValueType;
import org.elasticsearch.search.aggregations.support.ValuesSource;
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
import org.elasticsearch.search.internal.SearchContext;
import java.io.IOException;
import java.util.Objects;
public class GeoTileGridValuesSourceBuilder extends CompositeValuesSourceBuilder<GeoTileGridValuesSourceBuilder> {
static final String TYPE = "geotile_grid";
private static final ObjectParser<GeoTileGridValuesSourceBuilder, Void> PARSER;
static {
PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE);
PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision"));
CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER, ValueType.NUMERIC);
}
static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException {
return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null);
}
private int precision = GeoTileGridAggregationBuilder.DEFAULT_PRECISION;
GeoTileGridValuesSourceBuilder(String name) {
super(name);
}
GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException {
super(in);
this.precision = in.readInt();
}
public GeoTileGridValuesSourceBuilder precision(int precision) {
this.precision = GeoTileUtils.checkPrecisionRange(precision);
return this;
}
@Override
public GeoTileGridValuesSourceBuilder format(String format) {
throw new IllegalArgumentException("[format] is not supported for [" + TYPE + "]");
}
@Override
protected void innerWriteTo(StreamOutput out) throws IOException {
out.writeInt(precision);
}
@Override
protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
builder.field("precision", precision);
}
@Override
String type() {
return TYPE;
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), precision);
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null || getClass() != obj.getClass()) return false;
if (super.equals(obj) == false) return false;
GeoTileGridValuesSourceBuilder other = (GeoTileGridValuesSourceBuilder) obj;
return precision == other.precision;
}
@Override
protected CompositeValuesSourceConfig innerBuild(SearchContext context, ValuesSourceConfig<?> config) throws IOException {
ValuesSource orig = config.toValuesSource(context.getQueryShardContext());
if (orig == null) {
orig = ValuesSource.GeoPoint.EMPTY;
}
if (orig instanceof ValuesSource.GeoPoint) {
ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) orig;
// is specified in the builder.
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
CellIdSource cellIdSource = new CellIdSource(geoPoint, precision, GeoTileUtils::longEncode);
return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket());
} else {
throw new IllegalArgumentException("invalid source, expected geo_point, got " + orig.getClass().getSimpleName());
}
}
}

View File

@ -0,0 +1,61 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.bucket.composite;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.SortedNumericDocValues;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.util.BigArrays;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import java.io.IOException;
import java.util.function.LongUnaryOperator;
/**
* A {@link SingleDimensionValuesSource} for geotile values.
*
* Since geotile values can be represented as long values, this class is almost the same as {@link LongValuesSource}
* The main differences is {@link GeoTileValuesSource#setAfter(Comparable)} as it needs to accept geotile string values i.e. "zoom/x/y".
*/
class GeoTileValuesSource extends LongValuesSource {
GeoTileValuesSource(BigArrays bigArrays,
MappedFieldType fieldType,
CheckedFunction<LeafReaderContext, SortedNumericDocValues, IOException> docValuesFunc,
LongUnaryOperator rounding,
DocValueFormat format,
boolean missingBucket,
int size,
int reverseMul) {
super(bigArrays, fieldType, docValuesFunc, rounding, format, missingBucket, size, reverseMul);
}
@Override
void setAfter(Comparable value) {
if (missingBucket && value == null) {
afterValue = null;
} else if (value instanceof Number) {
afterValue = ((Number) value).longValue();
} else {
afterValue = GeoTileUtils.longEncode(value.toString());
}
}
}

View File

@ -32,12 +32,12 @@ import java.io.IOException;
* Wrapper class to help convert {@link MultiGeoPointValues}
* to numeric long values for bucketing.
*/
class CellIdSource extends ValuesSource.Numeric {
public class CellIdSource extends ValuesSource.Numeric {
private final ValuesSource.GeoPoint valuesSource;
private final int precision;
private final GeoPointLongEncoder encoder;
CellIdSource(GeoPoint valuesSource, int precision, GeoPointLongEncoder encoder) {
public CellIdSource(GeoPoint valuesSource, int precision, GeoPointLongEncoder encoder) {
this.valuesSource = valuesSource;
//different GeoPoints could map to the same or different hashing cells.
this.precision = precision;

View File

@ -35,7 +35,7 @@ import java.util.Map;
public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder {
public static final String NAME = "geotile_grid";
private static final int DEFAULT_PRECISION = 7;
public static final int DEFAULT_PRECISION = 7;
private static final int DEFAULT_MAX_NUM_CELLS = 10000;
private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoTileUtils::parsePrecision);

View File

@ -38,7 +38,7 @@ import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
* bits 29..57 -- X tile index (0..2^zoom)
* bits 0..28 -- Y tile index (0..2^zoom)
*/
final class GeoTileUtils {
public final class GeoTileUtils {
private GeoTileUtils() {}
@ -80,7 +80,7 @@ final class GeoTileUtils {
/**
* Assert the precision value is within the allowed range, and return it if ok, or throw.
*/
static int checkPrecisionRange(int precision) {
public static int checkPrecisionRange(int precision) {
if (precision < 0 || precision > MAX_ZOOM) {
throw new IllegalArgumentException("Invalid geotile_grid precision of " +
precision + ". Must be between 0 and " + MAX_ZOOM + ".");
@ -93,7 +93,7 @@ final class GeoTileUtils {
* The resulting hash contains interleaved tile X and Y coordinates.
* The precision itself is also encoded as a few high bits.
*/
static long longEncode(double longitude, double latitude, int precision) {
public static long longEncode(double longitude, double latitude, int precision) {
// Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java
// Number of tiles for the current zoom level along the X and Y axis
@ -119,10 +119,18 @@ final class GeoTileUtils {
yTile = tiles - 1;
}
// Zoom value is placed in front of all the bits used for the geotile
// e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th),
// leaving 5 bits unused for zoom. See MAX_ZOOM comment above.
return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile;
return longEncode((long) precision, xTile, yTile);
}
/**
* Encode a geotile hash style string to a long.
*
* @param hashAsString String in format "zoom/x/y"
* @return long encoded value of the given string hash
*/
public static long longEncode(String hashAsString) {
int[] parsed = parseHash(hashAsString);
return longEncode((long)parsed[0], (long)parsed[1], (long)parsed[2]);
}
/**
@ -135,10 +143,34 @@ final class GeoTileUtils {
return new int[]{zoom, xTile, yTile};
}
private static long longEncode(long precision, long xTile, long yTile) {
// Zoom value is placed in front of all the bits used for the geotile
// e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th),
// leaving 5 bits unused for zoom. See MAX_ZOOM comment above.
return (precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile;
}
/**
* Parse geotile String hash format in "zoom/x/y" into an array of integers
*/
private static int[] parseHash(String hashAsString) {
final String[] parts = hashAsString.split("/", 4);
if (parts.length != 3) {
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
hashAsString + ". Must be three integers in a form \"zoom/x/y\".");
}
try {
return new int[]{Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2])};
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
hashAsString + ". Must be three integers in a form \"zoom/x/y\".", e);
}
}
/**
* Encode to a geotile string from the geotile based long format
*/
static String stringEncode(long hash) {
public static String stringEncode(long hash) {
int[] res = parseHash(hash);
validateZXY(res[0], res[1], res[2]);
return "" + res[0] + "/" + res[1] + "/" + res[2];
@ -156,18 +188,8 @@ final class GeoTileUtils {
* Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile)
*/
static GeoPoint keyToGeoPoint(String hashAsString) {
final String[] parts = hashAsString.split("/", 4);
if (parts.length != 3) {
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
hashAsString + ". Must be three integers in a form \"zoom/x/y\".");
}
try {
return zxyToGeoPoint(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2]));
} catch (NumberFormatException e) {
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
hashAsString + ". Must be three integers in a form \"zoom/x/y\".", e);
}
int[] hashAsInts = parseHash(hashAsString);
return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]);
}
/**

View File

@ -35,6 +35,8 @@ import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode;
public class DocValueFormatTests extends ESTestCase {
public void testSerialization() throws Exception {
@ -43,6 +45,7 @@ public class DocValueFormatTests extends ESTestCase {
entries.add(new Entry(DocValueFormat.class, DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOTILE.getWriteableName(), in -> DocValueFormat.GEOTILE));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
@ -87,6 +90,11 @@ public class DocValueFormatTests extends ESTestCase {
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.GEOHASH, in.readNamedWriteable(DocValueFormat.class));
out = new BytesStreamOutput();
out.writeNamedWriteable(DocValueFormat.GEOTILE);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
assertSame(DocValueFormat.GEOTILE, in.readNamedWriteable(DocValueFormat.class));
out = new BytesStreamOutput();
out.writeNamedWriteable(DocValueFormat.IP);
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
@ -147,6 +155,16 @@ public class DocValueFormatTests extends ESTestCase {
assertEquals("859,802.354", formatter.format(0.8598023539251286d * 1_000_000));
}
public void testGeoTileFormat() {
assertEquals("0/0/0", DocValueFormat.GEOTILE.format(longEncode(0, 0, 0)));
assertEquals("15/19114/7333", DocValueFormat.GEOTILE.format(longEncode(30, 70, 15)));
assertEquals("29/536869420/0", DocValueFormat.GEOTILE.format(longEncode(179.999, 89.999, 29)));
assertEquals("29/1491/536870911", DocValueFormat.GEOTILE.format(longEncode(-179.999, -89.999, 29)));
assertEquals("2/2/1", DocValueFormat.GEOTILE.format(longEncode(1, 1, 2)));
assertEquals("1/1/0", DocValueFormat.GEOTILE.format(longEncode(13,95, 1)));
assertEquals("1/1/1", DocValueFormat.GEOTILE.format(longEncode(13,-95, 1)));
}
public void testRawParse() {
assertEquals(-1L, DocValueFormat.RAW.parseLong("-1", randomBoolean(), null));
assertEquals(1L, DocValueFormat.RAW.parseLong("1", randomBoolean(), null));

View File

@ -71,7 +71,12 @@ public class SearchSortValuesTests extends AbstractSerializingTestCase<SearchSor
}
private static DocValueFormat randomDocValueFormat() {
return randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.RAW, DocValueFormat.IP, DocValueFormat.BINARY, DocValueFormat.GEOHASH);
return randomFrom(DocValueFormat.BOOLEAN,
DocValueFormat.RAW,
DocValueFormat.IP,
DocValueFormat.BINARY,
DocValueFormat.GEOHASH,
DocValueFormat.GEOTILE);
}
@Override

View File

@ -51,6 +51,14 @@ public class CompositeAggregationBuilderTests extends BaseAggregationTestCase<Co
return histo;
}
private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() {
GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10));
if (randomBoolean()) {
geoTile.precision(randomIntBetween(1, 12));
}
return geoTile;
}
private TermsValuesSourceBuilder randomTermsSourceBuilder() {
TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10));
if (randomBoolean()) {
@ -84,7 +92,7 @@ public class CompositeAggregationBuilderTests extends BaseAggregationTestCase<Co
int numSources = randomIntBetween(1, 10);
List<CompositeValuesSourceBuilder<?>> sources = new ArrayList<>();
for (int i = 0; i < numSources; i++) {
int type = randomIntBetween(0, 2);
int type = randomIntBetween(0, 3);
switch (type) {
case 0:
sources.add(randomTermsSourceBuilder());
@ -95,6 +103,9 @@ public class CompositeAggregationBuilderTests extends BaseAggregationTestCase<Co
case 2:
sources.add(randomHistogramSourceBuilder());
break;
case 3:
sources.add(randomGeoTileGridValuesSourceBuilder());
break;
default:
throw new AssertionError("wrong branch");
}

View File

@ -24,6 +24,7 @@ import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LatLonPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
@ -39,9 +40,11 @@ import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.index.mapper.ContentPath;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.GeoPointFieldMapper;
import org.elasticsearch.index.mapper.IpFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
@ -49,6 +52,8 @@ import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.NumberFieldMapper;
import org.elasticsearch.search.aggregations.Aggregator;
import org.elasticsearch.search.aggregations.AggregatorTestCase;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
@ -89,7 +94,7 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
@Before
public void setUp() throws Exception {
super.setUp();
FIELD_TYPES = new MappedFieldType[7];
FIELD_TYPES = new MappedFieldType[8];
FIELD_TYPES[0] = new KeywordFieldMapper.KeywordFieldType();
FIELD_TYPES[0].setName("keyword");
FIELD_TYPES[0].setHasDocValues(true);
@ -119,6 +124,10 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
FIELD_TYPES[6] = new IpFieldMapper.IpFieldType();
FIELD_TYPES[6].setName("ip");
FIELD_TYPES[6].setHasDocValues(true);
FIELD_TYPES[7] = new GeoPointFieldMapper.GeoPointFieldType();
FIELD_TYPES[7].setName("geo_point");
FIELD_TYPES[7].setHasDocValues(true);
}
@Override
@ -1788,6 +1797,47 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
);
}
public void testWithGeoPoint() throws Exception {
final List<Map<String, List<Object>>> dataset = new ArrayList<>();
dataset.addAll(
Arrays.asList(
createDocument("geo_point", new GeoPoint(48.934059, 41.610741)),
createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)),
createDocument("geo_point", new GeoPoint(90.0, 0.0)),
createDocument("geo_point", new GeoPoint(37.2343, -115.8067)),
createDocument("geo_point", new GeoPoint(90.0, 0.0))
)
);
testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset,
() -> {
GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point")
.field("geo_point");
return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile));
}, (result) -> {
assertEquals(2, result.getBuckets().size());
assertEquals("{geo_point=7/64/56}", result.afterKey().toString());
assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString());
assertEquals(2L, result.getBuckets().get(0).getDocCount());
assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString());
assertEquals(3L, result.getBuckets().get(1).getDocCount());
}
);
testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset,
() -> {
GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point")
.field("geo_point");
return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile))
.aggregateAfter(Collections.singletonMap("geo_point", "7/32/56"));
}, (result) -> {
assertEquals(1, result.getBuckets().size());
assertEquals("{geo_point=7/64/56}", result.afterKey().toString());
assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString());
assertEquals(3L, result.getBuckets().get(0).getDocCount());
}
);
}
private void testSearchCase(List<Query> queries,
List<Map<String, List<Object>>> dataset,
Supplier<CompositeAggregationBuilder> create,
@ -1845,6 +1895,11 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
} else if (value instanceof InetAddress) {
doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value))));
doc.add(new InetAddressPoint(name, (InetAddress) value));
} else if (value instanceof GeoPoint) {
GeoPoint point = (GeoPoint)value;
doc.add(new SortedNumericDocValuesField(name,
GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION)));
doc.add(new LatLonPoint(name, point.lat(), point.lon()));
} else {
throw new AssertionError("invalid object: " + value.getClass().getSimpleName());
}

View File

@ -0,0 +1,31 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.search.aggregations.bucket.composite;
import org.elasticsearch.test.ESTestCase;
public class GeoTileGridValuesSourceBuilderTests extends ESTestCase {
public void testSetFormat() {
CompositeValuesSourceBuilder<?> builder = new GeoTileGridValuesSourceBuilder("name");
expectThrows(IllegalArgumentException.class, () -> builder.format("format"));
}
}

View File

@ -76,6 +76,31 @@ public class GeoTileUtilsTests extends ESTestCase {
expectThrows(IllegalArgumentException.class, () -> longEncode(-1, 0, MAX_ZOOM + 1));
}
public void testLongEncodeFromString() {
assertEquals(0x0000000000000000L, longEncode(stringEncode(longEncode(0, 0, 0))));
assertEquals(0x3C00095540001CA5L, longEncode(stringEncode(longEncode(30, 70, 15))));
assertEquals(0x77FFFF4580000000L, longEncode(stringEncode(longEncode(179.999, 89.999, 29))));
assertEquals(0x740000BA7FFFFFFFL, longEncode(stringEncode(longEncode(-179.999, -89.999, 29))));
assertEquals(0x0800000040000001L, longEncode(stringEncode(longEncode(1, 1, 2))));
assertEquals(0x0C00000060000000L, longEncode(stringEncode(longEncode(-20, 100, 3))));
assertEquals(0x71127D27C8ACA67AL, longEncode(stringEncode(longEncode(13, -15, 28))));
assertEquals(0x4C0077776003A9ACL, longEncode(stringEncode(longEncode(-12, 15, 19))));
assertEquals(0x140000024000000EL, longEncode(stringEncode(longEncode(-328.231870,16.064082, 5))));
assertEquals(0x6436F96B60000000L, longEncode(stringEncode(longEncode(-590.769588,89.549167, 25))));
assertEquals(0x6411BD6BA0A98359L, longEncode(stringEncode(longEncode(999.787079,51.830093, 25))));
assertEquals(0x751BD6BBCA983596L, longEncode(stringEncode(longEncode(999.787079,51.830093, 29))));
assertEquals(0x77CF880A20000000L, longEncode(stringEncode(longEncode(-557.039740,-632.103969, 29))));
assertEquals(0x7624FA4FA0000000L, longEncode(stringEncode(longEncode(13,88, 29))));
assertEquals(0x7624FA4FBFFFFFFFL, longEncode(stringEncode(longEncode(13,-88, 29))));
assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13,89, 1))));
assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13,-89, 1))));
assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13,95, 1))));
assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13,-95, 1))));
expectThrows(IllegalArgumentException.class, () -> longEncode("12/asdf/1"));
expectThrows(IllegalArgumentException.class, () -> longEncode("foo"));
}
private void assertGeoPointEquals(GeoPoint gp, final double longitude, final double latitude) {
assertThat(gp.lon(), closeTo(longitude, GEOTILE_TOLERANCE));
assertThat(gp.lat(), closeTo(latitude, GEOTILE_TOLERANCE));