* Support geotile_grid aggregation in composite agg sources (#45810) Adds support for `geotile_grid` as a source in composite aggs. Part of this change includes adding a new docFormat of `GEOTILE` that formats a hashed `long` value into a geotile formatting string `zoom/x/y`.
This commit is contained in:
parent
828ff01515
commit
d912a49c6f
rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation
server/src
main/java/org/elasticsearch/search
test/java/org/elasticsearch/search
|
@ -12,6 +12,8 @@ setup:
|
|||
type: keyword
|
||||
long:
|
||||
type: long
|
||||
geo_point:
|
||||
type: geo_point
|
||||
nested:
|
||||
type: nested
|
||||
properties:
|
||||
|
@ -38,25 +40,25 @@ setup:
|
|||
index:
|
||||
index: test
|
||||
id: 1
|
||||
body: { "keyword": "foo", "long": [10, 20], "nested": [{"nested_long": 10}, {"nested_long": 20}] }
|
||||
body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
id: 2
|
||||
body: { "keyword": ["foo", "bar"] }
|
||||
body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
id: 3
|
||||
body: { "keyword": "bar", "long": [100, 0], "nested": [{"nested_long": 10}, {"nested_long": 0}] }
|
||||
body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
id: 4
|
||||
body: { "keyword": "bar", "long": [1000, 0], "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
|
||||
body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] }
|
||||
|
||||
- do:
|
||||
index:
|
||||
|
@ -650,3 +652,87 @@ setup:
|
|||
}
|
||||
]
|
||||
|
||||
---
|
||||
"Simple Composite aggregation with GeoTile grid":
|
||||
- skip:
|
||||
version: " - 7.4.99"
|
||||
reason: geotile_grid is not supported until 7.5.0
|
||||
- do:
|
||||
search:
|
||||
rest_total_hits_as_int: true
|
||||
index: test
|
||||
body:
|
||||
aggregations:
|
||||
test:
|
||||
composite:
|
||||
sources: [
|
||||
"geo": {
|
||||
"geotile_grid": {
|
||||
"field": "geo_point",
|
||||
"precision": 12
|
||||
}
|
||||
},
|
||||
{
|
||||
"kw": {
|
||||
"terms": {
|
||||
"field": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
- match: {hits.total: 6}
|
||||
- length: { aggregations.test.buckets: 4 }
|
||||
- match: { aggregations.test.buckets.0.key.geo: "12/730/1590" }
|
||||
- match: { aggregations.test.buckets.0.key.kw: "foo" }
|
||||
- match: { aggregations.test.buckets.0.doc_count: 1 }
|
||||
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
|
||||
- match: { aggregations.test.buckets.1.key.kw: "bar" }
|
||||
- match: { aggregations.test.buckets.1.doc_count: 2 }
|
||||
- match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" }
|
||||
- match: { aggregations.test.buckets.2.key.kw: "foo" }
|
||||
- match: { aggregations.test.buckets.2.doc_count: 1 }
|
||||
- match: { aggregations.test.buckets.3.key.geo: "12/2048/0" }
|
||||
- match: { aggregations.test.buckets.3.key.kw: "bar" }
|
||||
- match: { aggregations.test.buckets.3.doc_count: 1 }
|
||||
---
|
||||
"Simple Composite aggregation with geotile grid add aggregate after":
|
||||
- skip:
|
||||
version: " - 7.4.99"
|
||||
reason: geotile_grid is not supported until 7.5.0
|
||||
- do:
|
||||
search:
|
||||
rest_total_hits_as_int: true
|
||||
index: test
|
||||
body:
|
||||
aggregations:
|
||||
test:
|
||||
composite:
|
||||
sources: [
|
||||
"geo": {
|
||||
"geotile_grid": {
|
||||
"field": "geo_point",
|
||||
"precision": 12
|
||||
}
|
||||
},
|
||||
{
|
||||
"kw": {
|
||||
"terms": {
|
||||
"field": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
after: { "geo": "12/730/1590", "kw": "foo" }
|
||||
|
||||
- match: {hits.total: 6}
|
||||
- length: { aggregations.test.buckets: 3 }
|
||||
- match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" }
|
||||
- match: { aggregations.test.buckets.0.key.kw: "bar" }
|
||||
- match: { aggregations.test.buckets.0.doc_count: 2 }
|
||||
- match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" }
|
||||
- match: { aggregations.test.buckets.1.key.kw: "foo" }
|
||||
- match: { aggregations.test.buckets.1.doc_count: 1 }
|
||||
- match: { aggregations.test.buckets.2.key.geo: "12/2048/0" }
|
||||
- match: { aggregations.test.buckets.2.key.kw: "bar" }
|
||||
- match: { aggregations.test.buckets.2.doc_count: 1 }
|
||||
|
|
|
@ -32,6 +32,7 @@ import org.elasticsearch.common.time.DateMathParser;
|
|||
import org.elasticsearch.common.time.DateUtils;
|
||||
import org.elasticsearch.geometry.utils.Geohash;
|
||||
import org.elasticsearch.index.mapper.DateFieldMapper;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetAddress;
|
||||
|
@ -257,6 +258,28 @@ public interface DocValueFormat extends NamedWriteable {
|
|||
}
|
||||
};
|
||||
|
||||
DocValueFormat GEOTILE = new DocValueFormat() {
|
||||
|
||||
@Override
|
||||
public String getWriteableName() {
|
||||
return "geo_tile";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeTo(StreamOutput out) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String format(long value) {
|
||||
return GeoTileUtils.stringEncode(value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String format(double value) {
|
||||
return format((long) value);
|
||||
}
|
||||
};
|
||||
|
||||
DocValueFormat BOOLEAN = new DocValueFormat() {
|
||||
|
||||
@Override
|
||||
|
|
|
@ -704,6 +704,7 @@ public class SearchModule {
|
|||
registerValueFormat(DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new);
|
||||
registerValueFormat(DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new);
|
||||
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
|
||||
registerValueFormat(DocValueFormat.GEOTILE.getWriteableName(), in -> DocValueFormat.GEOTILE);
|
||||
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
|
||||
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
|
||||
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);
|
||||
|
|
|
@ -42,6 +42,7 @@ import org.elasticsearch.search.aggregations.LeafBucketCollector;
|
|||
import org.elasticsearch.search.aggregations.MultiBucketCollector;
|
||||
import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
|
||||
import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
|
||||
import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
@ -299,6 +300,17 @@ final class CompositeAggregator extends BucketsAggregator {
|
|||
reverseMul
|
||||
);
|
||||
|
||||
} else if (config.valuesSource() instanceof CellIdSource) {
|
||||
final CellIdSource cis = (CellIdSource) config.valuesSource();
|
||||
return new GeoTileValuesSource(
|
||||
bigArrays,
|
||||
config.fieldType(),
|
||||
cis::longValues,
|
||||
LongUnaryOperator.identity(),
|
||||
config.format(),
|
||||
config.missingBucket(),
|
||||
size,
|
||||
reverseMul);
|
||||
} else if (config.valuesSource() instanceof ValuesSource.Numeric) {
|
||||
final ValuesSource.Numeric vs = (ValuesSource.Numeric) config.valuesSource();
|
||||
if (vs.isFloatingPoint()) {
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
|
||||
package org.elasticsearch.search.aggregations.bucket.composite;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.common.ParsingException;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
|
@ -67,6 +68,12 @@ public class CompositeValuesSourceParserHelper {
|
|||
code = 1;
|
||||
} else if (builder.getClass() == HistogramValuesSourceBuilder.class) {
|
||||
code = 2;
|
||||
} else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) {
|
||||
if (out.getVersion().before(Version.V_7_5_0)) {
|
||||
throw new IOException("Attempting to serialize [" + builder.getClass().getSimpleName()
|
||||
+ "] to a node with unsupported version [" + out.getVersion() + "]");
|
||||
}
|
||||
code = 3;
|
||||
} else {
|
||||
throw new IOException("invalid builder type: " + builder.getClass().getSimpleName());
|
||||
}
|
||||
|
@ -83,6 +90,8 @@ public class CompositeValuesSourceParserHelper {
|
|||
return new DateHistogramValuesSourceBuilder(in);
|
||||
case 2:
|
||||
return new HistogramValuesSourceBuilder(in);
|
||||
case 3:
|
||||
return new GeoTileGridValuesSourceBuilder(in);
|
||||
default:
|
||||
throw new IOException("Invalid code " + code);
|
||||
}
|
||||
|
@ -112,6 +121,9 @@ public class CompositeValuesSourceParserHelper {
|
|||
case HistogramValuesSourceBuilder.TYPE:
|
||||
builder = HistogramValuesSourceBuilder.parse(name, parser);
|
||||
break;
|
||||
case GeoTileGridValuesSourceBuilder.TYPE:
|
||||
builder = GeoTileGridValuesSourceBuilder.parse(name, parser);
|
||||
break;
|
||||
default:
|
||||
throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type);
|
||||
}
|
||||
|
|
|
@ -0,0 +1,122 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.bucket.composite;
|
||||
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.xcontent.ObjectParser;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentParser;
|
||||
import org.elasticsearch.index.mapper.MappedFieldType;
|
||||
import org.elasticsearch.search.DocValueFormat;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
|
||||
import org.elasticsearch.search.aggregations.support.ValueType;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSource;
|
||||
import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
|
||||
import org.elasticsearch.search.internal.SearchContext;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Objects;
|
||||
|
||||
public class GeoTileGridValuesSourceBuilder extends CompositeValuesSourceBuilder<GeoTileGridValuesSourceBuilder> {
|
||||
static final String TYPE = "geotile_grid";
|
||||
|
||||
private static final ObjectParser<GeoTileGridValuesSourceBuilder, Void> PARSER;
|
||||
static {
|
||||
PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE);
|
||||
PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision"));
|
||||
CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER, ValueType.NUMERIC);
|
||||
}
|
||||
|
||||
static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException {
|
||||
return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null);
|
||||
}
|
||||
|
||||
private int precision = GeoTileGridAggregationBuilder.DEFAULT_PRECISION;
|
||||
|
||||
GeoTileGridValuesSourceBuilder(String name) {
|
||||
super(name);
|
||||
}
|
||||
|
||||
GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException {
|
||||
super(in);
|
||||
this.precision = in.readInt();
|
||||
}
|
||||
|
||||
public GeoTileGridValuesSourceBuilder precision(int precision) {
|
||||
this.precision = GeoTileUtils.checkPrecisionRange(precision);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GeoTileGridValuesSourceBuilder format(String format) {
|
||||
throw new IllegalArgumentException("[format] is not supported for [" + TYPE + "]");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void innerWriteTo(StreamOutput out) throws IOException {
|
||||
out.writeInt(precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void doXContentBody(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.field("precision", precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
String type() {
|
||||
return TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(super.hashCode(), precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object obj) {
|
||||
if (this == obj) return true;
|
||||
if (obj == null || getClass() != obj.getClass()) return false;
|
||||
if (super.equals(obj) == false) return false;
|
||||
GeoTileGridValuesSourceBuilder other = (GeoTileGridValuesSourceBuilder) obj;
|
||||
return precision == other.precision;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CompositeValuesSourceConfig innerBuild(SearchContext context, ValuesSourceConfig<?> config) throws IOException {
|
||||
ValuesSource orig = config.toValuesSource(context.getQueryShardContext());
|
||||
if (orig == null) {
|
||||
orig = ValuesSource.GeoPoint.EMPTY;
|
||||
}
|
||||
if (orig instanceof ValuesSource.GeoPoint) {
|
||||
ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) orig;
|
||||
// is specified in the builder.
|
||||
final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null;
|
||||
CellIdSource cellIdSource = new CellIdSource(geoPoint, precision, GeoTileUtils::longEncode);
|
||||
return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket());
|
||||
} else {
|
||||
throw new IllegalArgumentException("invalid source, expected geo_point, got " + orig.getClass().getSimpleName());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,61 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.bucket.composite;
|
||||
|
||||
import org.apache.lucene.index.LeafReaderContext;
|
||||
import org.apache.lucene.index.SortedNumericDocValues;
|
||||
import org.elasticsearch.common.CheckedFunction;
|
||||
import org.elasticsearch.common.util.BigArrays;
|
||||
import org.elasticsearch.index.mapper.MappedFieldType;
|
||||
import org.elasticsearch.search.DocValueFormat;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.function.LongUnaryOperator;
|
||||
|
||||
/**
|
||||
* A {@link SingleDimensionValuesSource} for geotile values.
|
||||
*
|
||||
* Since geotile values can be represented as long values, this class is almost the same as {@link LongValuesSource}
|
||||
* The main differences is {@link GeoTileValuesSource#setAfter(Comparable)} as it needs to accept geotile string values i.e. "zoom/x/y".
|
||||
*/
|
||||
class GeoTileValuesSource extends LongValuesSource {
|
||||
GeoTileValuesSource(BigArrays bigArrays,
|
||||
MappedFieldType fieldType,
|
||||
CheckedFunction<LeafReaderContext, SortedNumericDocValues, IOException> docValuesFunc,
|
||||
LongUnaryOperator rounding,
|
||||
DocValueFormat format,
|
||||
boolean missingBucket,
|
||||
int size,
|
||||
int reverseMul) {
|
||||
super(bigArrays, fieldType, docValuesFunc, rounding, format, missingBucket, size, reverseMul);
|
||||
}
|
||||
|
||||
@Override
|
||||
void setAfter(Comparable value) {
|
||||
if (missingBucket && value == null) {
|
||||
afterValue = null;
|
||||
} else if (value instanceof Number) {
|
||||
afterValue = ((Number) value).longValue();
|
||||
} else {
|
||||
afterValue = GeoTileUtils.longEncode(value.toString());
|
||||
}
|
||||
}
|
||||
}
|
|
@ -32,12 +32,12 @@ import java.io.IOException;
|
|||
* Wrapper class to help convert {@link MultiGeoPointValues}
|
||||
* to numeric long values for bucketing.
|
||||
*/
|
||||
class CellIdSource extends ValuesSource.Numeric {
|
||||
public class CellIdSource extends ValuesSource.Numeric {
|
||||
private final ValuesSource.GeoPoint valuesSource;
|
||||
private final int precision;
|
||||
private final GeoPointLongEncoder encoder;
|
||||
|
||||
CellIdSource(GeoPoint valuesSource, int precision, GeoPointLongEncoder encoder) {
|
||||
public CellIdSource(GeoPoint valuesSource, int precision, GeoPointLongEncoder encoder) {
|
||||
this.valuesSource = valuesSource;
|
||||
//different GeoPoints could map to the same or different hashing cells.
|
||||
this.precision = precision;
|
||||
|
|
|
@ -35,7 +35,7 @@ import java.util.Map;
|
|||
|
||||
public class GeoTileGridAggregationBuilder extends GeoGridAggregationBuilder {
|
||||
public static final String NAME = "geotile_grid";
|
||||
private static final int DEFAULT_PRECISION = 7;
|
||||
public static final int DEFAULT_PRECISION = 7;
|
||||
private static final int DEFAULT_MAX_NUM_CELLS = 10000;
|
||||
|
||||
private static final ObjectParser<GeoGridAggregationBuilder, Void> PARSER = createParser(NAME, GeoTileUtils::parsePrecision);
|
||||
|
|
|
@ -38,7 +38,7 @@ import static org.elasticsearch.common.geo.GeoUtils.normalizeLon;
|
|||
* bits 29..57 -- X tile index (0..2^zoom)
|
||||
* bits 0..28 -- Y tile index (0..2^zoom)
|
||||
*/
|
||||
final class GeoTileUtils {
|
||||
public final class GeoTileUtils {
|
||||
|
||||
private GeoTileUtils() {}
|
||||
|
||||
|
@ -80,7 +80,7 @@ final class GeoTileUtils {
|
|||
/**
|
||||
* Assert the precision value is within the allowed range, and return it if ok, or throw.
|
||||
*/
|
||||
static int checkPrecisionRange(int precision) {
|
||||
public static int checkPrecisionRange(int precision) {
|
||||
if (precision < 0 || precision > MAX_ZOOM) {
|
||||
throw new IllegalArgumentException("Invalid geotile_grid precision of " +
|
||||
precision + ". Must be between 0 and " + MAX_ZOOM + ".");
|
||||
|
@ -93,7 +93,7 @@ final class GeoTileUtils {
|
|||
* The resulting hash contains interleaved tile X and Y coordinates.
|
||||
* The precision itself is also encoded as a few high bits.
|
||||
*/
|
||||
static long longEncode(double longitude, double latitude, int precision) {
|
||||
public static long longEncode(double longitude, double latitude, int precision) {
|
||||
// Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java
|
||||
|
||||
// Number of tiles for the current zoom level along the X and Y axis
|
||||
|
@ -119,10 +119,18 @@ final class GeoTileUtils {
|
|||
yTile = tiles - 1;
|
||||
}
|
||||
|
||||
// Zoom value is placed in front of all the bits used for the geotile
|
||||
// e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th),
|
||||
// leaving 5 bits unused for zoom. See MAX_ZOOM comment above.
|
||||
return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile;
|
||||
return longEncode((long) precision, xTile, yTile);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a geotile hash style string to a long.
|
||||
*
|
||||
* @param hashAsString String in format "zoom/x/y"
|
||||
* @return long encoded value of the given string hash
|
||||
*/
|
||||
public static long longEncode(String hashAsString) {
|
||||
int[] parsed = parseHash(hashAsString);
|
||||
return longEncode((long)parsed[0], (long)parsed[1], (long)parsed[2]);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -135,10 +143,34 @@ final class GeoTileUtils {
|
|||
return new int[]{zoom, xTile, yTile};
|
||||
}
|
||||
|
||||
private static long longEncode(long precision, long xTile, long yTile) {
|
||||
// Zoom value is placed in front of all the bits used for the geotile
|
||||
// e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th),
|
||||
// leaving 5 bits unused for zoom. See MAX_ZOOM comment above.
|
||||
return (precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse geotile String hash format in "zoom/x/y" into an array of integers
|
||||
*/
|
||||
private static int[] parseHash(String hashAsString) {
|
||||
final String[] parts = hashAsString.split("/", 4);
|
||||
if (parts.length != 3) {
|
||||
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
|
||||
hashAsString + ". Must be three integers in a form \"zoom/x/y\".");
|
||||
}
|
||||
try {
|
||||
return new int[]{Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2])};
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
|
||||
hashAsString + ". Must be three integers in a form \"zoom/x/y\".", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode to a geotile string from the geotile based long format
|
||||
*/
|
||||
static String stringEncode(long hash) {
|
||||
public static String stringEncode(long hash) {
|
||||
int[] res = parseHash(hash);
|
||||
validateZXY(res[0], res[1], res[2]);
|
||||
return "" + res[0] + "/" + res[1] + "/" + res[2];
|
||||
|
@ -156,18 +188,8 @@ final class GeoTileUtils {
|
|||
* Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile)
|
||||
*/
|
||||
static GeoPoint keyToGeoPoint(String hashAsString) {
|
||||
final String[] parts = hashAsString.split("/", 4);
|
||||
if (parts.length != 3) {
|
||||
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
|
||||
hashAsString + ". Must be three integers in a form \"zoom/x/y\".");
|
||||
}
|
||||
|
||||
try {
|
||||
return zxyToGeoPoint(Integer.parseInt(parts[0]), Integer.parseInt(parts[1]), Integer.parseInt(parts[2]));
|
||||
} catch (NumberFormatException e) {
|
||||
throw new IllegalArgumentException("Invalid geotile_grid hash string of " +
|
||||
hashAsString + ". Must be three integers in a form \"zoom/x/y\".", e);
|
||||
}
|
||||
int[] hashAsInts = parseHash(hashAsString);
|
||||
return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -35,6 +35,8 @@ import java.time.ZoneOffset;
|
|||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode;
|
||||
|
||||
public class DocValueFormatTests extends ESTestCase {
|
||||
|
||||
public void testSerialization() throws Exception {
|
||||
|
@ -43,6 +45,7 @@ public class DocValueFormatTests extends ESTestCase {
|
|||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.DateTime.NAME, DocValueFormat.DateTime::new));
|
||||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.Decimal.NAME, DocValueFormat.Decimal::new));
|
||||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
|
||||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOTILE.getWriteableName(), in -> DocValueFormat.GEOTILE));
|
||||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
|
||||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
|
||||
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
|
||||
|
@ -87,6 +90,11 @@ public class DocValueFormatTests extends ESTestCase {
|
|||
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
|
||||
assertSame(DocValueFormat.GEOHASH, in.readNamedWriteable(DocValueFormat.class));
|
||||
|
||||
out = new BytesStreamOutput();
|
||||
out.writeNamedWriteable(DocValueFormat.GEOTILE);
|
||||
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
|
||||
assertSame(DocValueFormat.GEOTILE, in.readNamedWriteable(DocValueFormat.class));
|
||||
|
||||
out = new BytesStreamOutput();
|
||||
out.writeNamedWriteable(DocValueFormat.IP);
|
||||
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
|
||||
|
@ -147,6 +155,16 @@ public class DocValueFormatTests extends ESTestCase {
|
|||
assertEquals("859,802.354", formatter.format(0.8598023539251286d * 1_000_000));
|
||||
}
|
||||
|
||||
public void testGeoTileFormat() {
|
||||
assertEquals("0/0/0", DocValueFormat.GEOTILE.format(longEncode(0, 0, 0)));
|
||||
assertEquals("15/19114/7333", DocValueFormat.GEOTILE.format(longEncode(30, 70, 15)));
|
||||
assertEquals("29/536869420/0", DocValueFormat.GEOTILE.format(longEncode(179.999, 89.999, 29)));
|
||||
assertEquals("29/1491/536870911", DocValueFormat.GEOTILE.format(longEncode(-179.999, -89.999, 29)));
|
||||
assertEquals("2/2/1", DocValueFormat.GEOTILE.format(longEncode(1, 1, 2)));
|
||||
assertEquals("1/1/0", DocValueFormat.GEOTILE.format(longEncode(13,95, 1)));
|
||||
assertEquals("1/1/1", DocValueFormat.GEOTILE.format(longEncode(13,-95, 1)));
|
||||
}
|
||||
|
||||
public void testRawParse() {
|
||||
assertEquals(-1L, DocValueFormat.RAW.parseLong("-1", randomBoolean(), null));
|
||||
assertEquals(1L, DocValueFormat.RAW.parseLong("1", randomBoolean(), null));
|
||||
|
|
|
@ -71,7 +71,12 @@ public class SearchSortValuesTests extends AbstractSerializingTestCase<SearchSor
|
|||
}
|
||||
|
||||
private static DocValueFormat randomDocValueFormat() {
|
||||
return randomFrom(DocValueFormat.BOOLEAN, DocValueFormat.RAW, DocValueFormat.IP, DocValueFormat.BINARY, DocValueFormat.GEOHASH);
|
||||
return randomFrom(DocValueFormat.BOOLEAN,
|
||||
DocValueFormat.RAW,
|
||||
DocValueFormat.IP,
|
||||
DocValueFormat.BINARY,
|
||||
DocValueFormat.GEOHASH,
|
||||
DocValueFormat.GEOTILE);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -51,6 +51,14 @@ public class CompositeAggregationBuilderTests extends BaseAggregationTestCase<Co
|
|||
return histo;
|
||||
}
|
||||
|
||||
private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() {
|
||||
GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10));
|
||||
if (randomBoolean()) {
|
||||
geoTile.precision(randomIntBetween(1, 12));
|
||||
}
|
||||
return geoTile;
|
||||
}
|
||||
|
||||
private TermsValuesSourceBuilder randomTermsSourceBuilder() {
|
||||
TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10));
|
||||
if (randomBoolean()) {
|
||||
|
@ -84,7 +92,7 @@ public class CompositeAggregationBuilderTests extends BaseAggregationTestCase<Co
|
|||
int numSources = randomIntBetween(1, 10);
|
||||
List<CompositeValuesSourceBuilder<?>> sources = new ArrayList<>();
|
||||
for (int i = 0; i < numSources; i++) {
|
||||
int type = randomIntBetween(0, 2);
|
||||
int type = randomIntBetween(0, 3);
|
||||
switch (type) {
|
||||
case 0:
|
||||
sources.add(randomTermsSourceBuilder());
|
||||
|
@ -95,6 +103,9 @@ public class CompositeAggregationBuilderTests extends BaseAggregationTestCase<Co
|
|||
case 2:
|
||||
sources.add(randomHistogramSourceBuilder());
|
||||
break;
|
||||
case 3:
|
||||
sources.add(randomGeoTileGridValuesSourceBuilder());
|
||||
break;
|
||||
default:
|
||||
throw new AssertionError("wrong branch");
|
||||
}
|
||||
|
|
|
@ -24,6 +24,7 @@ import org.apache.lucene.document.DoublePoint;
|
|||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.document.InetAddressPoint;
|
||||
import org.apache.lucene.document.IntPoint;
|
||||
import org.apache.lucene.document.LatLonPoint;
|
||||
import org.apache.lucene.document.LongPoint;
|
||||
import org.apache.lucene.document.SortedNumericDocValuesField;
|
||||
import org.apache.lucene.document.SortedSetDocValuesField;
|
||||
|
@ -39,9 +40,11 @@ import org.apache.lucene.store.Directory;
|
|||
import org.apache.lucene.util.BytesRef;
|
||||
import org.apache.lucene.util.NumericUtils;
|
||||
import org.elasticsearch.ElasticsearchParseException;
|
||||
import org.elasticsearch.common.geo.GeoPoint;
|
||||
import org.elasticsearch.common.time.DateFormatters;
|
||||
import org.elasticsearch.index.mapper.ContentPath;
|
||||
import org.elasticsearch.index.mapper.DateFieldMapper;
|
||||
import org.elasticsearch.index.mapper.GeoPointFieldMapper;
|
||||
import org.elasticsearch.index.mapper.IpFieldMapper;
|
||||
import org.elasticsearch.index.mapper.KeywordFieldMapper;
|
||||
import org.elasticsearch.index.mapper.MappedFieldType;
|
||||
|
@ -49,6 +52,8 @@ import org.elasticsearch.index.mapper.Mapper;
|
|||
import org.elasticsearch.index.mapper.NumberFieldMapper;
|
||||
import org.elasticsearch.search.aggregations.Aggregator;
|
||||
import org.elasticsearch.search.aggregations.AggregatorTestCase;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
|
||||
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
|
||||
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
|
||||
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
|
||||
|
@ -89,7 +94,7 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
|
|||
@Before
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
FIELD_TYPES = new MappedFieldType[7];
|
||||
FIELD_TYPES = new MappedFieldType[8];
|
||||
FIELD_TYPES[0] = new KeywordFieldMapper.KeywordFieldType();
|
||||
FIELD_TYPES[0].setName("keyword");
|
||||
FIELD_TYPES[0].setHasDocValues(true);
|
||||
|
@ -119,6 +124,10 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
|
|||
FIELD_TYPES[6] = new IpFieldMapper.IpFieldType();
|
||||
FIELD_TYPES[6].setName("ip");
|
||||
FIELD_TYPES[6].setHasDocValues(true);
|
||||
|
||||
FIELD_TYPES[7] = new GeoPointFieldMapper.GeoPointFieldType();
|
||||
FIELD_TYPES[7].setName("geo_point");
|
||||
FIELD_TYPES[7].setHasDocValues(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -1788,6 +1797,47 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
|
|||
);
|
||||
}
|
||||
|
||||
public void testWithGeoPoint() throws Exception {
|
||||
final List<Map<String, List<Object>>> dataset = new ArrayList<>();
|
||||
dataset.addAll(
|
||||
Arrays.asList(
|
||||
createDocument("geo_point", new GeoPoint(48.934059, 41.610741)),
|
||||
createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)),
|
||||
createDocument("geo_point", new GeoPoint(90.0, 0.0)),
|
||||
createDocument("geo_point", new GeoPoint(37.2343, -115.8067)),
|
||||
createDocument("geo_point", new GeoPoint(90.0, 0.0))
|
||||
)
|
||||
);
|
||||
testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset,
|
||||
() -> {
|
||||
GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point")
|
||||
.field("geo_point");
|
||||
return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile));
|
||||
}, (result) -> {
|
||||
assertEquals(2, result.getBuckets().size());
|
||||
assertEquals("{geo_point=7/64/56}", result.afterKey().toString());
|
||||
assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString());
|
||||
assertEquals(2L, result.getBuckets().get(0).getDocCount());
|
||||
assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString());
|
||||
assertEquals(3L, result.getBuckets().get(1).getDocCount());
|
||||
}
|
||||
);
|
||||
|
||||
testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset,
|
||||
() -> {
|
||||
GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point")
|
||||
.field("geo_point");
|
||||
return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile))
|
||||
.aggregateAfter(Collections.singletonMap("geo_point", "7/32/56"));
|
||||
}, (result) -> {
|
||||
assertEquals(1, result.getBuckets().size());
|
||||
assertEquals("{geo_point=7/64/56}", result.afterKey().toString());
|
||||
assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString());
|
||||
assertEquals(3L, result.getBuckets().get(0).getDocCount());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private void testSearchCase(List<Query> queries,
|
||||
List<Map<String, List<Object>>> dataset,
|
||||
Supplier<CompositeAggregationBuilder> create,
|
||||
|
@ -1845,6 +1895,11 @@ public class CompositeAggregatorTests extends AggregatorTestCase {
|
|||
} else if (value instanceof InetAddress) {
|
||||
doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value))));
|
||||
doc.add(new InetAddressPoint(name, (InetAddress) value));
|
||||
} else if (value instanceof GeoPoint) {
|
||||
GeoPoint point = (GeoPoint)value;
|
||||
doc.add(new SortedNumericDocValuesField(name,
|
||||
GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION)));
|
||||
doc.add(new LatLonPoint(name, point.lat(), point.lon()));
|
||||
} else {
|
||||
throw new AssertionError("invalid object: " + value.getClass().getSimpleName());
|
||||
}
|
||||
|
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.search.aggregations.bucket.composite;
|
||||
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
public class GeoTileGridValuesSourceBuilderTests extends ESTestCase {
|
||||
|
||||
public void testSetFormat() {
|
||||
CompositeValuesSourceBuilder<?> builder = new GeoTileGridValuesSourceBuilder("name");
|
||||
expectThrows(IllegalArgumentException.class, () -> builder.format("format"));
|
||||
}
|
||||
|
||||
}
|
|
@ -76,6 +76,31 @@ public class GeoTileUtilsTests extends ESTestCase {
|
|||
expectThrows(IllegalArgumentException.class, () -> longEncode(-1, 0, MAX_ZOOM + 1));
|
||||
}
|
||||
|
||||
public void testLongEncodeFromString() {
|
||||
assertEquals(0x0000000000000000L, longEncode(stringEncode(longEncode(0, 0, 0))));
|
||||
assertEquals(0x3C00095540001CA5L, longEncode(stringEncode(longEncode(30, 70, 15))));
|
||||
assertEquals(0x77FFFF4580000000L, longEncode(stringEncode(longEncode(179.999, 89.999, 29))));
|
||||
assertEquals(0x740000BA7FFFFFFFL, longEncode(stringEncode(longEncode(-179.999, -89.999, 29))));
|
||||
assertEquals(0x0800000040000001L, longEncode(stringEncode(longEncode(1, 1, 2))));
|
||||
assertEquals(0x0C00000060000000L, longEncode(stringEncode(longEncode(-20, 100, 3))));
|
||||
assertEquals(0x71127D27C8ACA67AL, longEncode(stringEncode(longEncode(13, -15, 28))));
|
||||
assertEquals(0x4C0077776003A9ACL, longEncode(stringEncode(longEncode(-12, 15, 19))));
|
||||
assertEquals(0x140000024000000EL, longEncode(stringEncode(longEncode(-328.231870,16.064082, 5))));
|
||||
assertEquals(0x6436F96B60000000L, longEncode(stringEncode(longEncode(-590.769588,89.549167, 25))));
|
||||
assertEquals(0x6411BD6BA0A98359L, longEncode(stringEncode(longEncode(999.787079,51.830093, 25))));
|
||||
assertEquals(0x751BD6BBCA983596L, longEncode(stringEncode(longEncode(999.787079,51.830093, 29))));
|
||||
assertEquals(0x77CF880A20000000L, longEncode(stringEncode(longEncode(-557.039740,-632.103969, 29))));
|
||||
assertEquals(0x7624FA4FA0000000L, longEncode(stringEncode(longEncode(13,88, 29))));
|
||||
assertEquals(0x7624FA4FBFFFFFFFL, longEncode(stringEncode(longEncode(13,-88, 29))));
|
||||
assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13,89, 1))));
|
||||
assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13,-89, 1))));
|
||||
assertEquals(0x0400000020000000L, longEncode(stringEncode(longEncode(13,95, 1))));
|
||||
assertEquals(0x0400000020000001L, longEncode(stringEncode(longEncode(13,-95, 1))));
|
||||
|
||||
expectThrows(IllegalArgumentException.class, () -> longEncode("12/asdf/1"));
|
||||
expectThrows(IllegalArgumentException.class, () -> longEncode("foo"));
|
||||
}
|
||||
|
||||
private void assertGeoPointEquals(GeoPoint gp, final double longitude, final double latitude) {
|
||||
assertThat(gp.lon(), closeTo(longitude, GEOTILE_TOLERANCE));
|
||||
assertThat(gp.lat(), closeTo(latitude, GEOTILE_TOLERANCE));
|
||||
|
|
Loading…
Reference in New Issue