mirror of https://github.com/apache/druid.git
changing names to be explicit about theta sketch algorithm
old names are still valid though so as to be backwards compatible for now
This commit is contained in:
parent
6c6a38cedb
commit
88ae3c43f9
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import io.druid.query.aggregation.Aggregator;
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import io.druid.query.aggregation.BufferAggregator;
|
||||
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.yahoo.sketches.Family;
|
||||
import com.yahoo.sketches.memory.Memory;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.base.Preconditions;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.yahoo.sketches.Family;
|
||||
import com.yahoo.sketches.memory.Memory;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
@ -64,6 +64,6 @@ public class SketchBuildAggregatorFactory extends SketchAggregatorFactory
|
|||
@Override
|
||||
public String getTypeName()
|
||||
{
|
||||
return SketchModule.SKETCH_BUILD;
|
||||
return SketchModule.THETA_SKETCH_BUILD_AGG;
|
||||
}
|
||||
}
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.metamx.common.IAE;
|
||||
import com.yahoo.sketches.memory.Memory;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonGenerator;
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
@ -84,7 +84,7 @@ public class SketchMergeAggregatorFactory extends SketchAggregatorFactory
|
|||
@Override
|
||||
public String getTypeName()
|
||||
{
|
||||
return SketchModule.SET_SKETCH;
|
||||
return SketchModule.THETA_SKETCH_MERGE_AGG;
|
||||
}
|
||||
|
||||
@Override
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.yahoo.sketches.memory.Memory;
|
||||
import com.yahoo.sketches.theta.Sketch;
|
||||
|
@ -40,7 +40,7 @@ public class SketchMergeComplexMetricSerde extends ComplexMetricSerde
|
|||
@Override
|
||||
public String getTypeName()
|
||||
{
|
||||
return SketchModule.SET_SKETCH;
|
||||
return SketchModule.THETA_SKETCH;
|
||||
}
|
||||
|
||||
@Override
|
|
@ -17,13 +17,12 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.databind.Module;
|
||||
import com.fasterxml.jackson.databind.jsontype.NamedType;
|
||||
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||
import com.google.inject.Binder;
|
||||
|
||||
import com.yahoo.sketches.theta.Sketch;
|
||||
import io.druid.initialization.DruidModule;
|
||||
import io.druid.segment.serde.ComplexMetrics;
|
||||
|
@ -33,21 +32,27 @@ import java.util.List;
|
|||
|
||||
public class SketchModule implements DruidModule
|
||||
{
|
||||
public static final String THETA_SKETCH = "thetaSketch";
|
||||
|
||||
public static final String SET_SKETCH = "setSketch";
|
||||
public static final String SKETCH_BUILD = "sketchBuild";
|
||||
public static final String THETA_SKETCH_MERGE_AGG = "thetaSketchMerge";
|
||||
public static final String THETA_SKETCH_BUILD_AGG = "thetaSketchBuild";
|
||||
|
||||
public static final String THETA_SKETCH_ESTIMATE_POST_AGG = "thetaSketchEstimate";
|
||||
public static final String THETA_SKETCH_SET_OP_POST_AGG = "thetaSketchSetOp";
|
||||
|
||||
@Override
|
||||
public void configure(Binder binder)
|
||||
{
|
||||
//gives the extractor to ingest non-sketch input data
|
||||
if (ComplexMetrics.getSerdeForType(SKETCH_BUILD) == null) {
|
||||
ComplexMetrics.registerSerde(SKETCH_BUILD, new SketchBuildComplexMetricSerde());
|
||||
if (ComplexMetrics.getSerdeForType(THETA_SKETCH) == null) {
|
||||
ComplexMetrics.registerSerde(THETA_SKETCH, new SketchMergeComplexMetricSerde());
|
||||
}
|
||||
|
||||
//gives the extractor to ingest sketch input data
|
||||
if (ComplexMetrics.getSerdeForType(SET_SKETCH) == null) {
|
||||
ComplexMetrics.registerSerde(SET_SKETCH, new SketchMergeComplexMetricSerde());
|
||||
if (ComplexMetrics.getSerdeForType(THETA_SKETCH_MERGE_AGG) == null) {
|
||||
ComplexMetrics.registerSerde(THETA_SKETCH_MERGE_AGG, new SketchMergeComplexMetricSerde());
|
||||
}
|
||||
|
||||
if (ComplexMetrics.getSerdeForType(THETA_SKETCH_BUILD_AGG) == null) {
|
||||
ComplexMetrics.registerSerde(THETA_SKETCH_BUILD_AGG, new SketchBuildComplexMetricSerde());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -55,12 +60,12 @@ public class SketchModule implements DruidModule
|
|||
public List<? extends Module> getJacksonModules()
|
||||
{
|
||||
return Arrays.<Module>asList(
|
||||
new SimpleModule("SketchModule")
|
||||
new SimpleModule("ThetaSketchModule")
|
||||
.registerSubtypes(
|
||||
new NamedType(SketchBuildAggregatorFactory.class, "sketchBuild"),
|
||||
new NamedType(SketchMergeAggregatorFactory.class, "sketchMerge"),
|
||||
new NamedType(SketchEstimatePostAggregator.class, "sketchEstimate"),
|
||||
new NamedType(SketchSetPostAggregator.class, "sketchSetOper")
|
||||
new NamedType(SketchBuildAggregatorFactory.class, THETA_SKETCH_BUILD_AGG),
|
||||
new NamedType(SketchMergeAggregatorFactory.class, THETA_SKETCH_MERGE_AGG),
|
||||
new NamedType(SketchEstimatePostAggregator.class, THETA_SKETCH_ESTIMATE_POST_AGG),
|
||||
new NamedType(SketchSetPostAggregator.class, THETA_SKETCH_SET_OP_POST_AGG)
|
||||
)
|
||||
.addSerializer(Sketch.class, new SketchJsonSerializer())
|
||||
);
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.google.common.primitives.Longs;
|
||||
import com.metamx.common.IAE;
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.google.common.base.Charsets;
|
||||
import com.metamx.common.logger.Logger;
|
|
@ -17,9 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
|
||||
import io.druid.query.aggregation.datasketches.SketchOperations.Func;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
|
@ -43,7 +41,7 @@ public class SketchSetPostAggregator implements PostAggregator
|
|||
|
||||
private final String name;
|
||||
private final List<PostAggregator> fields;
|
||||
private final Func func;
|
||||
private final SketchOperations.Func func;
|
||||
private final int maxSketchSize;
|
||||
|
||||
@JsonCreator
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.yahoo.sketches.memory.Memory;
|
||||
import com.yahoo.sketches.theta.CompactSketch;
|
|
@ -0,0 +1,87 @@
|
|||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches.theta.oldapi;
|
||||
|
||||
import com.fasterxml.jackson.databind.Module;
|
||||
import com.fasterxml.jackson.databind.jsontype.NamedType;
|
||||
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||
import com.google.inject.Binder;
|
||||
|
||||
import com.yahoo.sketches.theta.Sketch;
|
||||
import io.druid.initialization.DruidModule;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchBuildComplexMetricSerde;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchJsonSerializer;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchMergeComplexMetricSerde;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchModule;
|
||||
import io.druid.segment.serde.ComplexMetrics;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
public class OldApiSketchModule implements DruidModule
|
||||
{
|
||||
|
||||
public static final String SET_SKETCH = "setSketch";
|
||||
public static final String SKETCH_BUILD = "sketchBuild";
|
||||
public static final String SKETCH_MERGE = "sketchMerge";
|
||||
|
||||
@Override
|
||||
public void configure(Binder binder)
|
||||
{
|
||||
if (ComplexMetrics.getSerdeForType(SKETCH_BUILD) == null) {
|
||||
ComplexMetrics.registerSerde(SKETCH_BUILD, new SketchBuildComplexMetricSerde());
|
||||
}
|
||||
|
||||
if (ComplexMetrics.getSerdeForType(SET_SKETCH) == null) {
|
||||
ComplexMetrics.registerSerde(SET_SKETCH, new SketchMergeComplexMetricSerde());
|
||||
}
|
||||
|
||||
if (ComplexMetrics.getSerdeForType(SKETCH_MERGE) == null) {
|
||||
ComplexMetrics.registerSerde(SKETCH_MERGE, new SketchMergeComplexMetricSerde());
|
||||
}
|
||||
|
||||
if (ComplexMetrics.getSerdeForType(SketchModule.THETA_SKETCH) == null) {
|
||||
ComplexMetrics.registerSerde(SketchModule.THETA_SKETCH, new SketchMergeComplexMetricSerde());
|
||||
}
|
||||
|
||||
if (ComplexMetrics.getSerdeForType(SketchModule.THETA_SKETCH_MERGE_AGG) == null) {
|
||||
ComplexMetrics.registerSerde(SketchModule.THETA_SKETCH_MERGE_AGG, new SketchMergeComplexMetricSerde());
|
||||
}
|
||||
|
||||
if (ComplexMetrics.getSerdeForType(SketchModule.THETA_SKETCH_BUILD_AGG) == null) {
|
||||
ComplexMetrics.registerSerde(SketchModule.THETA_SKETCH_BUILD_AGG, new SketchBuildComplexMetricSerde());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<? extends Module> getJacksonModules()
|
||||
{
|
||||
return Arrays.<Module>asList(
|
||||
new SimpleModule("OldThetaSketchModule")
|
||||
.registerSubtypes(
|
||||
new NamedType(OldSketchBuildAggregatorFactory.class, SKETCH_BUILD),
|
||||
new NamedType(OldSketchMergeAggregatorFactory.class, SKETCH_MERGE),
|
||||
new NamedType(OldSketchEstimatePostAggregator.class, "sketchEstimate"),
|
||||
new NamedType(OldSketchSetPostAggregator.class, "sketchSetOper")
|
||||
)
|
||||
.addSerializer(Sketch.class, new SketchJsonSerializer())
|
||||
);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches.theta.oldapi;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchBuildAggregatorFactory;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class OldSketchBuildAggregatorFactory extends SketchBuildAggregatorFactory
|
||||
{
|
||||
@JsonCreator
|
||||
public OldSketchBuildAggregatorFactory(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("size") Integer size
|
||||
)
|
||||
{
|
||||
super(name, fieldName, size);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches.theta.oldapi;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.druid.query.aggregation.PostAggregator;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchEstimatePostAggregator;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class OldSketchEstimatePostAggregator extends SketchEstimatePostAggregator
|
||||
{
|
||||
@JsonCreator
|
||||
public OldSketchEstimatePostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("field") PostAggregator field
|
||||
)
|
||||
{
|
||||
super(name, field);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches.theta.oldapi;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class OldSketchMergeAggregatorFactory extends SketchMergeAggregatorFactory
|
||||
{
|
||||
@JsonCreator
|
||||
public OldSketchMergeAggregatorFactory(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("size") Integer size,
|
||||
@JsonProperty("shouldFinalize") Boolean shouldFinalize
|
||||
)
|
||||
{
|
||||
super(name, fieldName, size, shouldFinalize);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,43 @@
|
|||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches.theta.oldapi;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import io.druid.query.aggregation.PostAggregator;
|
||||
import io.druid.query.aggregation.datasketches.theta.SketchSetPostAggregator;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class OldSketchSetPostAggregator extends SketchSetPostAggregator
|
||||
{
|
||||
@JsonCreator
|
||||
public OldSketchSetPostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("func") String func,
|
||||
@JsonProperty("size") Integer maxSize,
|
||||
@JsonProperty("fields") List<PostAggregator> fields
|
||||
)
|
||||
{
|
||||
super(name, func, maxSize, fields);
|
||||
}
|
||||
}
|
|
@ -1 +1,2 @@
|
|||
io.druid.query.aggregation.datasketches.SketchModule
|
||||
io.druid.query.aggregation.datasketches.theta.SketchModule
|
||||
io.druid.query.aggregation.datasketches.theta.oldapi.OldApiSketchModule
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches;
|
||||
package io.druid.query.aggregation.datasketches.theta;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
|
@ -0,0 +1,196 @@
|
|||
/*
|
||||
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. Metamarkets licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package io.druid.query.aggregation.datasketches.theta.oldapi;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.io.Files;
|
||||
import com.metamx.common.guava.Sequence;
|
||||
import com.metamx.common.guava.Sequences;
|
||||
import io.druid.data.input.MapBasedRow;
|
||||
import io.druid.granularity.QueryGranularity;
|
||||
import io.druid.query.aggregation.AggregationTestHelper;
|
||||
import io.druid.query.aggregation.AggregatorFactory;
|
||||
import io.druid.query.aggregation.PostAggregator;
|
||||
import io.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||
import org.joda.time.DateTime;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
*/
|
||||
public class OldApiSketchAggregationTest
|
||||
{
|
||||
private final AggregationTestHelper helper;
|
||||
|
||||
@Rule
|
||||
public final TemporaryFolder tempFolder = new TemporaryFolder();
|
||||
|
||||
public OldApiSketchAggregationTest()
|
||||
{
|
||||
OldApiSketchModule sm = new OldApiSketchModule();
|
||||
sm.configure(null);
|
||||
|
||||
helper = new AggregationTestHelper(
|
||||
sm.getJacksonModules(),
|
||||
tempFolder
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSimpleDataIngestAndQuery() throws Exception
|
||||
{
|
||||
Sequence seq = helper.createIndexAndRunQueryOnSegment(
|
||||
new File(this.getClass().getClassLoader().getResource("simple_test_data.tsv").getFile()),
|
||||
readFileFromClasspathAsString("simple_test_data_record_parser.json"),
|
||||
readFileFromClasspathAsString("oldapi/old_simple_test_data_aggregators.json"),
|
||||
0,
|
||||
QueryGranularity.NONE,
|
||||
5,
|
||||
readFileFromClasspathAsString("oldapi/old_simple_test_data_group_by_query.json")
|
||||
);
|
||||
|
||||
List results = Sequences.toList(seq, Lists.newArrayList());
|
||||
Assert.assertEquals(1, results.size());
|
||||
Assert.assertEquals(
|
||||
new MapBasedRow(
|
||||
DateTime.parse("2014-10-19T00:00:00.000Z"),
|
||||
ImmutableMap
|
||||
.<String, Object>builder()
|
||||
.put("sketch_count", 50.0)
|
||||
.put("sketchEstimatePostAgg", 50.0)
|
||||
.put("sketchUnionPostAggEstimate", 50.0)
|
||||
.put("sketchIntersectionPostAggEstimate", 50.0)
|
||||
.put("sketchAnotBPostAggEstimate", 0.0)
|
||||
.put("non_existing_col_validation", 0.0)
|
||||
.build()
|
||||
),
|
||||
results.get(0)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSketchDataIngestAndQuery() throws Exception
|
||||
{
|
||||
Sequence seq = helper.createIndexAndRunQueryOnSegment(
|
||||
new File(OldApiSketchAggregationTest.class.getClassLoader().getResource("sketch_test_data.tsv").getFile()),
|
||||
readFileFromClasspathAsString("sketch_test_data_record_parser.json"),
|
||||
readFileFromClasspathAsString("oldapi/old_sketch_test_data_aggregators.json"),
|
||||
0,
|
||||
QueryGranularity.NONE,
|
||||
5,
|
||||
readFileFromClasspathAsString("oldapi/old_sketch_test_data_group_by_query.json")
|
||||
);
|
||||
|
||||
List results = Sequences.toList(seq, Lists.newArrayList());
|
||||
Assert.assertEquals(1, results.size());
|
||||
Assert.assertEquals(
|
||||
new MapBasedRow(
|
||||
DateTime.parse("2014-10-19T00:00:00.000Z"),
|
||||
ImmutableMap
|
||||
.<String, Object>builder()
|
||||
.put("sids_sketch_count", 50.0)
|
||||
.put("sketchEstimatePostAgg", 50.0)
|
||||
.put("sketchUnionPostAggEstimate", 50.0)
|
||||
.put("sketchIntersectionPostAggEstimate", 50.0)
|
||||
.put("sketchAnotBPostAggEstimate", 0.0)
|
||||
.put("non_existing_col_validation", 0.0)
|
||||
.build()
|
||||
),
|
||||
results.get(0)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSketchMergeAggregatorFactorySerde() throws Exception
|
||||
{
|
||||
assertAggregatorFactorySerde(new OldSketchMergeAggregatorFactory("name", "fieldName", 16, null));
|
||||
assertAggregatorFactorySerde(new OldSketchMergeAggregatorFactory("name", "fieldName", 16, false));
|
||||
assertAggregatorFactorySerde(new OldSketchMergeAggregatorFactory("name", "fieldName", 16, true));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSketchBuildAggregatorFactorySerde() throws Exception
|
||||
{
|
||||
assertAggregatorFactorySerde(new OldSketchBuildAggregatorFactory("name", "fieldName", 16));
|
||||
}
|
||||
|
||||
private void assertAggregatorFactorySerde(AggregatorFactory agg) throws Exception{
|
||||
Assert.assertEquals(
|
||||
agg,
|
||||
helper.getObjectMapper().readValue(
|
||||
helper.getObjectMapper().writeValueAsString(agg),
|
||||
AggregatorFactory.class
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSketchEstimatePostAggregatorSerde() throws Exception
|
||||
{
|
||||
assertPostAggregatorSerde(
|
||||
new OldSketchEstimatePostAggregator(
|
||||
"name",
|
||||
new FieldAccessPostAggregator("name", "fieldName")
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSketchSetPostAggregatorSerde() throws Exception
|
||||
{
|
||||
assertPostAggregatorSerde(
|
||||
new OldSketchSetPostAggregator(
|
||||
"name",
|
||||
"INTERSECT",
|
||||
null,
|
||||
Lists.<PostAggregator>newArrayList(
|
||||
new FieldAccessPostAggregator("name1", "fieldName1"),
|
||||
new FieldAccessPostAggregator("name2", "fieldName2")
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private void assertPostAggregatorSerde(PostAggregator agg) throws Exception{
|
||||
Assert.assertEquals(
|
||||
agg,
|
||||
helper.getObjectMapper().readValue(
|
||||
helper.getObjectMapper().writeValueAsString(agg),
|
||||
PostAggregator.class
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
public final static String readFileFromClasspathAsString(String fileName) throws IOException
|
||||
{
|
||||
return Files.asCharSource(
|
||||
new File(OldApiSketchAggregationTest.class.getClassLoader().getResource(fileName).getFile()),
|
||||
Charset.forName("UTF-8")
|
||||
).read();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
[
|
||||
{
|
||||
"type": "sketchBuild",
|
||||
"name": "pty_country",
|
||||
"fieldName": "pty_country"
|
||||
},
|
||||
{
|
||||
"type": "sketchBuild",
|
||||
"name": "non_existing_col_validation",
|
||||
"fieldName": "non_existing_col"
|
||||
}
|
||||
]
|
|
@ -0,0 +1,84 @@
|
|||
{
|
||||
"queryType": "groupBy",
|
||||
"dataSource": "test_datasource",
|
||||
"granularity": "ALL",
|
||||
"dimensions": [],
|
||||
"aggregations": [
|
||||
{ "type": "sketchMerge", "name": "sketch_count", "fieldName": "pty_country", "size": 16384 },
|
||||
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
|
||||
],
|
||||
"postAggregations": [
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchEstimatePostAgg",
|
||||
"field": {
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchIntersectionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"name": "sketchIntersectionPostAgg",
|
||||
"func": "INTERSECT",
|
||||
"size": 16384,
|
||||
"fields": [
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
},
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchAnotBPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"name": "sketchAnotBUnionPostAgg",
|
||||
"func": "NOT",
|
||||
"size": 16384,
|
||||
"fields": [
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
},
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchUnionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"name": "sketchUnionPostAgg",
|
||||
"func": "UNION",
|
||||
"size": 16384,
|
||||
"fields": [
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
},
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sketch_count"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"intervals": [ "2014-10-19T00:00:00.000Z/2014-10-22T00:00:00.000Z" ]
|
||||
}
|
|
@ -0,0 +1,14 @@
|
|||
[
|
||||
{
|
||||
"type": "sketchMerge",
|
||||
"name": "sids_sketch",
|
||||
"fieldName": "sketch",
|
||||
"size": 16384
|
||||
},
|
||||
{
|
||||
"type": "sketchMerge",
|
||||
"name": "non_existing_col_validation",
|
||||
"fieldName": "non_existing_col",
|
||||
"size": 16384
|
||||
}
|
||||
]
|
|
@ -0,0 +1,83 @@
|
|||
{
|
||||
"queryType": "groupBy",
|
||||
"dataSource": "test_datasource",
|
||||
"granularity": "ALL",
|
||||
"dimensions": [],
|
||||
"aggregations": [
|
||||
{ "type": "sketchMerge", "name": "sids_sketch_count", "fieldName": "sids_sketch", "size": 16384 },
|
||||
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
|
||||
],
|
||||
"postAggregations": [
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchEstimatePostAgg",
|
||||
"field": {
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchIntersectionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"name": "sketchIntersectionPostAgg",
|
||||
"func": "INTERSECT",
|
||||
"size": 16384,
|
||||
"fields": [
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
},
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchAnotBPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"name": "sketchAnotBUnionPostAgg",
|
||||
"func": "NOT",
|
||||
"fields": [
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
},
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"name": "sketchUnionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"name": "sketchUnionPostAgg",
|
||||
"func": "UNION",
|
||||
"size": 16384,
|
||||
"fields": [
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
},
|
||||
{
|
||||
"type": "fieldAccess",
|
||||
"fieldName": "sids_sketch_count"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"intervals": [ "2014-10-19T00:00:00.000Z/2014-10-22T00:00:00.000Z" ]
|
||||
}
|
|
@ -1,11 +1,11 @@
|
|||
[
|
||||
{
|
||||
"type": "sketchBuild",
|
||||
"type": "thetaSketchBuild",
|
||||
"name": "pty_country",
|
||||
"fieldName": "pty_country"
|
||||
},
|
||||
{
|
||||
"type": "sketchBuild",
|
||||
"type": "thetaSketchBuild",
|
||||
"name": "non_existing_col_validation",
|
||||
"fieldName": "non_existing_col"
|
||||
}
|
||||
|
|
|
@ -4,12 +4,12 @@
|
|||
"granularity": "ALL",
|
||||
"dimensions": [],
|
||||
"aggregations": [
|
||||
{ "type": "sketchMerge", "name": "sketch_count", "fieldName": "pty_country", "size": 16384 },
|
||||
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
|
||||
{ "type": "thetaSketchMerge", "name": "sketch_count", "fieldName": "pty_country", "size": 16384 },
|
||||
{ "type": "thetaSketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
|
||||
],
|
||||
"postAggregations": [
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchEstimatePostAgg",
|
||||
"field": {
|
||||
"type": "fieldAccess",
|
||||
|
@ -17,11 +17,11 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchIntersectionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"type": "thetaSketchSetOp",
|
||||
"name": "sketchIntersectionPostAgg",
|
||||
"func": "INTERSECT",
|
||||
"size": 16384,
|
||||
|
@ -38,11 +38,11 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchAnotBPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"type": "thetaSketchSetOp",
|
||||
"name": "sketchAnotBUnionPostAgg",
|
||||
"func": "NOT",
|
||||
"size": 16384,
|
||||
|
@ -59,11 +59,11 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchUnionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"type": "thetaSketchSetOp",
|
||||
"name": "sketchUnionPostAgg",
|
||||
"func": "UNION",
|
||||
"size": 16384,
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
[
|
||||
{
|
||||
"type": "sketchMerge",
|
||||
"type": "thetaSketchMerge",
|
||||
"name": "sids_sketch",
|
||||
"fieldName": "sketch",
|
||||
"size": 16384
|
||||
},
|
||||
{
|
||||
"type": "sketchMerge",
|
||||
"type": "thetaSketchMerge",
|
||||
"name": "non_existing_col_validation",
|
||||
"fieldName": "non_existing_col",
|
||||
"size": 16384
|
||||
|
|
|
@ -4,12 +4,12 @@
|
|||
"granularity": "ALL",
|
||||
"dimensions": [],
|
||||
"aggregations": [
|
||||
{ "type": "sketchMerge", "name": "sids_sketch_count", "fieldName": "sids_sketch", "size": 16384 },
|
||||
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
|
||||
{ "type": "thetaSketchMerge", "name": "sids_sketch_count", "fieldName": "sids_sketch", "size": 16384 },
|
||||
{ "type": "thetaSketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
|
||||
],
|
||||
"postAggregations": [
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchEstimatePostAgg",
|
||||
"field": {
|
||||
"type": "fieldAccess",
|
||||
|
@ -17,11 +17,11 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchIntersectionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"type": "thetaSketchSetOp",
|
||||
"name": "sketchIntersectionPostAgg",
|
||||
"func": "INTERSECT",
|
||||
"size": 16384,
|
||||
|
@ -38,11 +38,11 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchAnotBPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"type": "thetaSketchSetOp",
|
||||
"name": "sketchAnotBUnionPostAgg",
|
||||
"func": "NOT",
|
||||
"fields": [
|
||||
|
@ -58,11 +58,11 @@
|
|||
}
|
||||
},
|
||||
{
|
||||
"type": "sketchEstimate",
|
||||
"type": "thetaSketchEstimate",
|
||||
"name": "sketchUnionPostAggEstimate",
|
||||
"field":
|
||||
{
|
||||
"type": "sketchSetOper",
|
||||
"type": "thetaSketchSetOp",
|
||||
"name": "sketchUnionPostAgg",
|
||||
"func": "UNION",
|
||||
"size": 16384,
|
||||
|
|
Loading…
Reference in New Issue