changing names to be explicit about theta sketch algorithm

old names are still valid though so as to be backwards compatible for now
This commit is contained in:
Himanshu Gupta 2015-11-06 10:18:23 -06:00
parent 6c6a38cedb
commit 88ae3c43f9
32 changed files with 701 additions and 60 deletions

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import io.druid.query.aggregation.Aggregator;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import io.druid.query.aggregation.BufferAggregator;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.yahoo.sketches.Family;
import com.yahoo.sketches.memory.Memory;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.yahoo.sketches.Family;
import com.yahoo.sketches.memory.Memory;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -64,6 +64,6 @@ public class SketchBuildAggregatorFactory extends SketchAggregatorFactory
@Override
public String getTypeName()
{
return SketchModule.SKETCH_BUILD;
return SketchModule.THETA_SKETCH_BUILD_AGG;
}
}

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.metamx.common.IAE;
import com.yahoo.sketches.memory.Memory;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.JsonProcessingException;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -84,7 +84,7 @@ public class SketchMergeAggregatorFactory extends SketchAggregatorFactory
@Override
public String getTypeName()
{
return SketchModule.SET_SKETCH;
return SketchModule.THETA_SKETCH_MERGE_AGG;
}
@Override

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.yahoo.sketches.memory.Memory;
import com.yahoo.sketches.theta.Sketch;
@ -40,7 +40,7 @@ public class SketchMergeComplexMetricSerde extends ComplexMetricSerde
@Override
public String getTypeName()
{
return SketchModule.SET_SKETCH;
return SketchModule.THETA_SKETCH;
}
@Override

View File

@ -17,13 +17,12 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.databind.Module;
import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.inject.Binder;
import com.yahoo.sketches.theta.Sketch;
import io.druid.initialization.DruidModule;
import io.druid.segment.serde.ComplexMetrics;
@ -33,21 +32,27 @@ import java.util.List;
public class SketchModule implements DruidModule
{
public static final String THETA_SKETCH = "thetaSketch";
public static final String SET_SKETCH = "setSketch";
public static final String SKETCH_BUILD = "sketchBuild";
public static final String THETA_SKETCH_MERGE_AGG = "thetaSketchMerge";
public static final String THETA_SKETCH_BUILD_AGG = "thetaSketchBuild";
public static final String THETA_SKETCH_ESTIMATE_POST_AGG = "thetaSketchEstimate";
public static final String THETA_SKETCH_SET_OP_POST_AGG = "thetaSketchSetOp";
@Override
public void configure(Binder binder)
{
//gives the extractor to ingest non-sketch input data
if (ComplexMetrics.getSerdeForType(SKETCH_BUILD) == null) {
ComplexMetrics.registerSerde(SKETCH_BUILD, new SketchBuildComplexMetricSerde());
if (ComplexMetrics.getSerdeForType(THETA_SKETCH) == null) {
ComplexMetrics.registerSerde(THETA_SKETCH, new SketchMergeComplexMetricSerde());
}
//gives the extractor to ingest sketch input data
if (ComplexMetrics.getSerdeForType(SET_SKETCH) == null) {
ComplexMetrics.registerSerde(SET_SKETCH, new SketchMergeComplexMetricSerde());
if (ComplexMetrics.getSerdeForType(THETA_SKETCH_MERGE_AGG) == null) {
ComplexMetrics.registerSerde(THETA_SKETCH_MERGE_AGG, new SketchMergeComplexMetricSerde());
}
if (ComplexMetrics.getSerdeForType(THETA_SKETCH_BUILD_AGG) == null) {
ComplexMetrics.registerSerde(THETA_SKETCH_BUILD_AGG, new SketchBuildComplexMetricSerde());
}
}
@ -55,12 +60,12 @@ public class SketchModule implements DruidModule
public List<? extends Module> getJacksonModules()
{
return Arrays.<Module>asList(
new SimpleModule("SketchModule")
new SimpleModule("ThetaSketchModule")
.registerSubtypes(
new NamedType(SketchBuildAggregatorFactory.class, "sketchBuild"),
new NamedType(SketchMergeAggregatorFactory.class, "sketchMerge"),
new NamedType(SketchEstimatePostAggregator.class, "sketchEstimate"),
new NamedType(SketchSetPostAggregator.class, "sketchSetOper")
new NamedType(SketchBuildAggregatorFactory.class, THETA_SKETCH_BUILD_AGG),
new NamedType(SketchMergeAggregatorFactory.class, THETA_SKETCH_MERGE_AGG),
new NamedType(SketchEstimatePostAggregator.class, THETA_SKETCH_ESTIMATE_POST_AGG),
new NamedType(SketchSetPostAggregator.class, THETA_SKETCH_SET_OP_POST_AGG)
)
.addSerializer(Sketch.class, new SketchJsonSerializer())
);

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.google.common.primitives.Longs;
import com.metamx.common.IAE;

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.google.common.base.Charsets;
import com.metamx.common.logger.Logger;

View File

@ -17,9 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
import io.druid.query.aggregation.datasketches.SketchOperations.Func;
package io.druid.query.aggregation.datasketches.theta;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@ -43,7 +41,7 @@ public class SketchSetPostAggregator implements PostAggregator
private final String name;
private final List<PostAggregator> fields;
private final Func func;
private final SketchOperations.Func func;
private final int maxSketchSize;
@JsonCreator

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.yahoo.sketches.memory.Memory;
import com.yahoo.sketches.theta.CompactSketch;

View File

@ -0,0 +1,87 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.aggregation.datasketches.theta.oldapi;
import com.fasterxml.jackson.databind.Module;
import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.inject.Binder;
import com.yahoo.sketches.theta.Sketch;
import io.druid.initialization.DruidModule;
import io.druid.query.aggregation.datasketches.theta.SketchBuildComplexMetricSerde;
import io.druid.query.aggregation.datasketches.theta.SketchJsonSerializer;
import io.druid.query.aggregation.datasketches.theta.SketchMergeComplexMetricSerde;
import io.druid.query.aggregation.datasketches.theta.SketchModule;
import io.druid.segment.serde.ComplexMetrics;
import java.util.Arrays;
import java.util.List;
public class OldApiSketchModule implements DruidModule
{
public static final String SET_SKETCH = "setSketch";
public static final String SKETCH_BUILD = "sketchBuild";
public static final String SKETCH_MERGE = "sketchMerge";
@Override
public void configure(Binder binder)
{
if (ComplexMetrics.getSerdeForType(SKETCH_BUILD) == null) {
ComplexMetrics.registerSerde(SKETCH_BUILD, new SketchBuildComplexMetricSerde());
}
if (ComplexMetrics.getSerdeForType(SET_SKETCH) == null) {
ComplexMetrics.registerSerde(SET_SKETCH, new SketchMergeComplexMetricSerde());
}
if (ComplexMetrics.getSerdeForType(SKETCH_MERGE) == null) {
ComplexMetrics.registerSerde(SKETCH_MERGE, new SketchMergeComplexMetricSerde());
}
if (ComplexMetrics.getSerdeForType(SketchModule.THETA_SKETCH) == null) {
ComplexMetrics.registerSerde(SketchModule.THETA_SKETCH, new SketchMergeComplexMetricSerde());
}
if (ComplexMetrics.getSerdeForType(SketchModule.THETA_SKETCH_MERGE_AGG) == null) {
ComplexMetrics.registerSerde(SketchModule.THETA_SKETCH_MERGE_AGG, new SketchMergeComplexMetricSerde());
}
if (ComplexMetrics.getSerdeForType(SketchModule.THETA_SKETCH_BUILD_AGG) == null) {
ComplexMetrics.registerSerde(SketchModule.THETA_SKETCH_BUILD_AGG, new SketchBuildComplexMetricSerde());
}
}
@Override
public List<? extends Module> getJacksonModules()
{
return Arrays.<Module>asList(
new SimpleModule("OldThetaSketchModule")
.registerSubtypes(
new NamedType(OldSketchBuildAggregatorFactory.class, SKETCH_BUILD),
new NamedType(OldSketchMergeAggregatorFactory.class, SKETCH_MERGE),
new NamedType(OldSketchEstimatePostAggregator.class, "sketchEstimate"),
new NamedType(OldSketchSetPostAggregator.class, "sketchSetOper")
)
.addSerializer(Sketch.class, new SketchJsonSerializer())
);
}
}

View File

@ -0,0 +1,39 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.aggregation.datasketches.theta.oldapi;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.druid.query.aggregation.datasketches.theta.SketchBuildAggregatorFactory;
/**
*/
public class OldSketchBuildAggregatorFactory extends SketchBuildAggregatorFactory
{
@JsonCreator
public OldSketchBuildAggregatorFactory(
@JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName,
@JsonProperty("size") Integer size
)
{
super(name, fieldName, size);
}
}

View File

@ -0,0 +1,39 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.aggregation.datasketches.theta.oldapi;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.druid.query.aggregation.PostAggregator;
import io.druid.query.aggregation.datasketches.theta.SketchEstimatePostAggregator;
/**
*/
public class OldSketchEstimatePostAggregator extends SketchEstimatePostAggregator
{
@JsonCreator
public OldSketchEstimatePostAggregator(
@JsonProperty("name") String name,
@JsonProperty("field") PostAggregator field
)
{
super(name, field);
}
}

View File

@ -0,0 +1,40 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.aggregation.datasketches.theta.oldapi;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
/**
*/
public class OldSketchMergeAggregatorFactory extends SketchMergeAggregatorFactory
{
@JsonCreator
public OldSketchMergeAggregatorFactory(
@JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName,
@JsonProperty("size") Integer size,
@JsonProperty("shouldFinalize") Boolean shouldFinalize
)
{
super(name, fieldName, size, shouldFinalize);
}
}

View File

@ -0,0 +1,43 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.aggregation.datasketches.theta.oldapi;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.druid.query.aggregation.PostAggregator;
import io.druid.query.aggregation.datasketches.theta.SketchSetPostAggregator;
import java.util.List;
/**
*/
public class OldSketchSetPostAggregator extends SketchSetPostAggregator
{
@JsonCreator
public OldSketchSetPostAggregator(
@JsonProperty("name") String name,
@JsonProperty("func") String func,
@JsonProperty("size") Integer maxSize,
@JsonProperty("fields") List<PostAggregator> fields
)
{
super(name, func, maxSize, fields);
}
}

View File

@ -1 +1,2 @@
io.druid.query.aggregation.datasketches.SketchModule
io.druid.query.aggregation.datasketches.theta.SketchModule
io.druid.query.aggregation.datasketches.theta.oldapi.OldApiSketchModule

View File

@ -17,7 +17,7 @@
* under the License.
*/
package io.druid.query.aggregation.datasketches;
package io.druid.query.aggregation.datasketches.theta;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;

View File

@ -0,0 +1,196 @@
/*
* Licensed to Metamarkets Group Inc. (Metamarkets) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Metamarkets licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package io.druid.query.aggregation.datasketches.theta.oldapi;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import com.metamx.common.guava.Sequence;
import com.metamx.common.guava.Sequences;
import io.druid.data.input.MapBasedRow;
import io.druid.granularity.QueryGranularity;
import io.druid.query.aggregation.AggregationTestHelper;
import io.druid.query.aggregation.AggregatorFactory;
import io.druid.query.aggregation.PostAggregator;
import io.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;
/**
*/
public class OldApiSketchAggregationTest
{
private final AggregationTestHelper helper;
@Rule
public final TemporaryFolder tempFolder = new TemporaryFolder();
public OldApiSketchAggregationTest()
{
OldApiSketchModule sm = new OldApiSketchModule();
sm.configure(null);
helper = new AggregationTestHelper(
sm.getJacksonModules(),
tempFolder
);
}
@Test
public void testSimpleDataIngestAndQuery() throws Exception
{
Sequence seq = helper.createIndexAndRunQueryOnSegment(
new File(this.getClass().getClassLoader().getResource("simple_test_data.tsv").getFile()),
readFileFromClasspathAsString("simple_test_data_record_parser.json"),
readFileFromClasspathAsString("oldapi/old_simple_test_data_aggregators.json"),
0,
QueryGranularity.NONE,
5,
readFileFromClasspathAsString("oldapi/old_simple_test_data_group_by_query.json")
);
List results = Sequences.toList(seq, Lists.newArrayList());
Assert.assertEquals(1, results.size());
Assert.assertEquals(
new MapBasedRow(
DateTime.parse("2014-10-19T00:00:00.000Z"),
ImmutableMap
.<String, Object>builder()
.put("sketch_count", 50.0)
.put("sketchEstimatePostAgg", 50.0)
.put("sketchUnionPostAggEstimate", 50.0)
.put("sketchIntersectionPostAggEstimate", 50.0)
.put("sketchAnotBPostAggEstimate", 0.0)
.put("non_existing_col_validation", 0.0)
.build()
),
results.get(0)
);
}
@Test
public void testSketchDataIngestAndQuery() throws Exception
{
Sequence seq = helper.createIndexAndRunQueryOnSegment(
new File(OldApiSketchAggregationTest.class.getClassLoader().getResource("sketch_test_data.tsv").getFile()),
readFileFromClasspathAsString("sketch_test_data_record_parser.json"),
readFileFromClasspathAsString("oldapi/old_sketch_test_data_aggregators.json"),
0,
QueryGranularity.NONE,
5,
readFileFromClasspathAsString("oldapi/old_sketch_test_data_group_by_query.json")
);
List results = Sequences.toList(seq, Lists.newArrayList());
Assert.assertEquals(1, results.size());
Assert.assertEquals(
new MapBasedRow(
DateTime.parse("2014-10-19T00:00:00.000Z"),
ImmutableMap
.<String, Object>builder()
.put("sids_sketch_count", 50.0)
.put("sketchEstimatePostAgg", 50.0)
.put("sketchUnionPostAggEstimate", 50.0)
.put("sketchIntersectionPostAggEstimate", 50.0)
.put("sketchAnotBPostAggEstimate", 0.0)
.put("non_existing_col_validation", 0.0)
.build()
),
results.get(0)
);
}
@Test
public void testSketchMergeAggregatorFactorySerde() throws Exception
{
assertAggregatorFactorySerde(new OldSketchMergeAggregatorFactory("name", "fieldName", 16, null));
assertAggregatorFactorySerde(new OldSketchMergeAggregatorFactory("name", "fieldName", 16, false));
assertAggregatorFactorySerde(new OldSketchMergeAggregatorFactory("name", "fieldName", 16, true));
}
@Test
public void testSketchBuildAggregatorFactorySerde() throws Exception
{
assertAggregatorFactorySerde(new OldSketchBuildAggregatorFactory("name", "fieldName", 16));
}
private void assertAggregatorFactorySerde(AggregatorFactory agg) throws Exception{
Assert.assertEquals(
agg,
helper.getObjectMapper().readValue(
helper.getObjectMapper().writeValueAsString(agg),
AggregatorFactory.class
)
);
}
@Test
public void testSketchEstimatePostAggregatorSerde() throws Exception
{
assertPostAggregatorSerde(
new OldSketchEstimatePostAggregator(
"name",
new FieldAccessPostAggregator("name", "fieldName")
)
);
}
@Test
public void testSketchSetPostAggregatorSerde() throws Exception
{
assertPostAggregatorSerde(
new OldSketchSetPostAggregator(
"name",
"INTERSECT",
null,
Lists.<PostAggregator>newArrayList(
new FieldAccessPostAggregator("name1", "fieldName1"),
new FieldAccessPostAggregator("name2", "fieldName2")
)
)
);
}
private void assertPostAggregatorSerde(PostAggregator agg) throws Exception{
Assert.assertEquals(
agg,
helper.getObjectMapper().readValue(
helper.getObjectMapper().writeValueAsString(agg),
PostAggregator.class
)
);
}
public final static String readFileFromClasspathAsString(String fileName) throws IOException
{
return Files.asCharSource(
new File(OldApiSketchAggregationTest.class.getClassLoader().getResource(fileName).getFile()),
Charset.forName("UTF-8")
).read();
}
}

View File

@ -0,0 +1,12 @@
[
{
"type": "sketchBuild",
"name": "pty_country",
"fieldName": "pty_country"
},
{
"type": "sketchBuild",
"name": "non_existing_col_validation",
"fieldName": "non_existing_col"
}
]

View File

@ -0,0 +1,84 @@
{
"queryType": "groupBy",
"dataSource": "test_datasource",
"granularity": "ALL",
"dimensions": [],
"aggregations": [
{ "type": "sketchMerge", "name": "sketch_count", "fieldName": "pty_country", "size": 16384 },
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
],
"postAggregations": [
{
"type": "sketchEstimate",
"name": "sketchEstimatePostAgg",
"field": {
"type": "fieldAccess",
"fieldName": "sketch_count"
}
},
{
"type": "sketchEstimate",
"name": "sketchIntersectionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"name": "sketchIntersectionPostAgg",
"func": "INTERSECT",
"size": 16384,
"fields": [
{
"type": "fieldAccess",
"fieldName": "sketch_count"
},
{
"type": "fieldAccess",
"fieldName": "sketch_count"
}
]
}
},
{
"type": "sketchEstimate",
"name": "sketchAnotBPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"name": "sketchAnotBUnionPostAgg",
"func": "NOT",
"size": 16384,
"fields": [
{
"type": "fieldAccess",
"fieldName": "sketch_count"
},
{
"type": "fieldAccess",
"fieldName": "sketch_count"
}
]
}
},
{
"type": "sketchEstimate",
"name": "sketchUnionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"name": "sketchUnionPostAgg",
"func": "UNION",
"size": 16384,
"fields": [
{
"type": "fieldAccess",
"fieldName": "sketch_count"
},
{
"type": "fieldAccess",
"fieldName": "sketch_count"
}
]
}
}
],
"intervals": [ "2014-10-19T00:00:00.000Z/2014-10-22T00:00:00.000Z" ]
}

View File

@ -0,0 +1,14 @@
[
{
"type": "sketchMerge",
"name": "sids_sketch",
"fieldName": "sketch",
"size": 16384
},
{
"type": "sketchMerge",
"name": "non_existing_col_validation",
"fieldName": "non_existing_col",
"size": 16384
}
]

View File

@ -0,0 +1,83 @@
{
"queryType": "groupBy",
"dataSource": "test_datasource",
"granularity": "ALL",
"dimensions": [],
"aggregations": [
{ "type": "sketchMerge", "name": "sids_sketch_count", "fieldName": "sids_sketch", "size": 16384 },
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
],
"postAggregations": [
{
"type": "sketchEstimate",
"name": "sketchEstimatePostAgg",
"field": {
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
}
},
{
"type": "sketchEstimate",
"name": "sketchIntersectionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"name": "sketchIntersectionPostAgg",
"func": "INTERSECT",
"size": 16384,
"fields": [
{
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
},
{
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
}
]
}
},
{
"type": "sketchEstimate",
"name": "sketchAnotBPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"name": "sketchAnotBUnionPostAgg",
"func": "NOT",
"fields": [
{
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
},
{
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
}
]
}
},
{
"type": "sketchEstimate",
"name": "sketchUnionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"name": "sketchUnionPostAgg",
"func": "UNION",
"size": 16384,
"fields": [
{
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
},
{
"type": "fieldAccess",
"fieldName": "sids_sketch_count"
}
]
}
}
],
"intervals": [ "2014-10-19T00:00:00.000Z/2014-10-22T00:00:00.000Z" ]
}

View File

@ -1,11 +1,11 @@
[
{
"type": "sketchBuild",
"type": "thetaSketchBuild",
"name": "pty_country",
"fieldName": "pty_country"
},
{
"type": "sketchBuild",
"type": "thetaSketchBuild",
"name": "non_existing_col_validation",
"fieldName": "non_existing_col"
}

View File

@ -4,12 +4,12 @@
"granularity": "ALL",
"dimensions": [],
"aggregations": [
{ "type": "sketchMerge", "name": "sketch_count", "fieldName": "pty_country", "size": 16384 },
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
{ "type": "thetaSketchMerge", "name": "sketch_count", "fieldName": "pty_country", "size": 16384 },
{ "type": "thetaSketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
],
"postAggregations": [
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchEstimatePostAgg",
"field": {
"type": "fieldAccess",
@ -17,11 +17,11 @@
}
},
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchIntersectionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"type": "thetaSketchSetOp",
"name": "sketchIntersectionPostAgg",
"func": "INTERSECT",
"size": 16384,
@ -38,11 +38,11 @@
}
},
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchAnotBPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"type": "thetaSketchSetOp",
"name": "sketchAnotBUnionPostAgg",
"func": "NOT",
"size": 16384,
@ -59,11 +59,11 @@
}
},
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchUnionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"type": "thetaSketchSetOp",
"name": "sketchUnionPostAgg",
"func": "UNION",
"size": 16384,

View File

@ -1,12 +1,12 @@
[
{
"type": "sketchMerge",
"type": "thetaSketchMerge",
"name": "sids_sketch",
"fieldName": "sketch",
"size": 16384
},
{
"type": "sketchMerge",
"type": "thetaSketchMerge",
"name": "non_existing_col_validation",
"fieldName": "non_existing_col",
"size": 16384

View File

@ -4,12 +4,12 @@
"granularity": "ALL",
"dimensions": [],
"aggregations": [
{ "type": "sketchMerge", "name": "sids_sketch_count", "fieldName": "sids_sketch", "size": 16384 },
{ "type": "sketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
{ "type": "thetaSketchMerge", "name": "sids_sketch_count", "fieldName": "sids_sketch", "size": 16384 },
{ "type": "thetaSketchMerge", "name": "non_existing_col_validation", "fieldName": "non_existing_col", "size": 16384 }
],
"postAggregations": [
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchEstimatePostAgg",
"field": {
"type": "fieldAccess",
@ -17,11 +17,11 @@
}
},
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchIntersectionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"type": "thetaSketchSetOp",
"name": "sketchIntersectionPostAgg",
"func": "INTERSECT",
"size": 16384,
@ -38,11 +38,11 @@
}
},
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchAnotBPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"type": "thetaSketchSetOp",
"name": "sketchAnotBUnionPostAgg",
"func": "NOT",
"fields": [
@ -58,11 +58,11 @@
}
},
{
"type": "sketchEstimate",
"type": "thetaSketchEstimate",
"name": "sketchUnionPostAggEstimate",
"field":
{
"type": "sketchSetOper",
"type": "thetaSketchSetOp",
"name": "sketchUnionPostAgg",
"func": "UNION",
"size": 16384,