fix serde issues with time-min-max extension (#11146)

* fix serde issues with time-min-max extension

* fix pom dependencies
This commit is contained in:
Clint Wylie 2021-04-27 10:33:13 -07:00 committed by GitHub
parent 1b07d554a8
commit 57ddae782e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 158 additions and 81 deletions

View File

@ -54,6 +54,16 @@
<artifactId>jackson-annotations</artifactId> <artifactId>jackson-annotations</artifactId>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<scope>provided</scope>
</dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>
@ -69,11 +79,6 @@
<artifactId>guice</artifactId> <artifactId>guice</artifactId>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<scope>provided</scope>
</dependency>
<dependency> <dependency>
<groupId>it.unimi.dsi</groupId> <groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId> <artifactId>fastutil</artifactId>
@ -89,6 +94,11 @@
<artifactId>easymock</artifactId> <artifactId>easymock</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.druid</groupId> <groupId>org.apache.druid</groupId>
<artifactId>druid-core</artifactId> <artifactId>druid-core</artifactId>

View File

@ -38,7 +38,6 @@ public class TimestampAggregator implements Aggregator
} }
private final BaseObjectColumnValueSelector selector; private final BaseObjectColumnValueSelector selector;
private final String name;
private final TimestampSpec timestampSpec; private final TimestampSpec timestampSpec;
private final Comparator<Long> comparator; private final Comparator<Long> comparator;
private final Long initValue; private final Long initValue;
@ -46,14 +45,12 @@ public class TimestampAggregator implements Aggregator
private long most; private long most;
public TimestampAggregator( public TimestampAggregator(
String name,
BaseObjectColumnValueSelector selector, BaseObjectColumnValueSelector selector,
TimestampSpec timestampSpec, TimestampSpec timestampSpec,
Comparator<Long> comparator, Comparator<Long> comparator,
Long initValue Long initValue
) )
{ {
this.name = name;
this.selector = selector; this.selector = selector;
this.timestampSpec = timestampSpec; this.timestampSpec = timestampSpec;
this.comparator = comparator; this.comparator = comparator;

View File

@ -22,24 +22,25 @@ package org.apache.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueType;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.nio.ByteBuffer;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
public class TimestampAggregatorFactory extends AggregatorFactory public abstract class TimestampAggregatorFactory extends AggregatorFactory
{ {
final String name; final String name;
@Nullable
final String fieldName; final String fieldName;
@Nullable
final String timeFormat; final String timeFormat;
private final Comparator<Long> comparator; private final Comparator<Long> comparator;
private final Long initValue; private final Long initValue;
@ -48,8 +49,8 @@ public class TimestampAggregatorFactory extends AggregatorFactory
TimestampAggregatorFactory( TimestampAggregatorFactory(
String name, String name,
String fieldName, @Nullable String fieldName,
String timeFormat, @Nullable String timeFormat,
Comparator<Long> comparator, Comparator<Long> comparator,
Long initValue Long initValue
) )
@ -66,13 +67,23 @@ public class TimestampAggregatorFactory extends AggregatorFactory
@Override @Override
public Aggregator factorize(ColumnSelectorFactory metricFactory) public Aggregator factorize(ColumnSelectorFactory metricFactory)
{ {
return new TimestampAggregator(name, metricFactory.makeColumnValueSelector(fieldName), timestampSpec, comparator, initValue); return new TimestampAggregator(
metricFactory.makeColumnValueSelector(timestampSpec.getTimestampColumn()),
timestampSpec,
comparator,
initValue
);
} }
@Override @Override
public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory)
{ {
return new TimestampBufferAggregator(metricFactory.makeColumnValueSelector(fieldName), timestampSpec, comparator, initValue); return new TimestampBufferAggregator(
metricFactory.makeColumnValueSelector(timestampSpec.getTimestampColumn()),
timestampSpec,
comparator,
initValue
);
} }
@Override @Override
@ -129,12 +140,6 @@ public class TimestampAggregatorFactory extends AggregatorFactory
}; };
} }
@Override
public AggregatorFactory getCombiningFactory()
{
return new TimestampAggregatorFactory(name, name, timeFormat, comparator, initValue);
}
@Override @Override
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException
{ {
@ -145,14 +150,6 @@ public class TimestampAggregatorFactory extends AggregatorFactory
} }
} }
@Override
public List<AggregatorFactory> getRequiredColumns()
{
return Collections.singletonList(
new TimestampAggregatorFactory(fieldName, fieldName, timeFormat, comparator, initValue)
);
}
@Override @Override
public Object deserialize(Object object) public Object deserialize(Object object)
{ {
@ -173,12 +170,14 @@ public class TimestampAggregatorFactory extends AggregatorFactory
return name; return name;
} }
@Nullable
@JsonProperty @JsonProperty
public String getFieldName() public String getFieldName()
{ {
return fieldName; return fieldName;
} }
@Nullable
@JsonProperty @JsonProperty
public String getTimeFormat() public String getTimeFormat()
{ {
@ -188,16 +187,15 @@ public class TimestampAggregatorFactory extends AggregatorFactory
@Override @Override
public List<String> requiredFields() public List<String> requiredFields()
{ {
return Collections.singletonList(fieldName); return Collections.singletonList(timestampSpec.getTimestampColumn());
} }
@Override @Override
public byte[] getCacheKey() public byte[] getCacheKey()
{ {
byte[] fieldNameBytes = StringUtils.toUtf8(fieldName); return new CacheKeyBuilder(AggregatorUtil.TIMESTAMP_CACHE_TYPE_ID).appendString(timestampSpec.getTimestampColumn())
.appendString(timestampSpec.getTimestampFormat())
return ByteBuffer.allocate(1 + fieldNameBytes.length) .build();
.put(AggregatorUtil.TIMESTAMP_CACHE_TYPE_ID).put(fieldNameBytes).array();
} }
@Override @Override
@ -221,43 +219,6 @@ public class TimestampAggregatorFactory extends AggregatorFactory
return Long.BYTES; return Long.BYTES;
} }
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
TimestampAggregatorFactory that = (TimestampAggregatorFactory) o;
if (!Objects.equals(fieldName, that.fieldName)) {
return false;
}
if (!Objects.equals(name, that.name)) {
return false;
}
if (!Objects.equals(comparator, that.comparator)) {
return false;
}
if (!Objects.equals(initValue, that.initValue)) {
return false;
}
return true;
}
@Override
public int hashCode()
{
int result = fieldName != null ? fieldName.hashCode() : 0;
result = 31 * result + (name != null ? name.hashCode() : 0);
result = 31 * result + (comparator != null ? comparator.hashCode() : 0);
result = 31 * result + (initValue != null ? initValue.hashCode() : 0);
return result;
}
@Nullable @Nullable
static Long convertLong(TimestampSpec timestampSpec, Object input) static Long convertLong(TimestampSpec timestampSpec, Object input)
@ -274,4 +235,26 @@ public class TimestampAggregatorFactory extends AggregatorFactory
return null; return null;
} }
@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
TimestampAggregatorFactory that = (TimestampAggregatorFactory) o;
return name.equals(that.name) && Objects.equals(fieldName, that.fieldName) && Objects.equals(
timeFormat,
that.timeFormat
) && comparator.equals(that.comparator) && initValue.equals(that.initValue);
}
@Override
public int hashCode()
{
return Objects.hash(name, fieldName, timeFormat, comparator, initValue);
}
} }

View File

@ -22,28 +22,46 @@ package org.apache.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Ordering; import com.google.common.collect.Ordering;
import javax.annotation.Nullable;
import java.util.List;
public class TimestampMaxAggregatorFactory extends TimestampAggregatorFactory public class TimestampMaxAggregatorFactory extends TimestampAggregatorFactory
{ {
@JsonCreator @JsonCreator
public TimestampMaxAggregatorFactory( public TimestampMaxAggregatorFactory(
@JsonProperty("name") String name, @JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName, @JsonProperty("fieldName") @Nullable String fieldName,
@JsonProperty("timeFormat") String timeFormat @JsonProperty("timeFormat") @Nullable String timeFormat
) )
{ {
super(name, fieldName, timeFormat, Ordering.natural(), Long.MIN_VALUE); super(name, fieldName, timeFormat, Ordering.natural(), Long.MIN_VALUE);
Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name");
Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); }
@Override
public AggregatorFactory getCombiningFactory()
{
return new TimestampMaxAggregatorFactory(name, name, timeFormat);
}
@Override
public List<AggregatorFactory> getRequiredColumns()
{
return ImmutableList.of(
new TimestampMaxAggregatorFactory(name, fieldName, timeFormat)
);
} }
@Override @Override
public String toString() public String toString()
{ {
return "TimestampMaxAggregatorFactory{" + return "TimestampMaxAggregatorFactory{" +
"fieldName='" + fieldName + '\'' + "name='" + name + '\'' +
", name='" + name + '\'' + ", fieldName='" + fieldName + '\'' +
", timeFormat='" + timeFormat + '\'' +
'}'; '}';
} }
} }

View File

@ -22,8 +22,11 @@ package org.apache.druid.query.aggregation;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Ordering; import com.google.common.collect.Ordering;
import java.util.List;
public class TimestampMinAggregatorFactory extends TimestampAggregatorFactory public class TimestampMinAggregatorFactory extends TimestampAggregatorFactory
{ {
@JsonCreator @JsonCreator
@ -35,15 +38,29 @@ public class TimestampMinAggregatorFactory extends TimestampAggregatorFactory
{ {
super(name, fieldName, timeFormat, Ordering.natural().reverse(), Long.MAX_VALUE); super(name, fieldName, timeFormat, Ordering.natural().reverse(), Long.MAX_VALUE);
Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name"); Preconditions.checkNotNull(name, "Must have a valid, non-null aggregator name");
Preconditions.checkNotNull(fieldName, "Must have a valid, non-null fieldName"); }
@Override
public AggregatorFactory getCombiningFactory()
{
return new TimestampMinAggregatorFactory(name, name, timeFormat);
}
@Override
public List<AggregatorFactory> getRequiredColumns()
{
return ImmutableList.of(
new TimestampMinAggregatorFactory(name, fieldName, timeFormat)
);
} }
@Override @Override
public String toString() public String toString()
{ {
return "TimestampMinAggregatorFactory{" + return "TimestampMinAggregatorFactory{" +
"fieldName='" + fieldName + '\'' + "name='" + name + '\'' +
", name='" + name + '\'' + ", fieldName='" + fieldName + '\'' +
", timeFormat='" + timeFormat + '\'' +
'}'; '}';
} }
} }

View File

@ -19,12 +19,16 @@
package org.apache.druid.query.aggregation; package org.apache.druid.query.aggregation;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.Druids; import org.apache.druid.query.Druids;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator;
import org.apache.druid.query.timeseries.TimeseriesQuery; import org.apache.druid.query.timeseries.TimeseriesQuery;
import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest;
import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueType;
import org.junit.Assert; import org.junit.Assert;
@ -32,6 +36,54 @@ import org.junit.Test;
public class TimestampMinMaxAggregatorFactoryTest public class TimestampMinMaxAggregatorFactoryTest
{ {
private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper();
@Test
public void testSerde() throws JsonProcessingException
{
TimestampMaxAggregatorFactory maxAgg = new TimestampMaxAggregatorFactory("timeMax", "__time", null);
TimestampMinAggregatorFactory minAgg = new TimestampMinAggregatorFactory("timeMin", "__time", null);
Assert.assertEquals(
maxAgg,
JSON_MAPPER.readValue(JSON_MAPPER.writeValueAsString(maxAgg), TimestampMaxAggregatorFactory.class)
);
Assert.assertEquals(
maxAgg.getCombiningFactory(),
JSON_MAPPER.readValue(
JSON_MAPPER.writeValueAsString(maxAgg.getCombiningFactory()),
TimestampMaxAggregatorFactory.class
)
);
Assert.assertEquals(
minAgg,
JSON_MAPPER.readValue(JSON_MAPPER.writeValueAsString(minAgg), TimestampMinAggregatorFactory.class)
);
Assert.assertEquals(
minAgg.getCombiningFactory(),
JSON_MAPPER.readValue(
JSON_MAPPER.writeValueAsString(minAgg.getCombiningFactory()),
TimestampMinAggregatorFactory.class
)
);
}
@Test
public void testEqualsAndHashcode()
{
EqualsVerifier.forClass(TimestampMinAggregatorFactory.class)
.withNonnullFields("name", "comparator", "initValue")
.withIgnoredFields("timestampSpec")
.usingGetClass()
.verify();
EqualsVerifier.forClass(TimestampMaxAggregatorFactory.class)
.withNonnullFields("name", "comparator", "initValue")
.withIgnoredFields("timestampSpec")
.usingGetClass()
.verify();
}
@Test @Test
public void testResultArraySignature() public void testResultArraySignature()
{ {