diff --git a/.gitignore b/.gitignore index 816400909f0..a8b2a0bde5c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ NOTICE.BINARY README.BINARY README *.lock +**/.pmd +**/.pmdruleset.xml +.java-version diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java index 0a270cdca39..b6bdd073345 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/BoundFilterBenchmark.java @@ -26,16 +26,13 @@ import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.MutableBitmap; import org.apache.druid.collections.bitmap.RoaringBitmapFactory; -import org.apache.druid.collections.spatial.ImmutableRTree; import org.apache.druid.common.config.NullHandling; import org.apache.druid.extendedset.intset.ConciseSetUtils; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.BoundDimFilter; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.segment.column.BitmapIndex; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.data.BitmapSerdeFactory; -import org.apache.druid.segment.data.CloseableIndexed; import org.apache.druid.segment.data.GenericIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; import org.apache.druid.segment.filter.BoundFilter; @@ -187,50 +184,7 @@ public class BoundFilterBenchmark ), dictionary ).get(); - selector = new BitmapIndexSelector() - { - @Override - public CloseableIndexed getDimensionValues(String dimension) - { - return dictionary; - } - - @Override - public ColumnCapabilities.Capable hasMultipleValues(final String dimension) - { - throw new UnsupportedOperationException(); - } - - @Override - public int getNumRows() - { - throw new UnsupportedOperationException(); - } - - @Override - public BitmapFactory getBitmapFactory() - { - return bitmapFactory; - } - - @Override - public ImmutableBitmap getBitmapIndex(String dimension, String value) - { - return bitmapIndex.getBitmap(bitmapIndex.getIndex(value)); - } - - @Override - public BitmapIndex getBitmapIndex(String dimension) - { - return bitmapIndex; - } - - @Override - public ImmutableRTree getSpatialIndex(String dimension) - { - throw new UnsupportedOperationException(); - } - }; + selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex); } @Benchmark diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java index 54d9065a27a..9a69aa5fb7b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DimensionPredicateFilterBenchmark.java @@ -27,7 +27,6 @@ import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.MutableBitmap; import org.apache.druid.collections.bitmap.RoaringBitmapFactory; -import org.apache.druid.collections.spatial.ImmutableRTree; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.DruidDoublePredicate; @@ -35,9 +34,7 @@ import org.apache.druid.query.filter.DruidFloatPredicate; import org.apache.druid.query.filter.DruidLongPredicate; import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.segment.column.BitmapIndex; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.data.BitmapSerdeFactory; -import org.apache.druid.segment.data.CloseableIndexed; import org.apache.druid.segment.data.GenericIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; import org.apache.druid.segment.filter.DimensionPredicateFilter; @@ -158,50 +155,7 @@ public class DimensionPredicateFilterBenchmark ), dictionary ).get(); - selector = new BitmapIndexSelector() - { - @Override - public CloseableIndexed getDimensionValues(String dimension) - { - return dictionary; - } - - @Override - public ColumnCapabilities.Capable hasMultipleValues(final String dimension) - { - throw new UnsupportedOperationException(); - } - - @Override - public int getNumRows() - { - throw new UnsupportedOperationException(); - } - - @Override - public BitmapFactory getBitmapFactory() - { - return bitmapFactory; - } - - @Override - public ImmutableBitmap getBitmapIndex(String dimension, String value) - { - return bitmapIndex.getBitmap(bitmapIndex.getIndex(value)); - } - - @Override - public BitmapIndex getBitmapIndex(String dimension) - { - return bitmapIndex; - } - - @Override - public ImmutableRTree getSpatialIndex(String dimension) - { - throw new UnsupportedOperationException(); - } - }; + selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex); } @Benchmark diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java index ae9c22961da..8e162cb418b 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/LikeFilterBenchmark.java @@ -25,7 +25,6 @@ import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.collections.bitmap.MutableBitmap; import org.apache.druid.collections.bitmap.RoaringBitmapFactory; -import org.apache.druid.collections.spatial.ImmutableRTree; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.filter.BitmapIndexSelector; import org.apache.druid.query.filter.BoundDimFilter; @@ -35,9 +34,7 @@ import org.apache.druid.query.filter.RegexDimFilter; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.query.ordering.StringComparators; import org.apache.druid.segment.column.BitmapIndex; -import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.data.BitmapSerdeFactory; -import org.apache.druid.segment.data.CloseableIndexed; import org.apache.druid.segment.data.GenericIndexed; import org.apache.druid.segment.data.RoaringBitmapSerdeFactory; import org.apache.druid.segment.serde.StringBitmapIndexColumnPartSupplier; @@ -158,50 +155,7 @@ public class LikeFilterBenchmark ), dictionary ).get(); - selector = new BitmapIndexSelector() - { - @Override - public CloseableIndexed getDimensionValues(String dimension) - { - return dictionary; - } - - @Override - public ColumnCapabilities.Capable hasMultipleValues(final String dimension) - { - throw new UnsupportedOperationException(); - } - - @Override - public int getNumRows() - { - throw new UnsupportedOperationException(); - } - - @Override - public BitmapFactory getBitmapFactory() - { - return bitmapFactory; - } - - @Override - public ImmutableBitmap getBitmapIndex(String dimension, String value) - { - return bitmapIndex.getBitmap(bitmapIndex.getIndex(value)); - } - - @Override - public BitmapIndex getBitmapIndex(String dimension) - { - return bitmapIndex; - } - - @Override - public ImmutableRTree getSpatialIndex(String dimension) - { - throw new UnsupportedOperationException(); - } - }; + selector = new MockBitmapIndexSelector(dictionary, bitmapFactory, bitmapIndex); } @Benchmark diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java b/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java new file mode 100644 index 00000000000..55934fa67ff --- /dev/null +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/MockBitmapIndexSelector.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.benchmark; + +import org.apache.druid.collections.bitmap.BitmapFactory; +import org.apache.druid.collections.bitmap.ImmutableBitmap; +import org.apache.druid.collections.spatial.ImmutableRTree; +import org.apache.druid.query.filter.BitmapIndexSelector; +import org.apache.druid.segment.column.BitmapIndex; +import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.data.CloseableIndexed; +import org.apache.druid.segment.data.GenericIndexed; + +public class MockBitmapIndexSelector implements BitmapIndexSelector +{ + private final GenericIndexed dictionary; + private final BitmapFactory bitmapFactory; + private final BitmapIndex bitmapIndex; + + public MockBitmapIndexSelector( + GenericIndexed dictionary, + BitmapFactory bitmapFactory, + BitmapIndex bitmapIndex) + { + this.dictionary = dictionary; + this.bitmapFactory = bitmapFactory; + this.bitmapIndex = bitmapIndex; + } + + @Override + public CloseableIndexed getDimensionValues(String dimension) + { + return dictionary; + } + + @Override + public ColumnCapabilities.Capable hasMultipleValues(final String dimension) + { + throw new UnsupportedOperationException(); + } + + @Override + public int getNumRows() + { + throw new UnsupportedOperationException(); + } + + @Override + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } + + @Override + public ImmutableBitmap getBitmapIndex(String dimension, String value) + { + return bitmapIndex.getBitmap(bitmapIndex.getIndex(value)); + } + + @Override + public BitmapIndex getBitmapIndex(String dimension) + { + return bitmapIndex; + } + + @Override + public ImmutableRTree getSpatialIndex(String dimension) + { + throw new UnsupportedOperationException(); + } +} diff --git a/core/src/main/java/org/apache/druid/java/util/common/ISE.java b/core/src/main/java/org/apache/druid/java/util/common/ISE.java index 17164ff9b1d..db14706c1e5 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/ISE.java +++ b/core/src/main/java/org/apache/druid/java/util/common/ISE.java @@ -26,6 +26,7 @@ import java.util.function.Function; /** */ +@SuppressWarnings("serial") public class ISE extends IllegalStateException implements SanitizableException { public ISE(String formatText, Object... arguments) diff --git a/core/src/main/java/org/apache/druid/java/util/common/guava/ConcatSequence.java b/core/src/main/java/org/apache/druid/java/util/common/guava/ConcatSequence.java index 95ebe802323..577b3d6bb21 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/guava/ConcatSequence.java +++ b/core/src/main/java/org/apache/druid/java/util/common/guava/ConcatSequence.java @@ -28,6 +28,7 @@ public class ConcatSequence implements Sequence { private final Sequence> baseSequences; + @SuppressWarnings("unchecked") public ConcatSequence( Sequence> baseSequences ) diff --git a/core/src/main/java/org/apache/druid/java/util/common/guava/MergeSequence.java b/core/src/main/java/org/apache/druid/java/util/common/guava/MergeSequence.java index 90f9b9a48ff..5b65442fff5 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/guava/MergeSequence.java +++ b/core/src/main/java/org/apache/druid/java/util/common/guava/MergeSequence.java @@ -34,6 +34,7 @@ public class MergeSequence extends YieldingSequenceBase private final Ordering ordering; private final Sequence> baseSequences; + @SuppressWarnings("unchecked") public MergeSequence( Ordering ordering, Sequence> baseSequences diff --git a/core/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java b/core/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java index 749f3e64144..7bb80aae898 100644 --- a/core/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java +++ b/core/src/main/java/org/apache/druid/java/util/common/guava/ParallelMergeCombiningSequence.java @@ -77,7 +77,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase private final int queueSize; private final boolean hasTimeout; private final long timeoutAtNanos; - private final int queryPriority; // not currently used :( private final int yieldAfter; private final int batchSize; private final int parallelism; @@ -107,7 +106,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase this.combineFn = combineFn; this.hasTimeout = hasTimeout; this.timeoutAtNanos = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutMillis, TimeUnit.MILLISECONDS); - this.queryPriority = queryPriority; this.parallelism = parallelism; this.yieldAfter = yieldAfter; this.batchSize = batchSize; @@ -267,6 +265,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase * {@link MergeCombineAction} do a final merge combine of all the parallel computed results, again pushing * {@link ResultBatch} into a {@link BlockingQueue} with a {@link QueuePusher}. */ + @SuppressWarnings("serial") private static class MergeCombinePartitioningAction extends RecursiveAction { private final List> sequences; @@ -321,7 +320,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase try { final int parallelTaskCount = computeNumTasks(); - // if we have a small number of sequences to merge, or computed paralellism is too low, do not run in parallel, + // if we have a small number of sequences to merge, or computed parallelism is too low, do not run in parallel, // just serially perform the merge-combine with a single task if (parallelTaskCount < 2) { LOG.debug( @@ -360,6 +359,9 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase catch (Throwable t) { closeAllCursors(sequenceCursors); cancellationGizmo.cancel(t); + // Should be the following, but can' change due to lack of + // unit tests. + // out.offer((ParallelMergeCombiningSequence.ResultBatch) ResultBatch.TERMINAL); out.offer(ResultBatch.TERMINAL); } } @@ -449,14 +451,14 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase // running, minus 1 for the task that is running this calculation (as it will be replaced with the parallel tasks) final int utilizationEstimate = runningThreadCount + submissionCount - 1; - // 'computed parallelism' is the remaineder of the 'max parallelism' less current 'utilization estimate' + // 'computed parallelism' is the remainder of the 'max parallelism' less current 'utilization estimate' final int computedParallelismForUtilization = maxParallelism - utilizationEstimate; // try to balance partition size with partition count so we don't end up with layer 2 'final merge' task that has // significantly more work to do than the layer 1 'parallel' tasks. final int computedParallelismForSequences = (int) Math.floor(Math.sqrt(sequences.size())); - // compute total number of layer 1 'parallel' tasks, for the utilzation parallelism, subtract 1 as the final merge + // compute total number of layer 1 'parallel' tasks, for the utilization parallelism, subtract 1 as the final merge // task will take the remaining slot final int computedOptimalParallelism = Math.min( computedParallelismForSequences, @@ -484,7 +486,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * This {@link RecursiveAction} is the work-horse of the {@link ParallelMergeCombiningSequence}, it merge-combines * a set of {@link BatchedResultsCursor} and produces output to a {@link BlockingQueue} with the help of a @@ -501,6 +502,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase * how many times a task has continued executing, and utilized to compute a cumulative moving average of task run time * per amount yielded in order to 'smooth' out the continual adjustment. */ + @SuppressWarnings("serial") private static class MergeCombineAction extends RecursiveAction { private final PriorityQueue> pQueue; @@ -539,6 +541,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase this.cancellationGizmo = cancellationGizmo; } + @SuppressWarnings("unchecked") @Override protected void compute() { @@ -682,6 +685,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase * majority of its time will be spent managed blocking until results are ready for each cursor, or will be incredibly * short lived if all inputs are already available. */ + @SuppressWarnings("serial") private static class PrepareMergeCombineInputsAction extends RecursiveAction { private final List> partition; @@ -717,6 +721,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase this.cancellationGizmo = cancellationGizmo; } + @SuppressWarnings("unchecked") @Override protected void compute() { @@ -757,7 +762,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * {@link ForkJoinPool} friendly {@link BlockingQueue} feeder, adapted from 'QueueTaker' of Java documentation on * {@link ForkJoinPool.ManagedBlocker}, @@ -816,7 +820,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * Holder object for an ordered batch of results from a sequence. Batching the results vastly reduces the amount of * blocking that is needed to move results between stages of {@link MergeCombineAction} done in parallel, allowing @@ -824,6 +827,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase */ static class ResultBatch { + @SuppressWarnings("rawtypes") static final ResultBatch TERMINAL = new ResultBatch(); @Nullable @@ -893,7 +897,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * {@link ForkJoinPool} friendly conversion of {@link Sequence} to {@link Yielder< ResultBatch >} */ @@ -934,7 +937,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * Provides a higher level cursor interface to provide individual results out {@link ResultBatch} provided by * a {@link Yielder} or {@link BlockingQueue}. This is the mechanism that powers {@link MergeCombineAction}, where @@ -985,6 +987,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase return ordering.compare(get(), o.get()); } + @SuppressWarnings({ "rawtypes", "unchecked" }) @Override public boolean equals(Object o) { @@ -1001,7 +1004,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * {@link BatchedResultsCursor} that wraps a {@link Yielder} of {@link ResultBatch} to provide individual rows * of the result batch. @@ -1072,7 +1074,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * {@link BatchedResultsCursor} that wraps a {@link BlockingQueue} of {@link ResultBatch} to provide individual * rows from the result batch. @@ -1155,7 +1156,6 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } } - /** * Token to allow any {@link RecursiveAction} signal the others and the output sequence that something bad happened * and processing should cancel, such as a timeout or connection loss. @@ -1266,7 +1266,7 @@ public class ParallelMergeCombiningSequence extends YieldingSequenceBase } /** - * Holder to accumlate metrics for all work done {@link ParallelMergeCombiningSequence}, containing layer 1 task + * Holder to accumulate metrics for all work done {@link ParallelMergeCombiningSequence}, containing layer 1 task * metrics in {@link #partitionMetrics} and final merge task metrics in {@link #mergeMetrics}, in order to compute * {@link MergeCombineMetrics} after the {@link ParallelMergeCombiningSequence} is completely consumed. */ diff --git a/core/src/main/java/org/apache/druid/query/SegmentDescriptor.java b/core/src/main/java/org/apache/druid/query/SegmentDescriptor.java index 4f362cef0af..d52ea0d8191 100644 --- a/core/src/main/java/org/apache/druid/query/SegmentDescriptor.java +++ b/core/src/main/java/org/apache/druid/query/SegmentDescriptor.java @@ -29,7 +29,7 @@ import org.joda.time.Interval; * number. It's used where the data source, another essential part of {@link org.apache.druid.timeline.SegmentId} * is determined by the context (e. g. in org.apache.druid.client.CachingClusteredClient, where SegmentDescriptor is * used when Brokers tell data servers which segments to include for a particular query) and where having lean JSON - * representations is important, because it's actively transferred detween Druid nodes. It's also for this reason that + * representations is important, because it's actively transferred between Druid nodes. It's also for this reason that * the JSON field names of SegmentDescriptor are abbreviated. */ public class SegmentDescriptor diff --git a/core/src/test/java/org/apache/druid/guice/JsonConfiguratorTest.java b/core/src/test/java/org/apache/druid/guice/JsonConfiguratorTest.java index 5c95fd202fd..b9bfd126b4a 100644 --- a/core/src/test/java/org/apache/druid/guice/JsonConfiguratorTest.java +++ b/core/src/test/java/org/apache/druid/guice/JsonConfiguratorTest.java @@ -107,7 +107,7 @@ public class JsonConfiguratorTest } @Test - public void testsimpleConfigurate() + public void testSimpleConfigurate() { final JsonConfigurator configurator = new JsonConfigurator(mapper, validator); properties.setProperty(PROP_PREFIX + "prop1", "prop1"); diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java index 1fa16e72dc1..e7458c21d7d 100644 --- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java +++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/hll/sql/HllSketchSqlAggregatorTest.java @@ -28,6 +28,7 @@ import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.PeriodGranularity; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.Druids; import org.apache.druid.query.QueryDataSource; import org.apache.druid.query.aggregation.CountAggregatorFactory; @@ -295,7 +296,7 @@ public class HllSketchSqlAggregatorTest extends BaseCalciteQueryTest ImmutableMap.of( TimeseriesQuery.SKIP_EMPTY_BUCKETS, true, - "sqlQueryId", + BaseQuery.SQL_QUERY_ID, "dummy" ), "d0" diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java index 6b5105209b4..c0671d45cf8 100644 --- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java +++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTaskTest.java @@ -1332,6 +1332,7 @@ public class CompactionTaskTest RETRY_POLICY_FACTORY ); + @SuppressWarnings("unused") final CompactionTask task = builder .interval(Intervals.of("2000-01-01/2000-01-01")) .build(); @@ -2012,6 +2013,7 @@ public class CompactionTaskTest this.segments = segments; } + @SuppressWarnings("unchecked") @Override public RetType submit(TaskAction taskAction) { @@ -2160,7 +2162,7 @@ public class CompactionTaskTest } @Override - public SettableColumnValueSelector makeNewSettableColumnValueSelector() + public SettableColumnValueSelector makeNewSettableColumnValueSelector() { return null; } diff --git a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java index d77a9111588..905dcfed011 100644 --- a/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java +++ b/integration-tests/src/test/java/org/apache/druid/tests/query/ITSqlCancelTest.java @@ -25,6 +25,7 @@ import com.google.inject.Inject; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.http.client.response.StatusResponseHolder; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.QueryException; import org.apache.druid.query.QueryInterruptedException; import org.apache.druid.sql.http.SqlQuery; @@ -88,7 +89,7 @@ public class ITSqlCancelTest queryResponseFutures.add( sqlClient.queryAsync( sqlHelper.getQueryURL(config.getRouterUrl()), - new SqlQuery(QUERY, null, false, false, false, ImmutableMap.of("sqlQueryId", queryId), null) + new SqlQuery(QUERY, null, false, false, false, ImmutableMap.of(BaseQuery.SQL_QUERY_ID, queryId), null) ) ); } @@ -125,7 +126,7 @@ public class ITSqlCancelTest final Future queryResponseFuture = sqlClient .queryAsync( sqlHelper.getQueryURL(config.getRouterUrl()), - new SqlQuery(QUERY, null, false, false, false, ImmutableMap.of("sqlQueryId", "validId"), null) + new SqlQuery(QUERY, null, false, false, false, ImmutableMap.of(BaseQuery.SQL_QUERY_ID, "validId"), null) ); // Wait until the sqlLifecycle is authorized and registered diff --git a/processing/src/main/java/org/apache/druid/query/QueryContexts.java b/processing/src/main/java/org/apache/druid/query/QueryContexts.java index dd58dfc2e5a..22cb68b7960 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryContexts.java +++ b/processing/src/main/java/org/apache/druid/query/QueryContexts.java @@ -56,8 +56,8 @@ public class QueryContexts public static final String JOIN_FILTER_REWRITE_VALUE_COLUMN_FILTERS_ENABLE_KEY = "enableJoinFilterRewriteValueColumnFilters"; public static final String REWRITE_JOIN_TO_FILTER_ENABLE_KEY = "enableRewriteJoinToFilter"; public static final String JOIN_FILTER_REWRITE_MAX_SIZE_KEY = "joinFilterRewriteMaxSize"; - // This flag control whether a sql join query with left scan should be attempted to be run as direct table access - // instead of being wrapped inside a query. With direct table access enabled, druid can push down the join operation to + // This flag controls whether a SQL join query with left scan should be attempted to be run as direct table access + // instead of being wrapped inside a query. With direct table access enabled, Druid can push down the join operation to // data servers. public static final String SQL_JOIN_LEFT_SCAN_DIRECT = "enableJoinLeftTableScanDirect"; public static final String USE_FILTER_CNF_KEY = "useFilterCNF"; diff --git a/processing/src/main/java/org/apache/druid/query/QueryMetrics.java b/processing/src/main/java/org/apache/druid/query/QueryMetrics.java index 304ed6cd833..4d2406c697b 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryMetrics.java +++ b/processing/src/main/java/org/apache/druid/query/QueryMetrics.java @@ -262,31 +262,51 @@ public interface QueryMetrics> /** * Registers "query time" metric. + * + * Measures the time between a Jetty thread starting to handle a query, and the response being fully written to + * the response output stream. Does not include time spent waiting in a queue before the query runs. */ QueryMetrics reportQueryTime(long timeNs); /** * Registers "query bytes" metric. + * + * Measures the total number of bytes written by the query server thread to the response output stream. + * + * Emitted once per query. */ QueryMetrics reportQueryBytes(long byteCount); /** - * Registeres "segments queried count" metric. + * Registers "segments queried count" metric. */ QueryMetrics reportQueriedSegmentCount(long segmentCount); /** * Registers "wait time" metric. + * + * Measures the total time segment-processing runnables spent waiting for execution in the processing thread pool. + * + * Emitted once per segment. */ QueryMetrics reportWaitTime(long timeNs); /** * Registers "segment time" metric. + * + * Measures the total wall-clock time spent operating on segments in processing threads. + * + * Emitted once per segment. */ QueryMetrics reportSegmentTime(long timeNs); /** * Registers "segmentAndCache time" metric. + * + * Measures the total wall-clock time spent in processing threads, either operating on segments or retrieving items + * from cache. + * + * Emitted once per segment. */ QueryMetrics reportSegmentAndCacheTime(long timeNs); diff --git a/processing/src/main/java/org/apache/druid/query/filter/Filter.java b/processing/src/main/java/org/apache/druid/query/filter/Filter.java index c554e8f0280..2fe13e1abd7 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/Filter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/Filter.java @@ -83,7 +83,6 @@ public interface Filter */ double estimateSelectivity(BitmapIndexSelector indexSelector); - /** * Get a ValueMatcher that applies this filter to row values. * diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java index 10d39f9f3b5..b57abb741e1 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterTuning.java @@ -27,7 +27,7 @@ import javax.annotation.Nullable; import java.util.Objects; /** - * This class provides a mechansim to influence whether or not indexes are used for a {@link Filter} during processing + * This class provides a mechanism to influence whether or not indexes are used for a {@link Filter} during processing * by {@link org.apache.druid.segment.QueryableIndexStorageAdapter#analyzeFilter} (i.e. will a {@link Filter} be a "pre" * filter in which we union indexes for all values that match the filter to create a * {@link org.apache.druid.segment.BitmapOffset}/{@link org.apache.druid.segment.vector.BitmapVectorOffset}, or will it diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java index 5b97cea64be..71421366b4b 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentAnalysis.java @@ -34,11 +34,11 @@ public class SegmentAnalysis implements Comparable { /** * Segment id is stored as a String rather than {@link org.apache.druid.timeline.SegmentId}, because when a - * SegmentAnalysis object is sent across Druid nodes, on the reciever (deserialization) side it's impossible to + * SegmentAnalysis object is sent across Druid nodes, on the receiver (deserialization) side it's impossible to * unambiguously convert a segment id string (as transmitted in the JSON format) back into a {@code SegmentId} object * ({@link org.apache.druid.timeline.SegmentId#tryParse} javadoc explains that ambiguities in details). It would be * fine to have the type of this field of Object, setting it to {@code SegmentId} on the sender side and remaining as - * a String on the reciever side, but it's even less type-safe than always storing the segment id as a String. + * a String on the receiver side, but it's even less type-safe than always storing the segment id as a String. */ private final String id; private final List interval; diff --git a/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java b/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java index b3c7a0c21cb..e23e0cd0d1b 100644 --- a/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/QueryableIndex.java @@ -48,7 +48,7 @@ public interface QueryableIndex extends ColumnSelector, Closeable * The close method shouldn't actually be here as this is nasty. We will adjust it in the future. * @throws IOException if an exception was thrown closing the index */ - //@Deprecated // This is still required for SimpleQueryableIndex. It should not go away unitl SimpleQueryableIndex is fixed + //@Deprecated // This is still required for SimpleQueryableIndex. It should not go away until SimpleQueryableIndex is fixed @Override void close(); } diff --git a/processing/src/main/java/org/apache/druid/segment/data/Indexed.java b/processing/src/main/java/org/apache/druid/segment/data/Indexed.java index 1c4499f5054..aae7c377761 100644 --- a/processing/src/main/java/org/apache/druid/segment/data/Indexed.java +++ b/processing/src/main/java/org/apache/druid/segment/data/Indexed.java @@ -26,6 +26,13 @@ import org.apache.druid.query.monomorphicprocessing.HotLoopCallee; import javax.annotation.Nullable; import java.util.Comparator; +/** + * Indexed is a fixed-size, immutable, indexed set of values which allows + * locating a specific index via an exact match, the semantics of which are defined + * by the implementation. The indexed is ordered, and may contain duplicate values. + * + * @param the type of the value + */ @PublicApi public interface Indexed extends Iterable, HotLoopCallee { diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java index 72d04af336e..91ffe4da0d1 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java @@ -257,6 +257,7 @@ public class ListFilteredVirtualColumn implements VirtualColumn { return delegate.getBitmap(idMapping.getReverseId(idx)); } + @Override public int getCardinality() { diff --git a/processing/src/test/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryTest.java b/processing/src/test/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryTest.java index 5ac6b585220..7d6c493143e 100644 --- a/processing/src/test/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryTest.java +++ b/processing/src/test/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQueryTest.java @@ -62,12 +62,12 @@ public class DataSourceMetadataQueryTest @Test public void testQuerySerialization() throws IOException { - Query query = Druids.newDataSourceMetadataQueryBuilder() + Query query = Druids.newDataSourceMetadataQueryBuilder() .dataSource("testing") .build(); String json = JSON_MAPPER.writeValueAsString(query); - Query serdeQuery = JSON_MAPPER.readValue(json, Query.class); + Query serdeQuery = JSON_MAPPER.readValue(json, Query.class); Assert.assertEquals(query, serdeQuery); } @@ -93,7 +93,7 @@ public class DataSourceMetadataQueryTest final ObjectMapper mapper = new DefaultObjectMapper(); - final Query serdeQuery = mapper.readValue( + final Query serdeQuery = mapper.readValue( mapper.writeValueAsBytes( mapper.readValue( mapper.writeValueAsString( @@ -103,7 +103,7 @@ public class DataSourceMetadataQueryTest ), Query.class ); - Assert.assertEquals(1, serdeQuery.getContextValue(QueryContexts.PRIORITY_KEY)); + Assert.assertEquals((Integer) 1, serdeQuery.getContextValue(QueryContexts.PRIORITY_KEY)); Assert.assertEquals(true, serdeQuery.getContextValue("useCache")); Assert.assertEquals("true", serdeQuery.getContextValue("populateCache")); Assert.assertEquals(true, serdeQuery.getContextValue("finalize")); @@ -232,7 +232,7 @@ public class DataSourceMetadataQueryTest ); Assert.assertEquals(segments.size(), 2); - // should only have the latest segments. + // should only have the latest segments. List expected = Arrays.asList( new LogicalSegment() { diff --git a/server/src/main/java/org/apache/druid/client/selector/QueryableDruidServer.java b/server/src/main/java/org/apache/druid/client/selector/QueryableDruidServer.java index 3d94986c8f1..6bc317d359c 100644 --- a/server/src/main/java/org/apache/druid/client/selector/QueryableDruidServer.java +++ b/server/src/main/java/org/apache/druid/client/selector/QueryableDruidServer.java @@ -24,7 +24,7 @@ import org.apache.druid.query.QueryRunner; /** */ -public class QueryableDruidServer +public class QueryableDruidServer> { private final DruidServer server; private final T queryRunner; diff --git a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java index db9ad7b98a1..e0643288971 100644 --- a/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/org/apache/druid/client/selector/ServerSelector.java @@ -22,6 +22,7 @@ package org.apache.druid.client.selector; import it.unimi.dsi.fastutil.ints.Int2ObjectRBTreeMap; import org.apache.druid.client.DataSegmentInterner; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryRunner; import org.apache.druid.server.coordination.DruidServerMetadata; import org.apache.druid.server.coordination.ServerType; import org.apache.druid.timeline.DataSegment; @@ -161,7 +162,7 @@ public class ServerSelector implements Overshadowable } @Nullable - public QueryableDruidServer pick(@Nullable Query query) + public QueryableDruidServer> pick(@Nullable Query query) { synchronized (this) { if (!historicalServers.isEmpty()) { diff --git a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java index 0d993d632f7..5c1de5f64a1 100644 --- a/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java +++ b/server/src/main/java/org/apache/druid/server/log/FileRequestLoggerProvider.java @@ -49,7 +49,6 @@ public class FileRequestLoggerProvider implements RequestLoggerProvider @NotNull private ScheduledExecutorFactory factory = null; - @JacksonInject @NotNull @Json diff --git a/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java b/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java index bcb628a195b..4a8b946eecc 100644 --- a/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java +++ b/server/src/main/java/org/apache/druid/server/log/LoggingRequestLogger.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.Query; import org.apache.druid.query.TableDataSource; import org.apache.druid.server.RequestLogLine; @@ -60,7 +61,7 @@ public class LoggingRequestLogger implements RequestLogger try { final Query query = requestLogLine.getQuery(); MDC.put("queryId", query.getId()); - MDC.put("sqlQueryId", StringUtils.nullToEmptyNonDruidDataString(query.getSqlQueryId())); + MDC.put(BaseQuery.SQL_QUERY_ID, StringUtils.nullToEmptyNonDruidDataString(query.getSqlQueryId())); MDC.put("dataSource", String.join(",", query.getDataSource().getTableNames())); MDC.put("queryType", query.getType()); MDC.put("isNested", String.valueOf(!(query.getDataSource() instanceof TableDataSource))); diff --git a/server/src/main/java/org/apache/druid/server/log/RequestLoggerProvider.java b/server/src/main/java/org/apache/druid/server/log/RequestLoggerProvider.java index e4566286708..c4ed91356cb 100644 --- a/server/src/main/java/org/apache/druid/server/log/RequestLoggerProvider.java +++ b/server/src/main/java/org/apache/druid/server/log/RequestLoggerProvider.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.inject.Provider; /** - * A Marker interface for things that can provide a RequestLogger. This can be combined with jackson polymorphic serde + * A Marker interface for things that can provide a RequestLogger. This can be combined with Jackson polymorphic serde * to provide new RequestLogger implementations as plugins. */ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java index 3a5dbeb24fa..83bb89ac451 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/DruidPlanner.java @@ -153,7 +153,7 @@ public class DruidPlanner implements Closeable } /** - * Prepare an SQL query for execution, including some initial parsing and validation and any dyanmic parameter type + * Prepare an SQL query for execution, including some initial parsing and validation and any dynamic parameter type * resolution, to support prepared statements via JDBC. * * In some future this could perhaps re-use some of the work done by {@link #validate()} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java index 34cad6fdbcc..82e1ddc65c1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/PlannerContext.java @@ -32,6 +32,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Numbers; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.BaseQuery; import org.apache.druid.server.security.Access; import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.ResourceAction; @@ -57,7 +58,7 @@ import java.util.concurrent.CopyOnWriteArrayList; public class PlannerContext { // query context keys - public static final String CTX_SQL_QUERY_ID = "sqlQueryId"; + public static final String CTX_SQL_QUERY_ID = BaseQuery.SQL_QUERY_ID; public static final String CTX_SQL_CURRENT_TIMESTAMP = "sqlCurrentTimestamp"; public static final String CTX_SQL_TIME_ZONE = "sqlTimeZone"; public static final String CTX_SQL_STRINGIFY_ARRAYS = "sqlStringifyArrays"; diff --git a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java index 6b35daf1774..633fbce6805 100644 --- a/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java +++ b/sql/src/test/java/org/apache/druid/sql/avatica/DruidAvaticaHandlerTest.java @@ -48,6 +48,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.QueryScheduler; @@ -232,7 +233,7 @@ public abstract class DruidAvaticaHandlerTest extends CalciteTestBase final Properties propertiesLosAngeles = new Properties(); propertiesLosAngeles.setProperty("sqlTimeZone", "America/Los_Angeles"); propertiesLosAngeles.setProperty("user", "regularUserLA"); - propertiesLosAngeles.setProperty("sqlQueryId", DUMMY_SQL_QUERY_ID); + propertiesLosAngeles.setProperty(BaseQuery.SQL_QUERY_ID, DUMMY_SQL_QUERY_ID); clientLosAngeles = DriverManager.getConnection(url, propertiesLosAngeles); } diff --git a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java index 217645ef9d5..bc99f878d84 100644 --- a/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java +++ b/sql/src/test/java/org/apache/druid/sql/http/SqlResourceTest.java @@ -45,6 +45,7 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.Query; import org.apache.druid.query.QueryCapacityExceededException; import org.apache.druid.query.QueryContexts; @@ -1129,7 +1130,7 @@ public class SqlResourceTest extends CalciteTestBase false, false, false, - ImmutableMap.of("maxMergingDictionarySize", 1, "sqlQueryId", "id"), + ImmutableMap.of("maxMergingDictionarySize", 1, BaseQuery.SQL_QUERY_ID, "id"), null ) ).lhs; @@ -1147,7 +1148,7 @@ public class SqlResourceTest extends CalciteTestBase String errorMessage = "This will be support in Druid 9999"; SqlQuery badQuery = EasyMock.createMock(SqlQuery.class); EasyMock.expect(badQuery.getQuery()).andReturn("SELECT ANSWER TO LIFE"); - EasyMock.expect(badQuery.getContext()).andReturn(ImmutableMap.of("sqlQueryId", "id")); + EasyMock.expect(badQuery.getContext()).andReturn(ImmutableMap.of(BaseQuery.SQL_QUERY_ID, "id")); EasyMock.expect(badQuery.getParameterList()).andThrow(new QueryUnsupportedException(errorMessage)); EasyMock.replay(badQuery); final QueryException exception = doPost(badQuery).lhs; @@ -1291,7 +1292,7 @@ public class SqlResourceTest extends CalciteTestBase false, false, false, - ImmutableMap.of("priority", -5, "sqlQueryId", sqlQueryId), + ImmutableMap.of("priority", -5, BaseQuery.SQL_QUERY_ID, sqlQueryId), null ), makeRegularUserReq() @@ -1332,7 +1333,7 @@ public class SqlResourceTest extends CalciteTestBase public void testQueryTimeoutException() throws Exception { final String sqlQueryId = "timeoutTest"; - Map queryContext = ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 1, "sqlQueryId", sqlQueryId); + Map queryContext = ImmutableMap.of(QueryContexts.TIMEOUT_KEY, 1, BaseQuery.SQL_QUERY_ID, sqlQueryId); final QueryException timeoutException = doPost( new SqlQuery( "SELECT CAST(__time AS DATE), dim1, dim2, dim3 FROM druid.foo GROUP by __time, dim1, dim2, dim3 ORDER BY dim2 DESC", @@ -1483,7 +1484,7 @@ public class SqlResourceTest extends CalciteTestBase private static SqlQuery createSimpleQueryWithId(String sqlQueryId, String sql) { - return new SqlQuery(sql, null, false, false, false, ImmutableMap.of("sqlQueryId", sqlQueryId), null); + return new SqlQuery(sql, null, false, false, false, ImmutableMap.of(BaseQuery.SQL_QUERY_ID, sqlQueryId), null); } private Pair>> doPost(final SqlQuery query) throws Exception