Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable resultset validation of Drill tests (#15096) 

- introduces a test_X method for every testcase (995 testcases)
- added a resultset parser which reads the expected resultset based on the result schema
- loaded a few more datasets
- added a testcase to ensure that all files have a corresponding testcase
- renamed DecoupledIgnore to NegativeTest
- categorized the failing 268 tests
This commit is contained in:
Zoltan Haindrich 2023-10-10 11:10:50 +02:00 committed by GitHub
parent 48f35b3fdd
commit 23605c1edd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 7979 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java
View File

@ -126,6 +126,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
@ -1056,7 +1057,8 @@ public class BaseCalciteQueryTest extends CalciteTestBase
public void assertResultsEquals(String sql, List<Object[]> expectedResults, List<Object[]> results)
{
for (int i = 0; i < results.size(); i++) {
int minSize = Math.min(results.size(), expectedResults.size());
for (int i = 0; i < minSize; i++) {
Assert.assertArrayEquals(
StringUtils.format("result #%d: %s", i + 1, sql),
expectedResults.get(i),
@ -1368,7 +1370,7 @@ public class BaseCalciteQueryTest extends CalciteTestBase
assertResultsEquals(sql, expectedResults, results);
}
catch (AssertionError e) {
displayResults(results);
displayResults("Actual", results);
throw e;
}
}
@ -1380,10 +1382,10 @@ public class BaseCalciteQueryTest extends CalciteTestBase
* expected results: let the test fail with empty results. The actual results
* are printed to the console. Copy them into the test.
*/
public static void displayResults(List<Object[]> results)
public static void displayResults(String name, List<Object[]> results)
{
PrintStream out = System.out;
out.println("-- Actual results --");
out.printf(Locale.ENGLISH, "-- %s results --", name);
for (int rowIndex = 0; rowIndex < results.size(); rowIndex++) {
printArray(results.get(rowIndex), out);
if (rowIndex < results.size() - 1) {

46
sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java
View File

@ -109,7 +109,7 @@ import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.join.JoinType;
import org.apache.druid.sql.calcite.DecoupledIgnore.Modes;
import org.apache.druid.sql.calcite.NotYetSupported.Modes;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.filtration.Filtration;
import org.apache.druid.sql.calcite.planner.Calcites;
@ -2688,7 +2688,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSelectAndOrderByProjections()
{
@ -2773,7 +2773,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testTopNWithSelectAndOrderByProjections()
{
@ -2811,7 +2811,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore
@NotYetSupported
@Test
public void testUnionAllQueries()
{
@ -2845,7 +2845,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testUnionAllQueriesWithLimit()
{
@ -3203,7 +3203,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
* This test case should be in {@link CalciteUnionQueryTest}. However, there's a bug in the test framework that
* doesn't reset framework once the merge buffers
*/
@DecoupledIgnore
@NotYetSupported
@Test
public void testUnionAllSameTableThreeTimes()
{
@ -3247,7 +3247,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testExactCountDistinctUsingSubqueryOnUnionAllTables()
{
@ -4653,7 +4653,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSortOnPostAggregationDefault()
{
@ -4685,7 +4685,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSortOnPostAggregationNoTopNConfig()
{
@ -4729,7 +4729,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.CANNOT_CONVERT)
@NotYetSupported(Modes.CANNOT_CONVERT)
@Test
public void testGroupByWithSortOnPostAggregationNoTopNContext()
{
@ -5317,7 +5317,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.ERROR_HANDLING)
@NotYetSupported(Modes.ERROR_HANDLING)
@Test
public void testUnplannableQueries()
{
@ -5389,7 +5389,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.ERROR_HANDLING)
@NotYetSupported(Modes.ERROR_HANDLING)
@Test
public void testUnplannableExactCountDistinctOnSketch()
{
@ -6384,7 +6384,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testExactCountDistinctWithGroupingAndOtherAggregators()
{
@ -6439,7 +6439,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testMultipleExactCountDistinctWithGroupingAndOtherAggregatorsUsingJoin()
{
@ -7082,7 +7082,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testExactCountDistinctUsingSubqueryWithWherePushDown()
{
@ -7813,7 +7813,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testFilterOnCurrentTimestampWithIntervalArithmetic()
{
@ -7861,7 +7861,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testFilterOnCurrentTimestampOnView()
{
@ -10112,7 +10112,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim()
{
@ -11561,7 +11561,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore
@NotYetSupported
@Test
public void testRequireTimeConditionPositive()
{
@ -11765,7 +11765,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.ERROR_HANDLING)
@NotYetSupported(Modes.ERROR_HANDLING)
@Test
public void testRequireTimeConditionSemiJoinNegative()
{
@ -13744,7 +13744,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.PLAN_MISMATCH)
@NotYetSupported(Modes.PLAN_MISMATCH)
@Test
public void testPlanWithInFilterLessThanInSubQueryThreshold()
{
@ -14069,7 +14069,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
)
);
}
@DecoupledIgnore
@NotYetSupported
@Test
public void testOrderByAlongWithInternalScanQuery()
{
@ -14112,7 +14112,7 @@ public class CalciteQueryTest extends BaseCalciteQueryTest
);
}
@DecoupledIgnore(mode = Modes.NOT_ENOUGH_RULES)
@NotYetSupported(Modes.NOT_ENOUGH_RULES)
@Test
public void testOrderByAlongWithInternalScanQueryNoDistinct()
{

8
sql/src/test/java/org/apache/druid/sql/calcite/CalciteSysQueryTest.java
View File

@ -20,15 +20,15 @@
package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableList;
import org.apache.druid.sql.calcite.DecoupledIgnore.DecoupledIgnoreProcessor;
import org.apache.druid.sql.calcite.DecoupledIgnore.Modes;
import org.apache.druid.sql.calcite.NotYetSupported.Modes;
import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor;
import org.junit.Rule;
import org.junit.Test;
public class CalciteSysQueryTest extends BaseCalciteQueryTest
{
@Rule(order = 0)
public DecoupledIgnoreProcessor decoupledIgnoreProcessor = new DecoupledIgnoreProcessor();
public NotYetSupportedProcessor NegativeTestProcessor = new NotYetSupportedProcessor();
@Test
public void testTasksSum()
@ -46,7 +46,7 @@ public class CalciteSysQueryTest extends BaseCalciteQueryTest
.run();
}
@DecoupledIgnore(mode = Modes.EXPRESSION_NOT_GROUPED)
@NotYetSupported(Modes.EXPRESSION_NOT_GROUPED)
@Test
public void testTasksSumOver()
{

4
sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteQueryTest.java
View File

@ -22,7 +22,7 @@ package org.apache.druid.sql.calcite;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.query.QueryContexts;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.sql.calcite.DecoupledIgnore.DecoupledIgnoreProcessor;
import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.util.SqlTestFramework;
import org.junit.Rule;
@ -31,7 +31,7 @@ public class DecoupledPlanningCalciteQueryTest extends CalciteQueryTest
{
@Rule(order = 0)
public DecoupledIgnoreProcessor decoupledIgnoreProcessor = new DecoupledIgnoreProcessor();
public NotYetSupportedProcessor decoupledIgnoreProcessor = new NotYetSupportedProcessor();
private static final ImmutableMap<String, Object> CONTEXT_OVERRIDES = ImmutableMap.of(
PlannerConfig.CTX_NATIVE_QUERY_SQL_PLANNING_MODE, PlannerConfig.NATIVE_QUERY_SQL_PLANNING_MODE_DECOUPLED,

55
sql/src/test/java/org/apache/druid/sql/calcite/DisableUnless.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.druid.sql.calcite;
import com.google.common.base.Supplier;
import org.apache.druid.common.config.NullHandling;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import static org.junit.Assume.assumeTrue;
/**
* Collection of conditional disabler rules.
*/
class DisableUnless
{
public static final TestRule SQL_COMPATIBLE = new DisableUnlessRule("NullHandling::sqlCompatible", NullHandling::sqlCompatible);
public static class DisableUnlessRule implements TestRule
{
private Supplier<Boolean> predicate;
private String message;
public DisableUnlessRule(String message, Supplier<Boolean> predicate)
{
this.message = message;
this.predicate = predicate;
}
@Override
public Statement apply(Statement base, Description description)
{
assumeTrue("Testcase disabled; because condition not met: " + message, predicate.get());
return base;
}
}
}

7928
sql/src/test/java/org/apache/druid/sql/calcite/DrillWindowQueryTest.java
View File

File diff suppressed because it is too large Load Diff

69
sql/src/test/java/org/apache/druid/sql/calcite/DecoupledIgnore.java → sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java
View File

@ -21,6 +21,7 @@ package org.apache.druid.sql.calcite;
import com.google.common.base.Throwables;
import org.apache.druid.error.DruidException;
import org.apache.druid.java.util.common.UOE;
import org.junit.AssumptionViolatedException;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
@ -36,16 +37,35 @@ import java.util.regex.Pattern;
import static org.junit.Assert.assertThrows;
/**
* Can be used to mark tests which are not-yet supported in decoupled mode.
* Can be used to mark tests which are not-yet supported for some reason.
*
* In case a testcase marked with this annotation fails - it means that the
* testcase no longer fails with the annotated expectation. This means that a code change affected this test either
*
* <ol>
* <li>it suddenly passes: yay, assuming it makes sense that it suddenly passes, remove the annotation and move on</li>
* <li>it suddenly fails with a different error: validate that the new error is expected and either fix to continue failing with the old error or update the expected error.</li>
* </ol>
*
* During usage; the annotation process have to be added to the testclass.
* Ensure that it's loaded as the most outer-rule by using order=0 - otherwise
* it may interfere with other rules:
* <code>
* @Rule(order = 0)
* public TestRule notYetSupportedRule = new NotYetSupportedProcessor();
*
* @NotYetSupported(NOT_ENOUGH_RULES)
* @Test
* public void testA() {
* }
* </code>
*
* In case a testcase marked with this annotation fails - it may mean that the
* testcase no longer needs that annotation.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target({ElementType.METHOD})
public @interface DecoupledIgnore
public @interface NotYetSupported
{
Modes mode() default Modes.NOT_ENOUGH_RULES;
Modes value() default Modes.NOT_ENOUGH_RULES;
enum Modes
{
@ -53,7 +73,22 @@ public @interface DecoupledIgnore
NOT_ENOUGH_RULES(DruidException.class, "not enough rules"),
CANNOT_CONVERT(DruidException.class, "Cannot convert query parts"),
ERROR_HANDLING(AssertionError.class, "(is <ADMIN> was <OPERATOR>|is <INVALID_INPUT> was <UNCATEGORIZED>|with message a string containing)"),
EXPRESSION_NOT_GROUPED(DruidException.class, "Expression '[a-z]+' is not being grouped");
EXPRESSION_NOT_GROUPED(DruidException.class, "Expression '[a-z]+' is not being grouped"),
COLUMN_NOT_FOUND(DruidException.class, "CalciteContextException.*Column.*not found in any table"),
NULLS_FIRST_LAST(DruidException.class, "NULLS (FIRST|LAST)"),
BIGINT_TO_DATE(DruidException.class, "BIGINT to type (DATE|TIME)"),
NPE_PLAIN(NullPointerException.class, "java.lang.NullPointerException"),
NPE(DruidException.class, "java.lang.NullPointerException"),
AGGREGATION_NOT_SUPPORT_TYPE(DruidException.class, "Aggregation \\[(MIN|MAX)\\] does not support type"),
CANNOT_APPLY_VIRTUAL_COL(UOE.class, "apply virtual columns"),
MISSING_DESC(DruidException.class, "function signature DESC"),
RESULT_COUNT_MISMATCH(AssertionError.class, "result count:"),
ALLDATA_CSV(DruidException.class, "allData.csv"),
BIGINT_TIME_COMPARE(DruidException.class, "Cannot apply '.' to arguments of type"),
INCORRECT_SYNTAX(DruidException.class, "Incorrect syntax near the keyword"),
// at least c7 is represented oddly in the parquet file
T_ALLTYPES_ISSUES(AssertionError.class, "(t_alltype|allTypsUniq|fewRowsAllData).parquet.*Verifier.verify"),
RESULT_MISMATCH(AssertionError.class, "assertResultsEquals");
public Class<? extends Throwable> throwableClass;
public String regex;
@ -66,22 +101,23 @@ public @interface DecoupledIgnore
Pattern getPattern()
{
return Pattern.compile(regex);
return Pattern.compile(regex, Pattern.MULTILINE | Pattern.DOTALL);
}
};
/**
* Processes {@link DecoupledIgnore} annotations.
* Processes {@link NotYetSupported} annotations.
*
* Ensures that test cases disabled with that annotation can still not pass.
* If the error is as expected; the testcase is marked as "ignored".
*/
class DecoupledIgnoreProcessor implements TestRule
class NotYetSupportedProcessor implements TestRule
{
@Override
public Statement apply(Statement base, Description description)
{
DecoupledIgnore annotation = description.getAnnotation(DecoupledIgnore.class);
NotYetSupported annotation = description.getAnnotation(NotYetSupported.class);
if (annotation == null) {
return base;
}
@ -90,23 +126,22 @@ public @interface DecoupledIgnore
@Override
public void evaluate()
{
Modes ignoreMode = annotation.value();
Throwable e = assertThrows(
"Expected that this testcase will fail - it might got fixed?",
annotation.mode().throwableClass,
"Expected that this testcase will fail - it might got fixed; or failure have changed?",
ignoreMode.throwableClass,
base::evaluate
);
String trace = Throwables.getStackTraceAsString(e);
Matcher m = annotation.mode().getPattern().matcher(trace);
Matcher m = annotation.value().getPattern().matcher(trace);
if (!m.find()) {
throw new AssertionError("Exception stactrace doesn't match regex: " + annotation.mode().regex, e);
throw new AssertionError("Exception stactrace doesn't match regex: " + annotation.value().regex, e);
}
throw new AssumptionViolatedException("Test is not-yet supported in Decoupled mode");
throw new AssumptionViolatedException("Test is not-yet supported; ignored with:" + annotation);
}
};
}
}
}