run web-console e2e tests for java changes too (#12776)

* run web-console e2e tests for java changes too, fix travis stages for web e2e and docs jobs

* run the script test on script changes

.travis.yml

@@ -382,6 +382,35 @@ jobs:
       after_success:
         - (cd web-console && travis_retry npm run codecov)  # retry in case of network error
 
+    - name: "web console end-to-end test"
+      stage: Tests - phase 1
+      before_install: *setup_generate_license
+      install: web-console/script/druid build
+      before_script:
+        - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...'
+        - nvm install 14.19.3
+        - web-console/script/druid start
+      script: (cd web-console && npm run test-e2e)
+      after_script: web-console/script/druid stop
+
+    - name: "docs"
+      stage: Tests - phase 1
+      install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install)
+      script: |-
+        (cd website && npm run lint && npm run spellcheck) || { echo "
+
+        If there are spell check errors:
+
+        1) Suppressing False Positives: Edit website/.spelling to add suppressions. Instructions
+        are at the top of the file and explain how to suppress false positives either globally or
+        within a particular file.
+
+        2) Running Spell Check Locally: cd website && npm install && npm run spellcheck
+
+        For more information, refer to: https://www.npmjs.com/package/markdown-spellcheck
+
+        " && false; }
+
     - name: "Build and test on ARM64 CPU architecture (1)"
       stage: Tests - phase 2
       arch: arm64-graviton2
@@ -404,32 +433,6 @@ jobs:
         - MAVEN_PROJECTS='core,sql,server,services'
       script: ${MVN} test -B -pl ${MAVEN_PROJECTS} -Ddruid.console.skip=true -DargLine=-Xmx3000m
 
-    - name: "web console end-to-end test"
-      before_install: *setup_generate_license
-      install: web-console/script/druid build
-      before_script:
-        - ./check_test_suite.py && travis_terminate 0 || echo 'Starting nvm install...'
-        - nvm install 14.19.3
-        - web-console/script/druid start
-      script: (cd web-console && npm run test-e2e)
-      after_script: web-console/script/druid stop
-
-    - name: "docs"
-      install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install)
-      script: |-
-        (cd website && npm run lint && npm run spellcheck) || { echo "
-
-        If there are spell check errors:
-
-        1) Suppressing False Positives: Edit website/.spelling to add suppressions. Instructions
-        are at the top of the file and explain how to suppress false positives either globally or
-        within a particular file.
-
-        2) Running Spell Check Locally: cd website && npm install && npm run spellcheck
-
-        For more information, refer to: https://www.npmjs.com/package/markdown-spellcheck
-
-        " && false; }
 
     # Integration tests Java Compile version is set by the machine environment jdk (set by the jdk key)
     # Integration tests Java Runtime version is set by the JVM_RUNTIME env property (set env key to -Djvm.runtime=<JVM_RUNTIME_VERSION>)

check_test_suite.py

@@ -28,8 +28,11 @@ always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11)
 # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed
 ignore_prefixes = ['.github', '.idea', '.asf.yaml', '.backportrc.json', '.codecov.yml', '.dockerignore', '.gitignore',
                    '.lgtm.yml', 'CONTRIBUTING.md', 'setup-hooks.sh', 'upload.sh', 'dev', 'distribution/docker',
-                   'distribution/asf-release-process-guide.md', '.travis.yml', 'check_test_suite.py',
-                   'check_test_suite_test.py', 'owasp-dependency-check-suppressions.xml']
+                   'distribution/asf-release-process-guide.md', '.travis.yml',
+                   'owasp-dependency-check-suppressions.xml']
+
+script_prefixes = ['check_test_suite.py', 'check_test_suite_test.py']
+script_job = ['script checks']
 
 # these files are docs changes
 # if changes are limited to this set then we can skip web-console and java
@@ -43,6 +46,7 @@ docs_jobs = ['docs']
 web_console_prefixes = ['web-console/']
 # travis web-console job name
 web_console_jobs = ['web console', 'web console end-to-end test']
+web_console_still_run_for_java_jobs = ['web console end-to-end test']
 
 
 def check_ignore(file):
@@ -51,6 +55,11 @@ def check_ignore(file):
         print("found ignorable file change: {}".format(file))
     return is_always_ignore
 
+def check_testable_script(file):
+    is_script = True in (file.startswith(prefix) for prefix in script_prefixes)
+    if is_script:
+        print("found script file change: {}".format(file))
+    return is_script
 
 def check_docs(file):
     is_docs = True in (file.startswith(prefix) for prefix in docs_prefixes)
@@ -85,6 +94,8 @@ def check_should_run_suite(suite, diff_files):
     any_console = False
     all_console = True
     any_java = False
+    any_testable_script = False
+    all_testable_script = True
 
     # go over all of the files in the diff and collect some information about the diff contents, we'll use this later
     # to decide whether or not to run the suite
@@ -96,8 +107,11 @@ def check_should_run_suite(suite, diff_files):
         all_docs = all_docs and is_docs
         is_console = check_console(f)
         any_console = any_console or is_console
-        all_console = any_console and is_console
-        any_java = any_java or (not is_ignore and not is_docs and not is_console)
+        all_console = all_console and is_console
+        is_script = check_testable_script(f)
+        any_testable_script = any_testable_script or is_script
+        all_testable_script = all_testable_script and is_script
+        any_java = any_java or (not is_ignore and not is_docs and not is_console and not is_script)
 
     # if everything is ignorable, we can skip this suite
     if all_ignore:
@@ -108,12 +122,18 @@ def check_should_run_suite(suite, diff_files):
     # if all of the changes are docs paths, but the current suite is not a docs job, we can skip
     if all_docs:
         return False
+    if suite in web_console_still_run_for_java_jobs:
+        return any_console or any_java
     # if the test suite is a web console job, return true if any of the changes are web console files
     if suite in web_console_jobs:
         return any_console
     # if all of the changes are web console paths, but the current suite is not a web console job, we can skip
     if all_console:
         return False
+    if suite in script_job:
+        return any_testable_script
+    if all_testable_script:
+        return False
 
     # if all of the files belong to known non-java groups, we can also skip java
     # note that this should probably be reworked to much more selectively run the java jobs depending on the diff

check_test_suite_test.py

@@ -46,6 +46,8 @@ class CheckTestSuite(unittest.TestCase):
             )
 
     def test_web_console(self):
+        web_console_job = 'web console'
+        e2e_job = 'web console end-to-end test'
         self.assertEqual(False, check_test_suite.check_console('.travis.yml'))
         self.assertEqual(False, check_test_suite.check_console('check_test_suite_test.py'))
         self.assertEqual(False, check_test_suite.check_console('website/core/Footer.js'))
@@ -53,25 +55,74 @@ class CheckTestSuite(unittest.TestCase):
         self.assertEqual(True, check_test_suite.check_console('web-console/src/views/index.ts'))
         self.assertEqual(True, check_test_suite.check_console('web-console/unified-console.html'))
 
-        for job in check_test_suite.web_console_jobs:
-            self.assertEqual(
-                True,
-                check_test_suite.check_should_run_suite(
-                    job,
-                    ['check_test_suite_test.py', 'web-console/unified-console.html']
-                )
+        self.assertEqual(
+            True,
+            check_test_suite.check_should_run_suite(
+                web_console_job,
+                ['check_test_suite_test.py', 'web-console/unified-console.html']
             )
-            self.assertEqual(
-                False,
-                check_test_suite.check_should_run_suite(
-                    job,
-                    ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
-                )
+        )
+        self.assertEqual(
+            False,
+            check_test_suite.check_should_run_suite(
+                web_console_job,
+                ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
             )
+        )
+        self.assertEqual(
+            True,
+            check_test_suite.check_should_run_suite(
+                e2e_job,
+                ['check_test_suite_test.py', 'web-console/unified-console.html']
+            )
+        )
+        self.assertEqual(
+            True,
+            check_test_suite.check_should_run_suite(
+                e2e_job,
+                ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
+            )
+        )
+
+    def test_testable_script(self):
+        self.assertEqual(False, check_test_suite.check_testable_script('.travis.yml'))
+        self.assertEqual(True, check_test_suite.check_testable_script('check_test_suite.py'))
+        self.assertEqual(True, check_test_suite.check_testable_script('check_test_suite_test.py'))
+
+        script_job = 'script checks'
+        some_java_job = 'spotbugs checks'
+        self.assertEqual(
+            False,
+            check_test_suite.check_should_run_suite(
+                script_job,
+                ['core/src/main/java/org/apache/druid/math/expr/Expr.java']
+            )
+        )
+        self.assertEqual(
+            True,
+            check_test_suite.check_should_run_suite(
+                some_java_job,
+                ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
+            )
+        )
+        self.assertEqual(
+            True,
+            check_test_suite.check_should_run_suite(
+                some_java_job,
+                ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java']
+            )
+        )
+        self.assertEqual(
+            False,
+            check_test_suite.check_should_run_suite(
+                some_java_job,
+                ['check_test_suite_test.py']
+            )
+        )
 
     def test_some_java(self):
 
-        some_java_job = "spotbugs checks"
+        some_java_job = 'spotbugs checks'
         some_non_java_diffs = [
             ['.travis.yml'],
             ['check_test_suite_test.py'],