NIFI-8478 Addressed i18n test issues; CI supplies locale to surefire

This closes #5040

Signed-off-by: David Handermann <exceptionfactory@apache.org>

.github/workflows/ci-workflow.yml

@@ -91,9 +91,16 @@ jobs:
           df -h
       - name: Build with Maven
         env:
-          MAVEN_OPTS: -Xmx2g -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Dmaven.surefire.arguments="-Duser.language=en -Duser.region=AU -Duser.timezone=Australia/Melbourne" -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+          MAVEN_OPTS: -Xmx2g -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Duser.language=en -Duser.country=AU -Duser.region=AU -Duser.timezone=Australia/Melbourne -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
         run: |
           mvn -V -T 0.7C package verify -B -Pcontrib-check,include-grpc -Ddir-only -ntp -ff -pl -nifi-assembly,-nifi-toolkit/nifi-toolkit-assembly,-nifi-system-tests -nsu
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: surefire-reports-en
+          path: "**/surefire-reports/TEST-*.xml"
+          retention-days: 1
       - name: Disk After Build
         run: |
           df -h
@@ -167,9 +174,16 @@ jobs:
           df -h
       - name: Build with Maven
         env:
-          MAVEN_OPTS: -Xmx2g -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Dmaven.surefire.arguments="-Duser.language=ja -Duser.region=JP -Duser.timezone=Asia/Tokyo" -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+          MAVEN_OPTS: -Xmx2g -XX:ReservedCodeCacheSize=1g -XX:+UseG1GC -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Duser.language=ja -Duser.country=JP -Duser.region=JP -Duser.timezone=Asia/Tokyo -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
         run: |
           mvn -V -T 0.7C package verify -B -Pcontrib-check,include-grpc -Ddir-only -ntp -ff -pl -nifi-assembly,-nifi-toolkit/nifi-toolkit-assembly,-nifi-system-tests -nsu
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: surefire-reports-ja
+          path: "**/surefire-reports/TEST-*.xml"
+          retention-days: 1
       - name: Disk After Build
         run: |
           df -h
@@ -234,6 +248,16 @@ jobs:
           java-version: '8'
       - name: Build with Maven
         env:
-          MAVEN_OPTS: -Xmx2g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Dmaven.surefire.arguments="-Duser.language=fr -Duser.region=FR -Duser.timezone=Europe/Paris" -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
+          MAVEN_OPTS: -Xmx2g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN -Duser.language=fr -Duser.country=FR -Duser.region=FR -Duser.timezone=Europe/Paris -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
         run: |
           mvn -V -T 0.7C package -B -Ddir-only -ntp -ff -pl -nifi-assembly -pl -nifi-system-tests -nsu
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ false }}
+        with:
+          name: surefire-reports-fr
+          path: |
+            '**/surefire-reports/TEST-*.xml'
+            '!**/node_modules/**'
+            '!**/nifi-nar-bundles/nifi-standard-bundle/nifi-jolt-transform-json-ui/**'
+          retention-days: 1

nifi-commons/nifi-security-utils/src/test/groovy/org/apache/nifi/security/xml/XmlUtilsTest.groovy

@@ -94,7 +94,7 @@ class XmlUtilsTest {
 
         // Assert
         logger.expected(msg)
-        assert msg =~ "SAXParseException.* DOCTYPE is disallowed when the feature"
+        assert msg =~ "SAXParseException.*DOCTYPE"
     }
 }
 

nifi-commons/nifi-utils/src/test/java/org/apache/nifi/util/locale/TestLocaleOfTestSuite.java

@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.util.locale;
+
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+
+import java.text.DecimalFormatSymbols;
+import java.text.NumberFormat;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+/**
+ * Testing of the test suite environment {@link java.util.Locale}.  The locales specified
+ * in ".github/workflows/ci-workflow.yml" are exercised.  This test is inert when run in alternate locales.
+ */
+public class TestLocaleOfTestSuite {
+
+    /**
+     * Test behaviors associated with non-standard ".github/workflows/ci-workflow.yml" {@link java.util.Locale} "en-AU".
+     */
+    @Test
+    public void testLocaleCI_EN_AU() {
+        final Locale locale = Locale.getDefault();
+        Assume.assumeTrue(locale.toLanguageTag().equals("en-AU"));
+
+        final String runtimeJavaVersion = System.getProperty("java.version");
+        final boolean isJava8 = Pattern.compile("1\\.8.+").matcher(runtimeJavaVersion).matches();
+        final String expected = (isJava8 ? "E" : "e");  // tested in Java 8 and Java 11
+
+        Assert.assertEquals(expected, DecimalFormatSymbols.getInstance(locale).getExponentSeparator());
+        Assert.assertEquals("1,000", NumberFormat.getInstance(locale).format(1000));
+    }
+
+    /**
+     * Test behaviors associated with ".github/workflows/ci-workflow.yml" {@link java.util.Locale#JAPAN}.
+     */
+    @Test
+    public void testLocaleCI_JA_JP() {
+        final Locale locale = Locale.getDefault();
+        Assume.assumeTrue(locale.toLanguageTag().equals("ja-JP"));
+
+        Assert.assertEquals("E", DecimalFormatSymbols.getInstance(locale).getExponentSeparator());
+        Assert.assertEquals("1,000", NumberFormat.getInstance(locale).format(1000));
+    }
+
+    /**
+     * Test behaviors associated with ".github/workflows/ci-workflow.yml" {@link java.util.Locale#FRANCE}.
+     */
+    @Test
+    public void testLocaleCI_FR_FR() {
+        final Locale locale = Locale.getDefault();
+        Assume.assumeTrue(locale.toLanguageTag().equals("fr-FR"));
+
+        Assert.assertEquals("E", DecimalFormatSymbols.getInstance(locale).getExponentSeparator());
+        Assert.assertEquals("1\u00a0000", NumberFormat.getInstance(locale).format(1000));
+    }
+}

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/components/LongRunningTaskMonitorTest.java

@@ -29,7 +29,9 @@ import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.slf4j.Logger;
 
+import java.text.NumberFormat;
 import java.util.Arrays;
+import java.util.Locale;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -88,8 +90,7 @@ public class LongRunningTaskMonitorTest {
         assertEquals("Long running task detected on processor [id=Processor-1-ID, name=Processor-1-Name, type=Processor-1-Type]. Task time: 60 seconds. Stack trace:\n" + STACKTRACE,
                 logMessages.getAllValues().get(0));
         assertEquals("Long running task detected on processor [id=Processor-2-ID, name=Processor-2-Name, type=Processor-2-Type]. Task time: 1,000 seconds. Stack trace:\n" + STACKTRACE,
-                logMessages.getAllValues().get(1));
+                logMessages.getAllValues().get(1).replace(NumberFormat.getInstance(Locale.getDefault()).format(1000), NumberFormat.getInstance(Locale.US).format(1000)));
-
         ArgumentCaptor<String> controllerBulletinMessages = ArgumentCaptor.forClass(String.class);
         verify(eventReporter, times(2)).reportEvent(eq(Severity.WARNING), eq("Long Running Task"), controllerBulletinMessages.capture());
 

nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/test/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessorTest.java

@@ -137,7 +137,7 @@ public class ConvertExcelToCSVProcessorTest {
         DecimalFormatSymbols decimalFormatSymbols = DecimalFormatSymbols.getInstance();
         char decimalSeparator = decimalFormatSymbols.getDecimalSeparator();
         char groupingSeparator = decimalFormatSymbols.getGroupingSeparator();
-        ff.assertContentEquals("Numbers,Timestamps,Money\n" +
+        ff.assertContentEquals(("Numbers,Timestamps,Money\n" +
                 addQuotingIfNeeded(String.format("1234%1$s456", decimalSeparator)) + "," + DateTimeFormatter.ofPattern("d/M/yy").format(localDt) + "," +
                     addQuotingIfNeeded(String.format("$   123%1$s45", decimalSeparator)) + "\n" +
                 addQuotingIfNeeded(String.format("1234%1$s46", decimalSeparator)) + "," + DateTimeFormatter.ofPattern("hh:mm:ss a").format(localDt) + "," +
@@ -151,7 +151,15 @@ public class ConvertExcelToCSVProcessorTest {
                 addQuotingIfNeeded(String.format("9%1$s88E+08", decimalSeparator)) + "," + DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + "," +
                     addQuotingIfNeeded(String.format("¥   1%2$s023%1$s45", decimalSeparator, groupingSeparator)) + "\n" +
                 addQuotingIfNeeded(String.format("9%1$s877E+08", decimalSeparator)) + ",,\n" +
-                addQuotingIfNeeded(String.format("9%1$s8765E+08", decimalSeparator)) + ",,\n");
+                addQuotingIfNeeded(String.format("9%1$s8765E+08", decimalSeparator)) + ",,\n").replace("E+", getExponentSeparator(decimalFormatSymbols)));
+    }
+
+    /**
+     * Workaround for interaction between {@link DecimalFormatSymbols} and use of custom {@link java.util.Locale}.
+     */
+    private static String getExponentSeparator(final DecimalFormatSymbols decimalFormatSymbols) {
+        final String exponentSeparator = decimalFormatSymbols.getExponentSeparator();
+        return (exponentSeparator.equals("e") ? "e" : exponentSeparator + "+");
     }
 
     @Test
@@ -181,7 +189,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560," + DateTimeFormatter.ofPattern("hh:mm a").format(localDt) + ",£   1%2$s023%1$s45\n" +
                 "9%1$s88E+08," + DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + ",¥   1%2$s023%1$s45\n" +
                 "9%1$s877E+08,,\n" +
-                "9%1$s8765E+08,,\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08,,\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -213,7 +221,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560," + DateTimeFormatter.ofPattern("hh:mm a").format(localDt) + ",£   1%2$s023%1$s45\n" +
                 "9%1$s88E+08," + DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + ",¥   1%2$s023%1$s45\n" +
                 "9%1$s877E+08,,\n" +
-                "9%1$s8765E+08,,\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08,,\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -244,7 +252,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560,£   1%2$s023%1$s45\n" +
                 "9%1$s88E+08,¥   1%2$s023%1$s45\n" +
                 "9%1$s877E+08,\n" +
-                "9%1$s8765E+08,\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08,\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -277,7 +285,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560,£   1%2$s023%1$s45\n" +
                 "9%1$s88E+08,¥   1%2$s023%1$s45\n" +
                 "9%1$s877E+08,\n" +
-                "9%1$s8765E+08,\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08,\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -313,7 +321,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560|" + DateTimeFormatter.ofPattern("hh:mm a").format(localDt) + "|£   1%2$s023%1$s45\r\n" +
                 "9%1$s88E+08|" + DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + "|¥   1%2$s023%1$s45\r\n" +
                 "9%1$s877E+08||\r\n" +
-                "9%1$s8765E+08||\r\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08||\r\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -350,7 +358,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560|" + DateTimeFormatter.ofPattern("hh:mm a").format(localDt) + "|£   1%2$s023%1$s45\n" +
                 "9%1$s88E+08|" + DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + "|¥   1%2$s023%1$s45\n" +
                 "9%1$s877E+08||\n" +
-                "9%1$s8765E+08||\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08||\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -378,7 +386,7 @@ public class ConvertExcelToCSVProcessorTest {
         DecimalFormatSymbols decimalFormatSymbols = DecimalFormatSymbols.getInstance();
         char decimalSeparator = decimalFormatSymbols.getDecimalSeparator();
         char groupingSeparator = decimalFormatSymbols.getGroupingSeparator();
-        ff.assertContentEquals("'Numbers','Timestamps','Money'\n" +
+        ff.assertContentEquals(("'Numbers','Timestamps','Money'\n" +
                 addQuotingIfNeeded(String.format("1234%1$s456", decimalSeparator), ",", quoteCharValue, true) + "," + quoteCharValue +
                     DateTimeFormatter.ofPattern("d/M/yy").format(localDt) + quoteCharValue + "," +
                     addQuotingIfNeeded(String.format("$   123%1$s45", decimalSeparator), ",", quoteCharValue, true) + "\n" +
@@ -398,7 +406,7 @@ public class ConvertExcelToCSVProcessorTest {
                     DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + quoteCharValue + "," +
                     addQuotingIfNeeded(String.format("¥   1%2$s023%1$s45", decimalSeparator, groupingSeparator), ",", quoteCharValue, true) + "\n" +
                 addQuotingIfNeeded(String.format("9%1$s877E+08", decimalSeparator), ",", quoteCharValue, true) + ",,\n" +
-                addQuotingIfNeeded(String.format("9%1$s8765E+08", decimalSeparator), ",", quoteCharValue, true) + ",,\n");
+                addQuotingIfNeeded(String.format("9%1$s8765E+08", decimalSeparator), ",", quoteCharValue, true) + ",,\n").replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     @Test
@@ -435,7 +443,7 @@ public class ConvertExcelToCSVProcessorTest {
                 "1%2$s234%1$s4560," + DateTimeFormatter.ofPattern("hh:mm a").format(localDt) + ",£   1%2$s023%1$s45\n" +
                 "9%1$s88E+08," + DateTimeFormatter.ofPattern("yyyy/MM/dd/ HH:mm").format(localDt) + ",¥   1%2$s023%1$s45\n" +
                 "9%1$s877E+08,,\n" +
-                "9%1$s8765E+08,,\n", decimalSeparator, groupingSeparator));
+                "9%1$s8765E+08,,\n", decimalSeparator, groupingSeparator).replace("E+", getExponentSeparator(decimalFormatSymbols)));
     }
 
     /**

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java

@@ -40,6 +40,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
@@ -556,7 +557,8 @@ public class InvokeHTTP extends AbstractProcessor {
             REL_SUCCESS_REQ, REL_RESPONSE, REL_RETRY, REL_NO_RETRY, REL_FAILURE)));
 
     // RFC 2616 Date Time Formatter with hard-coded GMT Zone
-    private static final DateTimeFormatter RFC_2616_DATE_TIME = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'");
+    // https://tools.ietf.org/html/rfc2616#section-3.3 - date format header should not be localized
+    private static final DateTimeFormatter RFC_2616_DATE_TIME = DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US);
 
     // Multiple Header Delimiter
     private static final String MULTIPLE_HEADER_DELIMITER = ", ";

pom.xml

@@ -435,7 +435,10 @@
                         <redirectTestOutputToFile>true</redirectTestOutputToFile>
                         <argLine combine.children="append">-Xmx1G
                             -Djava.net.preferIPv4Stack=true
-                            ${maven.surefire.arguments}
+                            -Duser.language=${user.language}
+                            -Duser.country=${user.country}
+                            -Duser.region=${user.region}
+                            -Duser.timezone=${user.timezone}
                             -Dfile.encoding=UTF-8</argLine>
                     </configuration>
                     <dependencies>