Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9902 - Improve handling of charsets when performing string/byte conversions 

This closes #5954

Signed-off-by: Mike Thomsen <mthomsen@apache.org>
This commit is contained in:
Paul Grey 2022-04-11 11:35:37 -04:00 committed by Mike Thomsen
parent 157a5d342a
commit 900a34971c
No known key found for this signature in database
GPG Key ID: 88511C3D4CAD246F
5 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
nifi-mock/src/main/java/org/apache/nifi/util/MockFlowFile.java
View File

@ -272,7 +272,7 @@ public class MockFlowFile implements FlowFileRecord {
}
public void assertContentEquals(final String data) {
assertContentEquals(data, "UTF-8");
assertContentEquals(data, StandardCharsets.UTF_8);
}
public void assertContentEquals(final String data, final String charset) {

3
nifi-nar-bundles/nifi-asn1-bundle/nifi-asn1-services/src/test/java/org/apache/nifi/jasn1/TestJASN1RecordReaderWithSimpleTypes.java
View File

@ -46,6 +46,7 @@ import org.apache.nifi.serialization.record.RecordSchema;
import org.junit.jupiter.api.Test;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
@ -156,7 +157,7 @@ public class TestJASN1RecordReaderWithSimpleTypes implements JASN1ReadRecordTest
String dataFile = "target/bmpstring_wrapper.dat";
BMPStringWrapper berValue = new BMPStringWrapper();
berValue.setValue(new BerBMPString("Some UTF-8 String. こんにちは世界。".getBytes()));
berValue.setValue(new BerBMPString("Some UTF-8 String. こんにちは世界。".getBytes(StandardCharsets.UTF_8)));
Map<String, Object> expectedValues = new HashMap<String, Object>() {{
put("value", "Some UTF-8 String. こんにちは世界。");

9
nifi-nar-bundles/nifi-extension-utils/nifi-record-utils/nifi-mock-record-utils/src/main/java/org/apache/nifi/serialization/record/MockRecordWriter.java
View File

@ -28,6 +28,7 @@ import org.apache.nifi.serialization.WriteResult;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Map;
@ -85,7 +86,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
@Override
public WriteResult write(final RecordSet rs) throws IOException {
if (header != null && !headerWritten) {
out.write(header.getBytes());
out.write(header.getBytes(StandardCharsets.UTF_8));
out.write("\n".getBytes());
headerWritten = true;
}
@ -105,10 +106,10 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
if (val != null) {
if (quoteValues) {
out.write("\"".getBytes());
out.write(val.getBytes());
out.write(val.getBytes(StandardCharsets.UTF_8));
out.write("\"".getBytes());
} else {
out.write(val.getBytes());
out.write(val.getBytes(StandardCharsets.UTF_8));
}
}
@ -134,7 +135,7 @@ public class MockRecordWriter extends AbstractControllerService implements Recor
}
if (header != null && !headerWritten) {
out.write(header.getBytes());
out.write(header.getBytes(StandardCharsets.UTF_8));
out.write("\n".getBytes());
headerWritten = true;
}

5
nifi-nar-bundles/nifi-poi-bundle/nifi-poi-processors/src/main/java/org/apache/nifi/processors/poi/ConvertExcelToCSVProcessor.java
View File

@ -21,6 +21,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
@ -316,7 +317,7 @@ public class ConvertExcelToCSVProcessor
ff = session.write(ff, new OutputStreamCallback() {
@Override
public void process(OutputStream out) throws IOException {
PrintStream outPrint = new PrintStream(out);
PrintStream outPrint = new PrintStream(out, false, StandardCharsets.UTF_8.name());
sheetHandler.setOutput(outPrint);
try {
@ -386,7 +387,7 @@ public class ConvertExcelToCSVProcessor
}
public void setOutput(PrintStream output){
final OutputStreamWriter streamWriter = new OutputStreamWriter(output);
final OutputStreamWriter streamWriter = new OutputStreamWriter(output, StandardCharsets.UTF_8);
try {
printer = new CSVPrinter(streamWriter, csvFormat);

4
nifi-nar-bundles/nifi-slack-bundle/nifi-slack-processors/src/test/java/org/apache/nifi/processors/slack/PostSlackTextMessageTest.java
View File

@ -28,6 +28,7 @@ import javax.json.Json;
import javax.json.JsonObject;
import javax.json.JsonReader;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import static org.junit.jupiter.api.Assertions.assertEquals;
@ -282,7 +283,8 @@ public class PostSlackTextMessageTest {
private JsonObject getRequestBodyJson() {
try {
final RecordedRequest recordedRequest = mockWebServer.takeRequest();
try (final JsonReader reader = Json.createReader(new InputStreamReader(recordedRequest.getBody().inputStream()))) {
try (final JsonReader reader = Json.createReader(new InputStreamReader(
recordedRequest.getBody().inputStream(), StandardCharsets.UTF_8))) {
return reader.readObject();
}
} catch (final InterruptedException e) {