Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-2531: Fixed JdbcCommon handling of BigInteger objects for Avro 

This closes #823.

Signed-off-by: Bryan Bende <bbende@apache.org>
This commit is contained in:
Matt Burgess 2016-08-10 14:54:05 -04:00 committed by Bryan Bende
parent 2938a5353a
commit d544274881
No known key found for this signature in database
GPG Key ID: A0DDA9ED50711C39
1 changed files with 35 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/util/JdbcCommon.java
View File

@ -68,6 +68,8 @@ import org.apache.commons.lang3.StringUtils;
*/
public class JdbcCommon {
private static final int MAX_DIGITS_IN_BIGINT = 19;
public static long convertToAvroStream(final ResultSet rs, final OutputStream outStream) throws SQLException, IOException {
return convertToAvroStream(rs, outStream, null, null);
}
@ -114,10 +116,32 @@ public class JdbcCommon {
// org.apache.avro.AvroRuntimeException: Unknown datum type java.lang.Byte
rec.put(i - 1, ((Byte) value).intValue());
} else if (value instanceof BigDecimal || value instanceof BigInteger) {
// Avro can't handle BigDecimal and BigInteger as numbers - it will throw an AvroRuntimeException such as: "Unknown datum type: java.math.BigDecimal: 38"
} else if (value instanceof BigDecimal) {
// Avro can't handle BigDecimal as a number - it will throw an AvroRuntimeException such as: "Unknown datum type: java.math.BigDecimal: 38"
rec.put(i - 1, value.toString());
} else if (value instanceof BigInteger) {
// Check the precision of the BIGINT. Some databases allow arbitrary precision (> 19), but Avro won't handle that.
// It the SQL type is BIGINT and the precision is between 0 and 19 (inclusive); if so, the BigInteger is likely a
// long (and the schema says it will be), so try to get its value as a long.
// Otherwise, Avro can't handle BigInteger as a number - it will throw an AvroRuntimeException
// such as: "Unknown datum type: java.math.BigInteger: 38". In this case the schema is expecting a string.
int precision = meta.getPrecision(i);
if (javaSqlType == BIGINT) {
if (precision < 0 || precision > MAX_DIGITS_IN_BIGINT) {
rec.put(i - 1, value.toString());
} else {
try {
rec.put(i - 1, ((BigInteger) value).longValueExact());
} catch (ArithmeticException ae) {
// Since the value won't fit in a long, convert it to a string
rec.put(i - 1, value.toString());
}
}
} else {
rec.put(i - 1, value.toString());
}
} else if (value instanceof Number || value instanceof Boolean) {
rec.put(i - 1, value);
@ -196,7 +220,15 @@ public class JdbcCommon {
break;
case BIGINT:
builder.name(meta.getColumnName(i)).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault();
// Check the precision of the BIGINT. Some databases allow arbitrary precision (> 19), but Avro won't handle that.
// If the precision > 19 (or is negative), use a string for the type, otherwise use a long. The object(s) will be converted
// to strings as necessary
int precision = meta.getPrecision(i);
if (precision < 0 || precision > MAX_DIGITS_IN_BIGINT) {
builder.name(meta.getColumnName(i)).type().unionOf().nullBuilder().endNull().and().stringType().endUnion().noDefault();
} else {
builder.name(meta.getColumnName(i)).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault();
}
break;
// java.sql.RowId is interface, is seems to be database