Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-5475: Upgraded Hive 3 bundle to Apache Hive 3.1.0

This commit is contained in:
Matthew Burgess 2018-07-31 12:51:46 -04:00 committed by Matt Gilman
parent e62aa0252d
commit b4894c5572
No known key found for this signature in database
GPG Key ID: DF61EC19432AEE37
3 changed files with 16 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java
View File

@ -24,8 +24,8 @@ import org.apache.avro.util.Utf8;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
@ -113,10 +113,16 @@ public class NiFiOrcUtils {
return new BytesWritable(((ByteBuffer) o).array());
}
if (o instanceof Timestamp) {
return new TimestampWritable((Timestamp) o);
Timestamp t = (Timestamp) o;
org.apache.hadoop.hive.common.type.Timestamp timestamp = new org.apache.hadoop.hive.common.type.Timestamp();
timestamp.setTimeInMillis(t.getTime(), t.getNanos());
return new TimestampWritableV2(timestamp);
}
if (o instanceof Date) {
return new DateWritable((Date) o);
Date d = (Date) o;
org.apache.hadoop.hive.common.type.Date date = new org.apache.hadoop.hive.common.type.Date();
date.setTimeInMillis(d.getTime());
return new DateWritableV2(date);
}
if (o instanceof Object[]) {
Object[] objArray = (Object[]) o;

8
nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java
View File

@ -25,9 +25,9 @@ import org.apache.hadoop.hive.ql.io.orc.OrcFile;
import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
import org.apache.hadoop.hive.ql.io.orc.Reader;
import org.apache.hadoop.hive.ql.io.orc.RecordReader;
import org.apache.hadoop.hive.serde2.io.DateWritable;
import org.apache.hadoop.hive.serde2.io.DateWritableV2;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@ -246,8 +246,8 @@ public class PutORCTest {
verifyORCUsers(orcFile, 10, (x, currUser) -> {
assertEquals((int) currUser, ((IntWritable) x.get(0)).get());
assertEquals(timeMillis, ((IntWritable) x.get(1)).get());
assertEquals(timestampMillis, ((TimestampWritable) x.get(2)).getTimestamp());
assertEquals(dt, ((DateWritable) x.get(3)).get());
assertEquals(timestampMillis, ((TimestampWritableV2) x.get(2)).getTimestamp().toSqlTimestamp());
assertEquals(dt.toLocalDate().toEpochDay(), ((DateWritableV2) x.get(3)).get().toEpochDay());
assertEquals(dec, ((DoubleWritable) x.get(4)).get(), Double.MIN_VALUE);
return null;
}

5
nifi-nar-bundles/nifi-hive-bundle/pom.xml
View File

@ -38,9 +38,8 @@
<properties>
<hive.version>1.2.1</hive.version>
<hive.hadoop.version>2.6.2</hive.hadoop.version>
<hive3.version>3.0.0</hive3.version>
<hive3.hadoop.version>3.0.0</hive3.hadoop.version>
<orc.version>1.4.3</orc.version>
<hive3.version>3.1.0</hive3.version>
<hive3.hadoop.version>3.1.0</hive3.hadoop.version>
</properties>
<build>