hiveFields = recordFields.stream().map(
- recordField -> ("`" + (hiveFieldNames ? recordField.getFieldName().toLowerCase() : recordField.getFieldName()) + "`:"
- + getHiveTypeFromFieldType(recordField.getDataType(), hiveFieldNames))).collect(Collectors.toList());
- return "STRUCT<" + StringUtils.join(hiveFields, ", ") + ">";
- }
- return null;
- }
-
- throw new IllegalArgumentException("Error converting Avro type " + dataType.name() + " to Hive type");
- }
-
-
- public static OrcFlowFileWriter createWriter(OutputStream flowFileOutputStream,
- Path path,
- Configuration conf,
- TypeInfo orcSchema,
- long stripeSize,
- CompressionKind compress,
- int bufferSize) throws IOException {
-
- int rowIndexStride = HiveConf.getIntVar(conf, HIVE_ORC_DEFAULT_ROW_INDEX_STRIDE);
-
- boolean addBlockPadding = HiveConf.getBoolVar(conf, HIVE_ORC_DEFAULT_BLOCK_PADDING);
-
- String versionName = HiveConf.getVar(conf, HIVE_ORC_WRITE_FORMAT);
- OrcFile.Version versionValue = (versionName == null)
- ? OrcFile.Version.CURRENT
- : OrcFile.Version.byName(versionName);
-
- OrcFile.EncodingStrategy encodingStrategy;
- String enString = conf.get(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname);
- if (enString == null) {
- encodingStrategy = OrcFile.EncodingStrategy.SPEED;
- } else {
- encodingStrategy = OrcFile.EncodingStrategy.valueOf(enString);
- }
-
- OrcFile.CompressionStrategy compressionStrategy;
- String compString = conf.get(HiveConf.ConfVars.HIVE_ORC_COMPRESSION_STRATEGY.varname);
- if (compString == null) {
- compressionStrategy = OrcFile.CompressionStrategy.SPEED;
- } else {
- compressionStrategy = OrcFile.CompressionStrategy.valueOf(compString);
- }
-
- float paddingTolerance;
- paddingTolerance = conf.getFloat(HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.varname,
- HiveConf.ConfVars.HIVE_ORC_BLOCK_PADDING_TOLERANCE.defaultFloatVal);
-
- long blockSizeValue = HiveConf.getLongVar(conf, HIVE_ORC_DEFAULT_BLOCK_SIZE);
-
- double bloomFilterFpp = BloomFilterIO.DEFAULT_FPP;
-
- ObjectInspector inspector = OrcStruct.createObjectInspector(orcSchema);
-
- return new OrcFlowFileWriter(flowFileOutputStream,
- path,
- conf,
- inspector,
- stripeSize,
- compress,
- bufferSize,
- rowIndexStride,
- getMemoryManager(conf),
- addBlockPadding,
- versionValue,
- null, // no callback
- encodingStrategy,
- compressionStrategy,
- paddingTolerance,
- blockSizeValue,
- null, // no Bloom Filter column names
- bloomFilterFpp);
- }
-
- private static MemoryManager memoryManager = null;
-
- private static synchronized MemoryManager getMemoryManager(Configuration conf) {
- if (memoryManager == null) {
- memoryManager = new MemoryManager(conf);
- }
- return memoryManager;
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java
deleted file mode 100644
index c868b91a52..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/OrcFlowFileWriter.java
+++ /dev/null
@@ -1,2649 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.io.orc;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedOutputStream;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.io.IOConstants;
-import org.apache.hadoop.hive.ql.io.filters.BloomFilterIO;
-import org.apache.hadoop.hive.ql.io.orc.CompressionCodec.Modifier;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.CompressionStrategy;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile.EncodingStrategy;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndexEntry;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeStatistics;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.Type;
-import org.apache.hadoop.hive.ql.io.orc.OrcProto.UserMetadataItem;
-import org.apache.hadoop.hive.ql.util.JavaDataModel;
-import org.apache.hadoop.hive.serde2.io.DateWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.nifi.stream.io.ByteCountingOutputStream;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.management.ManagementFactory;
-import java.nio.ByteBuffer;
-import java.sql.Timestamp;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-
-/**
- * An ORC file writer. The file is divided into stripes, which is the natural
- * unit of work when reading. Each stripe is buffered in memory until the
- * memory reaches the stripe size and then it is written out broken down by
- * columns. Each column is written by a TreeWriter that is specific to that
- * type of column. TreeWriters may have children TreeWriters that handle the
- * sub-types. Each of the TreeWriters writes the column's data as a set of
- * streams.
- *
- * This class is synchronized so that multi-threaded access is ok. In
- * particular, because the MemoryManager is shared between writers, this class
- * assumes that checkMemory may be called from a separate thread.
- */
-public class OrcFlowFileWriter implements Writer, MemoryManager.Callback {
-
- private static final Log LOG = LogFactory.getLog(WriterImpl.class);
-
- private static final int HDFS_BUFFER_SIZE = 256 * 1024;
- private static final int MIN_ROW_INDEX_STRIDE = 1000;
-
- // threshold above which buffer size will be automatically resized
- private static final int COLUMN_COUNT_THRESHOLD = 1000;
-
- private final Path path;
- private final long defaultStripeSize;
- private long adjustedStripeSize;
- private final int rowIndexStride;
- private final CompressionKind compress;
- private final CompressionCodec codec;
- private final boolean addBlockPadding;
- private final int bufferSize;
- private final long blockSize;
- private final float paddingTolerance;
- // the streams that make up the current stripe
- private final Map streams = new TreeMap<>();
-
- private final OutputStream flowFileOutputStream;
-
- private ByteCountingOutputStream rawWriter = null;
- // the compressed metadata information outStream
- private OutStream writer = null;
- // a protobuf outStream around streamFactory
- private CodedOutputStream protobufWriter = null;
- private long headerLength;
- private int columnCount;
- private long rowCount = 0;
- private long rowsInStripe = 0;
- private long rawDataSize = 0;
- private int rowsInIndex = 0;
- private int stripesAtLastFlush = -1;
- private final List stripes = new ArrayList<>();
- private final Map userMetadata = new TreeMap<>();
- private final StreamFactory streamFactory = new StreamFactory();
- private final TreeWriter treeWriter;
- private final boolean buildIndex;
- private final MemoryManager memoryManager;
- private final OrcFile.Version version;
- private final Configuration conf;
- private final OrcFile.WriterCallback callback;
- private final OrcFile.WriterContext callbackContext;
- private final OrcFile.EncodingStrategy encodingStrategy;
- private final OrcFile.CompressionStrategy compressionStrategy;
- private final boolean[] bloomFilterColumns;
- private final double bloomFilterFpp;
-
-
- public OrcFlowFileWriter(OutputStream flowFileOutputStream,
- Path path,
- Configuration conf,
- ObjectInspector inspector,
- long stripeSize,
- CompressionKind compress,
- int bufferSize,
- int rowIndexStride,
- MemoryManager memoryManager,
- boolean addBlockPadding,
- OrcFile.Version version,
- OrcFile.WriterCallback callback,
- EncodingStrategy encodingStrategy,
- CompressionStrategy compressionStrategy,
- float paddingTolerance,
- long blockSizeValue,
- String bloomFilterColumnNames,
- double bloomFilterFpp) throws IOException {
- this.flowFileOutputStream = flowFileOutputStream;
- this.path = path;
- this.conf = conf;
- this.callback = callback;
- callbackContext = (callback != null) ? () -> OrcFlowFileWriter.this : null;
- this.adjustedStripeSize = stripeSize;
- this.defaultStripeSize = stripeSize;
- this.version = version;
- this.encodingStrategy = encodingStrategy;
- this.compressionStrategy = compressionStrategy;
- this.addBlockPadding = addBlockPadding;
- this.blockSize = blockSizeValue;
- this.paddingTolerance = paddingTolerance;
- this.compress = compress;
- this.rowIndexStride = rowIndexStride;
- this.memoryManager = memoryManager;
- buildIndex = rowIndexStride > 0;
- codec = createCodec(compress);
- String allColumns = conf.get(IOConstants.COLUMNS);
- if (allColumns == null) {
- allColumns = getColumnNamesFromInspector(inspector);
- }
- this.bufferSize = getEstimatedBufferSize(allColumns, bufferSize);
- if (version == OrcFile.Version.V_0_11) {
- /* do not write bloom filters for ORC v11 */
- this.bloomFilterColumns =
- OrcUtils.includeColumns(null, allColumns, inspector);
- } else {
- this.bloomFilterColumns =
- OrcUtils.includeColumns(bloomFilterColumnNames, allColumns, inspector);
- }
- this.bloomFilterFpp = bloomFilterFpp;
- treeWriter = createTreeWriter(inspector, streamFactory, false);
- if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE) {
- throw new IllegalArgumentException("Row stride must be at least " +
- MIN_ROW_INDEX_STRIDE);
- }
-
- // ensure that we are able to handle callbacks before we register ourselves
- memoryManager.addWriter(path, stripeSize, this);
- }
-
- private String getColumnNamesFromInspector(ObjectInspector inspector) {
- List fieldNames = Lists.newArrayList();
- Joiner joiner = Joiner.on(",");
- if (inspector instanceof StructObjectInspector) {
- StructObjectInspector soi = (StructObjectInspector) inspector;
- List fields = soi.getAllStructFieldRefs();
- fieldNames.addAll(fields.stream().map((Function) StructField::getFieldName).collect(Collectors.toList()));
- }
- return joiner.join(fieldNames);
- }
-
- @VisibleForTesting
- int getEstimatedBufferSize(int bs) {
- return getEstimatedBufferSize(conf.get(IOConstants.COLUMNS), bs);
- }
-
- int getEstimatedBufferSize(String colNames, int bs) {
- long availableMem = getMemoryAvailableForORC();
- if (colNames != null) {
- final int numCols = colNames.split(",").length;
- if (numCols > COLUMN_COUNT_THRESHOLD) {
- // In BufferedStream, there are 3 outstream buffers (compressed,
- // uncompressed and overflow) and list of previously compressed buffers.
- // Since overflow buffer is rarely used, lets consider only 2 allocation.
- // Also, initially, the list of compression buffers will be empty.
- final int outStreamBuffers = codec == null ? 1 : 2;
-
- // max possible streams per column is 5. For string columns, there is
- // ROW_INDEX, PRESENT, DATA, LENGTH, DICTIONARY_DATA streams.
- final int maxStreams = 5;
-
- // Lets assume 10% memory for holding dictionary in memory and other
- // object allocations
- final long miscAllocation = (long) (0.1f * availableMem);
-
- // compute the available memory
- final long remainingMem = availableMem - miscAllocation;
-
- int estBufferSize = (int) (remainingMem / (maxStreams * outStreamBuffers * numCols));
- estBufferSize = getClosestBufferSize(estBufferSize, bs);
- if (estBufferSize > bs) {
- estBufferSize = bs;
- }
-
- LOG.info("WIDE TABLE - Number of columns: " + numCols + " Chosen compression buffer size: " + estBufferSize);
- return estBufferSize;
- }
- }
- return bs;
- }
-
- private int getClosestBufferSize(int estBufferSize, int bs) {
- final int kb4 = 4 * 1024;
- final int kb8 = 8 * 1024;
- final int kb16 = 16 * 1024;
- final int kb32 = 32 * 1024;
- final int kb64 = 64 * 1024;
- final int kb128 = 128 * 1024;
- final int kb256 = 256 * 1024;
- if (estBufferSize <= kb4) {
- return kb4;
- } else if (estBufferSize > kb4 && estBufferSize <= kb8) {
- return kb8;
- } else if (estBufferSize > kb8 && estBufferSize <= kb16) {
- return kb16;
- } else if (estBufferSize > kb16 && estBufferSize <= kb32) {
- return kb32;
- } else if (estBufferSize > kb32 && estBufferSize <= kb64) {
- return kb64;
- } else if (estBufferSize > kb64 && estBufferSize <= kb128) {
- return kb128;
- } else {
- return kb256;
- }
- }
-
- // the assumption is only one ORC writer open at a time, which holds true for
- // most of the cases. HIVE-6455 forces single writer case.
- private long getMemoryAvailableForORC() {
- HiveConf.ConfVars poolVar = HiveConf.ConfVars.HIVE_ORC_FILE_MEMORY_POOL;
- double maxLoad = conf.getFloat(poolVar.varname, poolVar.defaultFloatVal);
- long totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax() * maxLoad);
- return totalMemoryPool;
- }
-
- public static CompressionCodec createCodec(CompressionKind kind) {
- switch (kind) {
- case NONE:
- return null;
- case ZLIB:
- return new ZlibCodec();
- case SNAPPY:
- return new SnappyCodec();
- case LZO:
- try {
- Class lzo =
- (Class)
- JavaUtils.loadClass("org.apache.hadoop.hive.ql.io.orc.LzoCodec");
- return lzo.newInstance();
- } catch (ClassNotFoundException e) {
- throw new IllegalArgumentException("LZO is not available.", e);
- } catch (InstantiationException e) {
- throw new IllegalArgumentException("Problem initializing LZO", e);
- } catch (IllegalAccessException e) {
- throw new IllegalArgumentException("Insufficient access to LZO", e);
- }
- default:
- throw new IllegalArgumentException("Unknown compression codec: " +
- kind);
- }
- }
-
- @Override
- public synchronized boolean checkMemory(double newScale) throws IOException {
- long limit = Math.round(adjustedStripeSize * newScale);
- long size = estimateStripeSize();
- if (LOG.isDebugEnabled()) {
- LOG.debug("ORC writer " + path + " size = " + size + " limit = " +
- limit);
- }
- if (size > limit) {
- flushStripe();
- return true;
- }
- return false;
- }
-
- /**
- * This class is used to hold the contents of streams as they are buffered.
- * The TreeWriters write to the outStream and the codec compresses the
- * data as buffers fill up and stores them in the output list. When the
- * stripe is being written, the whole stream is written to the file.
- */
- private class BufferedStream implements OutStream.OutputReceiver {
- private final OutStream outStream;
- private final List output = new ArrayList();
-
- BufferedStream(String name, int bufferSize,
- CompressionCodec codec) throws IOException {
- outStream = new OutStream(name, bufferSize, codec, this);
- }
-
- /**
- * Receive a buffer from the compression codec.
- *
- * @param buffer the buffer to save
- * @throws IOException if an error occurs while receiving a buffer
- */
- @Override
- public void output(ByteBuffer buffer) {
- output.add(buffer);
- }
-
- /**
- * Get the number of bytes in buffers that are allocated to this stream.
- *
- * @return number of bytes in buffers
- */
- public long getBufferSize() {
- long result = 0;
- for (ByteBuffer buf : output) {
- result += buf.capacity();
- }
- return outStream.getBufferSize() + result;
- }
-
- /**
- * Flush the stream to the codec.
- *
- * @throws IOException if an error occurs while flushing the stream
- */
- public void flush() throws IOException {
- outStream.flush();
- }
-
- /**
- * Clear all of the buffers.
- *
- * @throws IOException if an error occurs while clearing the buffers
- */
- public void clear() throws IOException {
- outStream.clear();
- output.clear();
- }
-
- /**
- * Check the state of suppress flag in output stream
- *
- * @return value of suppress flag
- */
- public boolean isSuppressed() {
- return outStream.isSuppressed();
- }
-
- /**
- * Get the number of bytes that will be written to the output. Assumes
- * the stream has already been flushed.
- *
- * @return the number of bytes
- */
- public long getOutputSize() {
- long result = 0;
- for (ByteBuffer buffer : output) {
- result += buffer.remaining();
- }
- return result;
- }
-
- /**
- * Write the saved compressed buffers to the OutputStream.
- *
- * @param out the stream to write to
- * @throws IOException if an error occurs during write
- */
- void spillTo(OutputStream out) throws IOException {
- for (ByteBuffer buffer : output) {
- out.write(buffer.array(), buffer.arrayOffset() + buffer.position(),
- buffer.remaining());
- }
- }
-
- @Override
- public String toString() {
- return outStream.toString();
- }
- }
-
- /**
- * An output receiver that writes the ByteBuffers to the output stream
- * as they are received.
- */
- private class DirectStream implements OutStream.OutputReceiver {
- private final OutputStream output;
-
- DirectStream(OutputStream output) {
- this.output = output;
- }
-
- @Override
- public void output(ByteBuffer buffer) throws IOException {
- output.write(buffer.array(), buffer.arrayOffset() + buffer.position(),
- buffer.remaining());
- }
- }
-
- private static class RowIndexPositionRecorder implements PositionRecorder {
- private final OrcProto.RowIndexEntry.Builder builder;
-
- RowIndexPositionRecorder(OrcProto.RowIndexEntry.Builder builder) {
- this.builder = builder;
- }
-
- @Override
- public void addPosition(long position) {
- builder.addPositions(position);
- }
- }
-
- /**
- * Interface from the Writer to the TreeWriters. This limits the visibility
- * that the TreeWriters have into the Writer.
- */
- private class StreamFactory {
- /**
- * Create a stream to store part of a column.
- *
- * @param column the column id for the stream
- * @param kind the kind of stream
- * @return The output outStream that the section needs to be written to.
- * @throws IOException if an error occurs while creating the stream
- */
- public OutStream createStream(int column,
- OrcProto.Stream.Kind kind
- ) throws IOException {
- final StreamName name = new StreamName(column, kind);
- final EnumSet modifiers;
-
- switch (kind) {
- case BLOOM_FILTER:
- case DATA:
- case DICTIONARY_DATA:
- if (getCompressionStrategy() == CompressionStrategy.SPEED) {
- modifiers = EnumSet.of(Modifier.FAST, Modifier.TEXT);
- } else {
- modifiers = EnumSet.of(Modifier.DEFAULT, Modifier.TEXT);
- }
- break;
- case LENGTH:
- case DICTIONARY_COUNT:
- case PRESENT:
- case ROW_INDEX:
- case SECONDARY:
- // easily compressed using the fastest modes
- modifiers = EnumSet.of(Modifier.FASTEST, Modifier.BINARY);
- break;
- default:
- LOG.warn("Missing ORC compression modifiers for " + kind);
- modifiers = null;
- break;
- }
-
- BufferedStream result = streams.get(name);
- if (result == null) {
- result = new BufferedStream(name.toString(), bufferSize,
- codec == null ? codec : codec.modify(modifiers));
- streams.put(name, result);
- }
- return result.outStream;
- }
-
- /**
- * Get the next column id.
- *
- * @return a number from 0 to the number of columns - 1
- */
- public int getNextColumnId() {
- return columnCount++;
- }
-
- /**
- * Get the current column id. After creating all tree writers this count should tell how many
- * columns (including columns within nested complex objects) are created in total.
- *
- * @return current column id
- */
- public int getCurrentColumnId() {
- return columnCount;
- }
-
- /**
- * Get the stride rate of the row index.
- */
- public int getRowIndexStride() {
- return rowIndexStride;
- }
-
- /**
- * Should be building the row index.
- *
- * @return true if we are building the index
- */
- public boolean buildIndex() {
- return buildIndex;
- }
-
- /**
- * Is the ORC file compressed?
- *
- * @return are the streams compressed
- */
- public boolean isCompressed() {
- return codec != null;
- }
-
- /**
- * Get the encoding strategy to use.
- *
- * @return encoding strategy
- */
- public EncodingStrategy getEncodingStrategy() {
- return encodingStrategy;
- }
-
- /**
- * Get the compression strategy to use.
- *
- * @return compression strategy
- */
- public CompressionStrategy getCompressionStrategy() {
- return compressionStrategy;
- }
-
- /**
- * Get the bloom filter columns
- *
- * @return bloom filter columns
- */
- public boolean[] getBloomFilterColumns() {
- return bloomFilterColumns;
- }
-
- /**
- * Get bloom filter false positive percentage.
- *
- * @return fpp
- */
- public double getBloomFilterFPP() {
- return bloomFilterFpp;
- }
-
- /**
- * Get the writer's configuration.
- *
- * @return configuration
- */
- public Configuration getConfiguration() {
- return conf;
- }
-
- /**
- * Get the version of the file to write.
- */
- public OrcFile.Version getVersion() {
- return version;
- }
- }
-
- /**
- * The parent class of all of the writers for each column. Each column
- * is written by an instance of this class. The compound types (struct,
- * list, map, and union) have children tree writers that write the children
- * types.
- */
- private abstract static class TreeWriter {
- protected final int id;
- protected final ObjectInspector inspector;
- private final BitFieldWriter isPresent;
- private final boolean isCompressed;
- protected final ColumnStatisticsImpl indexStatistics;
- protected final ColumnStatisticsImpl stripeColStatistics;
- private final ColumnStatisticsImpl fileStatistics;
- protected TreeWriter[] childrenWriters;
- protected final RowIndexPositionRecorder rowIndexPosition;
- private final OrcProto.RowIndex.Builder rowIndex;
- private final OrcProto.RowIndexEntry.Builder rowIndexEntry;
- private final PositionedOutputStream rowIndexStream;
- private final PositionedOutputStream bloomFilterStream;
- protected final BloomFilterIO bloomFilter;
- protected final boolean createBloomFilter;
- private final OrcProto.BloomFilterIndex.Builder bloomFilterIndex;
- private final OrcProto.BloomFilter.Builder bloomFilterEntry;
- private boolean foundNulls;
- private OutStream isPresentOutStream;
- private final List stripeStatsBuilders;
-
- /**
- * Create a tree writer.
- *
- * @param columnId the column id of the column to write
- * @param inspector the object inspector to use
- * @param streamFactory limited access to the Writer's data.
- * @param nullable can the value be null?
- * @throws IOException if an error occurs during creation
- */
- TreeWriter(int columnId, ObjectInspector inspector,
- StreamFactory streamFactory,
- boolean nullable) throws IOException {
- this.isCompressed = streamFactory.isCompressed();
- this.id = columnId;
- this.inspector = inspector;
- if (nullable) {
- isPresentOutStream = streamFactory.createStream(id,
- OrcProto.Stream.Kind.PRESENT);
- isPresent = new BitFieldWriter(isPresentOutStream, 1);
- } else {
- isPresent = null;
- }
- this.foundNulls = false;
- createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
- indexStatistics = ColumnStatisticsImpl.create(inspector);
- stripeColStatistics = ColumnStatisticsImpl.create(inspector);
- fileStatistics = ColumnStatisticsImpl.create(inspector);
- childrenWriters = new TreeWriter[0];
- rowIndex = OrcProto.RowIndex.newBuilder();
- rowIndexEntry = OrcProto.RowIndexEntry.newBuilder();
- rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
- stripeStatsBuilders = Lists.newArrayList();
- if (streamFactory.buildIndex()) {
- rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Kind.ROW_INDEX);
- } else {
- rowIndexStream = null;
- }
- if (createBloomFilter) {
- bloomFilterEntry = OrcProto.BloomFilter.newBuilder();
- bloomFilterIndex = OrcProto.BloomFilterIndex.newBuilder();
- bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Kind.BLOOM_FILTER);
- bloomFilter = new BloomFilterIO(streamFactory.getRowIndexStride(),
- streamFactory.getBloomFilterFPP());
- } else {
- bloomFilterEntry = null;
- bloomFilterIndex = null;
- bloomFilterStream = null;
- bloomFilter = null;
- }
- }
-
- protected OrcProto.RowIndex.Builder getRowIndex() {
- return rowIndex;
- }
-
- protected ColumnStatisticsImpl getStripeStatistics() {
- return stripeColStatistics;
- }
-
- protected ColumnStatisticsImpl getFileStatistics() {
- return fileStatistics;
- }
-
- protected OrcProto.RowIndexEntry.Builder getRowIndexEntry() {
- return rowIndexEntry;
- }
-
- IntegerWriter createIntegerWriter(PositionedOutputStream output,
- boolean signed, boolean isDirectV2,
- StreamFactory writer) {
- if (isDirectV2) {
- boolean alignedBitpacking = false;
- if (writer.getEncodingStrategy().equals(EncodingStrategy.SPEED)) {
- alignedBitpacking = true;
- }
- return new RunLengthIntegerWriterV2(output, signed, alignedBitpacking);
- } else {
- return new RunLengthIntegerWriter(output, signed);
- }
- }
-
- boolean isNewWriteFormat(StreamFactory writer) {
- return writer.getVersion() != OrcFile.Version.V_0_11;
- }
-
- /**
- * Add a new value to the column.
- *
- * @param obj The value to write to the column
- * @throws IOException if an error occurs during add
- */
- void write(Object obj) throws IOException {
- if (obj != null) {
- indexStatistics.increment();
- } else {
- indexStatistics.setNull();
- }
- if (isPresent != null) {
- isPresent.write(obj == null ? 0 : 1);
- if (obj == null) {
- foundNulls = true;
- }
- }
- }
-
- private void removeIsPresentPositions() {
- for (int i = 0; i < rowIndex.getEntryCount(); ++i) {
- RowIndexEntry.Builder entry = rowIndex.getEntryBuilder(i);
- List positions = entry.getPositionsList();
- // bit streams use 3 positions if uncompressed, 4 if compressed
- positions = positions.subList(isCompressed ? 4 : 3, positions.size());
- entry.clearPositions();
- entry.addAllPositions(positions);
- }
- }
-
- /**
- * Write the stripe out to the file.
- *
- * @param builder the stripe footer that contains the information about the
- * layout of the stripe. The TreeWriter is required to update
- * the footer with its information.
- * @param requiredIndexEntries the number of index entries that are
- * required. this is to check to make sure the
- * row index is well formed.
- * @throws IOException if an error occurs during write
- */
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- if (isPresent != null) {
- isPresent.flush();
-
- // if no nulls are found in a stream, then suppress the stream
- if (!foundNulls) {
- isPresentOutStream.suppress();
- // since isPresent bitstream is suppressed, update the index to
- // remove the positions of the isPresent stream
- if (rowIndexStream != null) {
- removeIsPresentPositions();
- }
- }
- }
-
- // merge stripe-level column statistics to file statistics and write it to
- // stripe statistics
- OrcProto.StripeStatistics.Builder stripeStatsBuilder = OrcProto.StripeStatistics.newBuilder();
- writeStripeStatistics(stripeStatsBuilder, this);
- stripeStatsBuilders.add(stripeStatsBuilder);
-
- // reset the flag for next stripe
- foundNulls = false;
-
- builder.addColumns(getEncoding());
- builder.setWriterTimezone(TimeZone.getDefault().getID());
- if (rowIndexStream != null) {
- if (rowIndex.getEntryCount() != requiredIndexEntries) {
- throw new IllegalArgumentException("Column has wrong number of " +
- "index entries found: " + rowIndex.getEntryCount() + " expected: " +
- requiredIndexEntries);
- }
- rowIndex.build().writeTo(rowIndexStream);
- rowIndexStream.flush();
- }
- rowIndex.clear();
- rowIndexEntry.clear();
-
- // write the bloom filter to out stream
- if (bloomFilterStream != null) {
- bloomFilterIndex.build().writeTo(bloomFilterStream);
- bloomFilterStream.flush();
- bloomFilterIndex.clear();
- bloomFilterEntry.clear();
- }
- }
-
- private void writeStripeStatistics(OrcProto.StripeStatistics.Builder builder,
- TreeWriter treeWriter) {
- treeWriter.fileStatistics.merge(treeWriter.stripeColStatistics);
- builder.addColStats(treeWriter.stripeColStatistics.serialize().build());
- treeWriter.stripeColStatistics.reset();
- for (TreeWriter child : treeWriter.getChildrenWriters()) {
- writeStripeStatistics(builder, child);
- }
- }
-
- TreeWriter[] getChildrenWriters() {
- return childrenWriters;
- }
-
- /**
- * Get the encoding for this column.
- *
- * @return the information about the encoding of this column
- */
- OrcProto.ColumnEncoding getEncoding() {
- return OrcProto.ColumnEncoding.newBuilder().setKind(
- OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- /**
- * Create a row index entry with the previous location and the current
- * index statistics. Also merges the index statistics into the file
- * statistics before they are cleared. Finally, it records the start of the
- * next index and ensures all of the children columns also create an entry.
- *
- * @throws IOException if an error occurs during create
- */
- void createRowIndexEntry() throws IOException {
- stripeColStatistics.merge(indexStatistics);
- rowIndexEntry.setStatistics(indexStatistics.serialize());
- indexStatistics.reset();
- rowIndex.addEntry(rowIndexEntry);
- rowIndexEntry.clear();
- addBloomFilterEntry();
- recordPosition(rowIndexPosition);
- for (TreeWriter child : childrenWriters) {
- child.createRowIndexEntry();
- }
- }
-
- void addBloomFilterEntry() {
- if (createBloomFilter) {
- bloomFilterEntry.setNumHashFunctions(bloomFilter.getNumHashFunctions());
- bloomFilterEntry.addAllBitset(Longs.asList(bloomFilter.getBitSet()));
- bloomFilterIndex.addBloomFilter(bloomFilterEntry.build());
- bloomFilter.reset();
- bloomFilterEntry.clear();
- }
- }
-
- /**
- * Record the current position in each of this column's streams.
- *
- * @param recorder where should the locations be recorded
- * @throws IOException if an error occurs during position recording
- */
- void recordPosition(PositionRecorder recorder) throws IOException {
- if (isPresent != null) {
- isPresent.getPosition(recorder);
- }
- }
-
- /**
- * Estimate how much memory the writer is consuming excluding the streams.
- *
- * @return the number of bytes.
- */
- long estimateMemory() {
- long result = 0;
- for (TreeWriter child : childrenWriters) {
- result += child.estimateMemory();
- }
- return result;
- }
-
- /**
- * Handle the top level object write.
- *
- * This default method is used for all types except structs, which are the
- * typical case. VectorizedRowBatch assumes the top level object is a
- * struct, so we use the first column for all other types.
- * @param batch the batch to write from
- * @param offset the row to start on
- * @param length the number of rows to write
- * @throws IOException if an error occurs during write
- */
- void writeRootBatch(VectorizedRowBatch batch, int offset,
- int length) throws IOException {
- writeBatch(batch.cols[0], offset, length);
- }
-
- /**
- * Write the values from the given vector from offset for length elements.
- * @param vector the vector to write from
- * @param offset the first value from the vector to write
- * @param length the number of values from the vector to write
- * @throws IOException if an error occurs during write
- */
- void writeBatch(ColumnVector vector, int offset,
- int length) throws IOException {
- if (vector.noNulls) {
- if (isPresent != null) {
- for (int i = 0; i < length; ++i) {
- isPresent.write(1);
- indexStatistics.increment();
- }
- }
- } else {
- if (vector.isRepeating) {
- boolean isNull = vector.isNull[0];
- if (isPresent != null) {
- for (int i = 0; i < length; ++i) {
- isPresent.write(isNull ? 0 : 1);
- }
- }
- if (isNull) {
- foundNulls = true;
- indexStatistics.setNull();
- } else {
- indexStatistics.increment();
- }
- } else {
- // count the number of non-null values
- int nonNullCount = 0;
- for(int i = 0; i < length; ++i) {
- boolean isNull = vector.isNull[i + offset];
- if (!isNull) {
- nonNullCount += 1;
- }
- if (isPresent != null) {
- isPresent.write(isNull ? 0 : 1);
- }
- }
- indexStatistics.increment();
- if (nonNullCount != length) {
- foundNulls = true;
- indexStatistics.setNull();
- }
- }
- }
- }
- }
-
- private static class BooleanTreeWriter extends TreeWriter {
- private final BitFieldWriter writer;
-
- BooleanTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- PositionedOutputStream out = writer.createStream(id,
- OrcProto.Stream.Kind.DATA);
- this.writer = new BitFieldWriter(out, 1);
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- boolean val = ((BooleanObjectInspector) inspector).get(obj);
- indexStatistics.updateBoolean(val);
- writer.write(val ? 1 : 0);
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- writer.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- writer.getPosition(recorder);
- }
- }
-
- private static class ByteTreeWriter extends TreeWriter {
- private final RunLengthByteWriter writer;
-
- ByteTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.writer = new RunLengthByteWriter(writer.createStream(id,
- OrcProto.Stream.Kind.DATA));
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- byte val = ((ByteObjectInspector) inspector).get(obj);
- indexStatistics.updateInteger(val);
- if (createBloomFilter) {
- bloomFilter.addLong(val);
- }
- writer.write(val);
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- writer.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- writer.getPosition(recorder);
- }
- }
-
- private static class IntegerTreeWriter extends TreeWriter {
- private final IntegerWriter writer;
- private final ShortObjectInspector shortInspector;
- private final IntObjectInspector intInspector;
- private final LongObjectInspector longInspector;
- private boolean isDirectV2 = true;
-
- IntegerTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- OutStream out = writer.createStream(id,
- OrcProto.Stream.Kind.DATA);
- this.isDirectV2 = isNewWriteFormat(writer);
- this.writer = createIntegerWriter(out, true, isDirectV2, writer);
- if (inspector instanceof IntObjectInspector) {
- intInspector = (IntObjectInspector) inspector;
- shortInspector = null;
- longInspector = null;
- } else {
- intInspector = null;
- if (inspector instanceof LongObjectInspector) {
- longInspector = (LongObjectInspector) inspector;
- shortInspector = null;
- } else {
- shortInspector = (ShortObjectInspector) inspector;
- longInspector = null;
- }
- }
- recordPosition(rowIndexPosition);
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- long val;
- if (intInspector != null) {
- val = intInspector.get(obj);
- } else if (longInspector != null) {
- val = longInspector.get(obj);
- } else {
- val = shortInspector.get(obj);
- }
- indexStatistics.updateInteger(val);
- if (createBloomFilter) {
- // integers are converted to longs in column statistics and during SARG evaluation
- bloomFilter.addLong(val);
- }
- writer.write(val);
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- writer.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- writer.getPosition(recorder);
- }
- }
-
- private static class FloatTreeWriter extends TreeWriter {
- private final PositionedOutputStream stream;
- private final SerializationUtils utils;
-
- FloatTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.stream = writer.createStream(id,
- OrcProto.Stream.Kind.DATA);
- this.utils = new SerializationUtils();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- float val = ((FloatObjectInspector) inspector).get(obj);
- indexStatistics.updateDouble(val);
- if (createBloomFilter) {
- // floats are converted to doubles in column statistics and during SARG evaluation
- bloomFilter.addDouble(val);
- }
- utils.writeFloat(stream, val);
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- stream.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- stream.getPosition(recorder);
- }
- }
-
- private static class DoubleTreeWriter extends TreeWriter {
- private final PositionedOutputStream stream;
- private final SerializationUtils utils;
-
- DoubleTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.stream = writer.createStream(id,
- OrcProto.Stream.Kind.DATA);
- this.utils = new SerializationUtils();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- double val = ((DoubleObjectInspector) inspector).get(obj);
- indexStatistics.updateDouble(val);
- if (createBloomFilter) {
- bloomFilter.addDouble(val);
- }
- utils.writeDouble(stream, val);
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- stream.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- stream.getPosition(recorder);
- }
- }
-
- private static class StringTreeWriter extends TreeWriter {
- private static final int INITIAL_DICTIONARY_SIZE = 4096;
- private final OutStream stringOutput;
- private final IntegerWriter lengthOutput;
- private final IntegerWriter rowOutput;
- private final StringRedBlackTree dictionary =
- new StringRedBlackTree(INITIAL_DICTIONARY_SIZE);
- private final DynamicIntArray rows = new DynamicIntArray();
- private final PositionedOutputStream directStreamOutput;
- private final IntegerWriter directLengthOutput;
- private final List savedRowIndex =
- new ArrayList();
- private final boolean buildIndex;
- private final List rowIndexValueCount = new ArrayList();
- // If the number of keys in a dictionary is greater than this fraction of
- //the total number of non-null rows, turn off dictionary encoding
- private final float dictionaryKeySizeThreshold;
- private boolean useDictionaryEncoding = true;
- private boolean isDirectV2 = true;
- private boolean doneDictionaryCheck;
- private final boolean strideDictionaryCheck;
-
- StringTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.isDirectV2 = isNewWriteFormat(writer);
- stringOutput = writer.createStream(id,
- OrcProto.Stream.Kind.DICTIONARY_DATA);
- lengthOutput = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
- rowOutput = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.DATA), false, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- rowIndexValueCount.add(0L);
- buildIndex = writer.buildIndex();
- directStreamOutput = writer.createStream(id, OrcProto.Stream.Kind.DATA);
- directLengthOutput = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
- dictionaryKeySizeThreshold = writer.getConfiguration().getFloat(
- HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname,
- HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.defaultFloatVal);
- strideDictionaryCheck = writer.getConfiguration().getBoolean(
- HiveConf.ConfVars.HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK.varname,
- HiveConf.ConfVars.HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK.defaultBoolVal);
- doneDictionaryCheck = false;
- }
-
- /**
- * Method to retrieve text values from the value object, which can be overridden
- * by subclasses.
- *
- * @param obj value
- * @return Text text value from obj
- */
- Text getTextValue(Object obj) {
- return ((StringObjectInspector) inspector).getPrimitiveWritableObject(obj);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- Text val = getTextValue(obj);
- if (useDictionaryEncoding || !strideDictionaryCheck) {
- rows.add(dictionary.add(val));
- } else {
- // write data and length
- directStreamOutput.write(val.getBytes(), 0, val.getLength());
- directLengthOutput.write(val.getLength());
- }
- indexStatistics.updateString(val);
- if (createBloomFilter) {
- bloomFilter.addBytes(val.getBytes(), val.getLength());
- }
- }
- }
-
- private boolean checkDictionaryEncoding() {
- if (!doneDictionaryCheck) {
- // Set the flag indicating whether or not to use dictionary encoding
- // based on whether or not the fraction of distinct keys over number of
- // non-null rows is less than the configured threshold
- float ratio = rows.size() > 0 ? (float) (dictionary.size()) / rows.size() : 0.0f;
- useDictionaryEncoding = !isDirectV2 || ratio <= dictionaryKeySizeThreshold;
- doneDictionaryCheck = true;
- }
- return useDictionaryEncoding;
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- // if rows in stripe is less than dictionaryCheckAfterRows, dictionary
- // checking would not have happened. So do it again here.
- checkDictionaryEncoding();
-
- if (useDictionaryEncoding) {
- flushDictionary();
- } else {
- // flushout any left over entries from dictionary
- if (rows.size() > 0) {
- flushDictionary();
- }
-
- // suppress the stream for every stripe if dictionary is disabled
- stringOutput.suppress();
- }
-
- // we need to build the rowindex before calling super, since it
- // writes it out.
- super.writeStripe(builder, requiredIndexEntries);
- stringOutput.flush();
- lengthOutput.flush();
- rowOutput.flush();
- directStreamOutput.flush();
- directLengthOutput.flush();
- // reset all of the fields to be ready for the next stripe.
- dictionary.clear();
- savedRowIndex.clear();
- rowIndexValueCount.clear();
- recordPosition(rowIndexPosition);
- rowIndexValueCount.add(0L);
-
- if (!useDictionaryEncoding) {
- // record the start positions of first index stride of next stripe i.e
- // beginning of the direct streams when dictionary is disabled
- recordDirectStreamPosition();
- }
- }
-
- private void flushDictionary() throws IOException {
- final int[] dumpOrder = new int[dictionary.size()];
-
- if (useDictionaryEncoding) {
- // Write the dictionary by traversing the red-black tree writing out
- // the bytes and lengths; and creating the map from the original order
- // to the final sorted order.
-
- dictionary.visit(new StringRedBlackTree.Visitor() {
- private int currentId = 0;
-
- @Override
- public void visit(StringRedBlackTree.VisitorContext context
- ) throws IOException {
- context.writeBytes(stringOutput);
- lengthOutput.write(context.getLength());
- dumpOrder[context.getOriginalPosition()] = currentId++;
- }
- });
- } else {
- // for direct encoding, we don't want the dictionary data stream
- stringOutput.suppress();
- }
- int length = rows.size();
- int rowIndexEntry = 0;
- OrcProto.RowIndex.Builder rowIndex = getRowIndex();
- Text text = new Text();
- // write the values translated into the dump order.
- for (int i = 0; i <= length; ++i) {
- // now that we are writing out the row values, we can finalize the
- // row index
- if (buildIndex) {
- while (i == rowIndexValueCount.get(rowIndexEntry) && rowIndexEntry < savedRowIndex.size()) {
- OrcProto.RowIndexEntry.Builder base =
- savedRowIndex.get(rowIndexEntry++).toBuilder();
- if (useDictionaryEncoding) {
- rowOutput.getPosition(new RowIndexPositionRecorder(base));
- } else {
- PositionRecorder posn = new RowIndexPositionRecorder(base);
- directStreamOutput.getPosition(posn);
- directLengthOutput.getPosition(posn);
- }
- rowIndex.addEntry(base.build());
- }
- }
- if (i != length) {
- if (useDictionaryEncoding) {
- rowOutput.write(dumpOrder[rows.get(i)]);
- } else {
- dictionary.getText(text, rows.get(i));
- directStreamOutput.write(text.getBytes(), 0, text.getLength());
- directLengthOutput.write(text.getLength());
- }
- }
- }
- rows.clear();
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- // Returns the encoding used for the last call to writeStripe
- if (useDictionaryEncoding) {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder().setKind(
- OrcProto.ColumnEncoding.Kind.DICTIONARY_V2).setDictionarySize(dictionary.size()).build();
- }
- return OrcProto.ColumnEncoding.newBuilder().setKind(
- OrcProto.ColumnEncoding.Kind.DICTIONARY).setDictionarySize(dictionary.size()).build();
- } else {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder().setKind(
- OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder().setKind(
- OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
- }
-
- /**
- * This method doesn't call the super method, because unlike most of the
- * other TreeWriters, this one can't record the position in the streams
- * until the stripe is being flushed. Therefore it saves all of the entries
- * and augments them with the final information as the stripe is written.
- *
- * @throws IOException if an error occurs on create
- */
- @Override
- void createRowIndexEntry() throws IOException {
- getStripeStatistics().merge(indexStatistics);
- OrcProto.RowIndexEntry.Builder rowIndexEntry = getRowIndexEntry();
- rowIndexEntry.setStatistics(indexStatistics.serialize());
- indexStatistics.reset();
- OrcProto.RowIndexEntry base = rowIndexEntry.build();
- savedRowIndex.add(base);
- rowIndexEntry.clear();
- addBloomFilterEntry();
- recordPosition(rowIndexPosition);
- rowIndexValueCount.add(Long.valueOf(rows.size()));
- if (strideDictionaryCheck) {
- checkDictionaryEncoding();
- }
- if (!useDictionaryEncoding) {
- if (rows.size() > 0) {
- flushDictionary();
- // just record the start positions of next index stride
- recordDirectStreamPosition();
- } else {
- // record the start positions of next index stride
- recordDirectStreamPosition();
- getRowIndex().addEntry(base);
- }
- }
- }
-
- private void recordDirectStreamPosition() throws IOException {
- directStreamOutput.getPosition(rowIndexPosition);
- directLengthOutput.getPosition(rowIndexPosition);
- }
-
- @Override
- long estimateMemory() {
- return rows.getSizeInBytes() + dictionary.getSizeInBytes();
- }
- }
-
- /**
- * Under the covers, char is written to ORC the same way as string.
- */
- private static class CharTreeWriter extends StringTreeWriter {
-
- CharTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- }
-
- /**
- * Override base class implementation to support char values.
- */
- @Override
- Text getTextValue(Object obj) {
- return (((HiveCharObjectInspector) inspector)
- .getPrimitiveWritableObject(obj)).getTextValue();
- }
- }
-
- /**
- * Under the covers, varchar is written to ORC the same way as string.
- */
- private static class VarcharTreeWriter extends StringTreeWriter {
-
- VarcharTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- }
-
- /**
- * Override base class implementation to support varchar values.
- */
- @Override
- Text getTextValue(Object obj) {
- return (((HiveVarcharObjectInspector) inspector)
- .getPrimitiveWritableObject(obj)).getTextValue();
- }
- }
-
- private static class BinaryTreeWriter extends TreeWriter {
- private final PositionedOutputStream stream;
- private final IntegerWriter length;
- private boolean isDirectV2 = true;
-
- BinaryTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.stream = writer.createStream(id,
- OrcProto.Stream.Kind.DATA);
- this.isDirectV2 = isNewWriteFormat(writer);
- this.length = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- BytesWritable val =
- ((BinaryObjectInspector) inspector).getPrimitiveWritableObject(obj);
- stream.write(val.getBytes(), 0, val.getLength());
- length.write(val.getLength());
- indexStatistics.updateBinary(val);
- if (createBloomFilter) {
- bloomFilter.addBytes(val.getBytes(), val.getLength());
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- stream.flush();
- length.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- stream.getPosition(recorder);
- length.getPosition(recorder);
- }
- }
-
- static final int MILLIS_PER_SECOND = 1000;
- static final String BASE_TIMESTAMP_STRING = "2015-01-01 00:00:00";
-
- private static class TimestampTreeWriter extends TreeWriter {
- private final IntegerWriter seconds;
- private final IntegerWriter nanos;
- private final boolean isDirectV2;
- private final long base_timestamp;
-
- TimestampTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.isDirectV2 = isNewWriteFormat(writer);
- this.seconds = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.DATA), true, isDirectV2, writer);
- this.nanos = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.SECONDARY), false, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- // for unit tests to set different time zones
- this.base_timestamp = Timestamp.valueOf(BASE_TIMESTAMP_STRING).getTime() / MILLIS_PER_SECOND;
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- Timestamp val =
- ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(obj);
- indexStatistics.updateTimestamp(val);
- seconds.write((val.getTime() / MILLIS_PER_SECOND) - base_timestamp);
- nanos.write(formatNanos(val.getNanos()));
- if (createBloomFilter) {
- bloomFilter.addLong(val.getTime());
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- seconds.flush();
- nanos.flush();
- recordPosition(rowIndexPosition);
- }
-
- private static long formatNanos(int nanos) {
- if (nanos == 0) {
- return 0;
- } else if (nanos % 100 != 0) {
- return ((long) nanos) << 3;
- } else {
- nanos /= 100;
- int trailingZeros = 1;
- while (nanos % 10 == 0 && trailingZeros < 7) {
- nanos /= 10;
- trailingZeros += 1;
- }
- return ((long) nanos) << 3 | trailingZeros;
- }
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- seconds.getPosition(recorder);
- nanos.getPosition(recorder);
- }
- }
-
- private static class DateTreeWriter extends TreeWriter {
- private final IntegerWriter writer;
- private final boolean isDirectV2;
-
- DateTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- OutStream out = writer.createStream(id,
- OrcProto.Stream.Kind.DATA);
- this.isDirectV2 = isNewWriteFormat(writer);
- this.writer = createIntegerWriter(out, true, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- // Using the Writable here as it's used directly for writing as well as for stats.
- DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj);
- indexStatistics.updateDate(val);
- writer.write(val.getDays());
- if (createBloomFilter) {
- bloomFilter.addLong(val.getDays());
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- writer.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- writer.getPosition(recorder);
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
- }
-
- private static class DecimalTreeWriter extends TreeWriter {
- private final PositionedOutputStream valueStream;
- private final IntegerWriter scaleStream;
- private final boolean isDirectV2;
-
- DecimalTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.isDirectV2 = isNewWriteFormat(writer);
- valueStream = writer.createStream(id, OrcProto.Stream.Kind.DATA);
- this.scaleStream = createIntegerWriter(writer.createStream(id,
- OrcProto.Stream.Kind.SECONDARY), true, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- HiveDecimal decimal = ((HiveDecimalObjectInspector) inspector).getPrimitiveJavaObject(obj);
- if (decimal == null) {
- return;
- }
- SerializationUtils.writeBigInteger(valueStream,
- decimal.unscaledValue());
- scaleStream.write(decimal.scale());
- indexStatistics.updateDecimal(decimal);
- if (createBloomFilter) {
- bloomFilter.addString(decimal.toString());
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- valueStream.flush();
- scaleStream.flush();
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- valueStream.getPosition(recorder);
- scaleStream.getPosition(recorder);
- }
- }
-
- private static class StructTreeWriter extends TreeWriter {
- private final List fields;
-
- StructTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- StructObjectInspector structObjectInspector =
- (StructObjectInspector) inspector;
- fields = structObjectInspector.getAllStructFieldRefs();
- childrenWriters = new TreeWriter[fields.size()];
- for (int i = 0; i < childrenWriters.length; ++i) {
- childrenWriters[i] = createTreeWriter(
- fields.get(i).getFieldObjectInspector(), writer, true);
- }
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- StructObjectInspector insp = (StructObjectInspector) inspector;
- for (int i = 0; i < fields.size(); ++i) {
- StructField field = fields.get(i);
- TreeWriter writer = childrenWriters[i];
- writer.write(insp.getStructFieldData(obj, field));
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- for (TreeWriter child : childrenWriters) {
- child.writeStripe(builder, requiredIndexEntries);
- }
- recordPosition(rowIndexPosition);
- }
- }
-
- private static class ListTreeWriter extends TreeWriter {
- private final IntegerWriter lengths;
- private final boolean isDirectV2;
-
- ListTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.isDirectV2 = isNewWriteFormat(writer);
- ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
- childrenWriters = new TreeWriter[1];
- childrenWriters[0] =
- createTreeWriter(listObjectInspector.getListElementObjectInspector(),
- writer, true);
- lengths = createIntegerWriter(writer.createStream(columnId,
- OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- ListObjectInspector insp = (ListObjectInspector) inspector;
- int len = insp.getListLength(obj);
- lengths.write(len);
- if (createBloomFilter) {
- bloomFilter.addLong(len);
- }
- for (int i = 0; i < len; ++i) {
- childrenWriters[0].write(insp.getListElement(obj, i));
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- lengths.flush();
- for (TreeWriter child : childrenWriters) {
- child.writeStripe(builder, requiredIndexEntries);
- }
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- lengths.getPosition(recorder);
- }
- }
-
- private static class MapTreeWriter extends TreeWriter {
- private final IntegerWriter lengths;
- private final boolean isDirectV2;
-
- MapTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- this.isDirectV2 = isNewWriteFormat(writer);
- MapObjectInspector insp = (MapObjectInspector) inspector;
- childrenWriters = new TreeWriter[2];
- childrenWriters[0] =
- createTreeWriter(insp.getMapKeyObjectInspector(), writer, true);
- childrenWriters[1] =
- createTreeWriter(insp.getMapValueObjectInspector(), writer, true);
- lengths = createIntegerWriter(writer.createStream(columnId,
- OrcProto.Stream.Kind.LENGTH), false, isDirectV2, writer);
- recordPosition(rowIndexPosition);
- }
-
- @Override
- OrcProto.ColumnEncoding getEncoding() {
- if (isDirectV2) {
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT_V2).build();
- }
- return OrcProto.ColumnEncoding.newBuilder()
- .setKind(OrcProto.ColumnEncoding.Kind.DIRECT).build();
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- MapObjectInspector insp = (MapObjectInspector) inspector;
- // this sucks, but it will have to do until we can get a better
- // accessor in the MapObjectInspector.
- Map valueMap = insp.getMap(obj);
- lengths.write(valueMap.size());
- if (createBloomFilter) {
- bloomFilter.addLong(valueMap.size());
- }
- for (Map.Entry entry : valueMap.entrySet()) {
- childrenWriters[0].write(entry.getKey());
- childrenWriters[1].write(entry.getValue());
- }
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- lengths.flush();
- for (TreeWriter child : childrenWriters) {
- child.writeStripe(builder, requiredIndexEntries);
- }
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- lengths.getPosition(recorder);
- }
- }
-
- private static class UnionTreeWriter extends TreeWriter {
- private final RunLengthByteWriter tags;
-
- UnionTreeWriter(int columnId,
- ObjectInspector inspector,
- StreamFactory writer,
- boolean nullable) throws IOException {
- super(columnId, inspector, writer, nullable);
- UnionObjectInspector insp = (UnionObjectInspector) inspector;
- List choices = insp.getObjectInspectors();
- childrenWriters = new TreeWriter[choices.size()];
- for (int i = 0; i < childrenWriters.length; ++i) {
- childrenWriters[i] = createTreeWriter(choices.get(i), writer, true);
- }
- tags =
- new RunLengthByteWriter(writer.createStream(columnId,
- OrcProto.Stream.Kind.DATA));
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void write(Object obj) throws IOException {
- super.write(obj);
- if (obj != null) {
- UnionObjectInspector insp = (UnionObjectInspector) inspector;
- byte tag = insp.getTag(obj);
- tags.write(tag);
- if (createBloomFilter) {
- bloomFilter.addLong(tag);
- }
- childrenWriters[tag].write(insp.getField(obj));
- }
- }
-
- @Override
- void writeStripe(OrcProto.StripeFooter.Builder builder,
- int requiredIndexEntries) throws IOException {
- super.writeStripe(builder, requiredIndexEntries);
- tags.flush();
- for (TreeWriter child : childrenWriters) {
- child.writeStripe(builder, requiredIndexEntries);
- }
- recordPosition(rowIndexPosition);
- }
-
- @Override
- void recordPosition(PositionRecorder recorder) throws IOException {
- super.recordPosition(recorder);
- tags.getPosition(recorder);
- }
- }
-
- private static TreeWriter createTreeWriter(ObjectInspector inspector,
- StreamFactory streamFactory,
- boolean nullable) throws IOException {
- switch (inspector.getCategory()) {
- case PRIMITIVE:
- switch (((PrimitiveObjectInspector) inspector).getPrimitiveCategory()) {
- case BOOLEAN:
- return new BooleanTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case BYTE:
- return new ByteTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case SHORT:
- case INT:
- case LONG:
- return new IntegerTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case FLOAT:
- return new FloatTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case DOUBLE:
- return new DoubleTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case STRING:
- return new StringTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case CHAR:
- return new CharTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case VARCHAR:
- return new VarcharTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case BINARY:
- return new BinaryTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case TIMESTAMP:
- return new TimestampTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case DATE:
- return new DateTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- case DECIMAL:
- return new DecimalTreeWriter(streamFactory.getNextColumnId(),
- inspector, streamFactory, nullable);
- default:
- throw new IllegalArgumentException("Bad primitive category " +
- ((PrimitiveObjectInspector) inspector).getPrimitiveCategory());
- }
- case STRUCT:
- return new StructTreeWriter(streamFactory.getNextColumnId(), inspector,
- streamFactory, nullable);
- case MAP:
- return new MapTreeWriter(streamFactory.getNextColumnId(), inspector,
- streamFactory, nullable);
- case LIST:
- return new ListTreeWriter(streamFactory.getNextColumnId(), inspector,
- streamFactory, nullable);
- case UNION:
- return new UnionTreeWriter(streamFactory.getNextColumnId(), inspector,
- streamFactory, nullable);
- default:
- throw new IllegalArgumentException("Bad category: " +
- inspector.getCategory());
- }
- }
-
- private static void writeTypes(OrcProto.Footer.Builder builder,
- TreeWriter treeWriter) {
- OrcProto.Type.Builder type = OrcProto.Type.newBuilder();
- switch (treeWriter.inspector.getCategory()) {
- case PRIMITIVE:
- switch (((PrimitiveObjectInspector) treeWriter.inspector).getPrimitiveCategory()) {
- case BOOLEAN:
- type.setKind(OrcProto.Type.Kind.BOOLEAN);
- break;
- case BYTE:
- type.setKind(OrcProto.Type.Kind.BYTE);
- break;
- case SHORT:
- type.setKind(OrcProto.Type.Kind.SHORT);
- break;
- case INT:
- type.setKind(OrcProto.Type.Kind.INT);
- break;
- case LONG:
- type.setKind(OrcProto.Type.Kind.LONG);
- break;
- case FLOAT:
- type.setKind(OrcProto.Type.Kind.FLOAT);
- break;
- case DOUBLE:
- type.setKind(OrcProto.Type.Kind.DOUBLE);
- break;
- case STRING:
- type.setKind(OrcProto.Type.Kind.STRING);
- break;
- case CHAR:
- // The char length needs to be written to file and should be available
- // from the object inspector
- CharTypeInfo charTypeInfo = (CharTypeInfo) ((PrimitiveObjectInspector) treeWriter.inspector).getTypeInfo();
- type.setKind(Type.Kind.CHAR);
- type.setMaximumLength(charTypeInfo.getLength());
- break;
- case VARCHAR:
- // The varchar length needs to be written to file and should be available
- // from the object inspector
- VarcharTypeInfo typeInfo = (VarcharTypeInfo) ((PrimitiveObjectInspector) treeWriter.inspector).getTypeInfo();
- type.setKind(Type.Kind.VARCHAR);
- type.setMaximumLength(typeInfo.getLength());
- break;
- case BINARY:
- type.setKind(OrcProto.Type.Kind.BINARY);
- break;
- case TIMESTAMP:
- type.setKind(OrcProto.Type.Kind.TIMESTAMP);
- break;
- case DATE:
- type.setKind(OrcProto.Type.Kind.DATE);
- break;
- case DECIMAL:
- DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) ((PrimitiveObjectInspector) treeWriter.inspector).getTypeInfo();
- type.setKind(OrcProto.Type.Kind.DECIMAL);
- type.setPrecision(decTypeInfo.precision());
- type.setScale(decTypeInfo.scale());
- break;
- default:
- throw new IllegalArgumentException("Unknown primitive category: "
- + ((PrimitiveObjectInspector) treeWriter.inspector).getPrimitiveCategory());
- }
- break;
- case LIST:
- type.setKind(OrcProto.Type.Kind.LIST);
- type.addSubtypes(treeWriter.childrenWriters[0].id);
- break;
- case MAP:
- type.setKind(OrcProto.Type.Kind.MAP);
- type.addSubtypes(treeWriter.childrenWriters[0].id);
- type.addSubtypes(treeWriter.childrenWriters[1].id);
- break;
- case STRUCT:
- type.setKind(OrcProto.Type.Kind.STRUCT);
- for (TreeWriter child : treeWriter.childrenWriters) {
- type.addSubtypes(child.id);
- }
- for (StructField field : ((StructTreeWriter) treeWriter).fields) {
- type.addFieldNames(field.getFieldName());
- }
- break;
- case UNION:
- type.setKind(OrcProto.Type.Kind.UNION);
- for (TreeWriter child : treeWriter.childrenWriters) {
- type.addSubtypes(child.id);
- }
- break;
- default:
- throw new IllegalArgumentException("Unknown category: " + treeWriter.inspector.getCategory());
- }
- builder.addTypes(type);
- for (TreeWriter child : treeWriter.childrenWriters) {
- writeTypes(builder, child);
- }
- }
-
- @VisibleForTesting
- public OutputStream getStream() throws IOException {
- if (rawWriter == null) {
- rawWriter = new ByteCountingOutputStream(flowFileOutputStream);
- rawWriter.write(OrcFile.MAGIC.getBytes());
- headerLength = rawWriter.getBytesWritten();
- writer = new OutStream("metadata", bufferSize, codec,
- new DirectStream(rawWriter));
- protobufWriter = CodedOutputStream.newInstance(writer);
- }
- return rawWriter;
- }
-
- private void createRowIndexEntry() throws IOException {
- treeWriter.createRowIndexEntry();
- rowsInIndex = 0;
- }
-
- private void flushStripe() throws IOException {
- getStream();
- if (buildIndex && rowsInIndex != 0) {
- createRowIndexEntry();
- }
- if (rowsInStripe != 0) {
- if (callback != null) {
- callback.preStripeWrite(callbackContext);
- }
- // finalize the data for the stripe
- int requiredIndexEntries = rowIndexStride == 0 ? 0 :
- (int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride);
- OrcProto.StripeFooter.Builder builder =
- OrcProto.StripeFooter.newBuilder();
- treeWriter.writeStripe(builder, requiredIndexEntries);
- long indexSize = 0;
- long dataSize = 0;
- for (Map.Entry pair : streams.entrySet()) {
- BufferedStream stream = pair.getValue();
- if (!stream.isSuppressed()) {
- stream.flush();
- StreamName name = pair.getKey();
- long streamSize = pair.getValue().getOutputSize();
- builder.addStreams(OrcProto.Stream.newBuilder()
- .setColumn(name.getColumn())
- .setKind(name.getKind())
- .setLength(streamSize));
- if (StreamName.Area.INDEX == name.getArea()) {
- indexSize += streamSize;
- } else {
- dataSize += streamSize;
- }
- }
- }
- OrcProto.StripeFooter footer = builder.build();
-
- // Do we need to pad the file so the stripe doesn't straddle a block
- // boundary?
- long start = rawWriter.getBytesWritten();
- final long currentStripeSize = indexSize + dataSize + footer.getSerializedSize();
- final long available = blockSize - (start % blockSize);
- final long overflow = currentStripeSize - adjustedStripeSize;
- final float availRatio = (float) available / (float) defaultStripeSize;
-
- if (availRatio > 0.0f && availRatio < 1.0f
- && availRatio > paddingTolerance) {
- // adjust default stripe size to fit into remaining space, also adjust
- // the next stripe for correction based on the current stripe size
- // and user specified padding tolerance. Since stripe size can overflow
- // the default stripe size we should apply this correction to avoid
- // writing portion of last stripe to next hdfs block.
- float correction = overflow > 0 ? (float) overflow
- / (float) adjustedStripeSize : 0.0f;
-
- // correction should not be greater than user specified padding
- // tolerance
- correction = correction > paddingTolerance ? paddingTolerance
- : correction;
-
- // adjust next stripe size based on current stripe estimate correction
- adjustedStripeSize = (long) ((1.0f - correction) * (availRatio * defaultStripeSize));
- } else if (availRatio >= 1.0) {
- adjustedStripeSize = defaultStripeSize;
- }
-
- if (availRatio < paddingTolerance && addBlockPadding) {
- long padding = blockSize - (start % blockSize);
- byte[] pad = new byte[(int) Math.min(HDFS_BUFFER_SIZE, padding)];
- LOG.info(String.format("Padding ORC by %d bytes (<= %.2f * %d)",
- padding, availRatio, defaultStripeSize));
- start += padding;
- while (padding > 0) {
- int writeLen = (int) Math.min(padding, pad.length);
- rawWriter.write(pad, 0, writeLen);
- padding -= writeLen;
- }
- adjustedStripeSize = defaultStripeSize;
- } else if (currentStripeSize < blockSize
- && (start % blockSize) + currentStripeSize > blockSize) {
- // even if you don't pad, reset the default stripe size when crossing a
- // block boundary
- adjustedStripeSize = defaultStripeSize;
- }
-
- // write out the data streams
- for (Map.Entry pair : streams.entrySet()) {
- BufferedStream stream = pair.getValue();
- if (!stream.isSuppressed()) {
- stream.spillTo(rawWriter);
- }
- stream.clear();
- }
- footer.writeTo(protobufWriter);
- protobufWriter.flush();
- writer.flush();
- long footerLength = rawWriter.getBytesWritten() - start - dataSize - indexSize;
- OrcProto.StripeInformation dirEntry =
- OrcProto.StripeInformation.newBuilder()
- .setOffset(start)
- .setNumberOfRows(rowsInStripe)
- .setIndexLength(indexSize)
- .setDataLength(dataSize)
- .setFooterLength(footerLength).build();
- stripes.add(dirEntry);
- rowCount += rowsInStripe;
- rowsInStripe = 0;
- }
- }
-
- private long computeRawDataSize() {
- long result = 0;
- for (TreeWriter child : treeWriter.getChildrenWriters()) {
- result += getRawDataSizeFromInspectors(child, child.inspector);
- }
- return result;
- }
-
- private long getRawDataSizeFromInspectors(TreeWriter child, ObjectInspector oi) {
- long total = 0;
- switch (oi.getCategory()) {
- case PRIMITIVE:
- total += getRawDataSizeFromPrimitives(child, oi);
- break;
- case LIST:
- case MAP:
- case UNION:
- case STRUCT:
- for (TreeWriter tw : child.childrenWriters) {
- total += getRawDataSizeFromInspectors(tw, tw.inspector);
- }
- break;
- default:
- LOG.debug("Unknown object inspector category.");
- break;
- }
- return total;
- }
-
- private long getRawDataSizeFromPrimitives(TreeWriter child, ObjectInspector oi) {
- long result = 0;
- long numVals = child.fileStatistics.getNumberOfValues();
- switch (((PrimitiveObjectInspector) oi).getPrimitiveCategory()) {
- case BOOLEAN:
- case BYTE:
- case SHORT:
- case INT:
- case FLOAT:
- return numVals * JavaDataModel.get().primitive1();
- case LONG:
- case DOUBLE:
- return numVals * JavaDataModel.get().primitive2();
- case STRING:
- case VARCHAR:
- case CHAR:
- // ORC strings are converted to java Strings. so use JavaDataModel to
- // compute the overall size of strings
- child = (StringTreeWriter) child;
- StringColumnStatistics scs = (StringColumnStatistics) child.fileStatistics;
- numVals = numVals == 0 ? 1 : numVals;
- int avgStringLen = (int) (scs.getSum() / numVals);
- return numVals * JavaDataModel.get().lengthForStringOfLength(avgStringLen);
- case DECIMAL:
- return numVals * JavaDataModel.get().lengthOfDecimal();
- case DATE:
- return numVals * JavaDataModel.get().lengthOfDate();
- case BINARY:
- // get total length of binary blob
- BinaryColumnStatistics bcs = (BinaryColumnStatistics) child.fileStatistics;
- return bcs.getSum();
- case TIMESTAMP:
- return numVals * JavaDataModel.get().lengthOfTimestamp();
- default:
- LOG.debug("Unknown primitive category.");
- break;
- }
-
- return result;
- }
-
- private OrcProto.CompressionKind writeCompressionKind(CompressionKind kind) {
- switch (kind) {
- case NONE:
- return OrcProto.CompressionKind.NONE;
- case ZLIB:
- return OrcProto.CompressionKind.ZLIB;
- case SNAPPY:
- return OrcProto.CompressionKind.SNAPPY;
- case LZO:
- return OrcProto.CompressionKind.LZO;
- default:
- throw new IllegalArgumentException("Unknown compression " + kind);
- }
- }
-
- private void writeFileStatistics(OrcProto.Footer.Builder builder,
- TreeWriter writer) throws IOException {
- builder.addStatistics(writer.fileStatistics.serialize());
- for (TreeWriter child : writer.getChildrenWriters()) {
- writeFileStatistics(builder, child);
- }
- }
-
- private int writeMetadata() throws IOException {
- getStream();
- OrcProto.Metadata.Builder builder = OrcProto.Metadata.newBuilder();
- for (OrcProto.StripeStatistics.Builder ssb : treeWriter.stripeStatsBuilders) {
- builder.addStripeStats(ssb.build());
- }
-
- long startPosn = rawWriter.getBytesWritten();
- OrcProto.Metadata metadata = builder.build();
- metadata.writeTo(protobufWriter);
- protobufWriter.flush();
- writer.flush();
- return (int) (rawWriter.getBytesWritten() - startPosn);
- }
-
- private int writeFooter(long bodyLength) throws IOException {
- getStream();
- OrcProto.Footer.Builder builder = OrcProto.Footer.newBuilder();
- builder.setContentLength(bodyLength);
- builder.setHeaderLength(headerLength);
- builder.setNumberOfRows(rowCount);
- builder.setRowIndexStride(rowIndexStride);
- // populate raw data size
- rawDataSize = computeRawDataSize();
- // serialize the types
- writeTypes(builder, treeWriter);
- // add the stripe information
- for (OrcProto.StripeInformation stripe : stripes) {
- builder.addStripes(stripe);
- }
- // add the column statistics
- writeFileStatistics(builder, treeWriter);
- // add all of the user metadata
- for (Map.Entry entry : userMetadata.entrySet()) {
- builder.addMetadata(OrcProto.UserMetadataItem.newBuilder()
- .setName(entry.getKey()).setValue(entry.getValue()));
- }
- long startPosn = rawWriter.getBytesWritten();
- OrcProto.Footer footer = builder.build();
- footer.writeTo(protobufWriter);
- protobufWriter.flush();
- writer.flush();
- return (int) (rawWriter.getBytesWritten() - startPosn);
- }
-
- private int writePostScript(int footerLength, int metadataLength) throws IOException {
- OrcProto.PostScript.Builder builder =
- OrcProto.PostScript.newBuilder()
- .setCompression(writeCompressionKind(compress))
- .setFooterLength(footerLength)
- .setMetadataLength(metadataLength)
- .setMagic(OrcFile.MAGIC)
- .addVersion(version.getMajor())
- .addVersion(version.getMinor())
- .setWriterVersion(OrcFile.WriterVersion.HIVE_8732.getId());
- if (compress != CompressionKind.NONE) {
- builder.setCompressionBlockSize(bufferSize);
- }
- OrcProto.PostScript ps = builder.build();
- // need to write this uncompressed
- long startPosn = rawWriter.getBytesWritten();
- ps.writeTo(rawWriter);
- long length = rawWriter.getBytesWritten() - startPosn;
- if (length > 255) {
- throw new IllegalArgumentException("PostScript too large at " + length);
- }
- return (int) length;
- }
-
- private long estimateStripeSize() {
- long result = 0;
- for (BufferedStream stream : streams.values()) {
- result += stream.getBufferSize();
- }
- result += treeWriter.estimateMemory();
- return result;
- }
-
- @Override
- public synchronized void addUserMetadata(String name, ByteBuffer value) {
- userMetadata.put(name, ByteString.copyFrom(value));
- }
-
- @Override
- public void addRow(Object row) throws IOException {
- synchronized (this) {
- treeWriter.write(row);
- rowsInStripe += 1;
- if (buildIndex) {
- rowsInIndex += 1;
-
- if (rowsInIndex >= rowIndexStride) {
- createRowIndexEntry();
- }
- }
- }
- memoryManager.addedRow();
- }
-
- public void addRowBatch(VectorizedRowBatch batch) throws IOException {
- if (buildIndex) {
- // Batch the writes up to the rowIndexStride so that we can get the
- // right size indexes.
- int posn = 0;
- while (posn < batch.size) {
- int chunkSize = Math.min(batch.size - posn,
- rowIndexStride - rowsInIndex);
- treeWriter.writeRootBatch(batch, posn, chunkSize);
- posn += chunkSize;
- rowsInIndex += chunkSize;
- rowsInStripe += chunkSize;
- if (rowsInIndex >= rowIndexStride) {
- createRowIndexEntry();
- }
- }
- } else {
- rowsInStripe += batch.size;
- treeWriter.writeRootBatch(batch, 0, batch.size);
- }
- memoryManager.addedRow();
- }
-
-
- @Override
- public void close() throws IOException {
- if (callback != null) {
- callback.preFooterWrite(callbackContext);
- }
- // remove us from the memory manager so that we don't get any callbacks
- memoryManager.removeWriter(path);
- // actually close the file
- flushStripe();
- int metadataLength = writeMetadata();
- int footerLength = writeFooter(rawWriter.getBytesWritten() - metadataLength);
- rawWriter.write(writePostScript(footerLength, metadataLength));
- rawWriter.close();
-
- }
-
- /**
- * Raw data size will be compute when writing the file footer. Hence raw data
- * size value will be available only after closing the writer.
- */
- @Override
- public long getRawDataSize() {
- return rawDataSize;
- }
-
- /**
- * Row count gets updated when flushing the stripes. To get accurate row
- * count call this method after writer is closed.
- */
- @Override
- public long getNumberOfRows() {
- return rowCount;
- }
-
- @Override
- public long writeIntermediateFooter() throws IOException {
- // flush any buffered rows
- flushStripe();
- // write a footer
- if (stripesAtLastFlush != stripes.size()) {
- if (callback != null) {
- callback.preFooterWrite(callbackContext);
- }
- int metaLength = writeMetadata();
- int footLength = writeFooter(rawWriter.getBytesWritten() - metaLength);
- rawWriter.write(writePostScript(footLength, metaLength));
- stripesAtLastFlush = stripes.size();
- rawWriter.flush();
- }
- return rawWriter.getBytesWritten();
- }
-
- @Override
- public void appendStripe(byte[] stripe, int offset, int length,
- StripeInformation stripeInfo,
- OrcProto.StripeStatistics stripeStatistics) throws IOException {
- Objects.requireNonNull(stripe, "Stripe must not be null");
- if (length > stripe.length) {
- throw new IllegalArgumentException("Specified length must not be greater specified array length");
- }
- Objects.requireNonNull(stripeInfo, "Stripe information must not be null");
- Objects.requireNonNull(stripeStatistics,
- "Stripe statistics must not be null");
-
- getStream();
- long start = rawWriter.getBytesWritten();
- long stripeLen = length;
- long availBlockSpace = blockSize - (start % blockSize);
-
- // see if stripe can fit in the current hdfs block, else pad the remaining
- // space in the block
- if (stripeLen < blockSize && stripeLen > availBlockSpace && addBlockPadding) {
- byte[] pad = new byte[(int) Math.min(HDFS_BUFFER_SIZE, availBlockSpace)];
- LOG.info(String.format("Padding ORC by %d bytes while merging..",
- availBlockSpace));
- start += availBlockSpace;
- while (availBlockSpace > 0) {
- int writeLen = (int) Math.min(availBlockSpace, pad.length);
- rawWriter.write(pad, 0, writeLen);
- availBlockSpace -= writeLen;
- }
- }
-
- rawWriter.write(stripe);
- rowsInStripe = stripeStatistics.getColStats(0).getNumberOfValues();
- rowCount += rowsInStripe;
-
- // since we have already written the stripe, just update stripe statistics
- treeWriter.stripeStatsBuilders.add(stripeStatistics.toBuilder());
-
- // update file level statistics
- updateFileStatistics(stripeStatistics);
-
- // update stripe information
- OrcProto.StripeInformation dirEntry = OrcProto.StripeInformation
- .newBuilder()
- .setOffset(start)
- .setNumberOfRows(rowsInStripe)
- .setIndexLength(stripeInfo.getIndexLength())
- .setDataLength(stripeInfo.getDataLength())
- .setFooterLength(stripeInfo.getFooterLength())
- .build();
- stripes.add(dirEntry);
-
- // reset it after writing the stripe
- rowsInStripe = 0;
- }
-
- private void updateFileStatistics(OrcProto.StripeStatistics stripeStatistics) {
- List cs = stripeStatistics.getColStatsList();
- List allWriters = getAllColumnTreeWriters(treeWriter);
- for (int i = 0; i < allWriters.size(); i++) {
- allWriters.get(i).fileStatistics.merge(ColumnStatisticsImpl.deserialize(cs.get(i)));
- }
- }
-
- private List getAllColumnTreeWriters(TreeWriter rootTreeWriter) {
- List result = Lists.newArrayList();
- getAllColumnTreeWritersImpl(rootTreeWriter, result);
- return result;
- }
-
- private void getAllColumnTreeWritersImpl(TreeWriter tw,
- List result) {
- result.add(tw);
- for (TreeWriter child : tw.childrenWriters) {
- getAllColumnTreeWritersImpl(child, result);
- }
- }
-
- @Override
- public void appendUserMetadata(List userMetadata) {
- if (userMetadata != null) {
- for (UserMetadataItem item : userMetadata) {
- this.userMetadata.put(item.getName(), item.getValue());
- }
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
deleted file mode 100644
index d9b2999378..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/dbcp/hive/HiveConnectionPool.java
+++ /dev/null
@@ -1,459 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.dbcp.hive;
-
-import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.jdbc.HiveDriver;
-import org.apache.nifi.annotation.behavior.RequiresInstanceClassLoading;
-import org.apache.nifi.annotation.documentation.CapabilityDescription;
-import org.apache.nifi.annotation.documentation.DeprecationNotice;
-import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnDisabled;
-import org.apache.nifi.annotation.lifecycle.OnEnabled;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.PropertyValue;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.resource.ResourceCardinality;
-import org.apache.nifi.components.resource.ResourceType;
-import org.apache.nifi.controller.AbstractControllerService;
-import org.apache.nifi.controller.ConfigurationContext;
-import org.apache.nifi.controller.ControllerServiceInitializationContext;
-import org.apache.nifi.dbcp.DBCPValidator;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.hadoop.KerberosProperties;
-import org.apache.nifi.hadoop.SecurityUtil;
-import org.apache.nifi.kerberos.KerberosCredentialsService;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.reporting.InitializationException;
-import org.apache.nifi.security.krb.KerberosKeytabUser;
-import org.apache.nifi.security.krb.KerberosLoginException;
-import org.apache.nifi.security.krb.KerberosPasswordUser;
-import org.apache.nifi.security.krb.KerberosUser;
-import org.apache.nifi.util.hive.AuthenticationFailedException;
-import org.apache.nifi.util.hive.HiveConfigurator;
-import org.apache.nifi.util.hive.ValidationResources;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * Implementation for Database Connection Pooling Service used for Apache Hive
- * connections. Apache DBCP is used for connection pooling functionality.
- */
-@RequiresInstanceClassLoading
-@Tags({"hive", "dbcp", "jdbc", "database", "connection", "pooling", "store"})
-@CapabilityDescription("Provides Database Connection Pooling Service for Apache Hive. Connections can be asked from pool and returned after usage.")
-@DeprecationNotice(classNames = "org.apache.nifi.dbcp.hive.Hive3ConnectionPool")
-public class HiveConnectionPool extends AbstractControllerService implements HiveDBCPService {
- private static final String ALLOW_EXPLICIT_KEYTAB = "NIFI_ALLOW_EXPLICIT_KEYTAB";
-
- private static final String DEFAULT_MAX_CONN_LIFETIME = "-1";
-
- public static final PropertyDescriptor DATABASE_URL = new PropertyDescriptor.Builder()
- .name("hive-db-connect-url")
- .displayName("Database Connection URL")
- .description("A database connection URL used to connect to a database. May contain database system name, host, port, database name and some parameters."
- + " The exact syntax of a database connection URL is specified by the Hive documentation. For example, the server principal is often included "
- + "as a connection parameter when connecting to a secure Hive server.")
- .defaultValue(null)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor HIVE_CONFIGURATION_RESOURCES = new PropertyDescriptor.Builder()
- .name("hive-config-resources")
- .displayName("Hive Configuration Resources")
- .description("A file or comma separated list of files which contains the Hive configuration (hive-site.xml, e.g.). Without this, Hadoop "
- + "will search the classpath for a 'hive-site.xml' file or will revert to a default configuration. Note that to enable authentication "
- + "with Kerberos e.g., the appropriate properties must be set in the configuration files. Please see the Hive documentation for more details.")
- .required(false)
- .identifiesExternalResource(ResourceCardinality.MULTIPLE, ResourceType.FILE)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor DB_USER = new PropertyDescriptor.Builder()
- .name("hive-db-user")
- .displayName("Database User")
- .description("Database user name")
- .defaultValue(null)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor DB_PASSWORD = new PropertyDescriptor.Builder()
- .name("hive-db-password")
- .displayName("Password")
- .description("The password for the database user")
- .defaultValue(null)
- .required(false)
- .sensitive(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor MAX_WAIT_TIME = new PropertyDescriptor.Builder()
- .name("hive-max-wait-time")
- .displayName("Max Wait Time")
- .description("The maximum amount of time that the pool will wait (when there are no available connections) "
- + " for a connection to be returned before failing, or -1 to wait indefinitely. ")
- .defaultValue("500 millis")
- .required(true)
- .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor MAX_TOTAL_CONNECTIONS = new PropertyDescriptor.Builder()
- .name("hive-max-total-connections")
- .displayName("Max Total Connections")
- .description("The maximum number of active connections that can be allocated from this pool at the same time, "
- + "or negative for no limit.")
- .defaultValue("8")
- .required(true)
- .addValidator(StandardValidators.INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor MAX_CONN_LIFETIME = new PropertyDescriptor.Builder()
- .displayName("Max Connection Lifetime")
- .name("hive-max-conn-lifetime")
- .description("The maximum lifetime in milliseconds of a connection. After this time is exceeded the " +
- "connection pool will invalidate the connection. A value of zero or -1 " +
- "means the connection has an infinite lifetime.")
- .defaultValue(DEFAULT_MAX_CONN_LIFETIME)
- .required(true)
- .addValidator(DBCPValidator.CUSTOM_TIME_PERIOD_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor VALIDATION_QUERY = new PropertyDescriptor.Builder()
- .name("Validation-query")
- .displayName("Validation query")
- .description("Validation query used to validate connections before returning them. "
- + "When a borrowed connection is invalid, it gets dropped and a new valid connection will be returned. "
- + "NOTE: Using validation may have a performance penalty.")
- .required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- static final PropertyDescriptor KERBEROS_CREDENTIALS_SERVICE = new PropertyDescriptor.Builder()
- .name("kerberos-credentials-service")
- .displayName("Kerberos Credentials Service")
- .description("Specifies the Kerberos Credentials Controller Service that should be used for authenticating with Kerberos")
- .identifiesControllerService(KerberosCredentialsService.class)
- .required(false)
- .build();
-
-
- private List properties;
-
- private String connectionUrl = "unknown";
-
- // Holder of cached Configuration information so validation does not reload the same config over and over
- private final AtomicReference validationResourceHolder = new AtomicReference<>();
-
- private volatile BasicDataSource dataSource;
-
- private volatile HiveConfigurator hiveConfigurator = new HiveConfigurator();
- private volatile UserGroupInformation ugi;
- private final AtomicReference kerberosUserReference = new AtomicReference<>();
- private volatile File kerberosConfigFile = null;
- private volatile KerberosProperties kerberosProperties;
-
- @Override
- protected void init(final ControllerServiceInitializationContext context) {
- List props = new ArrayList<>();
- props.add(DATABASE_URL);
- props.add(HIVE_CONFIGURATION_RESOURCES);
- props.add(DB_USER);
- props.add(DB_PASSWORD);
- props.add(MAX_WAIT_TIME);
- props.add(MAX_TOTAL_CONNECTIONS);
- props.add(MAX_CONN_LIFETIME);
- props.add(VALIDATION_QUERY);
- props.add(KERBEROS_CREDENTIALS_SERVICE);
-
- kerberosConfigFile = context.getKerberosConfigurationFile();
- kerberosProperties = new KerberosProperties(kerberosConfigFile);
- props.add(kerberosProperties.getKerberosPrincipal());
- props.add(kerberosProperties.getKerberosKeytab());
- props.add(kerberosProperties.getKerberosPassword());
- properties = props;
- }
-
- @Override
- protected List getSupportedPropertyDescriptors() {
- return properties;
- }
-
- @Override
- protected Collection customValidate(ValidationContext validationContext) {
- boolean confFileProvided = validationContext.getProperty(HIVE_CONFIGURATION_RESOURCES).isSet();
-
- final List problems = new ArrayList<>();
-
- if (confFileProvided) {
- final String explicitPrincipal = validationContext.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
- final String explicitKeytab = validationContext.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue();
- final String explicitPassword = validationContext.getProperty(kerberosProperties.getKerberosPassword()).getValue();
- final KerberosCredentialsService credentialsService = validationContext.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
-
- final String resolvedPrincipal;
- final String resolvedKeytab;
- if (credentialsService != null) {
- resolvedPrincipal = credentialsService.getPrincipal();
- resolvedKeytab = credentialsService.getKeytab();
- } else {
- resolvedPrincipal = explicitPrincipal;
- resolvedKeytab = explicitKeytab;
- }
-
-
- final String configFiles = validationContext.getProperty(HIVE_CONFIGURATION_RESOURCES).evaluateAttributeExpressions().getValue();
- problems.addAll(hiveConfigurator.validate(configFiles, resolvedPrincipal, resolvedKeytab, explicitPassword, validationResourceHolder, getLogger()));
-
- if (credentialsService != null && (explicitPrincipal != null || explicitKeytab != null || explicitPassword != null)) {
- problems.add(new ValidationResult.Builder()
- .subject("Kerberos Credentials")
- .valid(false)
- .explanation("Cannot specify a Kerberos Credentials Service while also specifying a Kerberos Principal, Kerberos Keytab, or Kerberos Password")
- .build());
- }
-
- if (!isAllowExplicitKeytab() && explicitKeytab != null) {
- problems.add(new ValidationResult.Builder()
- .subject("Kerberos Credentials")
- .valid(false)
- .explanation("The '" + ALLOW_EXPLICIT_KEYTAB + "' system environment variable is configured to forbid explicitly configuring Kerberos Keytab in processors. "
- + "The Kerberos Credentials Service should be used instead of setting the Kerberos Keytab or Kerberos Principal property.")
- .build());
- }
- }
-
- return problems;
- }
-
- /**
- * Configures connection pool by creating an instance of the
- * {@link BasicDataSource} based on configuration provided with
- * {@link ConfigurationContext}.
- *
- * This operation makes no guarantees that the actual connection could be
- * made since the underlying system may still go off-line during normal
- * operation of the connection pool.
- *
- * As of Apache NiFi 1.5.0, due to changes made to
- * {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this class invoking
- * {@link HiveConfigurator#authenticate(Configuration, String, String)}
- * to authenticate a principal with Kerberos, Hive controller services no longer use a separate thread to
- * relogin, and instead call {@link UserGroupInformation#checkTGTAndReloginFromKeytab()} from
- * {@link HiveConnectionPool#getConnection()}. The relogin request is performed in a synchronized block to prevent
- * threads from requesting concurrent relogins. For more information, please read the documentation for
- * {@link SecurityUtil#loginKerberos(Configuration, String, String)}.
- *
- * In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started by
- * {@link HiveConfigurator#authenticate(Configuration, String, String, long)} when the Hive
- * controller service was enabled. The use of a separate thread to explicitly relogin could cause race conditions
- * with the implicit relogin attempts made by hadoop/Hive code on a thread that references the same
- * {@link UserGroupInformation} instance. One of these threads could leave the
- * {@link javax.security.auth.Subject} in {@link UserGroupInformation} to be cleared or in an unexpected state
- * while the other thread is attempting to use the {@link javax.security.auth.Subject}, resulting in failed
- * authentication attempts that would leave the Hive controller service in an unrecoverable state.
- *
- * @see SecurityUtil#loginKerberos(Configuration, String, String)
- * @see HiveConfigurator#authenticate(Configuration, String, String)
- * @see HiveConfigurator#authenticate(Configuration, String, String, long)
- * @param context the configuration context
- * @throws InitializationException if unable to create a database connection
- */
- @OnEnabled
- public void onConfigured(final ConfigurationContext context) throws InitializationException {
-
- ComponentLog log = getLogger();
-
- final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).evaluateAttributeExpressions().getValue();
- final Configuration hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles);
- final String validationQuery = context.getProperty(VALIDATION_QUERY).evaluateAttributeExpressions().getValue();
-
- // add any dynamic properties to the Hive configuration
- for (final Map.Entry entry : context.getProperties().entrySet()) {
- final PropertyDescriptor descriptor = entry.getKey();
- if (descriptor.isDynamic()) {
- hiveConfig.set(descriptor.getName(), context.getProperty(descriptor).evaluateAttributeExpressions().getValue());
- }
- }
-
- final String drv = HiveDriver.class.getName();
- if (SecurityUtil.isSecurityEnabled(hiveConfig)) {
- final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
- final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue();
- final String explicitPassword = context.getProperty(kerberosProperties.getKerberosPassword()).getValue();
- final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
-
- final String resolvedPrincipal;
- final String resolvedKeytab;
- if (credentialsService != null) {
- resolvedPrincipal = credentialsService.getPrincipal();
- resolvedKeytab = credentialsService.getKeytab();
- } else {
- resolvedPrincipal = explicitPrincipal;
- resolvedKeytab = explicitKeytab;
- }
-
- if (resolvedKeytab != null) {
- kerberosUserReference.set(new KerberosKeytabUser(resolvedPrincipal, resolvedKeytab));
- log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab});
- } else if (explicitPassword != null) {
- kerberosUserReference.set(new KerberosPasswordUser(resolvedPrincipal, explicitPassword));
- log.info("Hive Security Enabled, logging in as principal {} with password", new Object[] {resolvedPrincipal});
- } else {
- throw new InitializationException("Unable to authenticate with Kerberos, no keytab or password was provided");
- }
-
- try {
- ugi = hiveConfigurator.authenticate(hiveConfig, kerberosUserReference.get());
- } catch (AuthenticationFailedException ae) {
- log.error(ae.getMessage(), ae);
- throw new InitializationException(ae);
- }
-
- getLogger().info("Successfully logged in as principal " + resolvedPrincipal);
- }
-
- final String user = context.getProperty(DB_USER).evaluateAttributeExpressions().getValue();
- final String passw = context.getProperty(DB_PASSWORD).evaluateAttributeExpressions().getValue();
- final Long maxWaitMillis = context.getProperty(MAX_WAIT_TIME).evaluateAttributeExpressions().asTimePeriod(TimeUnit.MILLISECONDS);
- final Integer maxTotal = context.getProperty(MAX_TOTAL_CONNECTIONS).evaluateAttributeExpressions().asInteger();
- final long maxConnectionLifetimeMillis = extractMillisWithInfinite(context.getProperty(MAX_CONN_LIFETIME).evaluateAttributeExpressions());
-
- dataSource = new BasicDataSource();
- dataSource.setDriverClassName(drv);
-
- connectionUrl = context.getProperty(DATABASE_URL).evaluateAttributeExpressions().getValue();
-
- dataSource.setMaxWaitMillis(maxWaitMillis);
- dataSource.setMaxTotal(maxTotal);
- dataSource.setMaxConnLifetimeMillis(maxConnectionLifetimeMillis);
-
- if (validationQuery != null && !validationQuery.isEmpty()) {
- dataSource.setValidationQuery(validationQuery);
- dataSource.setTestOnBorrow(true);
- }
-
- dataSource.setUrl(connectionUrl);
- dataSource.setUsername(user);
- dataSource.setPassword(passw);
- }
-
- /**
- * Shutdown pool, close all open connections.
- */
- @OnDisabled
- public void shutdown() {
- try {
- dataSource.close();
- } catch (final SQLException e) {
- throw new ProcessException(e);
- }
- }
-
- @Override
- public Connection getConnection() throws ProcessException {
- try {
- if (ugi != null) {
- /*
- * Explicitly check the TGT and relogin if necessary with the KerberosUser instance. No synchronization
- * is necessary in the client code, since AbstractKerberosUser's checkTGTAndRelogin method is synchronized.
- */
- getLogger().trace("getting UGI instance");
- if (kerberosUserReference.get() != null) {
- // if there's a KerberosUser associated with this UGI, check the TGT and relogin if it is close to expiring
- KerberosUser kerberosUser = kerberosUserReference.get();
- getLogger().debug("kerberosUser is " + kerberosUser);
- try {
- getLogger().debug("checking TGT on kerberosUser [{}]", new Object[]{kerberosUser});
- kerberosUser.checkTGTAndRelogin();
- } catch (final KerberosLoginException e) {
- throw new ProcessException("Unable to relogin with kerberos credentials for " + kerberosUser.getPrincipal(), e);
- }
- } else {
- getLogger().debug("kerberosUser was null, will not refresh TGT with KerberosUser");
- // no synchronization is needed for UserGroupInformation.checkTGTAndReloginFromKeytab; UGI handles the synchronization internally
- ugi.checkTGTAndReloginFromKeytab();
- }
- try {
- return ugi.doAs((PrivilegedExceptionAction) () -> dataSource.getConnection());
- } catch (UndeclaredThrowableException e) {
- Throwable cause = e.getCause();
- if (cause instanceof SQLException) {
- throw (SQLException) cause;
- } else {
- throw e;
- }
- }
- } else {
- getLogger().info("Simple Authentication");
- return dataSource.getConnection();
- }
- } catch (SQLException | IOException | InterruptedException e) {
- getLogger().error("Error getting Hive connection", e);
- throw new ProcessException(e);
- }
- }
-
- @Override
- public String toString() {
- return "HiveConnectionPool[id=" + getIdentifier() + "]";
- }
-
- @Override
- public String getConnectionURL() {
- return connectionUrl;
- }
-
- /*
- * Overridable by subclasses in the same package, mainly intended for testing purposes to allow verification without having to set environment variables.
- */
- boolean isAllowExplicitKeytab() {
- return Boolean.parseBoolean(System.getenv(ALLOW_EXPLICIT_KEYTAB));
- }
-
- private long extractMillisWithInfinite(PropertyValue prop) {
- if (prop.getValue() == null || DEFAULT_MAX_CONN_LIFETIME.equals(prop.getValue())) {
- return -1;
- } else {
- return prop.asTimePeriod(TimeUnit.MILLISECONDS);
- }
- }
-
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java
deleted file mode 100644
index f1ef5fd7e7..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/AbstractHiveQLProcessor.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import org.antlr.runtime.tree.CommonTree;
-import org.apache.hadoop.hive.ql.parse.ASTNode;
-import org.apache.hadoop.hive.ql.parse.ParseDriver;
-import org.apache.hadoop.hive.ql.parse.ParseException;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.dbcp.hive.HiveDBCPService;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.io.InputStreamCallback;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.stream.io.StreamUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.math.BigDecimal;
-import java.nio.charset.Charset;
-import java.sql.Date;
-import java.sql.PreparedStatement;
-import java.sql.SQLDataException;
-import java.sql.SQLException;
-import java.sql.Time;
-import java.sql.Timestamp;
-import java.sql.Types;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * An abstract base class for HiveQL processors to share common data, methods, etc.
- */
-public abstract class AbstractHiveQLProcessor extends AbstractSessionFactoryProcessor {
-
- protected static final Pattern HIVEQL_TYPE_ATTRIBUTE_PATTERN = Pattern.compile("hiveql\\.args\\.(\\d+)\\.type");
- protected static final Pattern NUMBER_PATTERN = Pattern.compile("-?\\d+");
- static String ATTR_INPUT_TABLES = "query.input.tables";
- static String ATTR_OUTPUT_TABLES = "query.output.tables";
-
-
- public static final PropertyDescriptor HIVE_DBCP_SERVICE = new PropertyDescriptor.Builder()
- .name("Hive Database Connection Pooling Service")
- .description("The Hive Controller Service that is used to obtain connection(s) to the Hive database")
- .required(true)
- .identifiesControllerService(HiveDBCPService.class)
- .build();
-
- public static final PropertyDescriptor CHARSET = new PropertyDescriptor.Builder()
- .name("hive-charset")
- .displayName("Character Set")
- .description("Specifies the character set of the record data.")
- .required(true)
- .defaultValue("UTF-8")
- .addValidator(StandardValidators.CHARACTER_SET_VALIDATOR)
- .build();
-
- /**
- * Determines the HiveQL statement that should be executed for the given FlowFile
- *
- * @param session the session that can be used to access the given FlowFile
- * @param flowFile the FlowFile whose HiveQL statement should be executed
- * @return the HiveQL that is associated with the given FlowFile
- */
- protected String getHiveQL(final ProcessSession session, final FlowFile flowFile, final Charset charset) {
- // Read the HiveQL from the FlowFile's content
- final byte[] buffer = new byte[(int) flowFile.getSize()];
- session.read(flowFile, new InputStreamCallback() {
- @Override
- public void process(final InputStream in) throws IOException {
- StreamUtils.fillBuffer(in, buffer);
- }
- });
-
- // Create the PreparedStatement to use for this FlowFile.
- return new String(buffer, charset);
- }
-
- private class ParameterHolder {
- String attributeName;
- int jdbcType;
- String value;
- }
-
- /**
- * Sets all of the appropriate parameters on the given PreparedStatement, based on the given FlowFile attributes.
- *
- * @param stmt the statement to set the parameters on
- * @param attributes the attributes from which to derive parameter indices, values, and types
- * @throws SQLException if the PreparedStatement throws a SQLException when the appropriate setter is called
- */
- protected int setParameters(int base, final PreparedStatement stmt, int paramCount, final Map attributes) throws SQLException {
-
- Map parmMap = new TreeMap();
-
- for (final Map.Entry entry : attributes.entrySet()) {
- final String key = entry.getKey();
- final Matcher matcher = HIVEQL_TYPE_ATTRIBUTE_PATTERN.matcher(key);
- if (matcher.matches()) {
- final int parameterIndex = Integer.parseInt(matcher.group(1));
- if (parameterIndex >= base && parameterIndex < base + paramCount) {
- final boolean isNumeric = NUMBER_PATTERN.matcher(entry.getValue()).matches();
- if (!isNumeric) {
- throw new SQLDataException("Value of the " + key + " attribute is '" + entry.getValue() + "', which is not a valid JDBC numeral jdbcType");
- }
-
- final String valueAttrName = "hiveql.args." + parameterIndex + ".value";
-
- ParameterHolder ph = new ParameterHolder();
- int realIndexLoc = parameterIndex - base +1;
-
- ph.jdbcType = Integer.parseInt(entry.getValue());
- ph.value = attributes.get(valueAttrName);
- ph.attributeName = valueAttrName;
-
- parmMap.put(realIndexLoc, ph);
-
- }
- }
- }
-
-
- // Now that's we've retrieved the correct number of parameters and it's sorted, let's set them.
- for (final Map.Entry entry : parmMap.entrySet()) {
- final Integer index = entry.getKey();
- final ParameterHolder ph = entry.getValue();
-
- try {
- setParameter(stmt, ph.attributeName, index, ph.value, ph.jdbcType);
- } catch (final NumberFormatException nfe) {
- throw new SQLDataException("The value of the " + ph.attributeName + " is '" + ph.value + "', which cannot be converted into the necessary data jdbcType", nfe);
- }
- }
- return base + paramCount;
- }
-
- /**
- * Determines how to map the given value to the appropriate JDBC data jdbcType and sets the parameter on the
- * provided PreparedStatement
- *
- * @param stmt the PreparedStatement to set the parameter on
- * @param attrName the name of the attribute that the parameter is coming from - for logging purposes
- * @param parameterIndex the index of the HiveQL parameter to set
- * @param parameterValue the value of the HiveQL parameter to set
- * @param jdbcType the JDBC Type of the HiveQL parameter to set
- * @throws SQLException if the PreparedStatement throws a SQLException when calling the appropriate setter
- */
- protected void setParameter(final PreparedStatement stmt, final String attrName, final int parameterIndex, final String parameterValue, final int jdbcType) throws SQLException {
- if (parameterValue == null) {
- stmt.setNull(parameterIndex, jdbcType);
- } else {
- try {
- switch (jdbcType) {
- case Types.BIT:
- case Types.BOOLEAN:
- stmt.setBoolean(parameterIndex, Boolean.parseBoolean(parameterValue));
- break;
- case Types.TINYINT:
- stmt.setByte(parameterIndex, Byte.parseByte(parameterValue));
- break;
- case Types.SMALLINT:
- stmt.setShort(parameterIndex, Short.parseShort(parameterValue));
- break;
- case Types.INTEGER:
- stmt.setInt(parameterIndex, Integer.parseInt(parameterValue));
- break;
- case Types.BIGINT:
- stmt.setLong(parameterIndex, Long.parseLong(parameterValue));
- break;
- case Types.REAL:
- stmt.setFloat(parameterIndex, Float.parseFloat(parameterValue));
- break;
- case Types.FLOAT:
- case Types.DOUBLE:
- stmt.setDouble(parameterIndex, Double.parseDouble(parameterValue));
- break;
- case Types.DECIMAL:
- case Types.NUMERIC:
- stmt.setBigDecimal(parameterIndex, new BigDecimal(parameterValue));
- break;
- case Types.DATE:
- stmt.setDate(parameterIndex, new Date(Long.parseLong(parameterValue)));
- break;
- case Types.TIME:
- stmt.setTime(parameterIndex, new Time(Long.parseLong(parameterValue)));
- break;
- case Types.TIMESTAMP:
- stmt.setTimestamp(parameterIndex, new Timestamp(Long.parseLong(parameterValue)));
- break;
- case Types.CHAR:
- case Types.VARCHAR:
- case Types.LONGNVARCHAR:
- case Types.LONGVARCHAR:
- stmt.setString(parameterIndex, parameterValue);
- break;
- default:
- stmt.setObject(parameterIndex, parameterValue, jdbcType);
- break;
- }
- } catch (SQLException e) {
- // Log which attribute/parameter had an error, then rethrow to be handled at the top level
- getLogger().error("Error setting parameter {} to value from {} ({})", new Object[]{parameterIndex, attrName, parameterValue}, e);
- throw e;
- }
- }
- }
-
- protected static class TableName {
- private final String database;
- private final String table;
- private final boolean input;
-
- TableName(String database, String table, boolean input) {
- this.database = database;
- this.table = table;
- this.input = input;
- }
-
- public String getDatabase() {
- return database;
- }
-
- public String getTable() {
- return table;
- }
-
- public boolean isInput() {
- return input;
- }
-
- @Override
- public String toString() {
- return database == null || database.isEmpty() ? table : database + '.' + table;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TableName tableName = (TableName) o;
-
- if (input != tableName.input) return false;
- if (database != null ? !database.equals(tableName.database) : tableName.database != null) return false;
- return table.equals(tableName.table);
- }
-
- @Override
- public int hashCode() {
- int result = database != null ? database.hashCode() : 0;
- result = 31 * result + table.hashCode();
- result = 31 * result + (input ? 1 : 0);
- return result;
- }
- }
-
- protected Set findTableNames(final String query) {
- final ASTNode node;
- try {
- node = new ParseDriver().parse(normalize(query));
- } catch (ParseException e) {
- // If failed to parse the query, just log a message, but continue.
- getLogger().debug("Failed to parse query: {} due to {}", new Object[]{query, e}, e);
- return Collections.emptySet();
- }
-
- final HashSet tableNames = new HashSet<>();
- findTableNames(node, tableNames);
- return tableNames;
- }
-
- /**
- * Normalize query.
- * Hive resolves prepared statement parameters before executing a query,
- * see {@link org.apache.hive.jdbc.HivePreparedStatement#updateSql(String, HashMap)} for detail.
- * HiveParser does not expect '?' to be in a query string, and throws an Exception if there is one.
- * In this normalize method, '?' is replaced to 'x' to avoid that.
- */
- private String normalize(String query) {
- return query.replace('?', 'x');
- }
-
- private void findTableNames(final Object obj, final Set tableNames) {
- if (!(obj instanceof CommonTree)) {
- return;
- }
- final CommonTree tree = (CommonTree) obj;
- final int childCount = tree.getChildCount();
- if ("TOK_TABNAME".equals(tree.getText())) {
- final TableName tableName;
- final boolean isInput = "TOK_TABREF".equals(tree.getParent().getText());
- switch (childCount) {
- case 1 :
- tableName = new TableName(null, tree.getChild(0).getText(), isInput);
- break;
- case 2:
- tableName = new TableName(tree.getChild(0).getText(), tree.getChild(1).getText(), isInput);
- break;
- default:
- throw new IllegalStateException("TOK_TABNAME does not have expected children, childCount=" + childCount);
- }
- // If parent is TOK_TABREF, then it is an input table.
- tableNames.add(tableName);
- return;
- }
- for (int i = 0; i < childCount; i++) {
- findTableNames(tree.getChild(i), tableNames);
- }
- }
-
- protected Map toQueryTableAttributes(Set tableNames) {
- final Map attributes = new HashMap<>();
- for (TableName tableName : tableNames) {
- final String attributeName = tableName.isInput() ? ATTR_INPUT_TABLES : ATTR_OUTPUT_TABLES;
- if (attributes.containsKey(attributeName)) {
- attributes.put(attributeName, attributes.get(attributeName) + "," + tableName);
- } else {
- attributes.put(attributeName, tableName.toString());
- }
- }
- return attributes;
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java
deleted file mode 100644
index d918365a37..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/ConvertAvroToORC.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.io.orc.CompressionKind;
-import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils;
-import org.apache.hadoop.hive.ql.io.orc.OrcFlowFileWriter;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.nifi.annotation.behavior.InputRequirement;
-import org.apache.nifi.annotation.behavior.SideEffectFree;
-import org.apache.nifi.annotation.behavior.SupportsBatching;
-import org.apache.nifi.annotation.behavior.WritesAttribute;
-import org.apache.nifi.annotation.behavior.WritesAttributes;
-import org.apache.nifi.annotation.documentation.CapabilityDescription;
-import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnScheduled;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.resource.ResourceCardinality;
-import org.apache.nifi.components.resource.ResourceType;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.processor.AbstractProcessor;
-import org.apache.nifi.processor.DataUnit;
-import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.util.hive.HiveJdbcCommon;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * The ConvertAvroToORC processor takes an Avro-formatted flow file as input and converts it into ORC format.
- */
-@SideEffectFree
-@SupportsBatching
-@Tags({"avro", "orc", "hive", "convert"})
-@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
-@CapabilityDescription("Converts an Avro record into ORC file format. This processor provides a direct mapping of an Avro record to an ORC record, such "
- + "that the resulting ORC file will have the same hierarchical structure as the Avro document. If an incoming FlowFile contains a stream of "
- + "multiple Avro records, the resultant FlowFile will contain a ORC file containing all of the Avro records. If an incoming FlowFile does "
- + "not contain any records, an empty ORC file is the output. NOTE: Many Avro datatypes (collections, primitives, and unions of primitives, e.g.) can "
- + "be converted to ORC, but unions of collections and other complex datatypes may not be able to be converted to ORC.")
-@WritesAttributes({
- @WritesAttribute(attribute = "mime.type", description = "Sets the mime type to application/octet-stream"),
- @WritesAttribute(attribute = "filename", description = "Sets the filename to the existing filename with the extension replaced by / added to by .orc"),
- @WritesAttribute(attribute = "record.count", description = "Sets the number of records in the ORC file."),
- @WritesAttribute(attribute = "hive.ddl", description = "Creates a partial Hive DDL statement for creating a table in Hive from this ORC file. "
- + "This can be used in ReplaceText for setting the content to the DDL. To make it valid DDL, add \"LOCATION ''\", where "
- + "the path is the directory that contains this ORC file on HDFS. For example, ConvertAvroToORC can send flow files to a PutHDFS processor to send the file to "
- + "HDFS, then to a ReplaceText to set the content to this DDL (plus the LOCATION clause as described), then to PutHiveQL processor to create the table "
- + "if it doesn't exist.")
-})
-public class ConvertAvroToORC extends AbstractProcessor {
-
- // Attributes
- public static final String ORC_MIME_TYPE = "application/octet-stream";
- public static final String HIVE_DDL_ATTRIBUTE = "hive.ddl";
- public static final String RECORD_COUNT_ATTRIBUTE = "record.count";
-
-
- // Properties
- public static final PropertyDescriptor ORC_CONFIGURATION_RESOURCES = new PropertyDescriptor.Builder()
- .name("orc-config-resources")
- .displayName("ORC Configuration Resources")
- .description("A file or comma separated list of files which contains the ORC configuration (hive-site.xml, e.g.). Without this, Hadoop "
- + "will search the classpath for a 'hive-site.xml' file or will revert to a default configuration. Please see the ORC documentation for more details.")
- .required(false)
- .identifiesExternalResource(ResourceCardinality.MULTIPLE, ResourceType.FILE)
- .build();
-
- public static final PropertyDescriptor STRIPE_SIZE = new PropertyDescriptor.Builder()
- .name("orc-stripe-size")
- .displayName("Stripe Size")
- .description("The size of the memory buffer (in bytes) for writing stripes to an ORC file")
- .required(true)
- .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
- .defaultValue("64 MB")
- .build();
-
- public static final PropertyDescriptor BUFFER_SIZE = new PropertyDescriptor.Builder()
- .name("orc-buffer-size")
- .displayName("Buffer Size")
- .description("The maximum size of the memory buffers (in bytes) used for compressing and storing a stripe in memory. This is a hint to the ORC writer, "
- + "which may choose to use a smaller buffer size based on stripe size and number of columns for efficient stripe writing and memory utilization.")
- .required(true)
- .addValidator(StandardValidators.DATA_SIZE_VALIDATOR)
- .defaultValue("10 KB")
- .build();
-
- public static final PropertyDescriptor COMPRESSION_TYPE = new PropertyDescriptor.Builder()
- .name("orc-compression-type")
- .displayName("Compression Type")
- .required(true)
- .allowableValues("NONE", "ZLIB", "SNAPPY", "LZO")
- .defaultValue("NONE")
- .build();
-
- public static final PropertyDescriptor HIVE_TABLE_NAME = new PropertyDescriptor.Builder()
- .name("orc-hive-table-name")
- .displayName("Hive Table Name")
- .description("An optional table name to insert into the hive.ddl attribute. The generated DDL can be used by "
- + "a PutHiveQL processor (presumably after a PutHDFS processor) to create a table backed by the converted ORC file. "
- + "If this property is not provided, the full name (including namespace) of the incoming Avro record will be normalized "
- + "and used as the table name.")
- .required(false)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
- .build();
-
- // Relationships
- static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("A FlowFile is routed to this relationship after it has been converted to ORC format.")
- .build();
- static final Relationship REL_FAILURE = new Relationship.Builder()
- .name("failure")
- .description("A FlowFile is routed to this relationship if it cannot be parsed as Avro or cannot be converted to ORC for any reason")
- .build();
-
- private final static List propertyDescriptors;
- private final static Set relationships;
-
- private volatile Configuration orcConfig;
-
- /*
- * Will ensure that the list of property descriptors is built only once.
- * Will also create a Set of relationships
- */
- static {
- List _propertyDescriptors = new ArrayList<>();
- _propertyDescriptors.add(ORC_CONFIGURATION_RESOURCES);
- _propertyDescriptors.add(STRIPE_SIZE);
- _propertyDescriptors.add(BUFFER_SIZE);
- _propertyDescriptors.add(COMPRESSION_TYPE);
- _propertyDescriptors.add(HIVE_TABLE_NAME);
- propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors);
-
- Set _relationships = new HashSet<>();
- _relationships.add(REL_SUCCESS);
- _relationships.add(REL_FAILURE);
- relationships = Collections.unmodifiableSet(_relationships);
- }
-
- @Override
- protected List getSupportedPropertyDescriptors() {
- return propertyDescriptors;
- }
-
- @Override
- public Set getRelationships() {
- return relationships;
- }
-
- @OnScheduled
- public void setup(ProcessContext context) {
- boolean confFileProvided = context.getProperty(ORC_CONFIGURATION_RESOURCES).isSet();
- if (confFileProvided) {
- final String configFiles = context.getProperty(ORC_CONFIGURATION_RESOURCES).getValue();
- orcConfig = HiveJdbcCommon.getConfigurationFromFiles(configFiles);
- }
- }
-
- @Override
- public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
- FlowFile flowFile = session.get();
- if (flowFile == null) {
- return;
- }
-
- try {
- long startTime = System.currentTimeMillis();
- final long stripeSize = context.getProperty(STRIPE_SIZE).asDataSize(DataUnit.B).longValue();
- final int bufferSize = context.getProperty(BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
- final CompressionKind compressionType = CompressionKind.valueOf(context.getProperty(COMPRESSION_TYPE).getValue());
- final AtomicReference hiveAvroSchema = new AtomicReference<>(null);
- final AtomicInteger totalRecordCount = new AtomicInteger(0);
- final String fileName = flowFile.getAttribute(CoreAttributes.FILENAME.key());
- flowFile = session.write(flowFile, (rawIn, rawOut) -> {
- try (final InputStream in = new BufferedInputStream(rawIn);
- final OutputStream out = new BufferedOutputStream(rawOut);
- final DataFileStream reader = new DataFileStream<>(in, new GenericDatumReader<>())) {
-
- // Create ORC schema from Avro schema
- Schema avroSchema = reader.getSchema();
-
- TypeInfo orcSchema = NiFiOrcUtils.getOrcField(avroSchema);
-
- if (orcConfig == null) {
- orcConfig = new Configuration();
- }
-
- OrcFlowFileWriter orcWriter = NiFiOrcUtils.createWriter(
- out,
- new Path(fileName),
- orcConfig,
- orcSchema,
- stripeSize,
- compressionType,
- bufferSize);
- try {
-
- int recordCount = 0;
- while (reader.hasNext()) {
- GenericRecord currRecord = reader.next();
- List fields = currRecord.getSchema().getFields();
- if (fields != null) {
- Object[] row = new Object[fields.size()];
- for (int i = 0; i < fields.size(); i++) {
- Schema.Field field = fields.get(i);
- Schema fieldSchema = field.schema();
- Object o = currRecord.get(field.name());
- try {
- row[i] = NiFiOrcUtils.convertToORCObject(NiFiOrcUtils.getOrcField(fieldSchema), o);
- } catch (ArrayIndexOutOfBoundsException aioobe) {
- getLogger().error("Index out of bounds at record {} for column {}, type {}, and object {}",
- new Object[]{recordCount, i, fieldSchema.getType().getName(), o.toString()},
- aioobe);
- throw new IOException(aioobe);
- }
- }
- orcWriter.addRow(NiFiOrcUtils.createOrcStruct(orcSchema, row));
- recordCount++;
- }
- }
- hiveAvroSchema.set(avroSchema);
- totalRecordCount.set(recordCount);
- } finally {
- // finished writing this record, close the writer (which will flush to the flow file)
- orcWriter.close();
- }
- }
- });
-
- final String hiveTableName = context.getProperty(HIVE_TABLE_NAME).isSet()
- ? context.getProperty(HIVE_TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue()
- : NiFiOrcUtils.normalizeHiveTableName(hiveAvroSchema.get().getFullName());
- String hiveDDL = NiFiOrcUtils.generateHiveDDL(hiveAvroSchema.get(), hiveTableName);
- // Add attributes and transfer to success
- flowFile = session.putAttribute(flowFile, RECORD_COUNT_ATTRIBUTE, Integer.toString(totalRecordCount.get()));
- flowFile = session.putAttribute(flowFile, HIVE_DDL_ATTRIBUTE, hiveDDL);
- StringBuilder newFilename = new StringBuilder();
- int extensionIndex = fileName.lastIndexOf(".");
- if (extensionIndex != -1) {
- newFilename.append(fileName.substring(0, extensionIndex));
- } else {
- newFilename.append(fileName);
- }
- newFilename.append(".orc");
- flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), ORC_MIME_TYPE);
- flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), newFilename.toString());
- session.transfer(flowFile, REL_SUCCESS);
- session.getProvenanceReporter().modifyContent(flowFile, "Converted " + totalRecordCount.get() + " records", System.currentTimeMillis() - startTime);
-
- } catch (ProcessException | IllegalArgumentException e) {
- getLogger().error("Failed to convert {} from Avro to ORC due to {}; transferring to failure", new Object[]{flowFile, e});
- session.transfer(flowFile, REL_FAILURE);
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java
deleted file mode 100644
index ffa0a3edc2..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveQL.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.nifi.annotation.behavior.InputRequirement;
-import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
-import org.apache.nifi.annotation.behavior.ReadsAttribute;
-import org.apache.nifi.annotation.behavior.ReadsAttributes;
-import org.apache.nifi.annotation.behavior.WritesAttribute;
-import org.apache.nifi.annotation.behavior.WritesAttributes;
-import org.apache.nifi.annotation.documentation.CapabilityDescription;
-import org.apache.nifi.annotation.documentation.DeprecationNotice;
-import org.apache.nifi.annotation.documentation.SeeAlso;
-import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnScheduled;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.dbcp.hive.HiveDBCPService;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.ProcessSessionFactory;
-import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processor.util.pattern.ErrorTypes;
-import org.apache.nifi.processor.util.pattern.ExceptionHandler;
-import org.apache.nifi.processor.util.pattern.ExceptionHandler.OnError;
-import org.apache.nifi.processor.util.pattern.PartialFunctions.FetchFlowFiles;
-import org.apache.nifi.processor.util.pattern.PartialFunctions.InitConnection;
-import org.apache.nifi.processor.util.pattern.Put;
-import org.apache.nifi.processor.util.pattern.RollbackOnFailure;
-import org.apache.nifi.processor.util.pattern.RoutingResult;
-
-import java.nio.charset.Charset;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.SQLNonTransientException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Pattern;
-
-@SeeAlso(SelectHiveQL.class)
-@InputRequirement(Requirement.INPUT_REQUIRED)
-@Tags({"sql", "hive", "put", "database", "update", "insert"})
-@CapabilityDescription("Executes a HiveQL DDL/DML command (UPDATE, INSERT, e.g.). The content of an incoming FlowFile is expected to be the HiveQL command "
- + "to execute. The HiveQL command may use the ? to escape parameters. In this case, the parameters to use must exist as FlowFile attributes "
- + "with the naming convention hiveql.args.N.type and hiveql.args.N.value, where N is a positive integer. The hiveql.args.N.type is expected to be "
- + "a number indicating the JDBC Type. The content of the FlowFile is expected to be in UTF-8 format.")
-@ReadsAttributes({
- @ReadsAttribute(attribute = "hiveql.args.N.type", description = "Incoming FlowFiles are expected to be parametrized HiveQL statements. The type of each Parameter is specified as an integer "
- + "that represents the JDBC Type of the parameter."),
- @ReadsAttribute(attribute = "hiveql.args.N.value", description = "Incoming FlowFiles are expected to be parametrized HiveQL statements. The value of the Parameters are specified as "
- + "hiveql.args.1.value, hiveql.args.2.value, hiveql.args.3.value, and so on. The type of the hiveql.args.1.value Parameter is specified by the hiveql.args.1.type attribute.")
-})
-@WritesAttributes({
- @WritesAttribute(attribute = "query.input.tables", description = "This attribute is written on the flow files routed to the 'success' relationships, "
- + "and contains input table names (if any) in comma delimited 'databaseName.tableName' format."),
- @WritesAttribute(attribute = "query.output.tables", description = "This attribute is written on the flow files routed to the 'success' relationships, "
- + "and contains the target table names in 'databaseName.tableName' format.")
-})
-@DeprecationNotice(classNames = "org.apache.nifi.processors.hive.PutHive3QL")
-public class PutHiveQL extends AbstractHiveQLProcessor {
-
- public static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder()
- .name("hive-batch-size")
- .displayName("Batch Size")
- .description("The preferred number of FlowFiles to put to the database in a single transaction")
- .required(true)
- .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR)
- .defaultValue("100")
- .build();
-
- public static final PropertyDescriptor STATEMENT_DELIMITER = new PropertyDescriptor.Builder()
- .name("statement-delimiter")
- .displayName("Statement Delimiter")
- .description("Statement Delimiter used to separate SQL statements in a multiple statement script")
- .required(true)
- .defaultValue(";")
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.NONE)
- .build();
-
- public static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("A FlowFile is routed to this relationship after the database is successfully updated")
- .build();
- public static final Relationship REL_RETRY = new Relationship.Builder()
- .name("retry")
- .description("A FlowFile is routed to this relationship if the database cannot be updated but attempting the operation again may succeed")
- .build();
- public static final Relationship REL_FAILURE = new Relationship.Builder()
- .name("failure")
- .description("A FlowFile is routed to this relationship if the database cannot be updated and retrying the operation will also fail, "
- + "such as an invalid query or an integrity constraint violation")
- .build();
-
-
- private final static List propertyDescriptors;
- private final static Set relationships;
-
- /*
- * Will ensure that the list of property descriptors is built only once.
- * Will also create a Set of relationships
- */
- static {
- List _propertyDescriptors = new ArrayList<>();
- _propertyDescriptors.add(HIVE_DBCP_SERVICE);
- _propertyDescriptors.add(BATCH_SIZE);
- _propertyDescriptors.add(CHARSET);
- _propertyDescriptors.add(STATEMENT_DELIMITER);
- _propertyDescriptors.add(RollbackOnFailure.ROLLBACK_ON_FAILURE);
- propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors);
-
- Set _relationships = new HashSet<>();
- _relationships.add(REL_SUCCESS);
- _relationships.add(REL_FAILURE);
- _relationships.add(REL_RETRY);
- relationships = Collections.unmodifiableSet(_relationships);
- }
-
- private Put process;
- private ExceptionHandler exceptionHandler;
-
- @OnScheduled
- public void constructProcess() {
- exceptionHandler = new ExceptionHandler<>();
- exceptionHandler.mapException(e -> {
- if (e instanceof SQLNonTransientException) {
- return ErrorTypes.InvalidInput;
- } else if (e instanceof SQLException) {
- // Use the SQLException's vendor code for guidance -- see Hive's ErrorMsg class for details on error codes
- int errorCode = ((SQLException) e).getErrorCode();
- getLogger().debug("Error occurred during Hive operation, Hive returned error code {}", new Object[]{errorCode});
- if (errorCode >= 10000 && errorCode < 20000) {
- return ErrorTypes.InvalidInput;
- } else if (errorCode >= 20000 && errorCode < 30000) {
- return ErrorTypes.InvalidInput;
- } else if (errorCode >= 30000 && errorCode < 40000) {
- return ErrorTypes.TemporalInputFailure;
- } else if (errorCode >= 40000 && errorCode < 50000) {
- // These are unknown errors (to include some parse errors), but rather than generating an UnknownFailure which causes
- // a ProcessException, we'll route to failure via an InvalidInput error type.
- return ErrorTypes.InvalidInput;
- } else {
- // Default unknown errors to TemporalFailure (as they were implemented originally), so they can be routed to failure
- // or rolled back depending on the user's setting of Rollback On Failure.
- return ErrorTypes.TemporalFailure;
- }
- } else {
- return ErrorTypes.UnknownFailure;
- }
- });
- exceptionHandler.adjustError(RollbackOnFailure.createAdjustError(getLogger()));
-
- process = new Put<>();
- process.setLogger(getLogger());
- process.initConnection(initConnection);
- process.fetchFlowFiles(fetchFlowFiles);
- process.putFlowFile(putFlowFile);
- process.adjustRoute(RollbackOnFailure.createAdjustRoute(REL_FAILURE, REL_RETRY));
- }
-
- @Override
- protected List getSupportedPropertyDescriptors() {
- return propertyDescriptors;
- }
-
- @Override
- public Set getRelationships() {
- return relationships;
- }
-
- private class FunctionContext extends RollbackOnFailure {
- final Charset charset;
- final String statementDelimiter;
- final long startNanos = System.nanoTime();
-
- String connectionUrl;
-
-
- private FunctionContext(boolean rollbackOnFailure, Charset charset, String statementDelimiter) {
- super(rollbackOnFailure, false);
- this.charset = charset;
- this.statementDelimiter = statementDelimiter;
- }
- }
-
- private InitConnection initConnection = (context, session, fc, ffs) -> {
- final HiveDBCPService dbcpService = context.getProperty(HIVE_DBCP_SERVICE).asControllerService(HiveDBCPService.class);
- final Connection connection = dbcpService.getConnection(ffs == null || ffs.isEmpty() ? Collections.emptyMap() : ffs.get(0).getAttributes());
- fc.connectionUrl = dbcpService.getConnectionURL();
- return connection;
- };
-
- private FetchFlowFiles fetchFlowFiles = (context, session, functionContext, result) -> {
- final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
- return session.get(batchSize);
- };
-
- private Put.PutFlowFile putFlowFile = (context, session, fc, conn, flowFile, result) -> {
- final String script = getHiveQL(session, flowFile, fc.charset);
- String regex = "(? tableNames = new HashSet<>();
- exceptionHandler.execute(fc, flowFile, input -> {
- int loc = 1;
- for (String hiveQLStr: hiveQLs) {
- getLogger().debug("HiveQL: {}", new Object[]{hiveQLStr});
-
- final String hiveQL = hiveQLStr.trim();
- if (!StringUtils.isEmpty(hiveQL)) {
- try (final PreparedStatement stmt = conn.prepareStatement(hiveQL)) {
-
- // Get ParameterMetadata
- // Hive JDBC Doesn't support this yet:
- // ParameterMetaData pmd = stmt.getParameterMetaData();
- // int paramCount = pmd.getParameterCount();
- int paramCount = StringUtils.countMatches(hiveQL, "?");
-
- if (paramCount > 0) {
- loc = setParameters(loc, stmt, paramCount, flowFile.getAttributes());
- }
-
- // Parse hiveQL and extract input/output tables
- try {
- tableNames.addAll(findTableNames(hiveQL));
- } catch (Exception e) {
- // If failed to parse the query, just log a warning message, but continue.
- getLogger().warn("Failed to parse hiveQL: {} due to {}", new Object[]{hiveQL, e}, e);
- }
-
- // Execute the statement
- stmt.execute();
- fc.proceed();
- }
- }
- }
-
- // Emit a Provenance SEND event
- final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - fc.startNanos);
-
- final FlowFile updatedFlowFile = session.putAllAttributes(flowFile, toQueryTableAttributes(tableNames));
- session.getProvenanceReporter().send(updatedFlowFile, fc.connectionUrl, transmissionMillis, true);
- result.routeTo(flowFile, REL_SUCCESS);
-
- }, onFlowFileError(context, session, result));
-
- };
-
- private OnError onFlowFileError(final ProcessContext context, final ProcessSession session, final RoutingResult result) {
- OnError onFlowFileError = ExceptionHandler.createOnError(context, session, result, REL_FAILURE, REL_RETRY);
- onFlowFileError = onFlowFileError.andThen((c, i, r, e) -> {
- switch (r.destination()) {
- case Failure:
- getLogger().error("Failed to update Hive for {} due to {}; routing to failure", new Object[] {i, e}, e);
- break;
- case Retry:
- getLogger().error("Failed to update Hive for {} due to {}; it is possible that retrying the operation will succeed, so routing to retry",
- new Object[] {i, e}, e);
- break;
- case Self:
- getLogger().error("Failed to update Hive for {} due to {};", new Object[] {i, e}, e);
- break;
- }
- });
- return RollbackOnFailure.createOnError(onFlowFileError);
- }
-
- @Override
- public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
- final Boolean rollbackOnFailure = context.getProperty(RollbackOnFailure.ROLLBACK_ON_FAILURE).asBoolean();
- final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
- final String statementDelimiter = context.getProperty(STATEMENT_DELIMITER).getValue();
- final FunctionContext functionContext = new FunctionContext(rollbackOnFailure, charset, statementDelimiter);
- RollbackOnFailure.onTrigger(context, sessionFactory, functionContext, getLogger(), session -> process.onTrigger(context, session, functionContext));
- }
-}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java
deleted file mode 100644
index 7511e44e2e..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/PutHiveStreaming.java
+++ /dev/null
@@ -1,1216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.file.CodecFactory;
-import org.apache.avro.file.DataFileConstants;
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.file.SeekableByteArrayInput;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.hcatalog.streaming.ConnectionError;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.SerializationError;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.nifi.annotation.behavior.RequiresInstanceClassLoading;
-import org.apache.nifi.annotation.behavior.WritesAttribute;
-import org.apache.nifi.annotation.behavior.WritesAttributes;
-import org.apache.nifi.annotation.documentation.CapabilityDescription;
-import org.apache.nifi.annotation.documentation.DeprecationNotice;
-import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnScheduled;
-import org.apache.nifi.annotation.lifecycle.OnStopped;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.Validator;
-import org.apache.nifi.components.resource.ResourceCardinality;
-import org.apache.nifi.components.resource.ResourceType;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.hadoop.KerberosProperties;
-import org.apache.nifi.hadoop.SecurityUtil;
-import org.apache.nifi.kerberos.KerberosCredentialsService;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.AbstractSessionFactoryProcessor;
-import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.ProcessSessionFactory;
-import org.apache.nifi.processor.ProcessorInitializationContext;
-import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.io.InputStreamCallback;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processor.util.pattern.DiscontinuedException;
-import org.apache.nifi.processor.util.pattern.ErrorTypes;
-import org.apache.nifi.processor.util.pattern.ExceptionHandler;
-import org.apache.nifi.processor.util.pattern.RollbackOnFailure;
-import org.apache.nifi.processor.util.pattern.RoutingResult;
-import org.apache.nifi.security.krb.KerberosKeytabUser;
-import org.apache.nifi.security.krb.KerberosLoginException;
-import org.apache.nifi.security.krb.KerberosPasswordUser;
-import org.apache.nifi.security.krb.KerberosUser;
-import org.apache.nifi.util.hive.AuthenticationFailedException;
-import org.apache.nifi.util.hive.HiveConfigurator;
-import org.apache.nifi.util.hive.HiveOptions;
-import org.apache.nifi.util.hive.HiveUtils;
-import org.apache.nifi.util.hive.HiveWriter;
-import org.apache.nifi.util.hive.ValidationResources;
-import org.xerial.snappy.Snappy;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.BiFunction;
-import java.util.regex.Pattern;
-
-@Tags({"hive", "streaming", "put", "database", "store"})
-@CapabilityDescription("This processor uses Hive Streaming to send flow file data to an Apache Hive table. The incoming flow file is expected to be in "
- + "Avro format and the table must exist in Hive. Please see the Hive documentation for requirements on the Hive table (format, partitions, etc.). "
- + "The partition values are extracted from the Avro record based on the names of the partition columns as specified in the processor. NOTE: If "
- + "multiple concurrent tasks are configured for this processor, only one table can be written to at any time by a single thread. Additional tasks "
- + "intending to write to the same table will wait for the current task to finish writing to the table.")
-@WritesAttributes({
- @WritesAttribute(attribute = "hivestreaming.record.count", description = "This attribute is written on the flow files routed to the 'success' "
- + "and 'failure' relationships, and contains the number of records from the incoming flow file written successfully and unsuccessfully, respectively."),
- @WritesAttribute(attribute = "query.output.tables", description = "This attribute is written on the flow files routed to the 'success' "
- + "and 'failure' relationships, and contains the target table name in 'databaseName.tableName' format.")
-})
-@RequiresInstanceClassLoading
-@DeprecationNotice(classNames = "org.apache.nifi.processors.hive.PutHive3Streaming")
-public class PutHiveStreaming extends AbstractSessionFactoryProcessor {
- private static final String ALLOW_EXPLICIT_KEYTAB = "NIFI_ALLOW_EXPLICIT_KEYTAB";
-
- // Attributes
- public static final String HIVE_STREAMING_RECORD_COUNT_ATTR = "hivestreaming.record.count";
-
- private static final String CLIENT_CACHE_DISABLED_PROPERTY = "hcatalog.hive.client.cache.disabled";
-
- // Validators
- private static final Validator GREATER_THAN_ONE_VALIDATOR = (subject, value, context) -> {
- if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(value)) {
- return new ValidationResult.Builder().subject(subject).input(value).explanation("Expression Language Present").valid(true).build();
- }
-
- String reason = null;
- try {
- final int intVal = Integer.parseInt(value);
-
- if (intVal < 2) {
- reason = "value is less than 2";
- }
- } catch (final NumberFormatException e) {
- reason = "value is not a valid integer";
- }
-
- return new ValidationResult.Builder().subject(subject).input(value).explanation(reason).valid(reason == null).build();
- };
-
- // Metadata keys that are not transferred to split files when output strategy is datafile
- // Avro will write this key/values pairs on its own
- private static final Set RESERVED_METADATA;
-
- static {
- // This is used to prevent a race condition in Snappy 1.0.5 where two classloaders could
- // try to define the native loader class at the same time, causing an error. Make a no-op
- // call here to ensure Snappy's static initializers are called. Note that this block is
- // called once by the extensions loader before any actual processor instances are created,
- // so the race condition will not occur, and for each other instance, this is a no-op
- try {
- Snappy.compress("");
- } catch (IOException ioe) {
- // Do nothing here, should never happen as it is intended to be a no-op
- }
-
- Set reservedMetadata = new HashSet<>();
- reservedMetadata.add("avro.schema");
- reservedMetadata.add("avro.codec");
- RESERVED_METADATA = Collections.unmodifiableSet(reservedMetadata);
- }
-
- // Properties
- public static final PropertyDescriptor METASTORE_URI = new PropertyDescriptor.Builder()
- .name("hive-stream-metastore-uri")
- .displayName("Hive Metastore URI")
- .description("The URI location for the Hive Metastore. Note that this is not the location of the Hive Server. The default port for the "
- + "Hive metastore is 9043.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.URI_VALIDATOR)
- .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("(^[^/]+.*[^/]+$|^[^/]+$|^$)"))) // no start with / or end with /
- .build();
-
- public static final PropertyDescriptor HIVE_CONFIGURATION_RESOURCES = new PropertyDescriptor.Builder()
- .name("hive-config-resources")
- .displayName("Hive Configuration Resources")
- .description("A file or comma separated list of files which contains the Hive configuration (hive-site.xml, e.g.). Without this, Hadoop "
- + "will search the classpath for a 'hive-site.xml' file or will revert to a default configuration. Note that to enable authentication "
- + "with Kerberos e.g., the appropriate properties must be set in the configuration files. Also note that if Max Concurrent Tasks is set "
- + "to a number greater than one, the 'hcatalog.hive.client.cache.disabled' property will be forced to 'true' to avoid concurrency issues. "
- + "Please see the Hive documentation for more details.")
- .required(false)
- .identifiesExternalResource(ResourceCardinality.MULTIPLE, ResourceType.FILE)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor DB_NAME = new PropertyDescriptor.Builder()
- .name("hive-stream-database-name")
- .displayName("Database Name")
- .description("The name of the database in which to put the data.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
-
- public static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder()
- .name("hive-stream-table-name")
- .displayName("Table Name")
- .description("The name of the database table in which to put the data.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
-
- public static final PropertyDescriptor PARTITION_COLUMNS = new PropertyDescriptor.Builder()
- .name("hive-stream-partition-cols")
- .displayName("Partition Columns")
- .description("A comma-delimited list of column names on which the table has been partitioned. The order of values in this list must "
- + "correspond exactly to the order of partition columns specified during the table creation.")
- .required(false)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("[^,]+(,[^,]+)*"))) // comma-separated list with non-empty entries
- .build();
-
- public static final PropertyDescriptor AUTOCREATE_PARTITIONS = new PropertyDescriptor.Builder()
- .name("hive-stream-autocreate-partition")
- .displayName("Auto-Create Partitions")
- .description("Flag indicating whether partitions should be automatically created")
- .required(true)
- .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
- .allowableValues("true", "false")
- .defaultValue("true")
- .build();
-
- public static final PropertyDescriptor MAX_OPEN_CONNECTIONS = new PropertyDescriptor.Builder()
- .name("hive-stream-max-open-connections")
- .displayName("Max Open Connections")
- .description("The maximum number of open connections that can be allocated from this pool at the same time, "
- + "or negative for no limit.")
- .defaultValue("8")
- .required(true)
- .addValidator(StandardValidators.INTEGER_VALIDATOR)
- .sensitive(false)
- .build();
-
- public static final PropertyDescriptor HEARTBEAT_INTERVAL = new PropertyDescriptor.Builder()
- .name("hive-stream-heartbeat-interval")
- .displayName("Heartbeat Interval")
- .description("Indicates that a heartbeat should be sent when the specified number of seconds has elapsed. "
- + "A value of 0 indicates that no heartbeat should be sent. "
- + "Note that although this property supports Expression Language, it will not be evaluated against incoming FlowFile attributes.")
- .defaultValue("60")
- .required(true)
- .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor TXNS_PER_BATCH = new PropertyDescriptor.Builder()
- .name("hive-stream-transactions-per-batch")
- .displayName("Transactions per Batch")
- .description("A hint to Hive Streaming indicating how many transactions the processor task will need. This value must be greater than 1.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(GREATER_THAN_ONE_VALIDATOR)
- .defaultValue("100")
- .build();
-
- public static final PropertyDescriptor RECORDS_PER_TXN = new PropertyDescriptor.Builder()
- .name("hive-stream-records-per-transaction")
- .displayName("Records per Transaction")
- .description("Number of records to process before committing the transaction. This value must be greater than 1.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(GREATER_THAN_ONE_VALIDATOR)
- .defaultValue("10000")
- .build();
-
- public static final PropertyDescriptor CALL_TIMEOUT = new PropertyDescriptor.Builder()
- .name("hive-stream-call-timeout")
- .displayName("Call Timeout")
- .description("The number of seconds allowed for a Hive Streaming operation to complete. A value of 0 indicates the processor should wait indefinitely on operations. "
- + "Note that although this property supports Expression Language, it will not be evaluated against incoming FlowFile attributes.")
- .defaultValue("0")
- .required(true)
- .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build();
-
- public static final PropertyDescriptor ROLLBACK_ON_FAILURE = RollbackOnFailure.createRollbackOnFailureProperty(
- "NOTE: When an error occurred after a Hive streaming transaction which is derived from the same input FlowFile is already committed," +
- " (i.e. a FlowFile contains more records than 'Records per Transaction' and a failure occurred at the 2nd transaction or later)" +
- " then the succeeded records will be transferred to 'success' relationship while the original input FlowFile stays in incoming queue." +
- " Duplicated records can be created for the succeeded ones when the same FlowFile is processed again.");
-
- static final PropertyDescriptor KERBEROS_CREDENTIALS_SERVICE = new PropertyDescriptor.Builder()
- .name("kerberos-credentials-service")
- .displayName("Kerberos Credentials Service")
- .description("Specifies the Kerberos Credentials Controller Service that should be used for authenticating with Kerberos")
- .identifiesControllerService(KerberosCredentialsService.class)
- .required(false)
- .build();
-
- // Relationships
- public static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("A FlowFile containing Avro records routed to this relationship after the record has been successfully transmitted to Hive.")
- .build();
-
- public static final Relationship REL_FAILURE = new Relationship.Builder()
- .name("failure")
- .description("A FlowFile containing Avro records routed to this relationship if the record could not be transmitted to Hive.")
- .build();
-
- public static final Relationship REL_RETRY = new Relationship.Builder()
- .name("retry")
- .description("The incoming FlowFile is routed to this relationship if its records cannot be transmitted to Hive. Note that "
- + "some records may have been processed successfully, they will be routed (as Avro flow files) to the success relationship. "
- + "The combination of the retry, success, and failure relationships indicate how many records succeeded and/or failed. This "
- + "can be used to provide a retry capability since full rollback is not possible.")
- .build();
-
- private List propertyDescriptors;
- private Set relationships;
-
- protected KerberosProperties kerberosProperties;
- private volatile File kerberosConfigFile = null;
-
- protected volatile HiveConfigurator hiveConfigurator = new HiveConfigurator();
- protected volatile UserGroupInformation ugi;
- final protected AtomicReference kerberosUserReference = new AtomicReference<>();
- protected volatile HiveConf hiveConfig;
-
- protected final AtomicBoolean sendHeartBeat = new AtomicBoolean(false);
-
- protected volatile int callTimeout;
- protected ExecutorService callTimeoutPool;
- protected transient Timer heartBeatTimer;
-
- protected volatile ConcurrentLinkedQueue> threadWriterList = new ConcurrentLinkedQueue<>();
- protected volatile ConcurrentHashMap tableSemaphoreMap = new ConcurrentHashMap<>();
-
- // Holder of cached Configuration information so validation does not reload the same config over and over
- private final AtomicReference validationResourceHolder = new AtomicReference<>();
-
- @Override
- protected void init(ProcessorInitializationContext context) {
- List props = new ArrayList<>();
- props.add(METASTORE_URI);
- props.add(HIVE_CONFIGURATION_RESOURCES);
- props.add(DB_NAME);
- props.add(TABLE_NAME);
- props.add(PARTITION_COLUMNS);
- props.add(AUTOCREATE_PARTITIONS);
- props.add(MAX_OPEN_CONNECTIONS);
- props.add(HEARTBEAT_INTERVAL);
- props.add(TXNS_PER_BATCH);
- props.add(RECORDS_PER_TXN);
- props.add(CALL_TIMEOUT);
- props.add(ROLLBACK_ON_FAILURE);
- props.add(KERBEROS_CREDENTIALS_SERVICE);
-
- kerberosConfigFile = context.getKerberosConfigurationFile();
- kerberosProperties = new KerberosProperties(kerberosConfigFile);
- props.add(kerberosProperties.getKerberosPrincipal());
- props.add(kerberosProperties.getKerberosKeytab());
- props.add(kerberosProperties.getKerberosPassword());
- propertyDescriptors = Collections.unmodifiableList(props);
-
- Set _relationships = new HashSet<>();
- _relationships.add(REL_SUCCESS);
- _relationships.add(REL_FAILURE);
- _relationships.add(REL_RETRY);
- relationships = Collections.unmodifiableSet(_relationships);
- }
-
- @Override
- protected List getSupportedPropertyDescriptors() {
- return propertyDescriptors;
- }
-
- @Override
- public Set getRelationships() {
- return relationships;
- }
-
-
- @Override
- protected Collection customValidate(final ValidationContext validationContext) {
- boolean confFileProvided = validationContext.getProperty(HIVE_CONFIGURATION_RESOURCES).isSet();
-
- final List problems = new ArrayList<>();
-
- if (confFileProvided) {
- final String explicitPrincipal = validationContext.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
- final String explicitKeytab = validationContext.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue();
- final String explicitPassword = validationContext.getProperty(kerberosProperties.getKerberosPassword()).getValue();
- final KerberosCredentialsService credentialsService = validationContext.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
-
- final String resolvedPrincipal;
- final String resolvedKeytab;
- if (credentialsService == null) {
- resolvedPrincipal = explicitPrincipal;
- resolvedKeytab = explicitKeytab;
- } else {
- resolvedPrincipal = credentialsService.getPrincipal();
- resolvedKeytab = credentialsService.getKeytab();
- }
-
-
- final String configFiles = validationContext.getProperty(HIVE_CONFIGURATION_RESOURCES).evaluateAttributeExpressions().getValue();
- problems.addAll(hiveConfigurator.validate(configFiles, resolvedPrincipal, resolvedKeytab, explicitPassword, validationResourceHolder, getLogger()));
-
- if (credentialsService != null && (explicitPrincipal != null || explicitKeytab != null || explicitPassword != null)) {
- problems.add(new ValidationResult.Builder()
- .subject("Kerberos Credentials")
- .valid(false)
- .explanation("Cannot specify a Kerberos Credentials Service while also specifying a Kerberos Principal, Kerberos Keytab, or Kerberos Password")
- .build());
- }
-
- if (!isAllowExplicitKeytab() && explicitKeytab != null) {
- problems.add(new ValidationResult.Builder()
- .subject("Kerberos Credentials")
- .valid(false)
- .explanation("The '" + ALLOW_EXPLICIT_KEYTAB + "' system environment variable is configured to forbid explicitly configuring Kerberos Keytab in processors. "
- + "The Kerberos Credentials Service should be used instead of setting the Kerberos Keytab or Kerberos Principal property.")
- .build());
- }
- }
-
- return problems;
- }
-
- @OnScheduled
- public void setup(final ProcessContext context) {
- ComponentLog log = getLogger();
-
- final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger();
- final String configFiles = context.getProperty(HIVE_CONFIGURATION_RESOURCES).evaluateAttributeExpressions().getValue();
- hiveConfig = hiveConfigurator.getConfigurationFromFiles(configFiles);
-
- // If more than one concurrent task, force 'hcatalog.hive.client.cache.disabled' to true
- if(context.getMaxConcurrentTasks() > 1) {
- hiveConfig.setBoolean(CLIENT_CACHE_DISABLED_PROPERTY, true);
- }
-
- // add any dynamic properties to the Hive configuration
- for (final Map.Entry entry : context.getProperties().entrySet()) {
- final PropertyDescriptor descriptor = entry.getKey();
- if (descriptor.isDynamic()) {
- hiveConfig.set(descriptor.getName(), entry.getValue());
- }
- }
-
- hiveConfigurator.preload(hiveConfig);
-
- if (SecurityUtil.isSecurityEnabled(hiveConfig)) {
- final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
- final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue();
- final String explicitPassword = context.getProperty(kerberosProperties.getKerberosPassword()).getValue();
- final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
-
- final String resolvedPrincipal;
- final String resolvedKeytab;
- if (credentialsService == null) {
- resolvedPrincipal = explicitPrincipal;
- resolvedKeytab = explicitKeytab;
- } else {
- resolvedPrincipal = credentialsService.getPrincipal();
- resolvedKeytab = credentialsService.getKeytab();
- }
-
- if (resolvedKeytab != null) {
- kerberosUserReference.set(new KerberosKeytabUser(resolvedPrincipal, resolvedKeytab));
- log.info("Hive Security Enabled, logging in as principal {} with keytab {}", new Object[] {resolvedPrincipal, resolvedKeytab});
- } else if (explicitPassword != null) {
- kerberosUserReference.set(new KerberosPasswordUser(resolvedPrincipal, explicitPassword));
- log.info("Hive Security Enabled, logging in as principal {} with password", new Object[] {resolvedPrincipal});
- } else {
- throw new ProcessException("Unable to authenticate with Kerberos, no keytab or password was provided");
- }
-
- try {
- ugi = hiveConfigurator.authenticate(hiveConfig, kerberosUserReference.get());
- } catch (AuthenticationFailedException ae) {
- throw new ProcessException("Kerberos authentication failed for Hive Streaming", ae);
- }
-
- log.info("Successfully logged in as principal " + resolvedPrincipal);
- } else {
- ugi = null;
- kerberosUserReference.set(null);
- }
-
- callTimeout = context.getProperty(CALL_TIMEOUT).evaluateAttributeExpressions().asInteger() * 1000; // milliseconds
- String timeoutName = "put-hive-streaming-%d";
- this.callTimeoutPool = Executors.newFixedThreadPool(1,
- new ThreadFactoryBuilder().setNameFormat(timeoutName).build());
-
- sendHeartBeat.set(true);
- heartBeatTimer = new Timer();
- setupHeartBeatTimer(heartbeatInterval);
- }
-
- private static class FunctionContext extends RollbackOnFailure {
-
- private AtomicReference successFlowFile;
- private AtomicReference failureFlowFile;
- private final DataFileWriter successAvroWriter = new DataFileWriter<>(new GenericDatumWriter());
- private final DataFileWriter failureAvroWriter = new DataFileWriter<>(new GenericDatumWriter());
- private byte[] successAvroHeader;
- private byte[] failureAvroHeader;
-
- private final AtomicInteger recordCount = new AtomicInteger(0);
- private final AtomicInteger successfulRecordCount = new AtomicInteger(0);
- private final AtomicInteger failedRecordCount = new AtomicInteger(0);
-
- private final ComponentLog logger;
-
- /**
- * It's possible that multiple Hive streaming transactions are committed within a single onTrigger.
- * PutHiveStreaming onTrigger is not 'transactional' in a sense of RollbackOnFailure.
- * Once a Hive streaming transaction is committed, processor session will not be rolled back.
- * @param rollbackOnFailure whether process session should be rolled back if failed
- */
- private FunctionContext(boolean rollbackOnFailure, ComponentLog logger) {
- super(rollbackOnFailure, false);
- this.logger = logger;
- }
-
- private void setFlowFiles(FlowFile successFlowFile, FlowFile failureFlowFile) {
- this.successFlowFile = new AtomicReference<>(successFlowFile);
- this.failureFlowFile = new AtomicReference<>(failureFlowFile);
- }
-
- private byte[] initAvroWriter(ProcessSession session, String codec, DataFileStream reader,
- DataFileWriter writer, AtomicReference flowFileRef) {
-
- writer.setCodec(CodecFactory.fromString(codec));
- // Transfer metadata (this is a subset of the incoming file)
- for (String metaKey : reader.getMetaKeys()) {
- if (!RESERVED_METADATA.contains(metaKey)) {
- writer.setMeta(metaKey, reader.getMeta(metaKey));
- }
- }
-
- final ByteArrayOutputStream avroHeader = new ByteArrayOutputStream();
- flowFileRef.set(session.append(flowFileRef.get(), (out) -> {
- // Create writer so that records can be appended later.
- writer.create(reader.getSchema(), avroHeader);
- writer.close();
-
- final byte[] header = avroHeader.toByteArray();
- out.write(header);
- }));
-
- // Capture the Avro header byte array that is just written to the FlowFile.
- // This is needed when Avro records are appended to the same FlowFile.
- return avroHeader.toByteArray();
- }
-
- private void initAvroWriters(ProcessSession session, String codec, DataFileStream reader) {
- successAvroHeader = initAvroWriter(session, codec, reader, successAvroWriter, successFlowFile);
- failureAvroHeader = initAvroWriter(session, codec, reader, failureAvroWriter, failureFlowFile);
- }
-
- private void appendAvroRecords(ProcessSession session, byte[] avroHeader, DataFileWriter writer,
- AtomicReference flowFileRef, List hRecords) {
-
- flowFileRef.set(session.append(flowFileRef.get(), (out) -> {
- if (hRecords != null) {
- // Initialize the writer again as append mode, so that Avro header is written only once.
- writer.appendTo(new SeekableByteArrayInput(avroHeader), out);
- try {
- for (HiveStreamingRecord hRecord : hRecords) {
- writer.append(hRecord.getRecord());
- }
- } catch (IOException ioe) {
- // The records were put to Hive Streaming successfully, but there was an error while writing the
- // Avro records to the flow file. Log as an error and move on.
- logger.error("Error writing Avro records (which were sent successfully to Hive Streaming) to the flow file, " + ioe, ioe);
- }
- }
- writer.close();
- }));
- }
-
- private void appendRecordsToSuccess(ProcessSession session, List records) {
- appendAvroRecords(session, successAvroHeader, successAvroWriter, successFlowFile, records);
- successfulRecordCount.addAndGet(records.size());
- }
-
- private void appendRecordsToFailure(ProcessSession session, List records) {
- appendAvroRecords(session, failureAvroHeader, failureAvroWriter, failureFlowFile, records);
- failedRecordCount.addAndGet(records.size());
- }
-
- private void transferFlowFiles(ProcessSession session, RoutingResult result, HiveOptions options) {
-
- if (successfulRecordCount.get() > 0) {
- // Transfer the flow file with successful records
- Map updateAttributes = new HashMap<>();
- updateAttributes.put(HIVE_STREAMING_RECORD_COUNT_ATTR, Integer.toString(successfulRecordCount.get()));
- updateAttributes.put(AbstractHiveQLProcessor.ATTR_OUTPUT_TABLES, options.getQualifiedTableName());
- successFlowFile.set(session.putAllAttributes(successFlowFile.get(), updateAttributes));
- session.getProvenanceReporter().send(successFlowFile.get(), options.getMetaStoreURI());
- result.routeTo(successFlowFile.get(), REL_SUCCESS);
- } else {
- session.remove(successFlowFile.get());
- }
-
- if (failedRecordCount.get() > 0) {
- // There were some failed records, so transfer that flow file to failure
- Map updateAttributes = new HashMap<>();
- updateAttributes.put(HIVE_STREAMING_RECORD_COUNT_ATTR, Integer.toString(failedRecordCount.get()));
- updateAttributes.put(AbstractHiveQLProcessor.ATTR_OUTPUT_TABLES, options.getQualifiedTableName());
- failureFlowFile.set(session.putAllAttributes(failureFlowFile.get(), updateAttributes));
- result.routeTo(failureFlowFile.get(), REL_FAILURE);
- } else {
- session.remove(failureFlowFile.get());
- }
-
- result.getRoutedFlowFiles().forEach((relationship, flowFiles) -> {
- session.transfer(flowFiles, relationship);
- });
- }
-
- }
-
- private static class ShouldRetryException extends RuntimeException {
- private ShouldRetryException(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
- private ExceptionHandler.OnError> onHiveRecordsError(ProcessContext context, ProcessSession session, Map writers) {
- return RollbackOnFailure.createOnError((fc, input, res, e) -> {
-
- if (res.penalty() == ErrorTypes.Penalty.Yield) {
- context.yield();
- }
-
- switch (res.destination()) {
- case Failure:
- // Add the failed record to the failure flow file
- getLogger().error(String.format("Error writing %s to Hive Streaming transaction due to %s", input, e), e);
- fc.appendRecordsToFailure(session, input);
- break;
-
- case Retry:
- // If we can't connect to the endpoint, exit the loop and let the outer exception handler route the original flow file to retry
- abortAndCloseWriters(writers);
- throw new ShouldRetryException("Hive Streaming connect/write error, flow file will be penalized and routed to retry. " + e, e);
-
- case Self:
- getLogger().error(String.format("Error writing %s to Hive Streaming transaction due to %s", input, e), e);
- abortAndCloseWriters(writers);
- break;
-
- default:
- abortAndCloseWriters(writers);
- if (e instanceof ProcessException) {
- throw (ProcessException) e;
- } else {
- throw new ProcessException(String.format("Error writing %s to Hive Streaming transaction due to %s", input, e), e);
- }
- }
- });
- }
-
- private ExceptionHandler.OnError onHiveRecordError(ProcessContext context, ProcessSession session, Map writers) {
- return (fc, input, res, e) -> onHiveRecordsError(context, session, writers).apply(fc, Collections.singletonList(input), res, e);
- }
-
- private ExceptionHandler.OnError onRecordError(ProcessContext context, ProcessSession session, Map writers) {
- return (fc, input, res, e) -> onHiveRecordError(context, session, writers).apply(fc, new HiveStreamingRecord(null, input), res, e);
- }
-
- @Override
- public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
- final FunctionContext functionContext = new FunctionContext(context.getProperty(ROLLBACK_ON_FAILURE).asBoolean(), getLogger());
- RollbackOnFailure.onTrigger(context, sessionFactory, functionContext, getLogger(), session -> onTrigger(context, session, functionContext));
- }
-
- private void onTrigger(ProcessContext context, ProcessSession session, FunctionContext functionContext) throws ProcessException {
- FlowFile flowFile = session.get();
- if (flowFile == null) {
- return;
- }
-
- final String dbName = context.getProperty(DB_NAME).evaluateAttributeExpressions(flowFile).getValue();
- final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
-
- // Only allow one thread to work on a DB/table at a time
- final Semaphore newSemaphore = new Semaphore(1);
- Semaphore semaphore = tableSemaphoreMap.putIfAbsent(dbName + "." + tableName, newSemaphore);
- if (semaphore == null) {
- semaphore = newSemaphore;
- }
-
- boolean gotSemaphore = false;
- try {
- gotSemaphore = semaphore.tryAcquire(0, TimeUnit.SECONDS);
- } catch (InterruptedException ie) {
- // Nothing to do, gotSemaphore defaults to false
- }
- if (!gotSemaphore) {
- // We didn't get a chance to acquire, so rollback the session and try again next time
- session.rollback();
- return;
- }
-
- final ComponentLog log = getLogger();
- final String metastoreUri = context.getProperty(METASTORE_URI).evaluateAttributeExpressions(flowFile).getValue();
-
- final boolean autoCreatePartitions = context.getProperty(AUTOCREATE_PARTITIONS).asBoolean();
- final Integer maxConnections = context.getProperty(MAX_OPEN_CONNECTIONS).asInteger();
- final Integer heartbeatInterval = context.getProperty(HEARTBEAT_INTERVAL).evaluateAttributeExpressions().asInteger();
- final Integer txnsPerBatch = context.getProperty(TXNS_PER_BATCH).evaluateAttributeExpressions(flowFile).asInteger();
- final Integer recordsPerTxn = context.getProperty(RECORDS_PER_TXN).evaluateAttributeExpressions(flowFile).asInteger();
-
- final Map myWriters = new ConcurrentHashMap<>();
- threadWriterList.add(myWriters);
-
- HiveOptions o = new HiveOptions(metastoreUri, dbName, tableName)
- .withTxnsPerBatch(txnsPerBatch)
- .withAutoCreatePartitions(autoCreatePartitions)
- .withMaxOpenConnections(maxConnections)
- .withHeartBeatInterval(heartbeatInterval)
- .withCallTimeout(callTimeout);
-
- if (SecurityUtil.isSecurityEnabled(hiveConfig)) {
- final String explicitPrincipal = context.getProperty(kerberosProperties.getKerberosPrincipal()).evaluateAttributeExpressions().getValue();
- final String explicitKeytab = context.getProperty(kerberosProperties.getKerberosKeytab()).evaluateAttributeExpressions().getValue();
- final KerberosCredentialsService credentialsService = context.getProperty(KERBEROS_CREDENTIALS_SERVICE).asControllerService(KerberosCredentialsService.class);
-
- final String resolvedPrincipal;
- final String resolvedKeytab;
- if (credentialsService == null) {
- resolvedPrincipal = explicitPrincipal;
- resolvedKeytab = explicitKeytab;
- } else {
- resolvedPrincipal = credentialsService.getPrincipal();
- resolvedKeytab = credentialsService.getKeytab();
- }
-
- o = o.withKerberosPrincipal(resolvedPrincipal).withKerberosKeytab(resolvedKeytab);
- }
-
- final HiveOptions options = o;
-
- // Store the original class loader, then explicitly set it to this class's classloader (for use by the Hive Metastore)
- ClassLoader originalClassloader = Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
-
- final List partitionColumnList;
- final String partitionColumns = context.getProperty(PARTITION_COLUMNS).evaluateAttributeExpressions().getValue();
- if (partitionColumns == null || partitionColumns.isEmpty()) {
- partitionColumnList = Collections.emptyList();
- } else {
- String[] partitionCols = partitionColumns.split(",");
- partitionColumnList = new ArrayList<>(partitionCols.length);
- for (String col : partitionCols) {
- partitionColumnList.add(col.trim());
- }
- }
-
- final AtomicReference> successfulRecords = new AtomicReference<>();
- successfulRecords.set(new ArrayList<>());
- final FlowFile inputFlowFile = flowFile;
-
- final RoutingResult result = new RoutingResult();
- final ExceptionHandler exceptionHandler = new ExceptionHandler<>();
- exceptionHandler.mapException(s -> {
- try {
- if (s == null) {
- return ErrorTypes.PersistentFailure;
- }
- throw s;
-
- } catch (IllegalArgumentException
- | AvroRuntimeException
- | HiveWriter.WriteFailure
- | SerializationError inputError) {
-
- return ErrorTypes.InvalidInput;
-
- } catch (HiveWriter.CommitFailure
- | HiveWriter.TxnBatchFailure
- | HiveWriter.TxnFailure writerTxError) {
-
- return ErrorTypes.TemporalInputFailure;
-
- } catch (ConnectionError
- | HiveWriter.ConnectFailure connectionError) {
- // Can't connect to Hive endpoint.
- log.error("Error connecting to Hive endpoint: table {} at {}",
- new Object[]{options.getTableName(), options.getMetaStoreURI()});
-
- return ErrorTypes.TemporalFailure;
-
- } catch (IOException
- | InterruptedException tempError) {
- return ErrorTypes.TemporalFailure;
-
- } catch (Exception t) {
- return ErrorTypes.UnknownFailure;
- }
- });
- final BiFunction adjustError = RollbackOnFailure.createAdjustError(getLogger());
- exceptionHandler.adjustError(adjustError);
-
- // Create output flow files and their Avro writers
- functionContext.setFlowFiles(session.create(inputFlowFile), session.create(inputFlowFile));
-
- try {
- session.read(inputFlowFile, new InputStreamCallback() {
- @Override
- public void process(InputStream in) throws IOException {
- try (final DataFileStream reader = new DataFileStream<>(in, new GenericDatumReader())) {
-
- GenericRecord currRecord = null;
-
- // Copy codec and schema information to all writers
- final String codec = reader.getMetaString(DataFileConstants.CODEC) == null
- ? DataFileConstants.NULL_CODEC
- : reader.getMetaString(DataFileConstants.CODEC);
-
- functionContext.initAvroWriters(session, codec, reader);
-
- Runnable flushSuccessfulRecords = () -> {
- // Now send the records to the successful FlowFile and update the success count
- functionContext.appendRecordsToSuccess(session, successfulRecords.get());
- // Clear the list of successful records, we'll use it at the end when we flush whatever records are left
- successfulRecords.set(new ArrayList<>());
- };
-
- while (reader.hasNext()) {
- // We can NOT reuse currRecord here, because currRecord is accumulated in successful records.
- // If we use the same GenericRecord instance, every record ends up having the same contents.
- // To avoid this, we need to create a brand new GenericRecord instance here each time.
- currRecord = reader.next();
- functionContext.recordCount.incrementAndGet();
-
- // Extract the partition values (they must be put separately into the Hive Streaming API)
- List partitionValues = new ArrayList<>();
-
- if (!exceptionHandler.execute(functionContext, currRecord, input -> {
- for (String partition : partitionColumnList) {
- Object partitionValue = input.get(partition);
- if (partitionValue == null) {
- throw new IllegalArgumentException("Partition column '" + partition + "' not found in Avro record");
- }
- partitionValues.add(partitionValue.toString());
- }
- }, onRecordError(context, session, myWriters))) {
- continue;
- }
-
- final HiveStreamingRecord record = new HiveStreamingRecord(partitionValues, currRecord);
- final AtomicReference hiveWriterRef = new AtomicReference<>();
-
- // Write record to Hive streaming
- if (!exceptionHandler.execute(functionContext, record, input -> {
-
- final HiveEndPoint endPoint = makeHiveEndPoint(record.getPartitionValues(), options);
- final HiveWriter hiveWriter = getOrCreateWriter(myWriters, options, endPoint);
- hiveWriterRef.set(hiveWriter);
-
- hiveWriter.write(record.getRecord().toString().getBytes(StandardCharsets.UTF_8));
- successfulRecords.get().add(record);
-
- }, onHiveRecordError(context, session, myWriters))) {
- continue;
- }
-
- // If we've reached the records-per-transaction limit, flush the Hive Writer and update the Avro Writer for successful records
- final HiveWriter hiveWriter = hiveWriterRef.get();
- if (hiveWriter.getTotalRecords() >= recordsPerTxn) {
- exceptionHandler.execute(functionContext, successfulRecords.get(), input -> {
-
- hiveWriter.flush(true);
- // Proceed function context. Process session can't be rollback anymore.
- functionContext.proceed();
-
- // Now send the records to the success relationship and update the success count
- flushSuccessfulRecords.run();
-
- }, onHiveRecordsError(context, session, myWriters).andThen((fc, input, res, commitException) -> {
- // Reset hiveWriter for succeeding records.
- switch (res.destination()) {
- case Retry:
- case Failure:
- try {
- // Abort current tx and move to next.
- hiveWriter.abort();
- } catch (Exception e) {
- // Can't even abort properly, throw a process exception
- throw new ProcessException(e);
- }
- }
- }));
- }
- }
-
- exceptionHandler.execute(functionContext, successfulRecords.get(), input -> {
- // Finish any transactions
- flushAllWriters(myWriters, true);
- closeAllWriters(myWriters);
-
- // Now send any remaining records to the success relationship and update the count
- flushSuccessfulRecords.run();
-
- // Append successfulRecords on failure.
- }, onHiveRecordsError(context, session, myWriters));
-
- } catch (IOException ioe) {
- // The Avro file is invalid (or may not be an Avro file at all), send it to failure
- final ErrorTypes.Result adjusted = adjustError.apply(functionContext, ErrorTypes.InvalidInput);
- final String msg = "The incoming flow file can not be read as an Avro file";
- switch (adjusted.destination()) {
- case Failure:
- log.error(msg, ioe);
- result.routeTo(inputFlowFile, REL_FAILURE);
- break;
- case ProcessException:
- throw new ProcessException(msg, ioe);
-
- }
- }
- }
- });
-
- // If we got here, we've processed the outgoing flow files correctly, so remove the incoming one if necessary
- if (result.getRoutedFlowFiles().values().stream().noneMatch(routed -> routed.contains(inputFlowFile))) {
- session.remove(inputFlowFile);
- }
-
- } catch (DiscontinuedException e) {
- // The input FlowFile processing is discontinued. Keep it in the input queue.
- getLogger().warn("Discontinued processing for {} due to {}", new Object[]{flowFile, e}, e);
- result.routeTo(flowFile, Relationship.SELF);
-
- } catch (ShouldRetryException e) {
- // This exception is already a result of adjusting an error, so simply transfer the FlowFile to retry.
- getLogger().error(e.getMessage(), e);
- flowFile = session.penalize(flowFile);
- result.routeTo(flowFile, REL_RETRY);
-
- } finally {
- threadWriterList.remove(myWriters);
- functionContext.transferFlowFiles(session, result, options);
- // Restore original class loader, might not be necessary but is good practice since the processor task changed it
- Thread.currentThread().setContextClassLoader(originalClassloader);
- semaphore.release();
- }
- }
-
- @OnStopped
- public void cleanup() {
- validationResourceHolder.set(null); // trigger re-validation of resources
-
- ComponentLog log = getLogger();
- sendHeartBeat.set(false);
- for(Map allWriters : threadWriterList) {
- for (Map.Entry entry : allWriters.entrySet()) {
- try {
- HiveWriter w = entry.getValue();
- w.flushAndClose();
- } catch (Exception ex) {
- log.warn("Error while closing writer to " + entry.getKey() + ". Exception follows.", ex);
- if (ex instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- }
- }
- allWriters.clear();
- }
-
- if (callTimeoutPool != null) {
- callTimeoutPool.shutdown();
- try {
- while (!callTimeoutPool.isTerminated()) {
- callTimeoutPool.awaitTermination(callTimeout, TimeUnit.MILLISECONDS);
- }
- } catch (Throwable t) {
- log.warn("shutdown interrupted on " + callTimeoutPool, t);
- }
- callTimeoutPool = null;
- }
-
- ugi = null;
- kerberosUserReference.set(null);
- }
-
- private void setupHeartBeatTimer(int heartbeatInterval) {
- if (heartbeatInterval > 0) {
- final ComponentLog log = getLogger();
- heartBeatTimer.schedule(new TimerTask() {
- @Override
- public void run() {
- try {
- if (sendHeartBeat.get()) {
- log.debug("Start sending heartbeat on all writers");
- sendHeartBeatOnAllWriters();
- setupHeartBeatTimer(heartbeatInterval);
- }
- } catch (Exception e) {
- log.warn("Failed to heartbeat on HiveWriter ", e);
- }
- }
- }, heartbeatInterval * 1000);
- }
- }
-
- private void sendHeartBeatOnAllWriters() throws InterruptedException {
- for(Map allWriters : threadWriterList) {
- for (HiveWriter writer : allWriters.values()) {
- writer.heartBeat();
- }
- }
- }
-
- private void flushAllWriters(Map writers, boolean rollToNext)
- throws HiveWriter.CommitFailure, HiveWriter.TxnBatchFailure, HiveWriter.TxnFailure, InterruptedException {
- for (HiveWriter writer : writers.values()) {
- writer.flush(rollToNext);
- }
- }
-
- private void abortAndCloseWriters(Map writers) {
- try {
- abortAllWriters(writers);
- closeAllWriters(writers);
- } catch (Exception ie) {
- getLogger().warn("unable to close hive connections. ", ie);
- }
- }
-
- /**
- * Abort current Txn on all writers
- */
- private void abortAllWriters(Map writers) throws InterruptedException, StreamingException, HiveWriter.TxnBatchFailure {
- for (Map.Entry entry : writers.entrySet()) {
- try {
- entry.getValue().abort();
- } catch (Exception e) {
- getLogger().error("Failed to abort hive transaction batch, HiveEndPoint " + entry.getValue() + " due to exception ", e);
- }
- }
- }
-
- /**
- * Closes all writers and remove them from cache
- */
- private void closeAllWriters(Map writers) {
- //1) Retire writers
- for (Map.Entry entry : writers.entrySet()) {
- try {
- entry.getValue().close();
- } catch (Exception e) {
- getLogger().warn("unable to close writers. ", e);
- }
- }
- //2) Clear cache
- writers.clear();
- }
-
- private HiveWriter getOrCreateWriter(Map writers, HiveOptions options, HiveEndPoint endPoint) throws HiveWriter.ConnectFailure, InterruptedException {
- ComponentLog log = getLogger();
- try {
- HiveWriter writer = writers.get(endPoint);
- if (writer == null) {
- log.debug("Creating Writer to Hive end point : " + endPoint);
- writer = makeHiveWriter(endPoint, callTimeoutPool, getUgi(), options);
- if (writers.size() > (options.getMaxOpenConnections() - 1)) {
- log.info("cached HiveEndPoint size {} exceeded maxOpenConnections {} ", new Object[]{writers.size(), options.getMaxOpenConnections()});
- int retired = retireIdleWriters(writers, options.getIdleTimeout());
- if (retired == 0) {
- retireEldestWriter(writers);
- }
- }
- writers.put(endPoint, writer);
- HiveUtils.logAllHiveEndPoints(writers);
- }
- return writer;
- } catch (HiveWriter.ConnectFailure e) {
- log.error("Failed to create HiveWriter for endpoint: " + endPoint, e);
- throw e;
- }
- }
-
- /**
- * Locate writer that has not been used for longest time and retire it
- */
- private void retireEldestWriter(Map writers) {
- ComponentLog log = getLogger();
-
- log.info("Attempting close eldest writers");
- long oldestTimeStamp = System.currentTimeMillis();
- HiveEndPoint eldest = null;
- for (Map.Entry entry : writers.entrySet()) {
- if (entry.getValue().getLastUsed() < oldestTimeStamp) {
- eldest = entry.getKey();
- oldestTimeStamp = entry.getValue().getLastUsed();
- }
- }
- try {
- log.info("Closing least used Writer to Hive end point : " + eldest);
- writers.remove(eldest).flushAndClose();
- } catch (IOException e) {
- log.warn("Failed to close writer for end point: " + eldest, e);
- } catch (InterruptedException e) {
- log.warn("Interrupted when attempting to close writer for end point: " + eldest, e);
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- log.warn("Interrupted when attempting to close writer for end point: " + eldest, e);
- }
- }
-
- /**
- * Locate all writers past idle timeout and retire them
- *
- * @return number of writers retired
- */
- private int retireIdleWriters(Map writers, int idleTimeout) {
- ComponentLog log = getLogger();
-
- log.info("Attempting to close idle HiveWriters");
- int count = 0;
- long now = System.currentTimeMillis();
- ArrayList retirees = new ArrayList<>();
-
- //1) Find retirement candidates
- for (Map.Entry entry : writers.entrySet()) {
- if (now - entry.getValue().getLastUsed() > idleTimeout) {
- ++count;
- retirees.add(entry.getKey());
- }
- }
- //2) Retire them
- for (HiveEndPoint ep : retirees) {
- try {
- log.info("Closing idle Writer to Hive end point : {}", new Object[]{ep});
- writers.remove(ep).flushAndClose();
- } catch (IOException e) {
- log.warn("Failed to close HiveWriter for end point: {}. Error: " + ep, e);
- } catch (InterruptedException e) {
- log.warn("Interrupted when attempting to close HiveWriter for end point: " + ep, e);
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- log.warn("Interrupted when attempting to close HiveWriter for end point: " + ep, e);
- }
- }
- return count;
- }
-
- protected HiveEndPoint makeHiveEndPoint(List partitionValues, HiveOptions options) throws ConnectionError {
- return HiveUtils.makeEndPoint(partitionValues, options);
- }
-
- protected HiveWriter makeHiveWriter(HiveEndPoint endPoint, ExecutorService callTimeoutPool, UserGroupInformation ugi, HiveOptions options)
- throws HiveWriter.ConnectFailure, InterruptedException {
- return HiveUtils.makeHiveWriter(endPoint, callTimeoutPool, ugi, options, hiveConfig);
- }
-
- protected KerberosProperties getKerberosProperties() {
- return kerberosProperties;
- }
-
- UserGroupInformation getUgi() {
- getLogger().trace("getting UGI instance");
- if (kerberosUserReference.get() != null) {
- // if there's a KerberosUser associated with this UGI, check the TGT and relogin if it is close to expiring
- KerberosUser kerberosUser = kerberosUserReference.get();
- getLogger().debug("kerberosUser is " + kerberosUser);
- try {
- getLogger().debug("checking TGT on kerberosUser [{}]", new Object[] {kerberosUser});
- kerberosUser.checkTGTAndRelogin();
- } catch (final KerberosLoginException e) {
- throw new ProcessException("Unable to relogin with kerberos credentials for " + kerberosUser.getPrincipal(), e);
- }
- } else {
- getLogger().debug("kerberosUser was null, will not refresh TGT with KerberosUser");
- }
- return ugi;
- }
-
- /*
- * Overridable by subclasses in the same package, mainly intended for testing purposes to allow verification without having to set environment variables.
- */
- boolean isAllowExplicitKeytab() {
- return Boolean.parseBoolean(System.getenv(ALLOW_EXPLICIT_KEYTAB));
- }
-
- protected class HiveStreamingRecord {
-
- private List partitionValues;
- private GenericRecord record;
-
- public HiveStreamingRecord(List partitionValues, GenericRecord record) {
- this.partitionValues = partitionValues;
- this.record = record;
- }
-
- public List getPartitionValues() {
- return partitionValues;
- }
-
- public GenericRecord getRecord() {
- return record;
- }
-
- }
-}
-
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/SelectHiveQL.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/SelectHiveQL.java
deleted file mode 100644
index be17f847df..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/SelectHiveQL.java
+++ /dev/null
@@ -1,572 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.nifi.annotation.behavior.EventDriven;
-import org.apache.nifi.annotation.behavior.InputRequirement;
-import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
-import org.apache.nifi.annotation.behavior.WritesAttribute;
-import org.apache.nifi.annotation.behavior.WritesAttributes;
-import org.apache.nifi.annotation.documentation.CapabilityDescription;
-import org.apache.nifi.annotation.documentation.DeprecationNotice;
-import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.annotation.lifecycle.OnScheduled;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.dbcp.hive.HiveDBCPService;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.ProcessSessionFactory;
-import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processor.util.pattern.PartialFunctions;
-import org.apache.nifi.util.StopWatch;
-import org.apache.nifi.util.hive.CsvOutputOptions;
-import org.apache.nifi.util.hive.HiveJdbcCommon;
-
-import java.nio.charset.Charset;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.nifi.util.hive.HiveJdbcCommon.AVRO;
-import static org.apache.nifi.util.hive.HiveJdbcCommon.CSV;
-import static org.apache.nifi.util.hive.HiveJdbcCommon.CSV_MIME_TYPE;
-import static org.apache.nifi.util.hive.HiveJdbcCommon.MIME_TYPE_AVRO_BINARY;
-import static org.apache.nifi.util.hive.HiveJdbcCommon.NORMALIZE_NAMES_FOR_AVRO;
-
-@EventDriven
-@InputRequirement(Requirement.INPUT_ALLOWED)
-@Tags({"hive", "sql", "select", "jdbc", "query", "database"})
-@CapabilityDescription("Execute provided HiveQL SELECT query against a Hive database connection. Query result will be converted to Avro or CSV format."
- + " Streaming is used so arbitrarily large result sets are supported. This processor can be scheduled to run on "
- + "a timer, or cron expression, using the standard scheduling methods, or it can be triggered by an incoming FlowFile. "
- + "If it is triggered by an incoming FlowFile, then attributes of that FlowFile will be available when evaluating the "
- + "select query. FlowFile attribute 'selecthiveql.row.count' indicates how many rows were selected.")
-@WritesAttributes({
- @WritesAttribute(attribute = "mime.type", description = "Sets the MIME type for the outgoing flowfile to application/avro-binary for Avro or text/csv for CSV."),
- @WritesAttribute(attribute = "filename", description = "Adds .avro or .csv to the filename attribute depending on which output format is selected."),
- @WritesAttribute(attribute = "selecthiveql.row.count", description = "Indicates how many rows were selected/returned by the query."),
- @WritesAttribute(attribute = "selecthiveql.query.duration", description = "Combined duration of the query execution time and fetch time in milliseconds. "
- + "If 'Max Rows Per Flow File' is set, then this number will reflect only the fetch time for the rows in the Flow File instead of the entire result set."),
- @WritesAttribute(attribute = "selecthiveql.query.executiontime", description = "Duration of the query execution time in milliseconds. "
- + "This number will reflect the query execution time regardless of the 'Max Rows Per Flow File' setting."),
- @WritesAttribute(attribute = "selecthiveql.query.fetchtime", description = "Duration of the result set fetch time in milliseconds. "
- + "If 'Max Rows Per Flow File' is set, then this number will reflect only the fetch time for the rows in the Flow File instead of the entire result set."),
- @WritesAttribute(attribute = "fragment.identifier", description = "If 'Max Rows Per Flow File' is set then all FlowFiles from the same query result set "
- + "will have the same value for the fragment.identifier attribute. This can then be used to correlate the results."),
- @WritesAttribute(attribute = "fragment.count", description = "If 'Max Rows Per Flow File' is set then this is the total number of "
- + "FlowFiles produced by a single ResultSet. This can be used in conjunction with the "
- + "fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet."),
- @WritesAttribute(attribute = "fragment.index", description = "If 'Max Rows Per Flow File' is set then the position of this FlowFile in the list of "
- + "outgoing FlowFiles that were all derived from the same result set FlowFile. This can be "
- + "used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same query result set and in what order "
- + "FlowFiles were produced"),
- @WritesAttribute(attribute = "query.input.tables", description = "Contains input table names in comma delimited 'databaseName.tableName' format.")
-})
-@DeprecationNotice(classNames = "org.apache.nifi.processors.hive.SelectHive3QL")
-public class SelectHiveQL extends AbstractHiveQLProcessor {
-
- public static final String RESULT_ROW_COUNT = "selecthiveql.row.count";
- public static final String RESULT_QUERY_DURATION = "selecthiveql.query.duration";
- public static final String RESULT_QUERY_EXECUTION_TIME = "selecthiveql.query.executiontime";
- public static final String RESULT_QUERY_FETCH_TIME = "selecthiveql.query.fetchtime";
-
- // Relationships
- public static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("Successfully created FlowFile from HiveQL query result set.")
- .build();
- public static final Relationship REL_FAILURE = new Relationship.Builder()
- .name("failure")
- .description("HiveQL query execution failed. Incoming FlowFile will be penalized and routed to this relationship.")
- .build();
-
-
- public static final PropertyDescriptor HIVEQL_PRE_QUERY = new PropertyDescriptor.Builder()
- .name("hive-pre-query")
- .displayName("HiveQL Pre-Query")
- .description("A semicolon-delimited list of queries executed before the main SQL query is executed. "
- + "Example: 'set tez.queue.name=queue1; set hive.exec.orc.split.strategy=ETL; set hive.exec.reducers.bytes.per.reducer=1073741824'. "
- + "Note, the results/outputs of these queries will be suppressed if successfully executed.")
- .required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor HIVEQL_SELECT_QUERY = new PropertyDescriptor.Builder()
- .name("hive-query")
- .displayName("HiveQL Select Query")
- .description("HiveQL SELECT query to execute. If this is not set, the query is assumed to be in the content of an incoming FlowFile.")
- .required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor HIVEQL_POST_QUERY = new PropertyDescriptor.Builder()
- .name("hive-post-query")
- .displayName("HiveQL Post-Query")
- .description("A semicolon-delimited list of queries executed after the main SQL query is executed. "
- + "Note, the results/outputs of these queries will be suppressed if successfully executed.")
- .required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor FETCH_SIZE = new PropertyDescriptor.Builder()
- .name("hive-fetch-size")
- .displayName("Fetch Size")
- .description("The number of result rows to be fetched from the result set at a time. This is a hint to the driver and may not be "
- + "honored and/or exact. If the value specified is zero, then the hint is ignored.")
- .defaultValue("0")
- .required(true)
- .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor MAX_ROWS_PER_FLOW_FILE = new PropertyDescriptor.Builder()
- .name("hive-max-rows")
- .displayName("Max Rows Per Flow File")
- .description("The maximum number of result rows that will be included in a single FlowFile. " +
- "This will allow you to break up very large result sets into multiple FlowFiles. If the value specified is zero, then all rows are returned in a single FlowFile.")
- .defaultValue("0")
- .required(true)
- .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor MAX_FRAGMENTS = new PropertyDescriptor.Builder()
- .name("hive-max-frags")
- .displayName("Maximum Number of Fragments")
- .description("The maximum number of fragments. If the value specified is zero, then all fragments are returned. " +
- "This prevents OutOfMemoryError when this processor ingests huge table.")
- .defaultValue("0")
- .required(true)
- .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor HIVEQL_CSV_HEADER = new PropertyDescriptor.Builder()
- .name("csv-header")
- .displayName("CSV Header")
- .description("Include Header in Output")
- .required(true)
- .allowableValues("true", "false")
- .defaultValue("true")
- .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
- .build();
-
- public static final PropertyDescriptor HIVEQL_CSV_ALT_HEADER = new PropertyDescriptor.Builder()
- .name("csv-alt-header")
- .displayName("Alternate CSV Header")
- .description("Comma separated list of header fields")
- .required(false)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor HIVEQL_CSV_DELIMITER = new PropertyDescriptor.Builder()
- .name("csv-delimiter")
- .displayName("CSV Delimiter")
- .description("CSV Delimiter used to separate fields")
- .required(true)
- .defaultValue(",")
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- public static final PropertyDescriptor HIVEQL_CSV_QUOTE = new PropertyDescriptor.Builder()
- .name("csv-quote")
- .displayName("CSV Quote")
- .description("Whether to force quoting of CSV fields. Note that this might conflict with the setting for CSV Escape.")
- .required(true)
- .allowableValues("true", "false")
- .defaultValue("true")
- .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
- .build();
- public static final PropertyDescriptor HIVEQL_CSV_ESCAPE = new PropertyDescriptor.Builder()
- .name("csv-escape")
- .displayName("CSV Escape")
- .description("Whether to escape CSV strings in output. Note that this might conflict with the setting for CSV Quote.")
- .required(true)
- .allowableValues("true", "false")
- .defaultValue("true")
- .addValidator(StandardValidators.BOOLEAN_VALIDATOR)
- .build();
-
- public static final PropertyDescriptor HIVEQL_OUTPUT_FORMAT = new PropertyDescriptor.Builder()
- .name("hive-output-format")
- .displayName("Output Format")
- .description("How to represent the records coming from Hive (Avro, CSV, e.g.)")
- .required(true)
- .allowableValues(AVRO, CSV)
- .defaultValue(AVRO)
- .expressionLanguageSupported(ExpressionLanguageScope.NONE)
- .build();
-
- private final static List propertyDescriptors;
- private final static Set relationships;
-
- /*
- * Will ensure that the list of property descriptors is built only once.
- * Will also create a Set of relationships
- */
- static {
- List _propertyDescriptors = new ArrayList<>();
- _propertyDescriptors.add(HIVE_DBCP_SERVICE);
- _propertyDescriptors.add(HIVEQL_PRE_QUERY);
- _propertyDescriptors.add(HIVEQL_SELECT_QUERY);
- _propertyDescriptors.add(HIVEQL_POST_QUERY);
- _propertyDescriptors.add(FETCH_SIZE);
- _propertyDescriptors.add(MAX_ROWS_PER_FLOW_FILE);
- _propertyDescriptors.add(MAX_FRAGMENTS);
- _propertyDescriptors.add(HIVEQL_OUTPUT_FORMAT);
- _propertyDescriptors.add(NORMALIZE_NAMES_FOR_AVRO);
- _propertyDescriptors.add(HIVEQL_CSV_HEADER);
- _propertyDescriptors.add(HIVEQL_CSV_ALT_HEADER);
- _propertyDescriptors.add(HIVEQL_CSV_DELIMITER);
- _propertyDescriptors.add(HIVEQL_CSV_QUOTE);
- _propertyDescriptors.add(HIVEQL_CSV_ESCAPE);
- _propertyDescriptors.add(CHARSET);
- propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors);
-
- Set _relationships = new HashSet<>();
- _relationships.add(REL_SUCCESS);
- _relationships.add(REL_FAILURE);
- relationships = Collections.unmodifiableSet(_relationships);
- }
-
- @Override
- protected List getSupportedPropertyDescriptors() {
- return propertyDescriptors;
- }
-
- @Override
- public Set getRelationships() {
- return relationships;
- }
-
- @OnScheduled
- public void setup(ProcessContext context) {
- // If the query is not set, then an incoming flow file is needed. Otherwise fail the initialization
- if (!context.getProperty(HIVEQL_SELECT_QUERY).isSet() && !context.hasIncomingConnection()) {
- final String errorString = "Either the Select Query must be specified or there must be an incoming connection "
- + "providing flowfile(s) containing a SQL select query";
- getLogger().error(errorString);
- throw new ProcessException(errorString);
- }
- }
-
- @Override
- public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
- PartialFunctions.onTrigger(context, sessionFactory, getLogger(), session -> onTrigger(context, session));
- }
-
- private void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
- FlowFile fileToProcess = (context.hasIncomingConnection() ? session.get() : null);
- FlowFile flowfile = null;
-
- // If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
- // However, if we have no FlowFile and we have connections coming from other Processors, then
- // we know that we should run only if we have a FlowFile.
- if (context.hasIncomingConnection()) {
- if (fileToProcess == null && context.hasNonLoopConnection()) {
- return;
- }
- }
-
- final ComponentLog logger = getLogger();
- final HiveDBCPService dbcpService = context.getProperty(HIVE_DBCP_SERVICE).asControllerService(HiveDBCPService.class);
- final Charset charset = Charset.forName(context.getProperty(CHARSET).getValue());
-
- List preQueries = getQueries(context.getProperty(HIVEQL_PRE_QUERY).evaluateAttributeExpressions(fileToProcess).getValue());
- List postQueries = getQueries(context.getProperty(HIVEQL_POST_QUERY).evaluateAttributeExpressions(fileToProcess).getValue());
-
- final boolean flowbased = !(context.getProperty(HIVEQL_SELECT_QUERY).isSet());
-
- // Source the SQL
- String hqlStatement;
-
- if (context.getProperty(HIVEQL_SELECT_QUERY).isSet()) {
- hqlStatement = context.getProperty(HIVEQL_SELECT_QUERY).evaluateAttributeExpressions(fileToProcess).getValue();
- } else {
- // If the query is not set, then an incoming flow file is required, and expected to contain a valid SQL select query.
- // If there is no incoming connection, onTrigger will not be called as the processor will fail when scheduled.
- final StringBuilder queryContents = new StringBuilder();
- session.read(fileToProcess, in -> queryContents.append(IOUtils.toString(in, charset)));
- hqlStatement = queryContents.toString();
- }
-
-
- final Integer fetchSize = context.getProperty(FETCH_SIZE).evaluateAttributeExpressions(fileToProcess).asInteger();
- final Integer maxRowsPerFlowFile = context.getProperty(MAX_ROWS_PER_FLOW_FILE).evaluateAttributeExpressions(fileToProcess).asInteger();
- final Integer maxFragments = context.getProperty(MAX_FRAGMENTS).isSet()
- ? context.getProperty(MAX_FRAGMENTS).evaluateAttributeExpressions(fileToProcess).asInteger()
- : 0;
- final String outputFormat = context.getProperty(HIVEQL_OUTPUT_FORMAT).getValue();
- final boolean convertNamesForAvro = context.getProperty(NORMALIZE_NAMES_FOR_AVRO).asBoolean();
- final StopWatch stopWatch = new StopWatch(true);
- final boolean header = context.getProperty(HIVEQL_CSV_HEADER).asBoolean();
- final String altHeader = context.getProperty(HIVEQL_CSV_ALT_HEADER).evaluateAttributeExpressions(fileToProcess).getValue();
- final String delimiter = context.getProperty(HIVEQL_CSV_DELIMITER).evaluateAttributeExpressions(fileToProcess).getValue();
- final boolean quote = context.getProperty(HIVEQL_CSV_QUOTE).asBoolean();
- final boolean escape = context.getProperty(HIVEQL_CSV_ESCAPE).asBoolean();
- final String fragmentIdentifier = UUID.randomUUID().toString();
-
- try (final Connection con = dbcpService.getConnection(fileToProcess == null ? Collections.emptyMap() : fileToProcess.getAttributes());
- final Statement st = (flowbased ? con.prepareStatement(hqlStatement) : con.createStatement())
- ) {
- Pair failure = executeConfigStatements(con, preQueries);
- if (failure != null) {
- // In case of failure, assigning config query to "hqlStatement" to follow current error handling
- hqlStatement = failure.getLeft();
- flowfile = (fileToProcess == null) ? session.create() : fileToProcess;
- fileToProcess = null;
- throw failure.getRight();
- }
- if (fetchSize != null && fetchSize > 0) {
- try {
- st.setFetchSize(fetchSize);
- } catch (SQLException se) {
- // Not all drivers support this, just log the error (at debug level) and move on
- logger.debug("Cannot set fetch size to {} due to {}", new Object[]{fetchSize, se.getLocalizedMessage()}, se);
- }
- }
-
- final List resultSetFlowFiles = new ArrayList<>();
- try {
- logger.debug("Executing query {}", new Object[]{hqlStatement});
- if (flowbased) {
- // Hive JDBC Doesn't Support this yet:
- // ParameterMetaData pmd = ((PreparedStatement)st).getParameterMetaData();
- // int paramCount = pmd.getParameterCount();
-
- // Alternate way to determine number of params in SQL.
- int paramCount = StringUtils.countMatches(hqlStatement, "?");
-
- if (paramCount > 0) {
- setParameters(1, (PreparedStatement) st, paramCount, fileToProcess.getAttributes());
- }
- }
-
- final StopWatch executionTime = new StopWatch(true);
- final ResultSet resultSet;
-
- try {
- resultSet = (flowbased ? ((PreparedStatement) st).executeQuery() : st.executeQuery(hqlStatement));
- } catch (SQLException se) {
- // If an error occurs during the query, a flowfile is expected to be routed to failure, so ensure one here
- flowfile = (fileToProcess == null) ? session.create() : fileToProcess;
- fileToProcess = null;
- throw se;
- }
- long executionTimeElapsed = executionTime.getElapsed(TimeUnit.MILLISECONDS);
-
- int fragmentIndex = 0;
- String baseFilename = (fileToProcess != null) ? fileToProcess.getAttribute(CoreAttributes.FILENAME.key()) : null;
- while (true) {
- final AtomicLong nrOfRows = new AtomicLong(0L);
- final StopWatch fetchTime = new StopWatch(true);
-
- flowfile = (fileToProcess == null) ? session.create() : session.create(fileToProcess);
- if (baseFilename == null) {
- baseFilename = flowfile.getAttribute(CoreAttributes.FILENAME.key());
- }
- try {
- flowfile = session.write(flowfile, out -> {
- try {
- if (AVRO.equals(outputFormat)) {
- nrOfRows.set(HiveJdbcCommon.convertToAvroStream(resultSet, out, maxRowsPerFlowFile, convertNamesForAvro));
- } else if (CSV.equals(outputFormat)) {
- CsvOutputOptions options = new CsvOutputOptions(header, altHeader, delimiter, quote, escape, maxRowsPerFlowFile);
- nrOfRows.set(HiveJdbcCommon.convertToCsvStream(resultSet, out, options));
- } else {
- nrOfRows.set(0L);
- throw new ProcessException("Unsupported output format: " + outputFormat);
- }
- } catch (final SQLException | RuntimeException e) {
- throw new ProcessException("Error during database query or conversion of records.", e);
- }
- });
- } catch (ProcessException e) {
- // Add flowfile to results before rethrowing so it will be removed from session in outer catch
- resultSetFlowFiles.add(flowfile);
- throw e;
- }
- long fetchTimeElapsed = fetchTime.getElapsed(TimeUnit.MILLISECONDS);
-
- if (nrOfRows.get() > 0 || resultSetFlowFiles.isEmpty()) {
- final Map attributes = new HashMap<>();
- // Set attribute for how many rows were selected
- attributes.put(RESULT_ROW_COUNT, String.valueOf(nrOfRows.get()));
-
- try {
- // Set input/output table names by parsing the query
- attributes.putAll(toQueryTableAttributes(findTableNames(hqlStatement)));
- } catch (Exception e) {
- // If failed to parse the query, just log a warning message, but continue.
- getLogger().warn("Failed to parse query: {} due to {}", new Object[]{hqlStatement, e}, e);
- }
-
- // Set MIME type on output document and add extension to filename
- if (AVRO.equals(outputFormat)) {
- attributes.put(CoreAttributes.MIME_TYPE.key(), MIME_TYPE_AVRO_BINARY);
- attributes.put(CoreAttributes.FILENAME.key(), baseFilename + "." + fragmentIndex + ".avro");
- } else if (CSV.equals(outputFormat)) {
- attributes.put(CoreAttributes.MIME_TYPE.key(), CSV_MIME_TYPE);
- attributes.put(CoreAttributes.FILENAME.key(), baseFilename + "." + fragmentIndex + ".csv");
- }
-
- if (maxRowsPerFlowFile > 0) {
- attributes.put("fragment.identifier", fragmentIdentifier);
- attributes.put("fragment.index", String.valueOf(fragmentIndex));
- }
-
- attributes.put(RESULT_QUERY_DURATION, String.valueOf(executionTimeElapsed + fetchTimeElapsed));
- attributes.put(RESULT_QUERY_EXECUTION_TIME, String.valueOf(executionTimeElapsed));
- attributes.put(RESULT_QUERY_FETCH_TIME, String.valueOf(fetchTimeElapsed));
-
- flowfile = session.putAllAttributes(flowfile, attributes);
-
- logger.info("{} contains {} " + outputFormat + " records; transferring to 'success'",
- new Object[]{flowfile, nrOfRows.get()});
-
- if (context.hasIncomingConnection()) {
- // If the flow file came from an incoming connection, issue a Fetch provenance event
- session.getProvenanceReporter().fetch(flowfile, dbcpService.getConnectionURL(),
- "Retrieved " + nrOfRows.get() + " rows", stopWatch.getElapsed(TimeUnit.MILLISECONDS));
- } else {
- // If we created a flow file from rows received from Hive, issue a Receive provenance event
- session.getProvenanceReporter().receive(flowfile, dbcpService.getConnectionURL(), stopWatch.getElapsed(TimeUnit.MILLISECONDS));
- }
- resultSetFlowFiles.add(flowfile);
- } else {
- // If there were no rows returned (and the first flow file has been sent, we're done processing, so remove the flowfile and carry on
- session.remove(flowfile);
- if (resultSetFlowFiles != null && resultSetFlowFiles.size()>0) {
- flowfile = resultSetFlowFiles.get(resultSetFlowFiles.size()-1);
- }
- break;
- }
-
- fragmentIndex++;
- if (maxFragments > 0 && fragmentIndex >= maxFragments) {
- break;
- }
- }
-
- for (int i = 0; i < resultSetFlowFiles.size(); i++) {
- // Set count on all FlowFiles
- if (maxRowsPerFlowFile > 0) {
- resultSetFlowFiles.set(i,
- session.putAttribute(resultSetFlowFiles.get(i), "fragment.count", Integer.toString(fragmentIndex)));
- }
- }
-
- } catch (final SQLException e) {
- throw e;
- }
-
- failure = executeConfigStatements(con, postQueries);
- if (failure != null) {
- hqlStatement = failure.getLeft();
- if (resultSetFlowFiles != null) {
- resultSetFlowFiles.forEach(ff -> session.remove(ff));
- }
- flowfile = (fileToProcess == null) ? session.create() : fileToProcess;
- fileToProcess = null;
- throw failure.getRight();
- }
-
- session.transfer(resultSetFlowFiles, REL_SUCCESS);
- if (fileToProcess != null) {
- session.remove(fileToProcess);
- }
- } catch (final ProcessException | SQLException e) {
- logger.error("Issue processing SQL {} due to {}.", new Object[]{hqlStatement, e});
- if (flowfile == null) {
- // This can happen if any exceptions occur while setting up the connection, statement, etc.
- logger.error("Unable to execute HiveQL select query {} due to {}. No FlowFile to route to failure",
- new Object[]{hqlStatement, e});
- context.yield();
- } else {
- if (context.hasIncomingConnection()) {
- logger.error("Unable to execute HiveQL select query {} for {} due to {}; routing to failure",
- new Object[]{hqlStatement, flowfile, e});
- flowfile = session.penalize(flowfile);
- } else {
- logger.error("Unable to execute HiveQL select query {} due to {}; routing to failure",
- new Object[]{hqlStatement, e});
- context.yield();
- }
- session.transfer(flowfile, REL_FAILURE);
- }
- }
- }
-
- /*
- * Executes given queries using pre-defined connection.
- * Returns null on success, or a query string if failed.
- */
- protected Pair executeConfigStatements(final Connection con, final List configQueries){
- if (configQueries == null || configQueries.isEmpty()) {
- return null;
- }
-
- for (String confSQL : configQueries) {
- try(final Statement st = con.createStatement()){
- st.execute(confSQL);
- } catch (SQLException e) {
- return Pair.of(confSQL, e);
- }
- }
- return null;
- }
-
- protected List getQueries(final String value) {
- if (value == null || value.length() == 0 || value.trim().length() == 0) {
- return null;
- }
- final List queries = new LinkedList<>();
- for (String query : value.split(";")) {
- if (query.trim().length() > 0) {
- queries.add(query.trim());
- }
- }
- return queries;
- }
-}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/UpdateHiveTable.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/UpdateHiveTable.java
deleted file mode 100644
index 61fe28d282..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/processors/hive/UpdateHiveTable.java
+++ /dev/null
@@ -1,769 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils;
-import org.apache.nifi.annotation.behavior.InputRequirement;
-import org.apache.nifi.annotation.behavior.ReadsAttribute;
-import org.apache.nifi.annotation.behavior.ReadsAttributes;
-import org.apache.nifi.annotation.behavior.RequiresInstanceClassLoading;
-import org.apache.nifi.annotation.behavior.WritesAttribute;
-import org.apache.nifi.annotation.behavior.WritesAttributes;
-import org.apache.nifi.annotation.documentation.CapabilityDescription;
-import org.apache.nifi.annotation.documentation.DeprecationNotice;
-import org.apache.nifi.annotation.documentation.Tags;
-import org.apache.nifi.components.AllowableValue;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.components.ValidationContext;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.components.Validator;
-import org.apache.nifi.dbcp.hive.HiveDBCPService;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.flowfile.FlowFile;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.AbstractProcessor;
-import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.ProcessSession;
-import org.apache.nifi.processor.ProcessorInitializationContext;
-import org.apache.nifi.processor.Relationship;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.processor.util.pattern.DiscontinuedException;
-import org.apache.nifi.processors.hadoop.exception.RecordReaderFactoryException;
-import org.apache.nifi.serialization.MalformedRecordException;
-import org.apache.nifi.serialization.RecordReader;
-import org.apache.nifi.serialization.RecordReaderFactory;
-import org.apache.nifi.serialization.RecordSetWriter;
-import org.apache.nifi.serialization.RecordSetWriterFactory;
-import org.apache.nifi.serialization.SimpleRecordSchema;
-import org.apache.nifi.serialization.WriteResult;
-import org.apache.nifi.serialization.record.MapRecord;
-import org.apache.nifi.serialization.record.Record;
-import org.apache.nifi.serialization.record.RecordField;
-import org.apache.nifi.serialization.record.RecordSchema;
-import org.apache.nifi.util.StringUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-
-@Tags({"hive", "metadata", "jdbc", "database", "table"})
-@CapabilityDescription("This processor uses a Hive JDBC connection and incoming records to generate any Hive 1.2 table changes needed to support the incoming records.")
-@ReadsAttributes({
- @ReadsAttribute(attribute = "hive.table.management.strategy", description = "This attribute is read if the 'Table Management Strategy' property is configured "
- + "to use the value of this attribute. The value of this attribute should correspond (ignoring case) to a valid option of the 'Table Management Strategy' property.")
-})
-@WritesAttributes({
- @WritesAttribute(attribute = "output.table", description = "This attribute is written on the flow files routed to the 'success' "
- + "and 'failure' relationships, and contains the target table name."),
- @WritesAttribute(attribute = "output.path", description = "This attribute is written on the flow files routed to the 'success' "
- + "and 'failure' relationships, and contains the path on the file system to the table (or partition location if the table is partitioned)."),
- @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Record Writer, only if a Record Writer is specified "
- + "and Update Field Names is 'true'."),
- @WritesAttribute(attribute = "record.count", description = "Sets the number of records in the FlowFile, only if a Record Writer is specified and Update Field Names is 'true'.")
-})
-@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED)
-@RequiresInstanceClassLoading
-@DeprecationNotice(classNames = "org.apache.nifi.processors.hive.UpdateHive3Table")
-public class UpdateHiveTable extends AbstractProcessor {
-
- static final String TEXTFILE = "TEXTFILE";
- static final String SEQUENCEFILE = "SEQUENCEFILE";
- static final String ORC = "ORC";
- static final String PARQUET = "PARQUET";
- static final String AVRO = "AVRO";
- static final String RCFILE = "RCFILE";
-
- static final AllowableValue TEXTFILE_STORAGE = new AllowableValue(TEXTFILE, TEXTFILE, "Stored as plain text files. TEXTFILE is the default file format, unless the configuration "
- + "parameter hive.default.fileformat has a different setting.");
- static final AllowableValue SEQUENCEFILE_STORAGE = new AllowableValue(SEQUENCEFILE, SEQUENCEFILE, "Stored as compressed Sequence Files.");
- static final AllowableValue ORC_STORAGE = new AllowableValue(ORC, ORC, "Stored as ORC file format. Supports ACID Transactions & Cost-based Optimizer (CBO). "
- + "Stores column-level metadata.");
- static final AllowableValue PARQUET_STORAGE = new AllowableValue(PARQUET, PARQUET, "Stored as Parquet format for the Parquet columnar storage format.");
- static final AllowableValue AVRO_STORAGE = new AllowableValue(AVRO, AVRO, "Stored as Avro format.");
- static final AllowableValue RCFILE_STORAGE = new AllowableValue(RCFILE, RCFILE, "Stored as Record Columnar File format.");
-
- static final AllowableValue CREATE_IF_NOT_EXISTS = new AllowableValue("Create If Not Exists", "Create If Not Exists",
- "Create a table with the given schema if it does not already exist");
- static final AllowableValue FAIL_IF_NOT_EXISTS = new AllowableValue("Fail If Not Exists", "Fail If Not Exists",
- "If the target does not already exist, log an error and route the flowfile to failure");
-
- static final String TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE = "hive.table.management.strategy";
- static final AllowableValue MANAGED_TABLE = new AllowableValue("Managed", "Managed",
- "Any tables created by this processor will be managed tables (see Hive documentation for details).");
- static final AllowableValue EXTERNAL_TABLE = new AllowableValue("External", "External",
- "Any tables created by this processor will be external tables located at the `External Table Location` property value.");
- static final AllowableValue ATTRIBUTE_DRIVEN_TABLE = new AllowableValue("Use '" + TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE + "' Attribute",
- "Use '" + TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE + "' Attribute",
- "Inspects the '" + TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE + "' FlowFile attribute to determine the table management strategy. The value "
- + "of this attribute must be a case-insensitive match to one of the other allowable values (Managed, External, e.g.).");
-
- static final String ATTR_OUTPUT_TABLE = "output.table";
- static final String ATTR_OUTPUT_PATH = "output.path";
-
- // Properties
- static final PropertyDescriptor RECORD_READER = new PropertyDescriptor.Builder()
- .name("record-reader")
- .displayName("Record Reader")
- .description("The service for reading incoming flow files. The reader is only used to determine the schema of the records, the actual records will not be processed.")
- .identifiesControllerService(RecordReaderFactory.class)
- .required(true)
- .build();
-
- static final PropertyDescriptor HIVE_DBCP_SERVICE = new PropertyDescriptor.Builder()
- .name("hive-dbcp-service")
- .displayName("Hive Database Connection Pooling Service")
- .description("The Hive Controller Service that is used to obtain connection(s) to the Hive database")
- .required(true)
- .identifiesControllerService(HiveDBCPService.class)
- .build();
-
- static final PropertyDescriptor TABLE_NAME = new PropertyDescriptor.Builder()
- .name("hive-table-name")
- .displayName("Table Name")
- .description("The name of the database table to update. If the table does not exist, then it will either be created or an error thrown, depending "
- + "on the value of the Create Table property.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
-
- static final PropertyDescriptor CREATE_TABLE = new PropertyDescriptor.Builder()
- .name("hive-create-table")
- .displayName("Create Table Strategy")
- .description("Specifies how to process the target table when it does not exist (create it, fail, e.g.).")
- .required(true)
- .addValidator(Validator.VALID)
- .allowableValues(CREATE_IF_NOT_EXISTS, FAIL_IF_NOT_EXISTS)
- .defaultValue(FAIL_IF_NOT_EXISTS.getValue())
- .build();
-
- static final PropertyDescriptor TABLE_MANAGEMENT_STRATEGY = new PropertyDescriptor.Builder()
- .name("hive-create-table-management")
- .displayName("Create Table Management Strategy")
- .description("Specifies (when a table is to be created) whether the table is a managed table or an external table. Note that when External is specified, the "
- + "'External Table Location' property must be specified. If the '" + TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE + "' value is selected, 'External Table Location' "
- + "must still be specified, but can contain Expression Language or be set to the empty string, and is ignored when the attribute evaluates to 'Managed'.")
- .required(true)
- .addValidator(Validator.VALID)
- .allowableValues(MANAGED_TABLE, EXTERNAL_TABLE, ATTRIBUTE_DRIVEN_TABLE)
- .defaultValue(MANAGED_TABLE.getValue())
- .dependsOn(CREATE_TABLE, CREATE_IF_NOT_EXISTS)
- .build();
-
- static final PropertyDescriptor UPDATE_FIELD_NAMES = new PropertyDescriptor.Builder()
- .name("hive-update-field-names")
- .displayName("Update Field Names")
- .description("This property indicates whether to update the output schema such that the field names are set to the exact column names from the specified "
- + "table. This should be used if the incoming record field names may not match the table's column names in terms of upper- and lower-case. For example, this property should be "
- + "set to true if the output FlowFile (and target table storage) is Avro format, as Hive/Impala expects the field names to match the column names exactly.")
- .allowableValues("true", "false")
- .defaultValue("false")
- .required(true)
- .build();
-
- static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
- .name("hive-record-writer")
- .displayName("Record Writer")
- .description("Specifies the Controller Service to use for writing results to a FlowFile. The Record Writer should use Inherit Schema to emulate the inferred schema behavior, i.e. "
- + "an explicit schema need not be defined in the writer, and will be supplied by the same logic used to infer the schema from the column types. If Create Table Strategy is set "
- + "'Create If Not Exists', the Record Writer's output format must match the Record Reader's format in order for the data to be placed in the created table location. Note that "
- + "this property is only used if 'Update Field Names' is set to true and the field names do not all match the column names exactly. If no "
- + "update is needed for any field names (or 'Update Field Names' is false), the Record Writer is not used and instead the input FlowFile is routed to success or failure "
- + "without modification.")
- .identifiesControllerService(RecordSetWriterFactory.class)
- .dependsOn(UPDATE_FIELD_NAMES, "true")
- .required(true)
- .build();
-
- static final PropertyDescriptor EXTERNAL_TABLE_LOCATION = new PropertyDescriptor.Builder()
- .name("hive-external-table-location")
- .displayName("External Table Location")
- .description("Specifies (when an external table is to be created) the file path (in HDFS, e.g.) to store table data.")
- .required(true)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
- .dependsOn(TABLE_MANAGEMENT_STRATEGY, EXTERNAL_TABLE, ATTRIBUTE_DRIVEN_TABLE)
- .build();
-
- static final PropertyDescriptor TABLE_STORAGE_FORMAT = new PropertyDescriptor.Builder()
- .name("hive-storage-format")
- .displayName("Create Table Storage Format")
- .description("If a table is to be created, the specified storage format will be used.")
- .required(true)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .allowableValues(TEXTFILE_STORAGE, SEQUENCEFILE_STORAGE, ORC_STORAGE, PARQUET_STORAGE, AVRO_STORAGE, RCFILE_STORAGE)
- .defaultValue(TEXTFILE)
- .dependsOn(CREATE_TABLE, CREATE_IF_NOT_EXISTS)
- .build();
-
- static final PropertyDescriptor QUERY_TIMEOUT = new PropertyDescriptor.Builder()
- .name("hive-query-timeout")
- .displayName("Query Timeout")
- .description("Sets the number of seconds the driver will wait for a query to execute. "
- + "A value of 0 means no timeout. NOTE: Non-zero values may not be supported by the driver.")
- .defaultValue("0")
- .required(true)
- .addValidator(StandardValidators.INTEGER_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .build();
-
- static final PropertyDescriptor PARTITION_CLAUSE = new PropertyDescriptor.Builder()
- .name("hive-partition-clause")
- .displayName("Partition Clause")
- .description("Specifies a comma-separated list of attribute names and optional data types corresponding to the partition columns of the target table. Simply put, if the table is "
- + "partitioned or is to be created with partitions, each partition name should be an attribute on the FlowFile and listed in this property. This assumes all incoming records "
- + "belong to the same partition and the partition columns are not fields in the record. An example of specifying this field is if PartitionRecord "
- + "is upstream and two partition columns 'name' (of type string) and 'age' (of type integer) are used, then this property can be set to 'name string, age int'. The data types "
- + "are optional and if partition(s) are to be created they will default to string type if not specified. For non-string primitive types, specifying the data type for existing "
- + "partition columns is helpful for interpreting the partition value(s). If the table exists, the data types need not be specified "
- + "(and are ignored in that case). This property must be set if the table is partitioned, and there must be an attribute for each partition column in the table. "
- + "The values of the attributes will be used as the partition values, and the resulting output.path attribute value will reflect the location of the partition in the filesystem "
- + "(for use downstream in processors such as PutHDFS).")
- .required(false)
- .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
- .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
- .build();
-
- // Relationships
- public static final Relationship REL_SUCCESS = new Relationship.Builder()
- .name("success")
- .description("A FlowFile containing records routed to this relationship after the record has been successfully transmitted to Hive.")
- .build();
-
- public static final Relationship REL_FAILURE = new Relationship.Builder()
- .name("failure")
- .description("A FlowFile containing records routed to this relationship if the record could not be transmitted to Hive.")
- .build();
-
- private List propertyDescriptors;
- private Set relationships;
-
- @Override
- protected void init(ProcessorInitializationContext context) {
- List props = new ArrayList<>();
- props.add(RECORD_READER);
- props.add(HIVE_DBCP_SERVICE);
- props.add(TABLE_NAME);
- props.add(PARTITION_CLAUSE);
- props.add(CREATE_TABLE);
- props.add(TABLE_MANAGEMENT_STRATEGY);
- props.add(EXTERNAL_TABLE_LOCATION);
- props.add(TABLE_STORAGE_FORMAT);
- props.add(UPDATE_FIELD_NAMES);
- props.add(RECORD_WRITER_FACTORY);
- props.add(QUERY_TIMEOUT);
-
- propertyDescriptors = Collections.unmodifiableList(props);
-
- Set _relationships = new HashSet<>();
- _relationships.add(REL_SUCCESS);
- _relationships.add(REL_FAILURE);
- relationships = Collections.unmodifiableSet(_relationships);
- }
-
- @Override
- protected List getSupportedPropertyDescriptors() {
- return propertyDescriptors;
- }
-
- @Override
- public Set getRelationships() {
- return relationships;
- }
-
- @Override
- protected Collection customValidate(ValidationContext validationContext) {
- List validationResults = new ArrayList<>(super.customValidate(validationContext));
- final boolean recordWriterFactorySet = validationContext.getProperty(RECORD_WRITER_FACTORY).isSet();
- final boolean createIfNotExists = validationContext.getProperty(CREATE_TABLE).getValue().equals(CREATE_IF_NOT_EXISTS.getValue());
- final boolean updateFieldNames = validationContext.getProperty(UPDATE_FIELD_NAMES).asBoolean();
-
- if (!recordWriterFactorySet && updateFieldNames) {
- validationResults.add(new ValidationResult.Builder().subject(RECORD_WRITER_FACTORY.getDisplayName())
- .explanation("Record Writer must be set if 'Update Field Names' is true").valid(false).build());
- }
- final String tableManagementStrategy = validationContext.getProperty(TABLE_MANAGEMENT_STRATEGY).getValue();
- final boolean managedTable;
- if (!ATTRIBUTE_DRIVEN_TABLE.getValue().equals(tableManagementStrategy)) {
- managedTable = MANAGED_TABLE.getValue().equals(tableManagementStrategy);
- // Ensure valid configuration for external tables
- if (createIfNotExists && !managedTable && !validationContext.getProperty(EXTERNAL_TABLE_LOCATION).isSet()) {
- validationResults.add(new ValidationResult.Builder().subject(EXTERNAL_TABLE_LOCATION.getDisplayName())
- .explanation("External Table Location must be set when Table Management Strategy is set to External").valid(false).build());
- }
- }
- return validationResults;
- }
-
- @Override
- public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
-
- FlowFile flowFile = session.get();
- if (flowFile == null) {
- return;
- }
-
- final RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
- final RecordSetWriterFactory recordWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
- final String tableName = context.getProperty(TABLE_NAME).evaluateAttributeExpressions(flowFile).getValue();
- final String partitionClauseString = context.getProperty(PARTITION_CLAUSE).evaluateAttributeExpressions(flowFile).getValue();
- List partitionClauseElements = null;
- if (!StringUtils.isEmpty(partitionClauseString)) {
- partitionClauseElements = Arrays.stream(partitionClauseString.split(",")).filter(Objects::nonNull).map(String::trim).collect(Collectors.toList());
- }
-
- final ComponentLog log = getLogger();
-
- try {
- final RecordReader reader;
-
- try (final InputStream in = session.read(flowFile)) {
- // if we fail to create the RecordReader then we want to route to failure, so we need to
- // handle this separately from the other IOExceptions which normally route to retry
- try {
- reader = recordReaderFactory.createRecordReader(flowFile, in, getLogger());
- } catch (Exception e) {
- throw new RecordReaderFactoryException("Unable to create RecordReader", e);
- }
- } catch (RecordReaderFactoryException rrfe) {
- log.error(
- "Failed to create {} for {} - routing to failure",
- new Object[]{RecordReader.class.getSimpleName(), flowFile},
- rrfe
- );
- // Since we are wrapping the exceptions above there should always be a cause
- // but it's possible it might not have a message. This handles that by logging
- // the name of the class thrown.
- Throwable c = rrfe.getCause();
- if (c != null) {
- session.putAttribute(flowFile, "record.error.message", (c.getLocalizedMessage() != null) ? c.getLocalizedMessage() : c.getClass().getCanonicalName() + " Thrown");
- } else {
- session.putAttribute(flowFile, "record.error.message", rrfe.getClass().getCanonicalName() + " Thrown");
- }
- session.transfer(flowFile, REL_FAILURE);
- return;
- }
-
- RecordSchema recordSchema = reader.getSchema();
-
- final boolean createIfNotExists = context.getProperty(CREATE_TABLE).getValue().equals(CREATE_IF_NOT_EXISTS.getValue());
- final boolean updateFieldNames = context.getProperty(UPDATE_FIELD_NAMES).asBoolean();
- if (recordWriterFactory == null && updateFieldNames) {
- throw new ProcessException("Record Writer must be set if 'Update Field Names' is true");
- }
- final String tableManagementStrategy = context.getProperty(TABLE_MANAGEMENT_STRATEGY).getValue();
- final boolean managedTable;
- if (ATTRIBUTE_DRIVEN_TABLE.getValue().equals(tableManagementStrategy)) {
- String tableManagementStrategyAttribute = flowFile.getAttribute(TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE);
- if (MANAGED_TABLE.getValue().equalsIgnoreCase(tableManagementStrategyAttribute)) {
- managedTable = true;
- } else if (EXTERNAL_TABLE.getValue().equalsIgnoreCase(tableManagementStrategyAttribute)) {
- managedTable = false;
- } else {
- log.error("The '{}' attribute either does not exist or has invalid value: {}. Must be one of (ignoring case): Managed, External. "
- + "Routing flowfile to failure",
- new Object[]{TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE, tableManagementStrategyAttribute});
- session.transfer(flowFile, REL_FAILURE);
- return;
- }
- } else {
- managedTable = MANAGED_TABLE.getValue().equals(tableManagementStrategy);
- }
-
- // Ensure valid configuration for external tables
- if (createIfNotExists && !managedTable && !context.getProperty(EXTERNAL_TABLE_LOCATION).isSet()) {
- throw new IOException("External Table Location must be set when Table Management Strategy is set to External");
- }
- final String externalTableLocation = managedTable ? null : context.getProperty(EXTERNAL_TABLE_LOCATION).evaluateAttributeExpressions(flowFile).getValue();
- if (!managedTable && StringUtils.isEmpty(externalTableLocation)) {
- log.error("External Table Location has invalid value: {}. Routing flowfile to failure", new Object[]{externalTableLocation});
- session.transfer(flowFile, REL_FAILURE);
- return;
- }
- final String storageFormat = context.getProperty(TABLE_STORAGE_FORMAT).getValue();
- final HiveDBCPService dbcpService = context.getProperty(HIVE_DBCP_SERVICE).asControllerService(HiveDBCPService.class);
- try (final Connection connection = dbcpService.getConnection()) {
- final Map attributes = new HashMap<>(flowFile.getAttributes());
- OutputMetadataHolder outputMetadataHolder = checkAndUpdateTableSchema(attributes, connection, recordSchema, tableName, partitionClauseElements,
- createIfNotExists, externalTableLocation, storageFormat, updateFieldNames);
- if (outputMetadataHolder != null) {
- // The output schema changed (i.e. field names were updated), so write out the corresponding FlowFile
- try {
- final FlowFile inputFlowFile = flowFile;
- flowFile = session.write(flowFile, (in, out) -> {
-
- // if we fail to create the RecordReader then we want to route to failure, so we need to
- // handle this separately from the other IOExceptions which normally route to retry
- final RecordReader recordReader;
- final RecordSetWriter recordSetWriter;
- try {
- recordReader = recordReaderFactory.createRecordReader(inputFlowFile, in, getLogger());
- recordSetWriter = recordWriterFactory.createWriter(getLogger(), outputMetadataHolder.getOutputSchema(), out, attributes);
- } catch (Exception e) {
- if(e instanceof IOException) {
- throw (IOException) e;
- }
- throw new IOException(new RecordReaderFactoryException("Unable to create RecordReader", e));
- }
-
- WriteResult writeResult = updateRecords(recordSchema, outputMetadataHolder, recordReader, recordSetWriter);
- recordSetWriter.flush();
- recordSetWriter.close();
- attributes.put("record.count", String.valueOf(writeResult.getRecordCount()));
- attributes.put(CoreAttributes.MIME_TYPE.key(), recordSetWriter.getMimeType());
- attributes.putAll(writeResult.getAttributes());
- });
- } catch (final Exception e) {
- getLogger().error("Failed to process {}; will route to failure", new Object[]{flowFile, e});
- // Since we are wrapping the exceptions above there should always be a cause
- // but it's possible it might not have a message. This handles that by logging
- // the name of the class thrown.
- Throwable c = e.getCause();
- if (c != null) {
- session.putAttribute(flowFile, "record.error.message", (c.getLocalizedMessage() != null) ? c.getLocalizedMessage() : c.getClass().getCanonicalName() + " Thrown");
- } else {
- session.putAttribute(flowFile, "record.error.message", e.getClass().getCanonicalName() + " Thrown");
- }
- session.transfer(flowFile, REL_FAILURE);
- return;
- }
-
- }
- attributes.put(ATTR_OUTPUT_TABLE, tableName);
- flowFile = session.putAllAttributes(flowFile, attributes);
- session.getProvenanceReporter().invokeRemoteProcess(flowFile, dbcpService.getConnectionURL());
- session.transfer(flowFile, REL_SUCCESS);
- }
- } catch (IOException | SQLException e) {
-
- flowFile = session.putAttribute(flowFile, ATTR_OUTPUT_TABLE, tableName);
- log.error("Exception while processing {} - routing to failure", new Object[]{flowFile}, e);
- session.transfer(flowFile, REL_FAILURE);
-
- } catch (DiscontinuedException e) {
- // The input FlowFile processing is discontinued. Keep it in the input queue.
- getLogger().warn("Discontinued processing for {} due to {}", new Object[]{flowFile, e}, e);
- session.transfer(flowFile, Relationship.SELF);
- } catch (Throwable t) {
- throw (t instanceof ProcessException) ? (ProcessException) t : new ProcessException(t);
- }
- }
-
- private synchronized OutputMetadataHolder checkAndUpdateTableSchema(Map attributes, final Connection conn, final RecordSchema schema,
- final String tableName, List partitionClause, final boolean createIfNotExists,
- final String externalTableLocation, final String storageFormat, final boolean updateFieldNames) throws IOException {
- // Read in the current table metadata, compare it to the reader's schema, and
- // add any columns from the schema that are missing in the table
- try (Statement s = conn.createStatement()) {
- // Determine whether the table exists
- ResultSet tables = s.executeQuery("SHOW TABLES");
- List tableNames = new ArrayList<>();
- String hiveTableName;
- while (tables.next() && StringUtils.isNotEmpty(hiveTableName = tables.getString(1))) {
- tableNames.add(hiveTableName);
- }
-
- List columnsToAdd = new ArrayList<>();
- String outputPath;
- boolean tableCreated = false;
- if (!tableNames.contains(tableName) && createIfNotExists) {
- StringBuilder createTableStatement = new StringBuilder();
- for (RecordField recordField : schema.getFields()) {
- String recordFieldName = recordField.getFieldName();
- // The field does not exist in the table, add it
- columnsToAdd.add("`" + recordFieldName + "` " + NiFiOrcUtils.getHiveTypeFromFieldType(recordField.getDataType(), true));
- getLogger().debug("Adding column " + recordFieldName + " to table " + tableName);
- }
-
- // Handle partition clause
- if (partitionClause == null) {
- partitionClause = Collections.emptyList();
- }
- List validatedPartitionClause = new ArrayList<>(partitionClause.size());
- for (String partition : partitionClause) {
- String[] partitionInfo = partition.split(" ");
- if (partitionInfo.length != 2) {
- validatedPartitionClause.add("`" + partitionInfo[0] + "` string");
- } else {
- validatedPartitionClause.add("`" + partitionInfo[0] + "` " + partitionInfo[1]);
- }
- }
-
- createTableStatement.append("CREATE ")
- .append(externalTableLocation == null ? "" : "EXTERNAL ")
- .append("TABLE IF NOT EXISTS `")
- .append(tableName)
- .append("` (")
- .append(String.join(", ", columnsToAdd))
- .append(") ")
- .append(validatedPartitionClause.isEmpty() ? "" : "PARTITIONED BY (" + String.join(", ", validatedPartitionClause) + ") ")
- .append("STORED AS ")
- .append(storageFormat)
- .append(externalTableLocation == null ? "" : " LOCATION '" + externalTableLocation + "'");
-
- String createTableSql = createTableStatement.toString();
-
- if (StringUtils.isNotEmpty(createTableSql)) {
- // Perform the table create
- getLogger().info("Executing Hive DDL: " + createTableSql);
- s.execute(createTableSql);
- }
-
- tableCreated = true;
- }
-
- // Process the table (columns, partitions, location, etc.)
- List hiveColumns = new ArrayList<>();
-
- String describeTable = "DESC FORMATTED `" + tableName + "`";
- ResultSet tableInfo = s.executeQuery(describeTable);
- // Result is 3 columns, col_name, data_type, comment. Check the first row for a header and skip if so, otherwise add column name
- tableInfo.next();
- String columnName = tableInfo.getString(1);
- if (StringUtils.isNotEmpty(columnName) && !columnName.startsWith("#")) {
- hiveColumns.add(columnName);
- }
- // If the column was a header, check for a blank line to follow and skip it, otherwise add the column name
- if (columnName.startsWith("#")) {
- tableInfo.next();
- columnName = tableInfo.getString(1);
- if (StringUtils.isNotEmpty(columnName)) {
- hiveColumns.add(columnName);
- }
- }
-
- // Collect all column names
- while (tableInfo.next() && StringUtils.isNotEmpty(columnName = tableInfo.getString(1))) {
- hiveColumns.add(columnName);
- }
-
- // Collect all partition columns
- boolean moreRows = true;
- boolean headerFound = false;
- while (moreRows && !headerFound) {
- String line = tableInfo.getString(1);
- if ("# Partition Information".equals(line)) {
- headerFound = true;
- } else if ("# Detailed Table Information".equals(line)) {
- // Not partitioned, exit the loop with headerFound = false
- break;
- }
- moreRows = tableInfo.next();
- }
-
- List partitionColumns = new ArrayList<>();
- List partitionColumnsEqualsValueList = new ArrayList<>();
- List partitionColumnsLocationList = new ArrayList<>();
- if (headerFound) {
- // If the table is partitioned, construct the partition=value strings for each partition column
- String partitionColumnName;
- columnName = tableInfo.getString(1);
- if (StringUtils.isNotEmpty(columnName) && !columnName.startsWith("#")) {
- partitionColumns.add(columnName);
- }
- // If the column was a header, check for a blank line to follow and skip it, otherwise add the column name
- if (columnName.startsWith("#")) {
- tableInfo.next();
- columnName = tableInfo.getString(1);
- if (StringUtils.isNotEmpty(columnName)) {
- partitionColumns.add(columnName);
- }
- }
- while (tableInfo.next() && StringUtils.isNotEmpty(partitionColumnName = tableInfo.getString(1))) {
- partitionColumns.add(partitionColumnName);
- }
-
- final int partitionColumnsSize = partitionColumns.size();
- final int partitionClauseSize = (partitionClause == null) ? 0 : partitionClause.size();
- if (partitionClauseSize != partitionColumnsSize) {
- throw new IOException("Found " + partitionColumnsSize + " partition columns but " + partitionClauseSize + " partition values were supplied");
- }
-
- for (int i = 0; i < partitionClauseSize; i++) {
- String partitionName = partitionClause.get(i).split(" ")[0];
- String partitionValue = attributes.get(partitionName);
- if (StringUtils.isEmpty(partitionValue)) {
- throw new IOException("No value found for partition value attribute '" + partitionName + "'");
- }
- if (!partitionColumns.contains(partitionName)) {
- throw new IOException("Cannot add partition '" + partitionName + "' to existing table");
- }
- partitionColumnsEqualsValueList.add("`" + partitionName + "`='" + partitionValue + "'");
- // Add unquoted version for the output path
- partitionColumnsLocationList.add(partitionName + "=" + partitionValue);
- }
- }
-
- // Get table location
- moreRows = true;
- headerFound = false;
- while (moreRows && !headerFound) {
- String line = tableInfo.getString(1);
- if (line.startsWith("Location:")) {
- headerFound = true;
- continue; // Don't do a next() here, need to get the second column value
- }
- moreRows = tableInfo.next();
- }
- String tableLocation = tableInfo.getString(2);
-
- String alterTableSql;
- // If the table wasn't newly created, alter it accordingly
- if (!tableCreated) {
- StringBuilder alterTableStatement = new StringBuilder();
- // Handle new columns
- for (RecordField recordField : schema.getFields()) {
- String recordFieldName = recordField.getFieldName().toLowerCase();
- if (!hiveColumns.contains(recordFieldName) && !partitionColumns.contains(recordFieldName)) {
- // The field does not exist in the table (and is not a partition column), add it
- columnsToAdd.add("`" + recordFieldName + "` " + NiFiOrcUtils.getHiveTypeFromFieldType(recordField.getDataType(), true));
- getLogger().info("Adding column " + recordFieldName + " to table " + tableName);
- }
- }
-
- if (!columnsToAdd.isEmpty()) {
- alterTableStatement.append("ALTER TABLE `")
- .append(tableName)
- .append("` ADD COLUMNS (")
- .append(String.join(", ", columnsToAdd))
- .append(")");
-
- alterTableSql = alterTableStatement.toString();
- if (StringUtils.isNotEmpty(alterTableSql)) {
- // Perform the table update
- getLogger().info("Executing Hive DDL: " + alterTableSql);
- s.execute(alterTableSql);
- }
- }
- }
-
- outputPath = tableLocation;
-
- // Handle new partition values
- if (!partitionColumnsEqualsValueList.isEmpty()) {
- alterTableSql = "ALTER TABLE `" +
- tableName +
- "` ADD IF NOT EXISTS PARTITION (" +
- String.join(", ", partitionColumnsEqualsValueList) +
- ")";
- if (StringUtils.isNotEmpty(alterTableSql)) {
- // Perform the table update
- getLogger().info("Executing Hive DDL: " + alterTableSql);
- s.execute(alterTableSql);
- }
- // Add attribute for HDFS location of the partition values
- outputPath = tableLocation + "/" + String.join("/", partitionColumnsLocationList);
- }
-
- // If updating field names, return a new RecordSchema, otherwise return null
- OutputMetadataHolder outputMetadataHolder;
- if (updateFieldNames) {
- List inputRecordFields = schema.getFields();
- List outputRecordFields = new ArrayList<>();
- Map fieldMap = new HashMap<>();
- boolean needsUpdating = false;
-
- for (RecordField inputRecordField : inputRecordFields) {
- final String inputRecordFieldName = inputRecordField.getFieldName();
- boolean found = false;
- for (String hiveColumnName : hiveColumns) {
- if (inputRecordFieldName.equalsIgnoreCase(hiveColumnName)) {
- // Set a flag if the field name doesn't match the column name exactly. This overall flag will determine whether
- // the records need updating (if true) or not (if false)
- if (!inputRecordFieldName.equals(hiveColumnName)) {
- needsUpdating = true;
- }
- fieldMap.put(inputRecordFieldName, hiveColumnName);
- outputRecordFields.add(new RecordField(hiveColumnName, inputRecordField.getDataType(), inputRecordField.getDefaultValue(), inputRecordField.isNullable()));
- found = true;
- break;
- }
- }
- if (!found) {
- // If the input field wasn't a Hive table column, add it back to the schema as-is
- fieldMap.put(inputRecordFieldName, inputRecordFieldName);
- }
- }
- outputMetadataHolder = needsUpdating ? new OutputMetadataHolder(new SimpleRecordSchema(outputRecordFields), fieldMap)
- : null;
- } else {
- outputMetadataHolder = null;
- }
- attributes.put(ATTR_OUTPUT_PATH, outputPath);
- return outputMetadataHolder;
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- private synchronized WriteResult updateRecords(final RecordSchema inputRecordSchema, final OutputMetadataHolder outputMetadataHolder,
- final RecordReader reader, final RecordSetWriter writer) throws IOException {
- try {
- writer.beginRecordSet();
- Record inputRecord;
- while((inputRecord = reader.nextRecord()) != null) {
- List inputRecordFields = inputRecordSchema.getFields();
- Map outputRecordFields = new HashMap<>(inputRecordFields.size());
- // Copy values from input field name to output field name
- for(Map.Entry mapping : outputMetadataHolder.getFieldMap().entrySet()) {
- outputRecordFields.put(mapping.getValue(), inputRecord.getValue(mapping.getKey()));
- }
- Record outputRecord = new MapRecord(outputMetadataHolder.getOutputSchema(), outputRecordFields);
- writer.write(outputRecord);
- }
- return writer.finishRecordSet();
-
- } catch (MalformedRecordException mre) {
- throw new IOException("Error reading records: "+mre.getMessage(), mre);
- }
- }
-
- private static class OutputMetadataHolder {
- private final RecordSchema outputSchema;
- private final Map fieldMap;
-
- public OutputMetadataHolder(RecordSchema outputSchema, Map fieldMap) {
- this.outputSchema = outputSchema;
- this.fieldMap = fieldMap;
- }
-
- public RecordSchema getOutputSchema() {
- return outputSchema;
- }
-
- public Map getFieldMap() {
- return fieldMap;
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/AuthenticationFailedException.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/AuthenticationFailedException.java
deleted file mode 100644
index 70cc6c13c4..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/AuthenticationFailedException.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.hive;
-
-public class AuthenticationFailedException extends Exception {
- public AuthenticationFailedException(String reason, Exception cause) {
- super(reason, cause);
- }
-}
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/CsvOutputOptions.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/CsvOutputOptions.java
deleted file mode 100644
index 36889129f3..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/CsvOutputOptions.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.hive;
-
-public class CsvOutputOptions {
-
- private boolean header = true;
- private String altHeader = null;
- private String delimiter = ",";
- private boolean quote = false;
- private boolean escape = true;
-
- private int maxRowsPerFlowFile = 0;
-
- public boolean isHeader() {
- return header;
- }
-
- public String getAltHeader() {
- return altHeader;
- }
-
-
- public String getDelimiter() {
- return delimiter;
- }
-
-
- public boolean isQuote() {
- return quote;
- }
-
- public boolean isEscape() {
- return escape;
- }
-
- public int getMaxRowsPerFlowFile() {
- return maxRowsPerFlowFile;
- }
-
- public CsvOutputOptions(boolean header, String altHeader, String delimiter, boolean quote, boolean escape, int maxRowsPerFlowFile) {
- this.header = header;
- this.altHeader = altHeader;
- this.delimiter = delimiter;
- this.quote = quote;
- this.escape = escape;
- this.maxRowsPerFlowFile = maxRowsPerFlowFile;
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java
deleted file mode 100644
index b212207ebb..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveConfigurator.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.hive;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.nifi.components.ValidationResult;
-import org.apache.nifi.hadoop.KerberosProperties;
-import org.apache.nifi.hadoop.SecurityUtil;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.security.krb.KerberosUser;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-public class HiveConfigurator {
-
- public Collection validate(String configFiles, String principal, String keyTab, String password,
- AtomicReference validationResourceHolder, ComponentLog log) {
-
- final List problems = new ArrayList<>();
- ValidationResources resources = validationResourceHolder.get();
-
- // if no resources in the holder, or if the holder has different resources loaded,
- // then load the Configuration and set the new resources in the holder
- if (resources == null || !configFiles.equals(resources.getConfigResources())) {
- log.debug("Reloading validation resources");
- resources = new ValidationResources(configFiles, getConfigurationFromFiles(configFiles));
- validationResourceHolder.set(resources);
- }
-
- final Configuration hiveConfig = resources.getConfiguration();
-
- problems.addAll(KerberosProperties.validatePrincipalWithKeytabOrPassword(this.getClass().getSimpleName(), hiveConfig, principal, keyTab, password, log));
-
- return problems;
- }
-
- public HiveConf getConfigurationFromFiles(final String configFiles) {
- final HiveConf hiveConfig = new HiveConf();
- if (StringUtils.isNotBlank(configFiles)) {
- for (final String configFile : configFiles.split(",")) {
- hiveConfig.addResource(new Path(configFile.trim()));
- }
- }
- return hiveConfig;
- }
-
- public void preload(Configuration configuration) {
- try {
- FileSystem.get(configuration).close();
- UserGroupInformation.setConfiguration(configuration);
- } catch (IOException ioe) {
- // Suppress exception as future uses of this configuration will fail
- }
- }
-
- /**
- * Acquires a {@link UserGroupInformation} using the given {@link Configuration} and {@link KerberosUser}.
- * @see SecurityUtil#getUgiForKerberosUser(Configuration, KerberosUser)
- * @param hiveConfig The Configuration to apply to the acquired UserGroupInformation
- * @param kerberosUser The KerberosUser to authenticate
- * @return A UserGroupInformation instance created using the Subject of the given KerberosUser
- * @throws AuthenticationFailedException if authentication fails
- */
- public UserGroupInformation authenticate(final Configuration hiveConfig, KerberosUser kerberosUser) throws AuthenticationFailedException {
- try {
- return SecurityUtil.getUgiForKerberosUser(hiveConfig, kerberosUser);
- } catch (IOException ioe) {
- throw new AuthenticationFailedException("Kerberos Authentication for Hive failed", ioe);
- }
- }
-
- /**
- * As of Apache NiFi 1.5.0, due to changes made to
- * {@link SecurityUtil#loginKerberos(Configuration, String, String)}, which is used by this
- * class to authenticate a principal with Kerberos, Hive controller services no longer
- * attempt relogins explicitly. For more information, please read the documentation for
- * {@link SecurityUtil#loginKerberos(Configuration, String, String)}.
- *
- * In previous versions of NiFi, a {@link org.apache.nifi.hadoop.KerberosTicketRenewer} was started by
- * {@link HiveConfigurator#authenticate(Configuration, String, String, long)} when the Hive
- * controller service was enabled. The use of a separate thread to explicitly relogin could cause race conditions
- * with the implicit relogin attempts made by hadoop/Hive code on a thread that references the same
- * {@link UserGroupInformation} instance. One of these threads could leave the
- * {@link javax.security.auth.Subject} in {@link UserGroupInformation} to be cleared or in an unexpected state
- * while the other thread is attempting to use the {@link javax.security.auth.Subject}, resulting in failed
- * authentication attempts that would leave the Hive controller service in an unrecoverable state.
- *
- * @see SecurityUtil#loginKerberos(Configuration, String, String)
- * @deprecated Use {@link SecurityUtil#getUgiForKerberosUser(Configuration, KerberosUser)}
- */
- @Deprecated
- public UserGroupInformation authenticate(final Configuration hiveConfig, String principal, String keyTab) throws AuthenticationFailedException {
- UserGroupInformation ugi;
- try {
- ugi = SecurityUtil.loginKerberos(hiveConfig, principal, keyTab);
- } catch (IOException ioe) {
- throw new AuthenticationFailedException("Kerberos Authentication for Hive failed", ioe);
- }
- return ugi;
- }
-
- /**
- * As of Apache NiFi 1.5.0, this method has been deprecated and is now a wrapper
- * method which invokes {@link HiveConfigurator#authenticate(Configuration, String, String)}. It will no longer start a
- * {@link org.apache.nifi.hadoop.KerberosTicketRenewer} to perform explicit relogins.
- *
- * @see HiveConfigurator#authenticate(Configuration, String, String)
- */
- @Deprecated
- public UserGroupInformation authenticate(final Configuration hiveConfig, String principal, String keyTab, long ticketRenewalPeriod) throws AuthenticationFailedException {
- return authenticate(hiveConfig, principal, keyTab);
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java
deleted file mode 100644
index fd1aaf4fad..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveJdbcCommon.java
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.hive;
-
-import org.apache.avro.Schema;
-import org.apache.avro.SchemaBuilder;
-import org.apache.avro.SchemaBuilder.FieldAssembler;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumWriter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.text.StringEscapeUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.nifi.components.PropertyDescriptor;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import static java.sql.Types.ARRAY;
-import static java.sql.Types.BIGINT;
-import static java.sql.Types.BINARY;
-import static java.sql.Types.BIT;
-import static java.sql.Types.BLOB;
-import static java.sql.Types.BOOLEAN;
-import static java.sql.Types.CHAR;
-import static java.sql.Types.CLOB;
-import static java.sql.Types.DATE;
-import static java.sql.Types.DECIMAL;
-import static java.sql.Types.DOUBLE;
-import static java.sql.Types.FLOAT;
-import static java.sql.Types.INTEGER;
-import static java.sql.Types.JAVA_OBJECT;
-import static java.sql.Types.LONGNVARCHAR;
-import static java.sql.Types.LONGVARBINARY;
-import static java.sql.Types.LONGVARCHAR;
-import static java.sql.Types.NCHAR;
-import static java.sql.Types.NUMERIC;
-import static java.sql.Types.NVARCHAR;
-import static java.sql.Types.OTHER;
-import static java.sql.Types.REAL;
-import static java.sql.Types.ROWID;
-import static java.sql.Types.SMALLINT;
-import static java.sql.Types.SQLXML;
-import static java.sql.Types.STRUCT;
-import static java.sql.Types.TIME;
-import static java.sql.Types.TIMESTAMP;
-import static java.sql.Types.TINYINT;
-import static java.sql.Types.VARBINARY;
-import static java.sql.Types.VARCHAR;
-
-/**
- * JDBC / HiveQL common functions.
- */
-public class HiveJdbcCommon {
-
- public static final String AVRO = "Avro";
- public static final String CSV = "CSV";
-
- public static final String MIME_TYPE_AVRO_BINARY = "application/avro-binary";
- public static final String CSV_MIME_TYPE = "text/csv";
-
-
- public static final PropertyDescriptor NORMALIZE_NAMES_FOR_AVRO = new PropertyDescriptor.Builder()
- .name("hive-normalize-avro")
- .displayName("Normalize Table/Column Names")
- .description("Whether to change non-Avro-compatible characters in column names to Avro-compatible characters. For example, colons and periods "
- + "will be changed to underscores in order to build a valid Avro record.")
- .allowableValues("true", "false")
- .defaultValue("false")
- .required(true)
- .build();
-
- public static long convertToAvroStream(final ResultSet rs, final OutputStream outStream, final int maxRows, boolean convertNames) throws SQLException, IOException {
- return convertToAvroStream(rs, outStream, null, maxRows, convertNames, null);
- }
-
-
- public static long convertToAvroStream(final ResultSet rs, final OutputStream outStream, String recordName, final int maxRows, boolean convertNames, ResultSetRowCallback callback)
- throws SQLException, IOException {
- final Schema schema = createSchema(rs, recordName, convertNames);
- final GenericRecord rec = new GenericData.Record(schema);
-
- final DatumWriter datumWriter = new GenericDatumWriter<>(schema);
- try (final DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter)) {
- dataFileWriter.create(schema, outStream);
-
- final ResultSetMetaData meta = rs.getMetaData();
- final int nrOfColumns = meta.getColumnCount();
- long nrOfRows = 0;
- while (rs.next()) {
- if (callback != null) {
- callback.processRow(rs);
- }
- for (int i = 1; i <= nrOfColumns; i++) {
- final int javaSqlType = meta.getColumnType(i);
- Object value = rs.getObject(i);
-
- if (value == null) {
- rec.put(i - 1, null);
-
- } else if (javaSqlType == BINARY || javaSqlType == VARBINARY || javaSqlType == LONGVARBINARY || javaSqlType == BLOB || javaSqlType == CLOB) {
- // bytes requires little bit different handling
- ByteBuffer bb = null;
- if (value instanceof byte[]) {
- bb = ByteBuffer.wrap((byte[]) value);
- } else if (value instanceof ByteBuffer) {
- bb = (ByteBuffer) value;
- }
- if (bb != null) {
- rec.put(i - 1, bb);
- } else {
- throw new IOException("Could not process binary object of type " + value.getClass().getName());
- }
-
- } else if (value instanceof Byte) {
- // tinyint(1) type is returned by JDBC driver as java.sql.Types.TINYINT
- // But value is returned by JDBC as java.lang.Byte
- // (at least H2 JDBC works this way)
- // direct put to avro record results:
- // org.apache.avro.AvroRuntimeException: Unknown datum type java.lang.Byte
- rec.put(i - 1, ((Byte) value).intValue());
-
- } else if (value instanceof BigDecimal || value instanceof BigInteger) {
- // Avro can't handle BigDecimal and BigInteger as numbers - it will throw an AvroRuntimeException such as: "Unknown datum type: java.math.BigDecimal: 38"
- rec.put(i - 1, value.toString());
-
- } else if (value instanceof Number) {
- // Need to call the right getXYZ() method (instead of the getObject() method above), since Doubles are sometimes returned
- // when the JDBC type is 6 (Float) for example.
- if (javaSqlType == FLOAT) {
- value = rs.getFloat(i);
- } else if (javaSqlType == DOUBLE) {
- value = rs.getDouble(i);
- } else if (javaSqlType == INTEGER || javaSqlType == TINYINT || javaSqlType == SMALLINT) {
- value = rs.getInt(i);
- }
-
- rec.put(i - 1, value);
-
- } else if (value instanceof Boolean) {
- rec.put(i - 1, value);
- } else if (value instanceof java.sql.SQLXML) {
- rec.put(i - 1, ((java.sql.SQLXML) value).getString());
- } else {
- // The different types that we support are numbers (int, long, double, float),
- // as well as boolean values and Strings. Since Avro doesn't provide
- // timestamp types, we want to convert those to Strings. So we will cast anything other
- // than numbers or booleans to strings by using the toString() method.
- rec.put(i - 1, value.toString());
- }
- }
- dataFileWriter.append(rec);
- nrOfRows += 1;
-
- if (maxRows > 0 && nrOfRows == maxRows)
- break;
- }
-
- return nrOfRows;
- }
- }
-
- public static Schema createSchema(final ResultSet rs, boolean convertNames) throws SQLException {
- return createSchema(rs, null, false);
- }
-
- /**
- * Creates an Avro schema from a result set. If the table/record name is known a priori and provided, use that as a
- * fallback for the record name if it cannot be retrieved from the result set, and finally fall back to a default value.
- *
- * @param rs The result set to convert to Avro
- * @param recordName The a priori record name to use if it cannot be determined from the result set.
- * @param convertNames Whether to convert column/table names to be legal Avro names
- * @return A Schema object representing the result set converted to an Avro record
- * @throws SQLException if any error occurs during conversion
- */
- public static Schema createSchema(final ResultSet rs, String recordName, boolean convertNames) throws SQLException {
- final ResultSetMetaData meta = rs.getMetaData();
- final int nrOfColumns = meta.getColumnCount();
- String tableName = StringUtils.isEmpty(recordName) ? "NiFi_SelectHiveQL_Record" : recordName;
- try {
- if (nrOfColumns > 0) {
- // Hive JDBC doesn't support getTableName, instead it returns table.column for column name. Grab the table name from the first column
- String firstColumnNameFromMeta = meta.getColumnName(1);
- int tableNameDelimiter = firstColumnNameFromMeta.lastIndexOf(".");
- if (tableNameDelimiter > -1) {
- String tableNameFromMeta = firstColumnNameFromMeta.substring(0, tableNameDelimiter);
- if (!StringUtils.isBlank(tableNameFromMeta)) {
- tableName = tableNameFromMeta;
- }
- }
- }
- } catch (SQLException se) {
- // Not all drivers support getTableName, so just use the previously-set default
- }
-
- if (convertNames) {
- tableName = normalizeNameForAvro(tableName);
- }
- final FieldAssembler builder = SchemaBuilder.record(tableName).namespace("any.data").fields();
-
- /**
- * Some missing Avro types - Decimal, Date types. May need some additional work.
- */
- for (int i = 1; i <= nrOfColumns; i++) {
- String columnNameFromMeta = meta.getColumnName(i);
- // Hive returns table.column for column name. Grab the column name as the string after the last period
- int columnNameDelimiter = columnNameFromMeta.lastIndexOf(".");
- String columnName = columnNameFromMeta.substring(columnNameDelimiter + 1);
- switch (meta.getColumnType(i)) {
- case CHAR:
- case LONGNVARCHAR:
- case LONGVARCHAR:
- case NCHAR:
- case NVARCHAR:
- case VARCHAR:
- case ARRAY:
- case STRUCT:
- case JAVA_OBJECT:
- case OTHER:
- case SQLXML:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().stringType().endUnion().noDefault();
- break;
-
- case BIT:
- case BOOLEAN:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().booleanType().endUnion().noDefault();
- break;
-
- case INTEGER:
- // Default to signed type unless otherwise noted. Some JDBC drivers don't implement isSigned()
- boolean signedType = true;
- try {
- signedType = meta.isSigned(i);
- } catch (SQLException se) {
- // Use signed types as default
- }
- if (signedType) {
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().intType().endUnion().noDefault();
- } else {
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault();
- }
- break;
-
- case SMALLINT:
- case TINYINT:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().intType().endUnion().noDefault();
- break;
-
- case BIGINT:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().longType().endUnion().noDefault();
- break;
-
- // java.sql.RowId is interface, is seems to be database
- // implementation specific, let's convert to String
- case ROWID:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().stringType().endUnion().noDefault();
- break;
-
- case FLOAT:
- case REAL:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().floatType().endUnion().noDefault();
- break;
-
- case DOUBLE:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().doubleType().endUnion().noDefault();
- break;
-
- // Did not find direct suitable type, need to be clarified!!!!
- case DECIMAL:
- case NUMERIC:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().stringType().endUnion().noDefault();
- break;
-
- // Did not find direct suitable type, need to be clarified!!!!
- case DATE:
- case TIME:
- case TIMESTAMP:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().stringType().endUnion().noDefault();
- break;
-
- case BINARY:
- case VARBINARY:
- case LONGVARBINARY:
- case BLOB:
- case CLOB:
- builder.name(columnName).type().unionOf().nullBuilder().endNull().and().bytesType().endUnion().noDefault();
- break;
-
-
- default:
- throw new IllegalArgumentException("createSchema: Unknown SQL type " + meta.getColumnType(i) + " cannot be converted to Avro type");
- }
- }
-
- return builder.endRecord();
- }
-
- public static long convertToCsvStream(final ResultSet rs, final OutputStream outStream, CsvOutputOptions outputOptions) throws SQLException, IOException {
- return convertToCsvStream(rs, outStream, null, null, outputOptions);
- }
-
- public static long convertToCsvStream(final ResultSet rs, final OutputStream outStream, String recordName, ResultSetRowCallback callback, CsvOutputOptions outputOptions)
- throws SQLException, IOException {
-
- final ResultSetMetaData meta = rs.getMetaData();
- final int nrOfColumns = meta.getColumnCount();
- List columnNames = new ArrayList<>(nrOfColumns);
-
- if (outputOptions.isHeader()) {
- if (outputOptions.getAltHeader() == null) {
- for (int i = 1; i <= nrOfColumns; i++) {
- String columnNameFromMeta = meta.getColumnName(i);
- // Hive returns table.column for column name. Grab the column name as the string after the last period
- int columnNameDelimiter = columnNameFromMeta.lastIndexOf(".");
- columnNames.add(columnNameFromMeta.substring(columnNameDelimiter + 1));
- }
- } else {
- String[] altHeaderNames = outputOptions.getAltHeader().split(",");
- columnNames = Arrays.asList(altHeaderNames);
- }
- }
-
- // Write column names as header row
- outStream.write(StringUtils.join(columnNames, outputOptions.getDelimiter()).getBytes(StandardCharsets.UTF_8));
- if (outputOptions.isHeader()) {
- outStream.write("\n".getBytes(StandardCharsets.UTF_8));
- }
-
- // Iterate over the rows
- int maxRows = outputOptions.getMaxRowsPerFlowFile();
- long nrOfRows = 0;
- while (rs.next()) {
- if (callback != null) {
- callback.processRow(rs);
- }
- List rowValues = new ArrayList<>(nrOfColumns);
- for (int i = 1; i <= nrOfColumns; i++) {
- final int javaSqlType = meta.getColumnType(i);
- final Object value = rs.getObject(i);
-
- switch (javaSqlType) {
- case CHAR:
- case LONGNVARCHAR:
- case LONGVARCHAR:
- case NCHAR:
- case NVARCHAR:
- case VARCHAR:
- String valueString = rs.getString(i);
- if (valueString != null) {
- // Removed extra quotes as those are a part of the escapeCsv when required.
- StringBuilder sb = new StringBuilder();
- if (outputOptions.isQuote()) {
- sb.append("\"");
- if (outputOptions.isEscape()) {
- sb.append(StringEscapeUtils.escapeCsv(valueString));
- } else {
- sb.append(valueString);
- }
- sb.append("\"");
- rowValues.add(sb.toString());
- } else {
- if (outputOptions.isEscape()) {
- rowValues.add(StringEscapeUtils.escapeCsv(valueString));
- } else {
- rowValues.add(valueString);
- }
- }
- } else {
- rowValues.add("");
- }
- break;
- case ARRAY:
- case STRUCT:
- case JAVA_OBJECT:
- String complexValueString = rs.getString(i);
- if (complexValueString != null) {
- rowValues.add(StringEscapeUtils.escapeCsv(complexValueString));
- } else {
- rowValues.add("");
- }
- break;
- case SQLXML:
- if (value != null) {
- rowValues.add(StringEscapeUtils.escapeCsv(((java.sql.SQLXML) value).getString()));
- } else {
- rowValues.add("");
- }
- default:
- if (value != null) {
- rowValues.add(value.toString());
- } else {
- rowValues.add("");
- }
- }
- }
- // Write row values
- outStream.write(StringUtils.join(rowValues, outputOptions.getDelimiter()).getBytes(StandardCharsets.UTF_8));
- outStream.write("\n".getBytes(StandardCharsets.UTF_8));
- nrOfRows++;
-
- if (maxRows > 0 && nrOfRows == maxRows)
- break;
- }
- return nrOfRows;
- }
-
- public static String normalizeNameForAvro(String inputName) {
- String normalizedName = inputName.replaceAll("[^A-Za-z0-9_]", "_");
- if (Character.isDigit(normalizedName.charAt(0))) {
- normalizedName = "_" + normalizedName;
- }
- return normalizedName;
- }
-
- /**
- * An interface for callback methods which allows processing of a row during the convertToXYZStream() processing.
- * IMPORTANT: This method should only work on the row pointed at by the current ResultSet reference.
- * Advancing the cursor (e.g.) can cause rows to be skipped during Avro transformation.
- */
- public interface ResultSetRowCallback {
- void processRow(ResultSet resultSet) throws IOException;
- }
-
- public static Configuration getConfigurationFromFiles(final String configFiles) {
- final Configuration hiveConfig = new HiveConf();
- if (StringUtils.isNotBlank(configFiles)) {
- for (final String configFile : configFiles.split(",")) {
- hiveConfig.addResource(new Path(configFile.trim()));
- }
- }
- return hiveConfig;
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveOptions.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveOptions.java
deleted file mode 100644
index ddc15673e5..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveOptions.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.util.hive;
-
-import java.io.Serializable;
-
-
-public class HiveOptions implements Serializable {
- /**
- * Half of the default Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS
- */
- private static final int DEFAULT_TICK_TUPLE_INTERVAL_SECS = 15;
-
- protected String databaseName;
- protected String tableName;
- protected String metaStoreURI;
- protected Integer txnsPerBatch = 100;
- protected Integer maxOpenConnections = 10;
- protected Integer batchSize = 15000;
- protected Integer idleTimeout = 60000;
- protected Integer callTimeout = 0;
- protected Integer heartBeatInterval = 60;
- protected Boolean autoCreatePartitions = true;
- protected String kerberosPrincipal;
- protected String kerberosKeytab;
- protected Integer tickTupleInterval = DEFAULT_TICK_TUPLE_INTERVAL_SECS;
-
- public HiveOptions(String metaStoreURI, String databaseName, String tableName) {
- this.metaStoreURI = metaStoreURI;
- this.databaseName = databaseName;
- this.tableName = tableName;
- }
-
- public HiveOptions withTickTupleInterval(Integer tickInterval) {
- this.tickTupleInterval = tickInterval;
- return this;
- }
-
- public HiveOptions withTxnsPerBatch(Integer txnsPerBatch) {
- this.txnsPerBatch = txnsPerBatch;
- return this;
- }
-
- public HiveOptions withMaxOpenConnections(Integer maxOpenConnections) {
- this.maxOpenConnections = maxOpenConnections;
- return this;
- }
-
- public HiveOptions withBatchSize(Integer batchSize) {
- this.batchSize = batchSize;
- return this;
- }
-
- public HiveOptions withIdleTimeout(Integer idleTimeout) {
- this.idleTimeout = idleTimeout;
- return this;
- }
-
- public HiveOptions withCallTimeout(Integer callTimeout) {
- this.callTimeout = callTimeout;
- return this;
- }
-
- public HiveOptions withHeartBeatInterval(Integer heartBeatInterval) {
- this.heartBeatInterval = heartBeatInterval;
- return this;
- }
-
- public HiveOptions withAutoCreatePartitions(Boolean autoCreatePartitions) {
- this.autoCreatePartitions = autoCreatePartitions;
- return this;
- }
-
- public HiveOptions withKerberosKeytab(String kerberosKeytab) {
- this.kerberosKeytab = kerberosKeytab;
- return this;
- }
-
- public HiveOptions withKerberosPrincipal(String kerberosPrincipal) {
- this.kerberosPrincipal = kerberosPrincipal;
- return this;
- }
-
- public String getMetaStoreURI() {
- return metaStoreURI;
- }
-
- public String getDatabaseName() {
- return databaseName;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public String getQualifiedTableName() {
- return databaseName + "." + tableName;
- }
-
- public Integer getBatchSize() {
- return batchSize;
- }
-
- public Integer getCallTimeOut() {
- return callTimeout;
- }
-
- public Integer getHeartBeatInterval() {
- return heartBeatInterval;
- }
-
- public Integer getMaxOpenConnections() {
- return maxOpenConnections;
- }
-
- public Integer getIdleTimeout() {
- return idleTimeout;
- }
-
- public Integer getTxnsPerBatch() {
- return txnsPerBatch;
- }
-
- public Boolean getAutoCreatePartitions() {
- return autoCreatePartitions;
- }
-
- public String getKerberosPrincipal() {
- return kerberosPrincipal;
- }
-
- public String getKerberosKeytab() {
- return kerberosKeytab;
- }
-
- public Integer getTickTupleInterval() {
- return tickTupleInterval;
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveUtils.java
deleted file mode 100644
index c58010c03d..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveUtils.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.util.hive;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.hcatalog.streaming.ConnectionError;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-
-public class HiveUtils {
- private static final Logger LOG = LoggerFactory.getLogger(HiveUtils.class);
-
- public static HiveEndPoint makeEndPoint(List partitionVals, HiveOptions options) throws ConnectionError {
- return new HiveEndPoint(options.getMetaStoreURI(), options.getDatabaseName(), options.getTableName(), partitionVals);
- }
-
- public static HiveWriter makeHiveWriter(HiveEndPoint endPoint, ExecutorService callTimeoutPool, UserGroupInformation ugi, HiveOptions options, HiveConf hiveConf)
- throws HiveWriter.ConnectFailure, InterruptedException {
- return new HiveWriter(endPoint, options.getTxnsPerBatch(), options.getAutoCreatePartitions(),
- options.getCallTimeOut(), callTimeoutPool, ugi, hiveConf);
- }
-
- public static void logAllHiveEndPoints(Map allWriters) {
- for (Map.Entry entry : allWriters.entrySet()) {
- LOG.info("cached writers {} ", entry.getValue());
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveWriter.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveWriter.java
deleted file mode 100644
index e5ab48ffb6..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/HiveWriter.java
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.util.hive;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.hcatalog.streaming.HiveEndPoint;
-import org.apache.hive.hcatalog.streaming.RecordWriter;
-import org.apache.hive.hcatalog.streaming.SerializationError;
-import org.apache.hive.hcatalog.streaming.StreamingConnection;
-import org.apache.hive.hcatalog.streaming.StreamingException;
-import org.apache.hive.hcatalog.streaming.StreamingIOFailure;
-import org.apache.hive.hcatalog.streaming.StrictJsonWriter;
-import org.apache.hive.hcatalog.streaming.TransactionBatch;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-
-public class HiveWriter {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveWriter.class);
-
- private final HiveEndPoint endPoint;
- private final StreamingConnection connection;
- private final int txnsPerBatch;
- private final RecordWriter recordWriter;
- private final ExecutorService callTimeoutPool;
- private final long callTimeout;
- private final Object txnBatchLock = new Object();
- private final UserGroupInformation ugi;
- private TransactionBatch txnBatch;
- private long lastUsed; // time of last flush on this writer
- protected boolean closed; // flag indicating HiveWriter was closed
- private int totalRecords = 0;
-
- public HiveWriter(HiveEndPoint endPoint, int txnsPerBatch, boolean autoCreatePartitions, long callTimeout, ExecutorService callTimeoutPool, UserGroupInformation ugi, HiveConf hiveConf)
- throws InterruptedException, ConnectFailure {
- try {
- this.ugi = ugi;
- this.callTimeout = callTimeout;
- this.callTimeoutPool = callTimeoutPool;
- this.endPoint = endPoint;
- this.connection = newConnection(endPoint, autoCreatePartitions, hiveConf, ugi);
- this.txnsPerBatch = txnsPerBatch;
- this.recordWriter = getRecordWriter(endPoint, ugi, hiveConf);
- this.txnBatch = nextTxnBatch(recordWriter);
- this.closed = false;
- this.lastUsed = System.currentTimeMillis();
- } catch (InterruptedException | RuntimeException | ConnectFailure e) {
- throw e;
- } catch (Exception e) {
- throw new ConnectFailure(endPoint, e);
- }
- }
-
- protected RecordWriter getRecordWriter(HiveEndPoint endPoint, UserGroupInformation ugi, HiveConf hiveConf) throws StreamingException, IOException, InterruptedException {
- if (ugi == null) {
- return new StrictJsonWriter(endPoint, hiveConf);
- } else {
- try {
- return ugi.doAs((PrivilegedExceptionAction) () -> new StrictJsonWriter(endPoint, hiveConf));
- } catch (UndeclaredThrowableException e) {
- Throwable cause = e.getCause();
- if (cause instanceof StreamingException) {
- throw (StreamingException) cause;
- } else {
- throw e;
- }
- }
- }
- }
-
- @Override
- public String toString() {
- return "{ endPoint = " + endPoint + ", TransactionBatch = " + txnBatch + " }";
- }
-
- /**
- * Write the record data to Hive
- *
- * @throws IOException if an error occurs during the write
- * @throws InterruptedException if the write operation is interrupted
- */
- public synchronized void write(final byte[] record)
- throws WriteFailure, SerializationError, InterruptedException {
- if (closed) {
- throw new IllegalStateException("This hive streaming writer was closed " +
- "and thus no longer able to write : " + endPoint);
- }
- // write the tuple
- try {
- LOG.debug("Writing event to {}", endPoint);
- callWithTimeout(new CallRunner() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.write(record);
- totalRecords++;
- return null;
- }
- });
- } catch (SerializationError se) {
- throw new SerializationError(endPoint.toString() + " SerializationError", se);
- } catch (StreamingException | TimeoutException e) {
- throw new WriteFailure(endPoint, txnBatch.getCurrentTxnId(), e);
- }
- }
-
- /**
- * Commits the current Txn if totalRecordsPerTransaction > 0 .
- * If 'rollToNext' is true, will switch to next Txn in batch or to a
- * new TxnBatch if current Txn batch is exhausted
- */
- public void flush(boolean rollToNext)
- throws CommitFailure, TxnBatchFailure, TxnFailure, InterruptedException {
- // if there are no records do not call flush
- if (totalRecords <= 0) return;
- try {
- synchronized (txnBatchLock) {
- commitTxn();
- nextTxn(rollToNext);
- totalRecords = 0;
- lastUsed = System.currentTimeMillis();
- }
- } catch (StreamingException e) {
- throw new TxnFailure(txnBatch, e);
- }
- }
-
- /** Queues up a heartbeat request on the current and remaining txns using the
- * heartbeatThdPool and returns immediately
- */
- public void heartBeat() throws InterruptedException {
- // 1) schedule the heartbeat on one thread in pool
- synchronized (txnBatchLock) {
- try {
- callWithTimeout(new CallRunner() {
- @Override
- public Void call() throws Exception {
- try {
- LOG.info("Sending heartbeat on batch " + txnBatch);
- txnBatch.heartbeat();
- } catch (StreamingException e) {
- LOG.warn("Heartbeat error on batch " + txnBatch, e);
- }
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw e;
- } catch (Exception e) {
- LOG.warn("Unable to send heartbeat on Txn Batch " + txnBatch, e);
- // Suppressing exceptions as we don't care for errors on heartbeats
- }
- }
- }
-
- /**
- * Returns totalRecords written so far in a transaction
- * @returns totalRecords
- */
- public int getTotalRecords() {
- return totalRecords;
- }
-
- /**
- * Flush and Close current transactionBatch.
- */
- public void flushAndClose() throws TxnBatchFailure, TxnFailure, CommitFailure,
- IOException, InterruptedException {
- flush(false);
- close();
- }
-
- /**
- * Close the Transaction Batch and connection
- * @throws IOException if an error occurs during close
- * @throws InterruptedException if the close operation is interrupted
- */
- public void close() throws IOException, InterruptedException {
- closeTxnBatch();
- closeConnection();
- closed = true;
- }
-
- protected void closeConnection() throws InterruptedException {
- LOG.info("Closing connection to end point : {}", endPoint);
- try {
- callWithTimeout(new CallRunner() {
- @Override
- public Void call() throws Exception {
- connection.close(); // could block
- return null;
- }
- });
- } catch (Exception e) {
- LOG.warn("Error closing connection to EndPoint : " + endPoint, e);
- // Suppressing exceptions as we don't care for errors on connection close
- }
- }
-
- protected void commitTxn() throws CommitFailure, InterruptedException {
- LOG.debug("Committing Txn id {} to {}", txnBatch.getCurrentTxnId(), endPoint);
- try {
- callWithTimeout(new CallRunner() {
- @Override
- public Void call() throws Exception {
- txnBatch.commit(); // could block
- return null;
- }
- });
- } catch (StreamingException | TimeoutException e) {
- throw new CommitFailure(endPoint, txnBatch.getCurrentTxnId(), e);
- }
- }
-
- protected StreamingConnection newConnection(HiveEndPoint endPoint, boolean autoCreatePartitions, HiveConf conf, UserGroupInformation ugi) throws InterruptedException, ConnectFailure {
- try {
- return callWithTimeout(() -> {
- return endPoint.newConnection(autoCreatePartitions, conf, ugi); // could block
- });
- } catch (StreamingException | TimeoutException e) {
- throw new ConnectFailure(endPoint, e);
- }
- }
-
- protected TransactionBatch nextTxnBatch(final RecordWriter recordWriter)
- throws InterruptedException, TxnBatchFailure {
- LOG.debug("Fetching new Txn Batch for {}", endPoint);
- TransactionBatch batch = null;
- try {
- batch = callWithTimeout(() -> {
- return connection.fetchTransactionBatch(txnsPerBatch, recordWriter); // could block
- });
- batch.beginNextTransaction();
- LOG.debug("Acquired {}. Switching to first txn", batch);
- } catch (TimeoutException | StreamingException e) {
- throw new TxnBatchFailure(endPoint, e);
- }
- return batch;
- }
-
- protected void closeTxnBatch() throws InterruptedException {
- try {
- LOG.debug("Closing Txn Batch {}", txnBatch);
- callWithTimeout(new CallRunner() {
- @Override
- public Void call() throws Exception {
- if (txnBatch != null) {
- txnBatch.close(); // could block
- }
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw e;
- } catch (Exception e) {
- LOG.warn("Error closing txn batch " + txnBatch, e);
- }
- }
-
- /**
- * Aborts the current Txn and switches to next Txn.
- * @throws StreamingException if could not get new Transaction Batch, or switch to next Txn
- */
- public void abort() throws StreamingException, TxnBatchFailure, InterruptedException {
- synchronized (txnBatchLock) {
- abortTxn();
- nextTxn(true); // roll to next
- }
- }
-
-
- /**
- * Aborts current Txn in the txnBatch.
- */
- protected void abortTxn() throws InterruptedException {
- LOG.info("Aborting Txn id {} on End Point {}", txnBatch.getCurrentTxnId(), endPoint);
- try {
- callWithTimeout(new CallRunner() {
- @Override
- public Void call() throws StreamingException, InterruptedException {
- txnBatch.abort(); // could block
- return null;
- }
- });
- } catch (InterruptedException e) {
- throw e;
- } catch (TimeoutException e) {
- LOG.warn("Timeout while aborting Txn " + txnBatch.getCurrentTxnId() + " on EndPoint: " + endPoint, e);
- } catch (Exception e) {
- LOG.warn("Error aborting Txn " + txnBatch.getCurrentTxnId() + " on EndPoint: " + endPoint, e);
- // Suppressing exceptions as we don't care for errors on abort
- }
- }
-
-
- /**
- * if there are remainingTransactions in current txnBatch, begins nextTransactions
- * otherwise creates new txnBatch.
- * @param rollToNext Whether to roll to the next transaction batch
- */
- protected void nextTxn(boolean rollToNext) throws StreamingException, InterruptedException, TxnBatchFailure {
- if (txnBatch.remainingTransactions() == 0) {
- closeTxnBatch();
- txnBatch = null;
- if (rollToNext) {
- txnBatch = nextTxnBatch(recordWriter);
- }
- } else if (rollToNext) {
- LOG.debug("Switching to next Txn for {}", endPoint);
- txnBatch.beginNextTransaction(); // does not block
- }
- }
-
- /**
- * If the current thread has been interrupted, then throws an
- * exception.
- * @throws InterruptedException uf the current thread has been interrupted
- */
- protected static void checkAndThrowInterruptedException()
- throws InterruptedException {
- if (Thread.currentThread().interrupted()) {
- throw new InterruptedException("Timed out before Hive call was made. "
- + "Your callTimeout might be set too low or Hive calls are "
- + "taking too long.");
- }
- }
-
- /**
- * Execute the callable on a separate thread and wait for the completion
- * for the specified amount of time in milliseconds. In case of timeout
- * cancel the callable and throw an IOException
- */
- private T callWithTimeout(final CallRunner callRunner)
- throws TimeoutException, StreamingException, InterruptedException {
- Future future = callTimeoutPool.submit(() -> {
- if (ugi == null) {
- return callRunner.call();
- }
- try {
- return ugi.doAs((PrivilegedExceptionAction) () -> callRunner.call());
- } catch (UndeclaredThrowableException e) {
- Throwable cause = e.getCause();
- // Unwrap exception so it is thrown the same way as without ugi
- if (!(cause instanceof Exception)) {
- throw e;
- }
- throw (Exception)cause;
- }
- });
- try {
- if (callTimeout > 0) {
- return future.get(callTimeout, TimeUnit.MILLISECONDS);
- } else {
- return future.get();
- }
- } catch (TimeoutException eT) {
- future.cancel(true);
- throw eT;
- } catch (ExecutionException e1) {
- Throwable cause = e1.getCause();
- if (cause instanceof IOException) {
- throw new StreamingIOFailure("I/O Failure", (IOException) cause);
- } else if (cause instanceof StreamingException) {
- throw (StreamingException) cause;
- } else if (cause instanceof InterruptedException) {
- throw (InterruptedException) cause;
- } else if (cause instanceof RuntimeException) {
- throw (RuntimeException) cause;
- } else if (cause instanceof TimeoutException) {
- throw new StreamingException("Operation Timed Out.", (TimeoutException) cause);
- } else {
- throw new RuntimeException(e1);
- }
- }
- }
-
- public long getLastUsed() {
- return lastUsed;
- }
-
- private byte[] generateRecord(List tuple) {
- StringBuilder buf = new StringBuilder();
- for (String o : tuple) {
- buf.append(o);
- buf.append(",");
- }
- return buf.toString().getBytes();
- }
-
- /**
- * Simple interface whose call method is called by
- * {#callWithTimeout} in a new thread inside a
- * {@linkplain java.security.PrivilegedExceptionAction#run()} call.
- * @param the type of object returned from the call
- */
- private interface CallRunner {
- T call() throws Exception;
- }
-
- public static class Failure extends Exception {
- public Failure(String message, Throwable cause) {
- super(message, cause);
- }
- }
-
- public static class WriteFailure extends Failure {
- public WriteFailure(HiveEndPoint endPoint, Long currentTxnId, Throwable cause) {
- super("Failed writing to : " + endPoint + ". TxnID : " + currentTxnId, cause);
- }
- }
-
- public static class CommitFailure extends Failure {
- public CommitFailure(HiveEndPoint endPoint, Long txnID, Throwable cause) {
- super("Commit of Txn " + txnID + " failed on EndPoint: " + endPoint, cause);
- }
- }
-
- public static class ConnectFailure extends Failure {
- public ConnectFailure(HiveEndPoint ep, Throwable cause) {
- super("Failed connecting to EndPoint " + ep, cause);
- }
- }
-
- public static class TxnBatchFailure extends Failure {
- public TxnBatchFailure(HiveEndPoint ep, Throwable cause) {
- super("Failed acquiring Transaction Batch from EndPoint: " + ep, cause);
- }
- }
-
- public static class TxnFailure extends Failure {
- public TxnFailure(TransactionBatch txnBatch, Throwable cause) {
- super("Failed switching to next Txn in TxnBatch " + txnBatch, cause);
- }
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/ValidationResources.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/ValidationResources.java
deleted file mode 100644
index 1014efb3c2..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/java/org/apache/nifi/util/hive/ValidationResources.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.util.hive;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * A helper class for maintaining loaded configurations (to avoid reloading on use unless necessary)
- */
-public class ValidationResources {
-
- private final String configResources;
- private final Configuration configuration;
-
- public ValidationResources(String configResources, Configuration configuration) {
- this.configResources = configResources;
- this.configuration = configuration;
- }
-
- public String getConfigResources() {
- return configResources;
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
deleted file mode 100644
index 6a2936e326..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService
+++ /dev/null
@@ -1,15 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-org.apache.nifi.dbcp.hive.HiveConnectionPool
\ No newline at end of file
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
deleted file mode 100644
index d86ee38fe7..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor
+++ /dev/null
@@ -1,19 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-org.apache.nifi.processors.hive.ConvertAvroToORC
-org.apache.nifi.processors.hive.SelectHiveQL
-org.apache.nifi.processors.hive.PutHiveQL
-org.apache.nifi.processors.hive.PutHiveStreaming
-org.apache.nifi.processors.hive.UpdateHiveTable
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java
deleted file mode 100644
index 77c72a5e2b..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/dbcp/hive/HiveConnectionPoolTest.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.nifi.dbcp.hive;
-
-import org.apache.commons.dbcp2.BasicDataSource;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.nifi.components.PropertyDescriptor;
-import org.apache.nifi.controller.AbstractControllerService;
-import org.apache.nifi.expression.ExpressionLanguageScope;
-import org.apache.nifi.hadoop.KerberosProperties;
-import org.apache.nifi.logging.ComponentLog;
-import org.apache.nifi.processor.exception.ProcessException;
-import org.apache.nifi.processor.util.StandardValidators;
-import org.apache.nifi.registry.VariableDescriptor;
-import org.apache.nifi.reporting.InitializationException;
-import org.apache.nifi.util.MockConfigurationContext;
-import org.apache.nifi.util.MockVariableRegistry;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.UndeclaredThrowableException;
-import java.security.PrivilegedExceptionAction;
-import java.sql.SQLException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertThrows;
-import static org.mockito.ArgumentMatchers.isA;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class HiveConnectionPoolTest {
- private UserGroupInformation userGroupInformation;
- private HiveConnectionPool hiveConnectionPool;
- private BasicDataSource basicDataSource;
- private ComponentLog componentLog;
- private KerberosProperties kerberosProperties;
- private File krb5conf = new File("src/test/resources/krb5.conf");
-
- @BeforeEach
- public void setup() throws Exception {
- // have to initialize this system property before anything else
- System.setProperty("java.security.krb5.conf", krb5conf.getAbsolutePath());
- System.setProperty("java.security.krb5.realm", "nifi.com");
- System.setProperty("java.security.krb5.kdc", "nifi.kdc");
-
- userGroupInformation = mock(UserGroupInformation.class);
- basicDataSource = mock(BasicDataSource.class);
- componentLog = mock(ComponentLog.class);
- kerberosProperties = mock(KerberosProperties.class);
-
- when(userGroupInformation.doAs(isA(PrivilegedExceptionAction.class))).thenAnswer(invocation -> {
- try {
- return ((PrivilegedExceptionAction) invocation.getArguments()[0]).run();
- } catch (IOException | Error | RuntimeException | InterruptedException e) {
- throw e;
- } catch (Throwable e) {
- throw new UndeclaredThrowableException(e);
- }
- });
-
- when(kerberosProperties.getKerberosKeytab()).thenReturn(new PropertyDescriptor.Builder()
- .name("Kerberos Principal")
- .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build());
-
- when(kerberosProperties.getKerberosPrincipal()).thenReturn(new PropertyDescriptor.Builder()
- .name("Kerberos Keytab")
- .addValidator(StandardValidators.ATTRIBUTE_EXPRESSION_LANGUAGE_VALIDATOR)
- .expressionLanguageSupported(ExpressionLanguageScope.VARIABLE_REGISTRY)
- .build());
-
- initPool();
- }
-
- private void initPool() throws Exception {
- hiveConnectionPool = new HiveConnectionPool();
-
- Field ugiField = HiveConnectionPool.class.getDeclaredField("ugi");
- ugiField.setAccessible(true);
- ugiField.set(hiveConnectionPool, userGroupInformation);
-
- Field dataSourceField = HiveConnectionPool.class.getDeclaredField("dataSource");
- dataSourceField.setAccessible(true);
- dataSourceField.set(hiveConnectionPool, basicDataSource);
-
- Field componentLogField = AbstractControllerService.class.getDeclaredField("logger");
- componentLogField.setAccessible(true);
- componentLogField.set(hiveConnectionPool, componentLog);
-
- Field kerberosPropertiesField = HiveConnectionPool.class.getDeclaredField("kerberosProperties");
- kerberosPropertiesField.setAccessible(true);
- kerberosPropertiesField.set(hiveConnectionPool, kerberosProperties);
- }
-
- @Test
- public void testGetConnectionSqlException() throws SQLException {
- SQLException sqlException = new SQLException("bad sql");
- when(basicDataSource.getConnection()).thenThrow(sqlException);
-
- ProcessException e = assertThrows(ProcessException.class, () -> hiveConnectionPool.getConnection());
- assertEquals(sqlException, e.getCause());
- }
-
- @Test
- public void testExpressionLanguageSupport() throws Exception {
- final String URL = "jdbc:hive2://localhost:10000/default";
- final String USER = "user";
- final String PASS = "pass";
- final int MAX_CONN = 7;
- final String MAX_CONN_LIFETIME = "1 sec";
- final String MAX_WAIT = "10 sec"; // 10000 milliseconds
- final String CONF = "/path/to/hive-site.xml";
- hiveConnectionPool = new HiveConnectionPool();
-
- Map props = new HashMap() {{
- put(HiveConnectionPool.DATABASE_URL, "${url}");
- put(HiveConnectionPool.DB_USER, "${username}");
- put(HiveConnectionPool.DB_PASSWORD, "${password}");
- put(HiveConnectionPool.MAX_TOTAL_CONNECTIONS, "${maxconn}");
- put(HiveConnectionPool.MAX_CONN_LIFETIME, "${maxconnlifetime}");
- put(HiveConnectionPool.MAX_WAIT_TIME, "${maxwait}");
- put(HiveConnectionPool.HIVE_CONFIGURATION_RESOURCES, "${hiveconf}");
- }};
-
- MockVariableRegistry registry = new MockVariableRegistry();
- registry.setVariable(new VariableDescriptor("url"), URL);
- registry.setVariable(new VariableDescriptor("username"), USER);
- registry.setVariable(new VariableDescriptor("password"), PASS);
- registry.setVariable(new VariableDescriptor("maxconn"), Integer.toString(MAX_CONN));
- registry.setVariable(new VariableDescriptor("maxconnlifetime"), MAX_CONN_LIFETIME);
- registry.setVariable(new VariableDescriptor("maxwait"), MAX_WAIT);
- registry.setVariable(new VariableDescriptor("hiveconf"), CONF);
-
-
- MockConfigurationContext context = new MockConfigurationContext(props, null, registry);
- hiveConnectionPool.onConfigured(context);
-
- Field dataSourceField = HiveConnectionPool.class.getDeclaredField("dataSource");
- dataSourceField.setAccessible(true);
- basicDataSource = (BasicDataSource) dataSourceField.get(hiveConnectionPool);
- assertEquals(URL, basicDataSource.getUrl());
- assertEquals(USER, basicDataSource.getUsername());
- assertEquals(PASS, basicDataSource.getPassword());
- assertEquals(MAX_CONN, basicDataSource.getMaxTotal());
- assertEquals(1000L, basicDataSource.getMaxConnLifetimeMillis());
- assertEquals(10000L, basicDataSource.getMaxWaitMillis());
- assertEquals(URL, hiveConnectionPool.getConnectionURL());
- }
-
- @EnabledIfSystemProperty(
- named = "nifi.test.unstable",
- matches = "true",
- disabledReason = "Kerberos does not seem to be properly handled in Travis build, but, locally, this test should successfully run")
- @Test
- public void testKerberosAuthException() {
- final String URL = "jdbc:hive2://localhost:10000/default";
- final String conf = "src/test/resources/hive-site-security.xml";
- final String ktab = "src/test/resources/fake.keytab";
- final String kprinc = "bad@PRINCIPAL.COM";
-
- KerberosProperties kerbProperties = new KerberosProperties(krb5conf);
-
- Map props = new HashMap() {{
- put(HiveConnectionPool.DATABASE_URL, "${url}");
- put(HiveConnectionPool.HIVE_CONFIGURATION_RESOURCES, "${conf}");
- put(kerbProperties.getKerberosKeytab(), "${ktab}");
- put(kerbProperties.getKerberosPrincipal(), "${kprinc}");
- }};
-
- MockVariableRegistry registry = new MockVariableRegistry();
- registry.setVariable(new VariableDescriptor("url"), URL);
- registry.setVariable(new VariableDescriptor("conf"), conf);
- registry.setVariable(new VariableDescriptor("ktab"), ktab);
- registry.setVariable(new VariableDescriptor("kprinc"), kprinc);
-
- MockConfigurationContext context = new MockConfigurationContext(props, null, registry);
- assertThrows(InitializationException.class, () -> hiveConnectionPool.onConfigured(context));
- }
-}
diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java
deleted file mode 100644
index da5f18bd2d..0000000000
--- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive-processors/src/test/java/org/apache/nifi/processors/hive/TestConvertAvroToORC.java
+++ /dev/null
@@ -1,568 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.processors.hive;
-
-import org.apache.avro.Schema;
-import org.apache.avro.file.DataFileStream;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.io.DatumWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.io.orc.NiFiOrcUtils;
-import org.apache.hadoop.hive.ql.io.orc.OrcFile;
-import org.apache.hadoop.hive.ql.io.orc.OrcStruct;
-import org.apache.hadoop.hive.ql.io.orc.Reader;
-import org.apache.hadoop.hive.ql.io.orc.RecordReader;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.io.DoubleWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.nifi.flowfile.attributes.CoreAttributes;
-import org.apache.nifi.util.MockFlowFile;
-import org.apache.nifi.util.TestRunner;
-import org.apache.nifi.util.TestRunners;
-import org.apache.nifi.util.orc.TestNiFiOrcUtils;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.junit.jupiter.api.Assertions.assertArrayEquals;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
-
-
-/**
- * Unit tests for ConvertAvroToORC processor
- */
-public class TestConvertAvroToORC {
-
- private ConvertAvroToORC processor;
- private TestRunner runner;
-
- @BeforeEach
- public void setUp() throws Exception {
- processor = new ConvertAvroToORC();
- runner = TestRunners.newTestRunner(processor);
- }
-
- @Test
- public void test_onTrigger_routing_to_failure_null_type() throws Exception {
- String testString = "Hello World";
- GenericData.Record record = TestNiFiOrcUtils.buildAvroRecordWithNull(testString);
-
- DatumWriter writer = new GenericDatumWriter<>(record.getSchema());
- DataFileWriter fileWriter = new DataFileWriter<>(writer);
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- fileWriter.create(record.getSchema(), out);
- fileWriter.append(record);
- fileWriter.flush();
- fileWriter.close();
- out.close();
-
- Map attributes = new HashMap() {{
- put(CoreAttributes.FILENAME.key(), "test.avro");
- }};
- runner.enqueue(out.toByteArray(), attributes);
- runner.run();
-
- runner.assertAllFlowFilesTransferred(ConvertAvroToORC.REL_SUCCESS, 1);
- MockFlowFile resultFlowFile = runner.getFlowFilesForRelationship(ConvertAvroToORC.REL_SUCCESS).get(0);
- assertEquals("test.orc", resultFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
- assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS test_record (string STRING, null BOOLEAN) STORED AS ORC",
- resultFlowFile.getAttribute(ConvertAvroToORC.HIVE_DDL_ATTRIBUTE));
- }
-
- @Test
- public void test_onTrigger_routing_to_failure_empty_array_type() throws Exception {
- String testString = "Hello World";
- GenericData.Record record = TestNiFiOrcUtils.buildAvroRecordWithEmptyArray(testString);
-
- DatumWriter writer = new GenericDatumWriter<>(record.getSchema());
- DataFileWriter fileWriter = new DataFileWriter<>(writer);
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- fileWriter.create(record.getSchema(), out);
- fileWriter.append(record);
- fileWriter.flush();
- fileWriter.close();
- out.close();
-
- Map attributes = new HashMap() {{
- put(CoreAttributes.FILENAME.key(), "test.avro");
- }};
- runner.enqueue(out.toByteArray(), attributes);
- runner.run();
-
- runner.assertAllFlowFilesTransferred(ConvertAvroToORC.REL_SUCCESS, 1);
- MockFlowFile resultFlowFile = runner.getFlowFilesForRelationship(ConvertAvroToORC.REL_SUCCESS).get(0);
- assertEquals("test.orc", resultFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
- assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS test_record (string STRING, emptyArray ARRAY) STORED AS ORC",
- resultFlowFile.getAttribute(ConvertAvroToORC.HIVE_DDL_ATTRIBUTE));
- }
-
- @Test
- public void test_onTrigger_routing_to_failure_fixed_type() throws Exception {
- String testString = "Hello!";
- GenericData.Record record = TestNiFiOrcUtils.buildAvroRecordWithFixed(testString);
-
- DatumWriter writer = new GenericDatumWriter<>(record.getSchema());
- DataFileWriter fileWriter = new DataFileWriter<>(writer);
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- fileWriter.create(record.getSchema(), out);
- fileWriter.append(record);
- fileWriter.flush();
- fileWriter.close();
- out.close();
-
- Map attributes = new HashMap() {{
- put(CoreAttributes.FILENAME.key(), "test.avro");
- }};
- runner.enqueue(out.toByteArray(), attributes);
- runner.run();
-
- runner.assertAllFlowFilesTransferred(ConvertAvroToORC.REL_FAILURE, 1);
- MockFlowFile resultFlowFile = runner.getFlowFilesForRelationship(ConvertAvroToORC.REL_FAILURE).get(0);
- assertEquals("test.avro", resultFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
-
- final InputStream in = new ByteArrayInputStream(resultFlowFile.toByteArray());
- final DatumReader datumReader = new GenericDatumReader<>();
- try (DataFileStream dataFileReader = new DataFileStream<>(in, datumReader)) {
- assertTrue(dataFileReader.hasNext());
- GenericRecord testedRecord = dataFileReader.next();
-
- assertNotNull(testedRecord.get("fixed"));
- assertArrayEquals(testString.getBytes(StandardCharsets.UTF_8), ((GenericData.Fixed) testedRecord.get("fixed")).bytes());
- }
- }
-
- @Test
- public void test_onTrigger_primitive_record() throws Exception {
- GenericData.Record record = TestNiFiOrcUtils.buildPrimitiveAvroRecord(10, 20L, true, 30.0f, 40, StandardCharsets.UTF_8.encode("Hello"), "World");
-
- DatumWriter writer = new GenericDatumWriter<>(record.getSchema());
- DataFileWriter fileWriter = new DataFileWriter<>(writer);
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- fileWriter.create(record.getSchema(), out);
- fileWriter.append(record);
- // Put another record in
- record = TestNiFiOrcUtils.buildPrimitiveAvroRecord(1, 2L, false, 3.0f, 4L, StandardCharsets.UTF_8.encode("I am"), "another record");
- fileWriter.append(record);
- // And one more
- record = TestNiFiOrcUtils.buildPrimitiveAvroRecord(100, 200L, true, 300.0f, 400L, StandardCharsets.UTF_8.encode("Me"), "too!");
- fileWriter.append(record);
- fileWriter.flush();
- fileWriter.close();
- out.close();
- Map attributes = new HashMap() {{
- put(CoreAttributes.FILENAME.key(), "test.avro");
- }};
- runner.enqueue(out.toByteArray(), attributes);
- runner.run();
-
- runner.assertAllFlowFilesTransferred(ConvertAvroToORC.REL_SUCCESS, 1);
-
- // Write the flow file out to disk, since the ORC Reader needs a path
- MockFlowFile resultFlowFile = runner.getFlowFilesForRelationship(ConvertAvroToORC.REL_SUCCESS).get(0);
- assertEquals("CREATE EXTERNAL TABLE IF NOT EXISTS test_record (int INT, long BIGINT, boolean BOOLEAN, float FLOAT, double DOUBLE, bytes BINARY, string STRING)"
- + " STORED AS ORC", resultFlowFile.getAttribute(ConvertAvroToORC.HIVE_DDL_ATTRIBUTE));
- assertEquals("3", resultFlowFile.getAttribute(ConvertAvroToORC.RECORD_COUNT_ATTRIBUTE));
- assertEquals("test.orc", resultFlowFile.getAttribute(CoreAttributes.FILENAME.key()));
- byte[] resultContents = runner.getContentAsByteArray(resultFlowFile);
- FileOutputStream fos = new FileOutputStream("target/test1.orc");
- fos.write(resultContents);
- fos.flush();
- fos.close();
-
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.getLocal(conf);
- Reader reader = OrcFile.createReader(new Path("target/test1.orc"), OrcFile.readerOptions(conf).filesystem(fs));
- RecordReader rows = reader.rows();
- Object o = rows.next(null);
- assertNotNull(o);
- assertTrue(o instanceof OrcStruct);
- TypeInfo resultSchema = TestNiFiOrcUtils.buildPrimitiveOrcSchema();
- StructObjectInspector inspector = (StructObjectInspector) OrcStruct.createObjectInspector(resultSchema);
-
- // Check some fields in the first row
- Object intFieldObject = inspector.getStructFieldData(o, inspector.getStructFieldRef("int"));
- assertTrue(intFieldObject instanceof IntWritable);
- assertEquals(10, ((IntWritable) intFieldObject).get());
- Object stringFieldObject = inspector.getStructFieldData(o, inspector.getStructFieldRef("string"));
- assertTrue(stringFieldObject instanceof Text);
- assertEquals("World", stringFieldObject.toString());
-
- }
-
- @Test
- public void test_onTrigger_complex_record() throws Exception {
-
- Map mapData1 = new TreeMap() {{
- put("key1", 1.0);
- put("key2", 2.0);
- }};
-
- BigDecimal sampleBigDecimal = new BigDecimal("12.34");
- ByteBuffer bigDecimalAsBytes = ByteBuffer.wrap(sampleBigDecimal.unscaledValue().toByteArray());
- GenericData.Record record = TestNiFiOrcUtils.buildComplexAvroRecord(10, mapData1, "DEF", 3.0f, Arrays.asList(10, 20), bigDecimalAsBytes);
-
- DatumWriter writer = new GenericDatumWriter<>(record.getSchema());
- DataFileWriter fileWriter = new DataFileWriter<>(writer);
- ByteArrayOutputStream out = new ByteArrayOutputStream();
- fileWriter.create(record.getSchema(), out);
- fileWriter.append(record);
-
- // Put another record in
- Map