org.apache.nifi.processors.hive

Class PutHiveStreaming.FunctionContext

java.lang.Object

org.apache.nifi.processor.util.pattern.RollbackOnFailure

org.apache.nifi.processors.hive.PutHiveStreaming.FunctionContext

Enclosing class:

PutHiveStreaming

private static class PutHiveStreaming.FunctionContext
extends RollbackOnFailure

Field Summary

Fields

Modifier and Type	Field and Description
private AtomicInteger	failedRecordCount
private byte[]	failureAvroHeader
private org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord>	failureAvroWriter
private AtomicReference<FlowFile>	failureFlowFile
private ComponentLog	logger
private AtomicInteger	recordCount
private byte[]	successAvroHeader
private org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord>	successAvroWriter
private AtomicReference<FlowFile>	successFlowFile
private AtomicInteger	successfulRecordCount

Fields inherited from class org.apache.nifi.processor.util.pattern.RollbackOnFailure

ROLLBACK_ON_FAILURE

Constructor Summary

Constructors

Modifier	Constructor and Description
private	FunctionContext(boolean rollbackOnFailure, ComponentLog logger) It's possible that multiple Hive streaming transactions are committed within a single onTrigger.

Method Summary

Modifier and Type	Method and Description
private void	appendAvroRecords(ProcessSession session, byte[] avroHeader, org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> writer, AtomicReference<FlowFile> flowFileRef, List<PutHiveStreaming.HiveStreamingRecord> hRecords)
private void	appendRecordsToFailure(ProcessSession session, List<PutHiveStreaming.HiveStreamingRecord> records)
private void	appendRecordsToSuccess(ProcessSession session, List<PutHiveStreaming.HiveStreamingRecord> records)
private byte[]	initAvroWriter(ProcessSession session, String codec, org.apache.avro.file.DataFileStream<org.apache.avro.generic.GenericRecord> reader, org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> writer, AtomicReference<FlowFile> flowFileRef)
private void	initAvroWriters(ProcessSession session, String codec, org.apache.avro.file.DataFileStream<org.apache.avro.generic.GenericRecord> reader)
private void	setFlowFiles(FlowFile successFlowFile, FlowFile failureFlowFile)
private void	transferFlowFiles(ProcessSession session, RoutingResult result, HiveOptions options)

Methods inherited from class org.apache.nifi.processor.util.pattern.RollbackOnFailure

canRollback, createAdjustError, createAdjustRoute, createOnError, createRollbackOnFailureProperty, discontinue, getProcessedCount, isRollbackOnFailure, isTransactional, onTrigger, proceed, shouldDiscontinue

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

successFlowFile

private AtomicReference<FlowFile> successFlowFile

failureFlowFile

private AtomicReference<FlowFile> failureFlowFile

successAvroWriter

private final org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> successAvroWriter

failureAvroWriter

private final org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> failureAvroWriter

successAvroHeader

private byte[] successAvroHeader

failureAvroHeader

private byte[] failureAvroHeader

recordCount

private final AtomicInteger recordCount

successfulRecordCount

private final AtomicInteger successfulRecordCount

failedRecordCount

private final AtomicInteger failedRecordCount

logger

private final ComponentLog logger

Constructor Detail

FunctionContext

private FunctionContext(boolean rollbackOnFailure,
                        ComponentLog logger)

It's possible that multiple Hive streaming transactions are committed within a single onTrigger. PutHiveStreaming onTrigger is not 'transactional' in a sense of RollbackOnFailure. Once a Hive streaming transaction is committed, processor session will not be rolled back.

Parameters:

rollbackOnFailure - whether process session should be rolled back if failed

Method Detail

setFlowFiles

private void setFlowFiles(FlowFile successFlowFile,
                          FlowFile failureFlowFile)

initAvroWriter

private byte[] initAvroWriter(ProcessSession session,
                              String codec,
                              org.apache.avro.file.DataFileStream<org.apache.avro.generic.GenericRecord> reader,
                              org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> writer,
                              AtomicReference<FlowFile> flowFileRef)

initAvroWriters

private void initAvroWriters(ProcessSession session,
                             String codec,
                             org.apache.avro.file.DataFileStream<org.apache.avro.generic.GenericRecord> reader)

appendAvroRecords

private void appendAvroRecords(ProcessSession session,
                               byte[] avroHeader,
                               org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> writer,
                               AtomicReference<FlowFile> flowFileRef,
                               List<PutHiveStreaming.HiveStreamingRecord> hRecords)

appendRecordsToSuccess

private void appendRecordsToSuccess(ProcessSession session,
                                    List<PutHiveStreaming.HiveStreamingRecord> records)

appendRecordsToFailure

private void appendRecordsToFailure(ProcessSession session,
                                    List<PutHiveStreaming.HiveStreamingRecord> records)

transferFlowFiles

private void transferFlowFiles(ProcessSession session,
                               RoutingResult result,
                               HiveOptions options)