org.apache.nifi.processors.hive

Class PutHiveStreaming

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processors.hive.PutHiveStreaming

All Implemented Interfaces:

ConfigurableComponent, Processor

@TriggerSerially
 @Tags(value={"hive","streaming","put","database","store"})
 @CapabilityDescription(value="This processor uses Hive Streaming to send flow file data to an Apache Hive table. The incoming flow file is expected to be in Avro format and the table must exist in Hive. Please see the Hive documentation for requirements on the Hive table (format, partitions, etc.). The partition values are extracted from the Avro record based on the names of the partition columns as specified in the processor.")
 @WritesAttributes(value=@WritesAttribute(attribute="hivestreaming.record.count",description="This attribute is written on the flow files routed to the \'success\' and \'failure\' relationships, and contains the number of records from the incoming flow file written successfully and unsuccessfully, respectively."))
public class PutHiveStreaming
extends AbstractProcessor

This processor utilizes the Hive Streaming capability to insert data from the flow into a Hive database table.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
protected class	PutHiveStreaming.HiveStreamingRecord

Field Summary

Fields

Modifier and Type	Field and Description
protected Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter>	allWriters
static PropertyDescriptor	AUTOCREATE_PARTITIONS
protected ExecutorService	callTimeoutPool
static PropertyDescriptor	DB_NAME
private static Validator	GREATER_THAN_ONE_VALIDATOR
static PropertyDescriptor	HEARTBEAT_INTERVAL
protected Timer	heartBeatTimer
static PropertyDescriptor	HIVE_CONFIGURATION_RESOURCES
static String	HIVE_STREAMING_RECORD_COUNT_ATTR
protected HiveConfigurator	hiveConfigurator
protected AtomicBoolean	isInitialized
private File	kerberosConfigFile
protected KerberosProperties	kerberosProperties
static PropertyDescriptor	MAX_OPEN_CONNECTIONS
static PropertyDescriptor	METASTORE_URI
protected HiveOptions	options
static PropertyDescriptor	PARTITION_COLUMNS
private List<PropertyDescriptor>	propertyDescriptors
static Relationship	REL_FAILURE
static Relationship	REL_RETRY
static Relationship	REL_SUCCESS
private Set<Relationship>	relationships
private static Set<String>	RESERVED_METADATA
protected AtomicBoolean	sendHeartBeat
static PropertyDescriptor	TABLE_NAME
private static long	TICKET_RENEWAL_PERIOD
static PropertyDescriptor	TXNS_PER_BATCH
protected org.apache.hadoop.security.UserGroupInformation	ugi

Constructor Summary

Constructors

Constructor and Description
PutHiveStreaming()

Method Summary

Modifier and Type	Method and Description
private void	abortAllWriters() Abort current Txn on all writers
private void	abortAndCloseWriters()
private void	appendRecordsToFlowFile(ProcessSession session, List<PutHiveStreaming.HiveStreamingRecord> records, AtomicReference<FlowFile> appendFlowFile, org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> avroWriter, org.apache.avro.file.DataFileStream<org.apache.avro.generic.GenericRecord> reader)
void	cleanup()
private void	closeAllWriters() Closes all writers and remove them from cache
private void	flushAllWriters(boolean rollToNext)
protected KerberosProperties	getKerberosProperties()
private HiveWriter	getOrCreateWriter(org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint)
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
protected void	init(ProcessorInitializationContext context)
protected org.apache.hive.hcatalog.streaming.HiveEndPoint	makeHiveEndPoint(List<String> partitionValues, HiveOptions options)
protected HiveWriter	makeHiveWriter(org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint, ExecutorService callTimeoutPool, org.apache.hadoop.security.UserGroupInformation ugi, HiveOptions options)
void	onTrigger(ProcessContext context, ProcessSession session)
private void	retireEldestWriter() Locate writer that has not been used for longest time and retire it
private int	retireIdleWriters() Locate all writers past idle timeout and retire them
private void	sendHeartBeatOnAllWriters()
void	setup(ProcessContext context)
private void	setupHeartBeatTimer()

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

customValidate, equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

HIVE_STREAMING_RECORD_COUNT_ATTR

public static final String HIVE_STREAMING_RECORD_COUNT_ATTR

See Also:

Constant Field Values

GREATER_THAN_ONE_VALIDATOR

private static final Validator GREATER_THAN_ONE_VALIDATOR

RESERVED_METADATA

private static final Set<String> RESERVED_METADATA

METASTORE_URI

public static final PropertyDescriptor METASTORE_URI

HIVE_CONFIGURATION_RESOURCES

public static final PropertyDescriptor HIVE_CONFIGURATION_RESOURCES

DB_NAME

public static final PropertyDescriptor DB_NAME

TABLE_NAME

public static final PropertyDescriptor TABLE_NAME

PARTITION_COLUMNS

public static final PropertyDescriptor PARTITION_COLUMNS

AUTOCREATE_PARTITIONS

public static final PropertyDescriptor AUTOCREATE_PARTITIONS

MAX_OPEN_CONNECTIONS

public static final PropertyDescriptor MAX_OPEN_CONNECTIONS

HEARTBEAT_INTERVAL

public static final PropertyDescriptor HEARTBEAT_INTERVAL

TXNS_PER_BATCH

public static final PropertyDescriptor TXNS_PER_BATCH

REL_SUCCESS

public static final Relationship REL_SUCCESS

REL_FAILURE

public static final Relationship REL_FAILURE

REL_RETRY

public static final Relationship REL_RETRY

propertyDescriptors

private List<PropertyDescriptor> propertyDescriptors

relationships

private Set<Relationship> relationships

TICKET_RENEWAL_PERIOD

private static final long TICKET_RENEWAL_PERIOD

See Also:

Constant Field Values

kerberosProperties

protected KerberosProperties kerberosProperties

kerberosConfigFile

private volatile File kerberosConfigFile

hiveConfigurator

protected volatile HiveConfigurator hiveConfigurator

ugi

protected volatile org.apache.hadoop.security.UserGroupInformation ugi

isInitialized

protected final AtomicBoolean isInitialized

options

protected HiveOptions options

callTimeoutPool

protected ExecutorService callTimeoutPool

heartBeatTimer

protected transient Timer heartBeatTimer

sendHeartBeat

protected AtomicBoolean sendHeartBeat

allWriters

protected Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> allWriters

Constructor Detail

PutHiveStreaming

public PutHiveStreaming()

Method Detail

init

protected void init(ProcessorInitializationContext context)

Overrides:

init in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

setup

@OnScheduled
public void setup(ProcessContext context)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSession session)
               throws ProcessException

Specified by:

onTrigger in class AbstractProcessor

Throws:

ProcessException

appendRecordsToFlowFile

private void appendRecordsToFlowFile(ProcessSession session,
                                     List<PutHiveStreaming.HiveStreamingRecord> records,
                                     AtomicReference<FlowFile> appendFlowFile,
                                     org.apache.avro.file.DataFileWriter<org.apache.avro.generic.GenericRecord> avroWriter,
                                     org.apache.avro.file.DataFileStream<org.apache.avro.generic.GenericRecord> reader)
                              throws IOException

Throws:

IOException

cleanup

@OnStopped
public void cleanup()

setupHeartBeatTimer

private void setupHeartBeatTimer()

sendHeartBeatOnAllWriters

private void sendHeartBeatOnAllWriters()
                                throws InterruptedException

Throws:

InterruptedException

flushAllWriters

private void flushAllWriters(boolean rollToNext)
                      throws HiveWriter.CommitFailure,
                             HiveWriter.TxnBatchFailure,
                             HiveWriter.TxnFailure,
                             InterruptedException

Throws:

HiveWriter.CommitFailure

HiveWriter.TxnBatchFailure

HiveWriter.TxnFailure

InterruptedException

abortAndCloseWriters

private void abortAndCloseWriters()

abortAllWriters

private void abortAllWriters()
                      throws InterruptedException,
                             org.apache.hive.hcatalog.streaming.StreamingException,
                             HiveWriter.TxnBatchFailure

Abort current Txn on all writers

Throws:

InterruptedException

org.apache.hive.hcatalog.streaming.StreamingException

HiveWriter.TxnBatchFailure

closeAllWriters

private void closeAllWriters()

Closes all writers and remove them from cache

getOrCreateWriter

private HiveWriter getOrCreateWriter(org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint)
                              throws HiveWriter.ConnectFailure,
                                     InterruptedException

Throws:

HiveWriter.ConnectFailure

InterruptedException

retireEldestWriter

private void retireEldestWriter()

Locate writer that has not been used for longest time and retire it

retireIdleWriters

private int retireIdleWriters()

Locate all writers past idle timeout and retire them

Returns:

number of writers retired

makeHiveEndPoint

protected org.apache.hive.hcatalog.streaming.HiveEndPoint makeHiveEndPoint(List<String> partitionValues,
                                                                           HiveOptions options)
                                                                    throws org.apache.hive.hcatalog.streaming.ConnectionError

Throws:

org.apache.hive.hcatalog.streaming.ConnectionError

makeHiveWriter

protected HiveWriter makeHiveWriter(org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint,
                                    ExecutorService callTimeoutPool,
                                    org.apache.hadoop.security.UserGroupInformation ugi,
                                    HiveOptions options)
                             throws HiveWriter.ConnectFailure,
                                    InterruptedException

Throws:

HiveWriter.ConnectFailure

InterruptedException

getKerberosProperties

protected KerberosProperties getKerberosProperties()