org.apache.nifi.processors.hive

Class PutHiveStreaming

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processors.hive.PutHiveStreaming

All Implemented Interfaces:

ConfigurableComponent, Processor

@Tags(value={"hive","streaming","put","database","store"})
 @CapabilityDescription(value="This processor uses Hive Streaming to send flow file data to an Apache Hive table. The incoming flow file is expected to be in Avro format and the table must exist in Hive. Please see the Hive documentation for requirements on the Hive table (format, partitions, etc.). The partition values are extracted from the Avro record based on the names of the partition columns as specified in the processor. NOTE: If multiple concurrent tasks are configured for this processor, only one table can be written to at any time by a single thread. Additional tasks intending to write to the same table will wait for the current task to finish writing to the table.")
 @WritesAttribute(attribute="hivestreaming.record.count",description="This attribute is written on the flow files routed to the \'success\' and \'failure\' relationships, and contains the number of records from the incoming flow file written successfully and unsuccessfully, respectively.") @WritesAttribute(attribute="query.output.tables",description="This attribute is written on the flow files routed to the \'success\' and \'failure\' relationships, and contains the target table name in \'databaseName.tableName\' format.")
 @RequiresInstanceClassLoading
 @DeprecationNotice(classNames="org.apache.nifi.processors.hive.PutHive3Streaming")
public class PutHiveStreaming
extends AbstractSessionFactoryProcessor

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	PutHiveStreaming.FunctionContext
protected class	PutHiveStreaming.HiveStreamingRecord
private static class	PutHiveStreaming.ShouldRetryException

Field Summary

Fields

Modifier and Type	Field and Description
private static String	ALLOW_EXPLICIT_KEYTAB
static PropertyDescriptor	AUTOCREATE_PARTITIONS
static PropertyDescriptor	CALL_TIMEOUT
protected int	callTimeout
protected ExecutorService	callTimeoutPool
private static String	CLIENT_CACHE_DISABLED_PROPERTY
static PropertyDescriptor	DB_NAME
private static Validator	GREATER_THAN_ONE_VALIDATOR
static PropertyDescriptor	HEARTBEAT_INTERVAL
protected Timer	heartBeatTimer
static PropertyDescriptor	HIVE_CONFIGURATION_RESOURCES
static String	HIVE_STREAMING_RECORD_COUNT_ATTR
protected org.apache.hadoop.hive.conf.HiveConf	hiveConfig
protected HiveConfigurator	hiveConfigurator
(package private) static PropertyDescriptor	KERBEROS_CREDENTIALS_SERVICE
private File	kerberosConfigFile
protected KerberosProperties	kerberosProperties
protected AtomicReference<KerberosUser>	kerberosUserReference
static PropertyDescriptor	MAX_OPEN_CONNECTIONS
static PropertyDescriptor	METASTORE_URI
static PropertyDescriptor	PARTITION_COLUMNS
private List<PropertyDescriptor>	propertyDescriptors
static PropertyDescriptor	RECORDS_PER_TXN
static Relationship	REL_FAILURE
static Relationship	REL_RETRY
static Relationship	REL_SUCCESS
private Set<Relationship>	relationships
private static Set<String>	RESERVED_METADATA
static PropertyDescriptor	ROLLBACK_ON_FAILURE
protected AtomicBoolean	sendHeartBeat
static PropertyDescriptor	TABLE_NAME
protected ConcurrentHashMap<String,Semaphore>	tableSemaphoreMap
protected ConcurrentLinkedQueue<Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter>>	threadWriterList
static PropertyDescriptor	TXNS_PER_BATCH
protected org.apache.hadoop.security.UserGroupInformation	ugi
private AtomicReference<ValidationResources>	validationResourceHolder

Constructor Summary

Constructors

Constructor and Description
PutHiveStreaming()

Method Summary

Modifier and Type	Method and Description
private void	abortAllWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers) Abort current Txn on all writers
private void	abortAndCloseWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)
void	cleanup()
private void	closeAllWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers) Closes all writers and remove them from cache
protected Collection<ValidationResult>	customValidate(ValidationContext validationContext)
private void	flushAllWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers, boolean rollToNext)
protected KerberosProperties	getKerberosProperties()
private HiveWriter	getOrCreateWriter(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers, HiveOptions options, org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint)
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
(package private) org.apache.hadoop.security.UserGroupInformation	getUgi()
protected void	init(ProcessorInitializationContext context)
(package private) boolean	isAllowExplicitKeytab()
protected org.apache.hive.hcatalog.streaming.HiveEndPoint	makeHiveEndPoint(List<String> partitionValues, HiveOptions options)
protected HiveWriter	makeHiveWriter(org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint, ExecutorService callTimeoutPool, org.apache.hadoop.security.UserGroupInformation ugi, HiveOptions options)
private ExceptionHandler.OnError<PutHiveStreaming.FunctionContext,PutHiveStreaming.HiveStreamingRecord>	onHiveRecordError(ProcessContext context, ProcessSession session, Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)
private ExceptionHandler.OnError<PutHiveStreaming.FunctionContext,List<PutHiveStreaming.HiveStreamingRecord>>	onHiveRecordsError(ProcessContext context, ProcessSession session, Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)
private ExceptionHandler.OnError<PutHiveStreaming.FunctionContext,org.apache.avro.generic.GenericRecord>	onRecordError(ProcessContext context, ProcessSession session, Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)
void	onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory)
private void	onTrigger(ProcessContext context, ProcessSession session, PutHiveStreaming.FunctionContext functionContext)
private void	retireEldestWriter(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers) Locate writer that has not been used for longest time and retire it
private int	retireIdleWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers, int idleTimeout) Locate all writers past idle timeout and retire them
private void	sendHeartBeatOnAllWriters()
void	setup(ProcessContext context)
private void	setupHeartBeatTimer(int heartbeatInterval)

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

ALLOW_EXPLICIT_KEYTAB

private static final String ALLOW_EXPLICIT_KEYTAB

See Also:

Constant Field Values

HIVE_STREAMING_RECORD_COUNT_ATTR

public static final String HIVE_STREAMING_RECORD_COUNT_ATTR

See Also:

Constant Field Values

CLIENT_CACHE_DISABLED_PROPERTY

private static final String CLIENT_CACHE_DISABLED_PROPERTY

See Also:

Constant Field Values

GREATER_THAN_ONE_VALIDATOR

private static final Validator GREATER_THAN_ONE_VALIDATOR

RESERVED_METADATA

private static final Set<String> RESERVED_METADATA

METASTORE_URI

public static final PropertyDescriptor METASTORE_URI

HIVE_CONFIGURATION_RESOURCES

public static final PropertyDescriptor HIVE_CONFIGURATION_RESOURCES

DB_NAME

public static final PropertyDescriptor DB_NAME

TABLE_NAME

public static final PropertyDescriptor TABLE_NAME

PARTITION_COLUMNS

public static final PropertyDescriptor PARTITION_COLUMNS

AUTOCREATE_PARTITIONS

public static final PropertyDescriptor AUTOCREATE_PARTITIONS

MAX_OPEN_CONNECTIONS

public static final PropertyDescriptor MAX_OPEN_CONNECTIONS

HEARTBEAT_INTERVAL

public static final PropertyDescriptor HEARTBEAT_INTERVAL

TXNS_PER_BATCH

public static final PropertyDescriptor TXNS_PER_BATCH

RECORDS_PER_TXN

public static final PropertyDescriptor RECORDS_PER_TXN

CALL_TIMEOUT

public static final PropertyDescriptor CALL_TIMEOUT

ROLLBACK_ON_FAILURE

public static final PropertyDescriptor ROLLBACK_ON_FAILURE

KERBEROS_CREDENTIALS_SERVICE

static final PropertyDescriptor KERBEROS_CREDENTIALS_SERVICE

REL_SUCCESS

public static final Relationship REL_SUCCESS

REL_FAILURE

public static final Relationship REL_FAILURE

REL_RETRY

public static final Relationship REL_RETRY

propertyDescriptors

private List<PropertyDescriptor> propertyDescriptors

relationships

private Set<Relationship> relationships

kerberosProperties

protected KerberosProperties kerberosProperties

kerberosConfigFile

private volatile File kerberosConfigFile

hiveConfigurator

protected volatile HiveConfigurator hiveConfigurator

ugi

protected volatile org.apache.hadoop.security.UserGroupInformation ugi

kerberosUserReference

protected final AtomicReference<KerberosUser> kerberosUserReference

hiveConfig

protected volatile org.apache.hadoop.hive.conf.HiveConf hiveConfig

sendHeartBeat

protected final AtomicBoolean sendHeartBeat

callTimeout

protected volatile int callTimeout

callTimeoutPool

protected ExecutorService callTimeoutPool

heartBeatTimer

protected transient Timer heartBeatTimer

threadWriterList

protected volatile ConcurrentLinkedQueue<Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter>> threadWriterList

tableSemaphoreMap

protected volatile ConcurrentHashMap<String,Semaphore> tableSemaphoreMap

validationResourceHolder

private final AtomicReference<ValidationResources> validationResourceHolder

Constructor Detail

PutHiveStreaming

public PutHiveStreaming()

Method Detail

init

protected void init(ProcessorInitializationContext context)

Overrides:

init in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext validationContext)

Overrides:

customValidate in class AbstractConfigurableComponent

setup

@OnScheduled
public void setup(ProcessContext context)

onHiveRecordsError

private ExceptionHandler.OnError<PutHiveStreaming.FunctionContext,List<PutHiveStreaming.HiveStreamingRecord>> onHiveRecordsError(ProcessContext context,
                                                                                                                                 ProcessSession session,
                                                                                                                                 Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)

onHiveRecordError

private ExceptionHandler.OnError<PutHiveStreaming.FunctionContext,PutHiveStreaming.HiveStreamingRecord> onHiveRecordError(ProcessContext context,
                                                                                                                          ProcessSession session,
                                                                                                                          Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)

onRecordError

private ExceptionHandler.OnError<PutHiveStreaming.FunctionContext,org.apache.avro.generic.GenericRecord> onRecordError(ProcessContext context,
                                                                                                                       ProcessSession session,
                                                                                                                       Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSessionFactory sessionFactory)
               throws ProcessException

Throws:

ProcessException

onTrigger

private void onTrigger(ProcessContext context,
                       ProcessSession session,
                       PutHiveStreaming.FunctionContext functionContext)
                throws ProcessException

Throws:

ProcessException

cleanup

@OnStopped
public void cleanup()

setupHeartBeatTimer

private void setupHeartBeatTimer(int heartbeatInterval)

sendHeartBeatOnAllWriters

private void sendHeartBeatOnAllWriters()
                                throws InterruptedException

Throws:

InterruptedException

flushAllWriters

private void flushAllWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers,
                             boolean rollToNext)
                      throws HiveWriter.CommitFailure,
                             HiveWriter.TxnBatchFailure,
                             HiveWriter.TxnFailure,
                             InterruptedException

Throws:

HiveWriter.CommitFailure

HiveWriter.TxnBatchFailure

HiveWriter.TxnFailure

InterruptedException

abortAndCloseWriters

private void abortAndCloseWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)

abortAllWriters

private void abortAllWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)
                      throws InterruptedException,
                             org.apache.hive.hcatalog.streaming.StreamingException,
                             HiveWriter.TxnBatchFailure

Abort current Txn on all writers

Throws:

InterruptedException

org.apache.hive.hcatalog.streaming.StreamingException

HiveWriter.TxnBatchFailure

closeAllWriters

private void closeAllWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)

Closes all writers and remove them from cache

getOrCreateWriter

private HiveWriter getOrCreateWriter(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers,
                                     HiveOptions options,
                                     org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint)
                              throws HiveWriter.ConnectFailure,
                                     InterruptedException

Throws:

HiveWriter.ConnectFailure

InterruptedException

retireEldestWriter

private void retireEldestWriter(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers)

Locate writer that has not been used for longest time and retire it

retireIdleWriters

private int retireIdleWriters(Map<org.apache.hive.hcatalog.streaming.HiveEndPoint,HiveWriter> writers,
                              int idleTimeout)

Locate all writers past idle timeout and retire them

Returns:

number of writers retired

makeHiveEndPoint

protected org.apache.hive.hcatalog.streaming.HiveEndPoint makeHiveEndPoint(List<String> partitionValues,
                                                                           HiveOptions options)
                                                                    throws org.apache.hive.hcatalog.streaming.ConnectionError

Throws:

org.apache.hive.hcatalog.streaming.ConnectionError

makeHiveWriter

protected HiveWriter makeHiveWriter(org.apache.hive.hcatalog.streaming.HiveEndPoint endPoint,
                                    ExecutorService callTimeoutPool,
                                    org.apache.hadoop.security.UserGroupInformation ugi,
                                    HiveOptions options)
                             throws HiveWriter.ConnectFailure,
                                    InterruptedException

Throws:

HiveWriter.ConnectFailure

InterruptedException

getKerberosProperties

protected KerberosProperties getKerberosProperties()

getUgi

org.apache.hadoop.security.UserGroupInformation getUgi()

isAllowExplicitKeytab

boolean isAllowExplicitKeytab()