org.apache.nifi.processors.hive

Class UpdateHiveTable

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processors.hive.UpdateHiveTable

All Implemented Interfaces:

ConfigurableComponent, Processor

@Tags(value={"hive","metadata","jdbc","database","table"})
 @CapabilityDescription(value="This processor uses a Hive JDBC connection and incoming records to generate any Hive 1.2 table changes needed to support the incoming records.")
 @ReadsAttributes(value=@ReadsAttribute(attribute="hive.table.management.strategy",description="This attribute is read if the \'Table Management Strategy\' property is configured to use the value of this attribute. The value of this attribute should correspond (ignoring case) to a valid option of the \'Table Management Strategy\' property."))
 @WritesAttribute(attribute="output.table",description="This attribute is written on the flow files routed to the \'success\' and \'failure\' relationships, and contains the target table name.") @WritesAttribute(attribute="output.path",description="This attribute is written on the flow files routed to the \'success\' and \'failure\' relationships, and contains the path on the file system to the table (or partition location if the table is partitioned).") @WritesAttribute(attribute="mime.type",description="Sets the mime.type attribute to the MIME Type specified by the Record Writer, only if a Record Writer is specified and Update Field Names is \'true\'.") @WritesAttribute(attribute="record.count",description="Sets the number of records in the FlowFile, only if a Record Writer is specified and Update Field Names is \'true\'.")
 @InputRequirement(value=INPUT_REQUIRED)
 @RequiresInstanceClassLoading
 @DeprecationNotice(classNames="org.apache.nifi.processors.hive.UpdateHive3Table")
public class UpdateHiveTable
extends AbstractProcessor

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	UpdateHiveTable.OutputMetadataHolder

Field Summary

Fields

Modifier and Type	Field and Description
(package private) static String	ATTR_OUTPUT_PATH
(package private) static String	ATTR_OUTPUT_TABLE
(package private) static AllowableValue	ATTRIBUTE_DRIVEN_TABLE
(package private) static String	AVRO
(package private) static AllowableValue	AVRO_STORAGE
(package private) static AllowableValue	CREATE_IF_NOT_EXISTS
(package private) static PropertyDescriptor	CREATE_TABLE
(package private) static AllowableValue	EXTERNAL_TABLE
(package private) static PropertyDescriptor	EXTERNAL_TABLE_LOCATION
(package private) static AllowableValue	FAIL_IF_NOT_EXISTS
(package private) static PropertyDescriptor	HIVE_DBCP_SERVICE
(package private) static AllowableValue	MANAGED_TABLE
(package private) static String	ORC
(package private) static AllowableValue	ORC_STORAGE
(package private) static String	PARQUET
(package private) static AllowableValue	PARQUET_STORAGE
(package private) static PropertyDescriptor	PARTITION_CLAUSE
private List<PropertyDescriptor>	propertyDescriptors
(package private) static PropertyDescriptor	QUERY_TIMEOUT
(package private) static String	RCFILE
(package private) static AllowableValue	RCFILE_STORAGE
(package private) static PropertyDescriptor	RECORD_READER
(package private) static PropertyDescriptor	RECORD_WRITER_FACTORY
static Relationship	REL_FAILURE
static Relationship	REL_SUCCESS
private Set<Relationship>	relationships
(package private) static String	SEQUENCEFILE
(package private) static AllowableValue	SEQUENCEFILE_STORAGE
(package private) static PropertyDescriptor	TABLE_MANAGEMENT_STRATEGY
(package private) static String	TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE
(package private) static PropertyDescriptor	TABLE_NAME
(package private) static PropertyDescriptor	TABLE_STORAGE_FORMAT
(package private) static String	TEXTFILE
(package private) static AllowableValue	TEXTFILE_STORAGE
(package private) static PropertyDescriptor	UPDATE_FIELD_NAMES

Constructor Summary

Constructors

Constructor and Description
UpdateHiveTable()

Method Summary

Modifier and Type	Method and Description
private UpdateHiveTable.OutputMetadataHolder	checkAndUpdateTableSchema(Map<String,String> attributes, Connection conn, RecordSchema schema, String tableName, List<String> partitionClause, boolean createIfNotExists, String externalTableLocation, String storageFormat, boolean updateFieldNames)
protected Collection<ValidationResult>	customValidate(ValidationContext validationContext)
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
protected void	init(ProcessorInitializationContext context)
void	onTrigger(ProcessContext context, ProcessSession session)
private WriteResult	updateRecords(RecordSchema inputRecordSchema, UpdateHiveTable.OutputMetadataHolder outputMetadataHolder, RecordReader reader, RecordSetWriter writer)

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

TEXTFILE

static final String TEXTFILE

See Also:

Constant Field Values

SEQUENCEFILE

static final String SEQUENCEFILE

See Also:

Constant Field Values

ORC

static final String ORC

See Also:

Constant Field Values

PARQUET

static final String PARQUET

See Also:

Constant Field Values

AVRO

static final String AVRO

See Also:

Constant Field Values

RCFILE

static final String RCFILE

See Also:

Constant Field Values

TEXTFILE_STORAGE

static final AllowableValue TEXTFILE_STORAGE

SEQUENCEFILE_STORAGE

static final AllowableValue SEQUENCEFILE_STORAGE

ORC_STORAGE

static final AllowableValue ORC_STORAGE

PARQUET_STORAGE

static final AllowableValue PARQUET_STORAGE

AVRO_STORAGE

static final AllowableValue AVRO_STORAGE

RCFILE_STORAGE

static final AllowableValue RCFILE_STORAGE

CREATE_IF_NOT_EXISTS

static final AllowableValue CREATE_IF_NOT_EXISTS

FAIL_IF_NOT_EXISTS

static final AllowableValue FAIL_IF_NOT_EXISTS

TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE

static final String TABLE_MANAGEMENT_STRATEGY_ATTRIBUTE

See Also:

Constant Field Values

MANAGED_TABLE

static final AllowableValue MANAGED_TABLE

EXTERNAL_TABLE

static final AllowableValue EXTERNAL_TABLE

ATTRIBUTE_DRIVEN_TABLE

static final AllowableValue ATTRIBUTE_DRIVEN_TABLE

ATTR_OUTPUT_TABLE

static final String ATTR_OUTPUT_TABLE

See Also:

Constant Field Values

ATTR_OUTPUT_PATH

static final String ATTR_OUTPUT_PATH

See Also:

Constant Field Values

RECORD_READER

static final PropertyDescriptor RECORD_READER

HIVE_DBCP_SERVICE

static final PropertyDescriptor HIVE_DBCP_SERVICE

TABLE_NAME

static final PropertyDescriptor TABLE_NAME

CREATE_TABLE

static final PropertyDescriptor CREATE_TABLE

TABLE_MANAGEMENT_STRATEGY

static final PropertyDescriptor TABLE_MANAGEMENT_STRATEGY

UPDATE_FIELD_NAMES

static final PropertyDescriptor UPDATE_FIELD_NAMES

RECORD_WRITER_FACTORY

static final PropertyDescriptor RECORD_WRITER_FACTORY

EXTERNAL_TABLE_LOCATION

static final PropertyDescriptor EXTERNAL_TABLE_LOCATION

TABLE_STORAGE_FORMAT

static final PropertyDescriptor TABLE_STORAGE_FORMAT

QUERY_TIMEOUT

static final PropertyDescriptor QUERY_TIMEOUT

PARTITION_CLAUSE

static final PropertyDescriptor PARTITION_CLAUSE

REL_SUCCESS

public static final Relationship REL_SUCCESS

REL_FAILURE

public static final Relationship REL_FAILURE

propertyDescriptors

private List<PropertyDescriptor> propertyDescriptors

relationships

private Set<Relationship> relationships

Constructor Detail

UpdateHiveTable

public UpdateHiveTable()

Method Detail

init

protected void init(ProcessorInitializationContext context)

Overrides:

init in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext validationContext)

Overrides:

customValidate in class AbstractConfigurableComponent

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSession session)
               throws ProcessException

Specified by:

onTrigger in class AbstractProcessor

Throws:

ProcessException

checkAndUpdateTableSchema

private UpdateHiveTable.OutputMetadataHolder checkAndUpdateTableSchema(Map<String,String> attributes,
                                                                       Connection conn,
                                                                       RecordSchema schema,
                                                                       String tableName,
                                                                       List<String> partitionClause,
                                                                       boolean createIfNotExists,
                                                                       String externalTableLocation,
                                                                       String storageFormat,
                                                                       boolean updateFieldNames)
                                                                throws IOException

Throws:

IOException

updateRecords

private WriteResult updateRecords(RecordSchema inputRecordSchema,
                                  UpdateHiveTable.OutputMetadataHolder outputMetadataHolder,
                                  RecordReader reader,
                                  RecordSetWriter writer)
                           throws IOException

Throws:

IOException