org.apache.nifi.processors.hive

Class SelectHiveQL

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processors.hive.AbstractHiveQLProcessor

org.apache.nifi.processors.hive.SelectHiveQL

All Implemented Interfaces:

ConfigurableComponent, Processor

@EventDriven
 @InputRequirement(value=INPUT_ALLOWED)
 @Tags(value={"hive","sql","select","jdbc","query","database"})
 @CapabilityDescription(value="Execute provided HiveQL SELECT query against a Hive database connection. Query result will be converted to Avro or CSV format. Streaming is used so arbitrarily large result sets are supported. This processor can be scheduled to run on a timer, or cron expression, using the standard scheduling methods, or it can be triggered by an incoming FlowFile. If it is triggered by an incoming FlowFile, then attributes of that FlowFile will be available when evaluating the select query. FlowFile attribute \'selecthiveql.row.count\' indicates how many rows were selected.")
 @WritesAttribute(attribute="mime.type",description="Sets the MIME type for the outgoing flowfile to application/avro-binary for Avro or text/csv for CSV.") @WritesAttribute(attribute="filename",description="Adds .avro or .csv to the filename attribute depending on which output format is selected.") @WritesAttribute(attribute="selecthiveql.row.count",description="Indicates how many rows were selected/returned by the query.") @WritesAttribute(attribute="selecthiveql.query.duration",description="Combined duration of the query execution time and fetch time in milliseconds. If \'Max Rows Per Flow File\' is set, then this number will reflect only the fetch time for the rows in the Flow File instead of the entire result set.") @WritesAttribute(attribute="selecthiveql.query.executiontime",description="Duration of the query execution time in milliseconds. This number will reflect the query execution time regardless of the \'Max Rows Per Flow File\' setting.") @WritesAttribute(attribute="selecthiveql.query.fetchtime",description="Duration of the result set fetch time in milliseconds. If \'Max Rows Per Flow File\' is set, then this number will reflect only the fetch time for the rows in the Flow File instead of the entire result set.") @WritesAttribute(attribute="fragment.identifier",description="If \'Max Rows Per Flow File\' is set then all FlowFiles from the same query result set will have the same value for the fragment.identifier attribute. This can then be used to correlate the results.") @WritesAttribute(attribute="fragment.count",description="If \'Max Rows Per Flow File\' is set then this is the total number of  FlowFiles produced by a single ResultSet. This can be used in conjunction with the fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet.") @WritesAttribute(attribute="fragment.index",description="If \'Max Rows Per Flow File\' is set then the position of this FlowFile in the list of outgoing FlowFiles that were all derived from the same result set FlowFile. This can be used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same query result set and in what order  FlowFiles were produced") @WritesAttribute(attribute="query.input.tables",description="Contains input table names in comma delimited \'databaseName.tableName\' format.")
 @DeprecationNotice(classNames="org.apache.nifi.processors.hive.SelectHive3QL")
public class SelectHiveQL
extends AbstractHiveQLProcessor

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.nifi.processors.hive.AbstractHiveQLProcessor

AbstractHiveQLProcessor.TableName

Field Summary

Fields

Modifier and Type	Field and Description
static PropertyDescriptor	FETCH_SIZE
static PropertyDescriptor	HIVEQL_CSV_ALT_HEADER
static PropertyDescriptor	HIVEQL_CSV_DELIMITER
static PropertyDescriptor	HIVEQL_CSV_ESCAPE
static PropertyDescriptor	HIVEQL_CSV_HEADER
static PropertyDescriptor	HIVEQL_CSV_QUOTE
static PropertyDescriptor	HIVEQL_OUTPUT_FORMAT
static PropertyDescriptor	HIVEQL_POST_QUERY
static PropertyDescriptor	HIVEQL_PRE_QUERY
static PropertyDescriptor	HIVEQL_SELECT_QUERY
static PropertyDescriptor	MAX_FRAGMENTS
static PropertyDescriptor	MAX_ROWS_PER_FLOW_FILE
private static List<PropertyDescriptor>	propertyDescriptors
static Relationship	REL_FAILURE
static Relationship	REL_SUCCESS
private static Set<Relationship>	relationships
static String	RESULT_QUERY_DURATION
static String	RESULT_QUERY_EXECUTION_TIME
static String	RESULT_QUERY_FETCH_TIME
static String	RESULT_ROW_COUNT

Fields inherited from class org.apache.nifi.processors.hive.AbstractHiveQLProcessor

ATTR_INPUT_TABLES, ATTR_OUTPUT_TABLES, CHARSET, HIVE_DBCP_SERVICE, HIVEQL_TYPE_ATTRIBUTE_PATTERN, NUMBER_PATTERN

Constructor Summary

Constructors

Constructor and Description
SelectHiveQL()

Method Summary

Modifier and Type	Method and Description
protected org.apache.commons.lang3.tuple.Pair<String,SQLException>	executeConfigStatements(Connection con, List<String> configQueries)
protected List<String>	getQueries(String value)
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private void	onTrigger(ProcessContext context, ProcessSession session)
void	onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory)
void	setup(ProcessContext context)

Methods inherited from class org.apache.nifi.processors.hive.AbstractHiveQLProcessor

findTableNames, getHiveQL, setParameter, setParameters, toQueryTableAttributes

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

customValidate, equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

RESULT_ROW_COUNT

public static final String RESULT_ROW_COUNT

See Also:

Constant Field Values

RESULT_QUERY_DURATION

public static final String RESULT_QUERY_DURATION

See Also:

Constant Field Values

RESULT_QUERY_EXECUTION_TIME

public static final String RESULT_QUERY_EXECUTION_TIME

See Also:

Constant Field Values

RESULT_QUERY_FETCH_TIME

public static final String RESULT_QUERY_FETCH_TIME

See Also:

Constant Field Values

REL_SUCCESS

public static final Relationship REL_SUCCESS

REL_FAILURE

public static final Relationship REL_FAILURE

HIVEQL_PRE_QUERY

public static final PropertyDescriptor HIVEQL_PRE_QUERY

HIVEQL_SELECT_QUERY

public static final PropertyDescriptor HIVEQL_SELECT_QUERY

HIVEQL_POST_QUERY

public static final PropertyDescriptor HIVEQL_POST_QUERY

FETCH_SIZE

public static final PropertyDescriptor FETCH_SIZE

MAX_ROWS_PER_FLOW_FILE

public static final PropertyDescriptor MAX_ROWS_PER_FLOW_FILE

MAX_FRAGMENTS

public static final PropertyDescriptor MAX_FRAGMENTS

HIVEQL_CSV_HEADER

public static final PropertyDescriptor HIVEQL_CSV_HEADER

HIVEQL_CSV_ALT_HEADER

public static final PropertyDescriptor HIVEQL_CSV_ALT_HEADER

HIVEQL_CSV_DELIMITER

public static final PropertyDescriptor HIVEQL_CSV_DELIMITER

HIVEQL_CSV_QUOTE

public static final PropertyDescriptor HIVEQL_CSV_QUOTE

HIVEQL_CSV_ESCAPE

public static final PropertyDescriptor HIVEQL_CSV_ESCAPE

HIVEQL_OUTPUT_FORMAT

public static final PropertyDescriptor HIVEQL_OUTPUT_FORMAT

propertyDescriptors

private static final List<PropertyDescriptor> propertyDescriptors

relationships

private static final Set<Relationship> relationships

Constructor Detail

SelectHiveQL

public SelectHiveQL()

Method Detail

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

setup

@OnScheduled
public void setup(ProcessContext context)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSessionFactory sessionFactory)
               throws ProcessException

Throws:

ProcessException

onTrigger

private void onTrigger(ProcessContext context,
                       ProcessSession session)
                throws ProcessException

Throws:

ProcessException

executeConfigStatements

protected org.apache.commons.lang3.tuple.Pair<String,SQLException> executeConfigStatements(Connection con,
                                                                                           List<String> configQueries)

getQueries

protected List<String> getQueries(String value)