org.apache.nifi.processors.kudu

Class PutKudu

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.processor.AbstractProcessor

org.apache.nifi.processors.kudu.AbstractKuduProcessor

org.apache.nifi.processors.kudu.PutKudu

All Implemented Interfaces:

ConfigurableComponent, Processor

@SystemResourceConsideration(resource=MEMORY)
 @EventDriven
 @SupportsBatching
 @RequiresInstanceClassLoading
 @InputRequirement(value=INPUT_REQUIRED)
 @Tags(value={"put","database","NoSQL","kudu","HDFS","record"})
 @CapabilityDescription(value="Reads records from an incoming FlowFile using the provided Record Reader, and writes those records to the specified Kudu\'s table. The schema for the Kudu table is inferred from the schema of the Record Reader. If any error occurs while reading records from the input, or writing records to Kudu, the FlowFile will be routed to failure")
 @WritesAttribute(attribute="record.count",
                 description="Number of records written to Kudu")
public class PutKudu
extends AbstractKuduProcessor

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	PutKudu.RecordPathOperationType

Field Summary

Fields

Modifier and Type	Field and Description
protected static PropertyDescriptor	BATCH_SIZE
private int	batchSize
(package private) static PropertyDescriptor	DATA_RECORD_PATH
private RecordPath	dataRecordPath
(package private) static PropertyDescriptor	FAILURE_STRATEGY
(package private) static AllowableValue	FAILURE_STRATEGY_ROLLBACK
(package private) static AllowableValue	FAILURE_STRATEGY_ROUTE
private String	failureStrategy
private int	ffbatch
protected static PropertyDescriptor	FLOWFILE_BATCH_SIZE
protected static PropertyDescriptor	FLUSH_MODE
private org.apache.kudu.client.SessionConfiguration.FlushMode	flushMode
protected static PropertyDescriptor	HANDLE_SCHEMA_DRIFT
protected static PropertyDescriptor	IGNORE_NULL
protected static PropertyDescriptor	INSERT_OPERATION
protected static PropertyDescriptor	LOWERCASE_FIELD_NAMES
(package private) static PropertyDescriptor	OPERATION_RECORD_PATH
protected static Validator	OperationTypeValidator
static String	RECORD_COUNT_ATTR
static PropertyDescriptor	RECORD_READER
private Function<Record,OperationType>	recordPathOperationType
protected static Relationship	REL_FAILURE
protected static Relationship	REL_SUCCESS
protected static PropertyDescriptor	SKIP_HEAD_LINE
private boolean	supportsInsertIgnoreOp
protected static PropertyDescriptor	TABLE_NAME

Fields inherited from class org.apache.nifi.processors.kudu.AbstractKuduProcessor

KERBEROS_CREDENTIALS_SERVICE, KERBEROS_PASSWORD, KERBEROS_PRINCIPAL, KERBEROS_USER_SERVICE, KUDU_KEEP_ALIVE_PERIOD_TIMEOUT_MS, KUDU_MASTERS, KUDU_OPERATION_TIMEOUT_MS, KUDU_SASL_PROTOCOL_NAME, WORKER_COUNT

Constructor Summary

Constructors

Constructor and Description
PutKudu()

Method Summary

Modifier and Type	Method and Description
protected org.apache.kudu.client.Operation	createKuduOperation(OperationType operationType, Record record, List<String> fieldNames, boolean ignoreNull, boolean lowercaseFields, org.apache.kudu.client.KuduTable kuduTable)
protected org.apache.kudu.client.KuduSession	createKuduSession(org.apache.kudu.client.KuduClient client)
private String	getEvaluatedProperty(PropertyDescriptor property, ProcessContext context, FlowFile flowFile)
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private boolean	handleSchemaDrift(org.apache.kudu.client.KuduTable kuduTable, org.apache.kudu.client.KuduClient kuduClient, FlowFile flowFile, Record record, boolean lowercaseFields)
private boolean	isRollbackOnFailure()
private void	logFailures(List<org.apache.kudu.client.RowError> pendingRowErrors, Map<org.apache.kudu.client.Operation,FlowFile> operationFlowFileMap)
void	onScheduled(ProcessContext context)
void	onTrigger(ProcessContext context, ProcessSession session)
private void	processFlowFiles(ProcessContext context, ProcessSession session, List<FlowFile> flowFiles, org.apache.kudu.client.KuduClient kuduClient)
private void	processRecords(List<FlowFile> flowFiles, Map<FlowFile,Integer> processedRecords, Map<FlowFile,Object> flowFileFailures, Map<org.apache.kudu.client.Operation,FlowFile> operationFlowFileMap, List<org.apache.kudu.client.RowError> pendingRowErrors, ProcessSession session, ProcessContext context, org.apache.kudu.client.KuduClient kuduClient, org.apache.kudu.client.KuduSession kuduSession)
private void	transferFlowFiles(List<FlowFile> flowFiles, Map<FlowFile,Integer> processedRecords, Map<FlowFile,Object> flowFileFailures, Map<org.apache.kudu.client.Operation,FlowFile> operationFlowFileMap, List<org.apache.kudu.client.RowError> pendingRowErrors, ProcessSession session)

Methods inherited from class org.apache.nifi.processors.kudu.AbstractKuduProcessor

buildClient, buildPartialRow, createKerberosKeytabUser, createKerberosPasswordUser, createKerberosUserAndOrKuduClient, createKuduClient, customValidate, executeOnKuduClient, flushKuduSession, getAddNullableColumnStatement, getKerberosUser, shutdown, supportsIgnoreOperations

Methods inherited from class org.apache.nifi.processor.AbstractProcessor

onTrigger

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

FAILURE_STRATEGY_ROUTE

static final AllowableValue FAILURE_STRATEGY_ROUTE

FAILURE_STRATEGY_ROLLBACK

static final AllowableValue FAILURE_STRATEGY_ROLLBACK

TABLE_NAME

protected static final PropertyDescriptor TABLE_NAME

RECORD_READER

public static final PropertyDescriptor RECORD_READER

FAILURE_STRATEGY

static final PropertyDescriptor FAILURE_STRATEGY

SKIP_HEAD_LINE

protected static final PropertyDescriptor SKIP_HEAD_LINE

LOWERCASE_FIELD_NAMES

protected static final PropertyDescriptor LOWERCASE_FIELD_NAMES

HANDLE_SCHEMA_DRIFT

protected static final PropertyDescriptor HANDLE_SCHEMA_DRIFT

DATA_RECORD_PATH

static final PropertyDescriptor DATA_RECORD_PATH

OPERATION_RECORD_PATH

static final PropertyDescriptor OPERATION_RECORD_PATH

OperationTypeValidator

protected static final Validator OperationTypeValidator

INSERT_OPERATION

protected static final PropertyDescriptor INSERT_OPERATION

FLUSH_MODE

protected static final PropertyDescriptor FLUSH_MODE

FLOWFILE_BATCH_SIZE

protected static final PropertyDescriptor FLOWFILE_BATCH_SIZE

BATCH_SIZE

protected static final PropertyDescriptor BATCH_SIZE

IGNORE_NULL

protected static final PropertyDescriptor IGNORE_NULL

REL_SUCCESS

protected static final Relationship REL_SUCCESS

REL_FAILURE

protected static final Relationship REL_FAILURE

RECORD_COUNT_ATTR

public static final String RECORD_COUNT_ATTR

See Also:

Constant Field Values

batchSize

private volatile int batchSize

ffbatch

private volatile int ffbatch

flushMode

private volatile org.apache.kudu.client.SessionConfiguration.FlushMode flushMode

recordPathOperationType

private volatile Function<Record,OperationType> recordPathOperationType

dataRecordPath

private volatile RecordPath dataRecordPath

failureStrategy

private volatile String failureStrategy

supportsInsertIgnoreOp

private volatile boolean supportsInsertIgnoreOp

Constructor Detail

PutKudu

public PutKudu()

Method Detail

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

onScheduled

@OnScheduled
public void onScheduled(ProcessContext context)
                              throws LoginException

Throws:

LoginException

isRollbackOnFailure

private boolean isRollbackOnFailure()

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSession session)
               throws ProcessException

Specified by:

onTrigger in class AbstractProcessor

Throws:

ProcessException

processFlowFiles

private void processFlowFiles(ProcessContext context,
                              ProcessSession session,
                              List<FlowFile> flowFiles,
                              org.apache.kudu.client.KuduClient kuduClient)

processRecords

private void processRecords(List<FlowFile> flowFiles,
                            Map<FlowFile,Integer> processedRecords,
                            Map<FlowFile,Object> flowFileFailures,
                            Map<org.apache.kudu.client.Operation,FlowFile> operationFlowFileMap,
                            List<org.apache.kudu.client.RowError> pendingRowErrors,
                            ProcessSession session,
                            ProcessContext context,
                            org.apache.kudu.client.KuduClient kuduClient,
                            org.apache.kudu.client.KuduSession kuduSession)

handleSchemaDrift

private boolean handleSchemaDrift(org.apache.kudu.client.KuduTable kuduTable,
                                  org.apache.kudu.client.KuduClient kuduClient,
                                  FlowFile flowFile,
                                  Record record,
                                  boolean lowercaseFields)

transferFlowFiles

private void transferFlowFiles(List<FlowFile> flowFiles,
                               Map<FlowFile,Integer> processedRecords,
                               Map<FlowFile,Object> flowFileFailures,
                               Map<org.apache.kudu.client.Operation,FlowFile> operationFlowFileMap,
                               List<org.apache.kudu.client.RowError> pendingRowErrors,
                               ProcessSession session)

logFailures

private void logFailures(List<org.apache.kudu.client.RowError> pendingRowErrors,
                         Map<org.apache.kudu.client.Operation,FlowFile> operationFlowFileMap)

getEvaluatedProperty

private String getEvaluatedProperty(PropertyDescriptor property,
                                    ProcessContext context,
                                    FlowFile flowFile)

createKuduSession

protected org.apache.kudu.client.KuduSession createKuduSession(org.apache.kudu.client.KuduClient client)

createKuduOperation

protected org.apache.kudu.client.Operation createKuduOperation(OperationType operationType,
                                                               Record record,
                                                               List<String> fieldNames,
                                                               boolean ignoreNull,
                                                               boolean lowercaseFields,
                                                               org.apache.kudu.client.KuduTable kuduTable)