org.apache.nifi.cdc.mysql.processors

Class CaptureChangeMySQL

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.cdc.mysql.processors.CaptureChangeMySQL

All Implemented Interfaces:

ConfigurableComponent, Processor

@TriggerSerially
 @PrimaryNodeOnly
 @InputRequirement(value=INPUT_FORBIDDEN)
 @Tags(value={"sql","jdbc","cdc","mysql","transaction","event"})
 @CapabilityDescription(value="Retrieves Change Data Capture (CDC) events from a MySQL database. CDC Events include INSERT, UPDATE, DELETE operations. Events are output as either a group of a specified number of events (the default is 1 so each event becomes its own flow file) or grouped as a full transaction (BEGIN to COMMIT). All events are ordered by the time at which the operation occurred. NOTE: If the processor is stopped before the specified number of events have been written to a flow file, the partial flow file will be output in order to maintain the consistency of the event stream.")
 @Stateful(scopes=CLUSTER,
          description="Information such as a \'pointer\' to the current CDC event in the database is stored by this processor, such that it can continue from the same location if restarted.")
 @WritesAttribute(attribute="cdc.sequence.id",description="A sequence identifier (i.e. strictly increasing integer value) specifying the order of the CDC event flow file relative to the other event flow file(s).") @WritesAttribute(attribute="cdc.event.type",description="A string indicating the type of CDC event that occurred, including (but not limited to) \'begin\', \'insert\', \'update\', \'delete\', \'ddl\' and \'commit\'.") @WritesAttribute(attribute="mime.type",description="The processor outputs flow file content in JSON format, and sets the mime.type attribute to application/json")
 @RequiresInstanceClassLoading
 @Restricted(restrictions=@Restriction(requiredPermission=REFERENCE_REMOTE_RESOURCES,explanation="Database Driver Location can reference resources over HTTP"))
public class CaptureChangeMySQL
extends AbstractSessionFactoryProcessor

A processor to retrieve Change Data Capture (CDC) events and send them as flow files.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	CaptureChangeMySQL.BinlogEventState
static class	CaptureChangeMySQL.BinlogResourceInfo
private static class	CaptureChangeMySQL.DriverShim
private class	CaptureChangeMySQL.JDBCConnectionHolder

Field Summary

Fields

Modifier and Type	Field and Description
private BeginEventHandler	beginEventHandler
private com.github.shyiko.mysql.binlog.BinaryLogClient	binlogClient
private CaptureChangeMySQL.BinlogEventState	binlogEventState
private CaptureChangeMySQL.BinlogResourceInfo	binlogResourceInfo
private CommitEventHandler	commitEventHandler
static PropertyDescriptor	CONNECT_TIMEOUT
private DataCaptureState	currentDataCaptureState
private int	currentHost
private ProcessSession	currentSession
static PropertyDescriptor	DATABASE_NAME_PATTERN
private Pattern	databaseNamePattern
private DDLEventHandler	ddlEventHandler
private static int	DEFAULT_MYSQL_PORT
private DeleteEventHandler	deleteEventHandler
static PropertyDescriptor	DIST_CACHE_CLIENT
private static int	DO_NOT_SET
static PropertyDescriptor	DRIVER_LOCATION
static PropertyDescriptor	DRIVER_NAME
private BinlogEventListener	eventListener
static PropertyDescriptor	EVENTS_PER_FLOWFILE_STRATEGY
private EventWriterConfiguration	eventWriterConfiguration
private com.github.shyiko.mysql.binlog.GtidSet	gtidSet
static PropertyDescriptor	HOSTS
static PropertyDescriptor	INCLUDE_BEGIN_COMMIT
static PropertyDescriptor	INCLUDE_DDL_EVENTS
static PropertyDescriptor	INIT_BINLOG_FILENAME
static PropertyDescriptor	INIT_BINLOG_GTID
static PropertyDescriptor	INIT_BINLOG_POSITION
static PropertyDescriptor	INIT_SEQUENCE_ID
private InsertEventHandler	insertEventHandler
private CaptureChangeMySQL.JDBCConnectionHolder	jdbcConnectionHolder
private BinlogLifecycleListener	lifecycleListener
private static Pattern	MULTI_COMMENT_PATTERN
static PropertyDescriptor	NUMBER_OF_EVENTS_PER_FLOWFILE
static PropertyDescriptor	PASSWORD
private static List<PropertyDescriptor>	propDescriptors
private BlockingQueue<RawBinlogEvent>	queue
static Relationship	REL_SUCCESS
protected static Set<Relationship>	relationships
static PropertyDescriptor	RETRIEVE_ALL_RECORDS
static PropertyDescriptor	SERVER_ID
private boolean	skipTable
static PropertyDescriptor	SSL_CONTEXT_SERVICE
static PropertyDescriptor	SSL_MODE
private static AllowableValue	SSL_MODE_DISABLED
private static AllowableValue	SSL_MODE_PREFERRED
private static AllowableValue	SSL_MODE_REQUIRED
private static AllowableValue	SSL_MODE_VERIFY_IDENTITY
static PropertyDescriptor	STATE_UPDATE_INTERVAL
static PropertyDescriptor	TABLE_NAME_PATTERN
private Map<TableInfoCacheKey,TableInfo>	tableInfoCache
private Pattern	tableNamePattern
private UpdateEventHandler	updateEventHandler
static PropertyDescriptor	USE_BINLOG_GTID
static PropertyDescriptor	USERNAME

Constructor Summary

Constructors

Constructor and Description
CaptureChangeMySQL()

Method Summary

Modifier and Type	Method and Description
protected void	clearState()
protected void	connect(List<InetSocketAddress> hosts, String username, String password, Long serverId, String driverLocation, String driverName, long connectTimeout, SSLContextService sslContextService, com.github.shyiko.mysql.binlog.network.SSLMode sslMode)
protected com.github.shyiko.mysql.binlog.BinaryLogClient	createBinlogClient(String hostname, int port, String username, String password)
(package private) BinlogEventListener	createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client, BlockingQueue<RawBinlogEvent> q) Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.
(package private) BinlogLifecycleListener	createBinlogLifecycleListener() Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.
protected Collection<ValidationResult>	customValidate(ValidationContext validationContext)
private List<InetSocketAddress>	getHosts(String hostsString) Get a list of hosts from a NiFi property, e.g.
protected Connection	getJdbcConnection()
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
private boolean	isQueryDDL(String sql)
protected TableInfo	loadTableInfo(TableInfoCacheKey key) Retrieves the column information for the specified database and table.
protected String	normalizeQuery(String sql)
void	onPrimaryNodeChange(PrimaryNodeState state)
void	onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory)
void	outputEvents(ProcessSession session, ProcessContext context, ComponentLog log)
protected void	registerDriver(String locationString, String drvName) using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.
void	setup(ProcessContext context)
void	stop()
private void	updateState(ProcessSession session, DataCaptureState dataCaptureState)
private void	updateState(ProcessSession session, DataCaptureState dataCaptureState, boolean inTransaction)

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

DO_NOT_SET

private static final int DO_NOT_SET

See Also:

Constant Field Values

DEFAULT_MYSQL_PORT

private static final int DEFAULT_MYSQL_PORT

See Also:

Constant Field Values

MULTI_COMMENT_PATTERN

private static final Pattern MULTI_COMMENT_PATTERN

REL_SUCCESS

public static final Relationship REL_SUCCESS

relationships

protected static Set<Relationship> relationships

SSL_MODE_DISABLED

private static final AllowableValue SSL_MODE_DISABLED

SSL_MODE_PREFERRED

private static final AllowableValue SSL_MODE_PREFERRED

SSL_MODE_REQUIRED

private static final AllowableValue SSL_MODE_REQUIRED

SSL_MODE_VERIFY_IDENTITY

private static final AllowableValue SSL_MODE_VERIFY_IDENTITY

DATABASE_NAME_PATTERN

public static final PropertyDescriptor DATABASE_NAME_PATTERN

TABLE_NAME_PATTERN

public static final PropertyDescriptor TABLE_NAME_PATTERN

CONNECT_TIMEOUT

public static final PropertyDescriptor CONNECT_TIMEOUT

HOSTS

public static final PropertyDescriptor HOSTS

DRIVER_NAME

public static final PropertyDescriptor DRIVER_NAME

DRIVER_LOCATION

public static final PropertyDescriptor DRIVER_LOCATION

USERNAME

public static final PropertyDescriptor USERNAME

PASSWORD

public static final PropertyDescriptor PASSWORD

EVENTS_PER_FLOWFILE_STRATEGY

public static final PropertyDescriptor EVENTS_PER_FLOWFILE_STRATEGY

NUMBER_OF_EVENTS_PER_FLOWFILE

public static final PropertyDescriptor NUMBER_OF_EVENTS_PER_FLOWFILE

SERVER_ID

public static final PropertyDescriptor SERVER_ID

DIST_CACHE_CLIENT

public static final PropertyDescriptor DIST_CACHE_CLIENT

RETRIEVE_ALL_RECORDS

public static final PropertyDescriptor RETRIEVE_ALL_RECORDS

INCLUDE_BEGIN_COMMIT

public static final PropertyDescriptor INCLUDE_BEGIN_COMMIT

INCLUDE_DDL_EVENTS

public static final PropertyDescriptor INCLUDE_DDL_EVENTS

STATE_UPDATE_INTERVAL

public static final PropertyDescriptor STATE_UPDATE_INTERVAL

INIT_SEQUENCE_ID

public static final PropertyDescriptor INIT_SEQUENCE_ID

INIT_BINLOG_FILENAME

public static final PropertyDescriptor INIT_BINLOG_FILENAME

INIT_BINLOG_POSITION

public static final PropertyDescriptor INIT_BINLOG_POSITION

USE_BINLOG_GTID

public static final PropertyDescriptor USE_BINLOG_GTID

INIT_BINLOG_GTID

public static final PropertyDescriptor INIT_BINLOG_GTID

SSL_MODE

public static final PropertyDescriptor SSL_MODE

SSL_CONTEXT_SERVICE

public static final PropertyDescriptor SSL_CONTEXT_SERVICE

propDescriptors

private static final List<PropertyDescriptor> propDescriptors

binlogClient

private volatile com.github.shyiko.mysql.binlog.BinaryLogClient binlogClient

eventListener

private volatile BinlogEventListener eventListener

lifecycleListener

private volatile BinlogLifecycleListener lifecycleListener

gtidSet

private volatile com.github.shyiko.mysql.binlog.GtidSet gtidSet

queue

private final BlockingQueue<RawBinlogEvent> queue

tableInfoCache

private final Map<TableInfoCacheKey,TableInfo> tableInfoCache

currentSession

private volatile ProcessSession currentSession

currentDataCaptureState

private DataCaptureState currentDataCaptureState

binlogResourceInfo

private volatile CaptureChangeMySQL.BinlogResourceInfo binlogResourceInfo

databaseNamePattern

private volatile Pattern databaseNamePattern

tableNamePattern

private volatile Pattern tableNamePattern

skipTable

private volatile boolean skipTable

currentHost

private int currentHost

jdbcConnectionHolder

private volatile CaptureChangeMySQL.JDBCConnectionHolder jdbcConnectionHolder

binlogEventState

private final CaptureChangeMySQL.BinlogEventState binlogEventState

beginEventHandler

private final BeginEventHandler beginEventHandler

commitEventHandler

private final CommitEventHandler commitEventHandler

ddlEventHandler

private final DDLEventHandler ddlEventHandler

insertEventHandler

private final InsertEventHandler insertEventHandler

deleteEventHandler

private final DeleteEventHandler deleteEventHandler

updateEventHandler

private final UpdateEventHandler updateEventHandler

eventWriterConfiguration

private volatile EventWriterConfiguration eventWriterConfiguration

Constructor Detail

CaptureChangeMySQL

public CaptureChangeMySQL()

Method Detail

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext validationContext)

Overrides:

customValidate in class AbstractConfigurableComponent

onPrimaryNodeChange

@OnPrimaryNodeStateChange
public void onPrimaryNodeChange(PrimaryNodeState state)
                                                   throws CDCException

Throws:

CDCException

setup

public void setup(ProcessContext context)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSessionFactory sessionFactory)
               throws ProcessException

Throws:

ProcessException

getHosts

private List<InetSocketAddress> getHosts(String hostsString)

Get a list of hosts from a NiFi property, e.g.

Parameters:

hostsString - A comma-separated list of hosts (host:port,host2:port2, etc.)

Returns:

List of InetSocketAddresses for the hosts

connect

protected void connect(List<InetSocketAddress> hosts,
                       String username,
                       String password,
                       Long serverId,
                       String driverLocation,
                       String driverName,
                       long connectTimeout,
                       SSLContextService sslContextService,
                       com.github.shyiko.mysql.binlog.network.SSLMode sslMode)
                throws IOException

Throws:

IOException

outputEvents

public void outputEvents(ProcessSession session,
                         ProcessContext context,
                         ComponentLog log)
                  throws IOException

Throws:

IOException

isQueryDDL

private boolean isQueryDDL(String sql)

clearState

protected void clearState()
                   throws IOException

Throws:

IOException

normalizeQuery

protected String normalizeQuery(String sql)

stop

@OnStopped
public void stop()
                     throws CDCException

Throws:

CDCException

updateState

private void updateState(ProcessSession session,
                         DataCaptureState dataCaptureState)
                  throws IOException

Throws:

IOException

updateState

private void updateState(ProcessSession session,
                         DataCaptureState dataCaptureState,
                         boolean inTransaction)
                  throws IOException

Throws:

IOException

createBinlogEventListener

BinlogEventListener createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client,
                                              BlockingQueue<RawBinlogEvent> q)

Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.

Parameters:

client - A reference to a BinaryLogClient. The listener is associated with the given client, such that the listener is notified when events are available to the given client.

q - A queue used to communicate events between the listener and the NiFi processor thread.

Returns:

A BinlogEventListener instance, which will be notified of events associated with the specified client

createBinlogLifecycleListener

BinlogLifecycleListener createBinlogLifecycleListener()

Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.

Returns:

A BinlogLifecycleListener instance, which will be notified of events associated with the specified client

createBinlogClient

protected com.github.shyiko.mysql.binlog.BinaryLogClient createBinlogClient(String hostname,
                                                                            int port,
                                                                            String username,
                                                                            String password)

loadTableInfo

protected TableInfo loadTableInfo(TableInfoCacheKey key)
                           throws SQLException

Retrieves the column information for the specified database and table. The column information can be used to enrich CDC events coming from the RDBMS.

Parameters:

key - A TableInfoCacheKey reference, which contains the database and table names

Returns:

A TableInfo instance with the ColumnDefinitions provided (if retrieved successfully from the database)

Throws:

SQLException

getJdbcConnection

protected Connection getJdbcConnection()
                                throws SQLException

Throws:

SQLException

registerDriver

protected void registerDriver(String locationString,
                              String drvName)
                       throws InitializationException

using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.

Throws:

InitializationException - if there is a problem obtaining the ClassLoader