org.apache.nifi.cdc.mysql.processors

Class CaptureChangeMySQL

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.cdc.mysql.processors.CaptureChangeMySQL

All Implemented Interfaces:

ConfigurableComponent, Processor

@TriggerSerially
 @PrimaryNodeOnly
 @InputRequirement(value=INPUT_FORBIDDEN)
 @Tags(value={"sql","jdbc","cdc","mysql","transaction","event"})
 @CapabilityDescription(value="Retrieves Change Data Capture (CDC) events from a MySQL database. CDC Events include INSERT, UPDATE, DELETE operations. Events are output as either a group of a specified number of events (the default is 1 so each event becomes its own flow file) or grouped as a full transaction (BEGIN to COMMIT). All events are ordered by the time at which the operation occurred. NOTE: If the processor is stopped before the specified number of events have been written to a flow file, the partial flow file will be output in order to maintain the consistency of the event stream.")
 @Stateful(scopes=CLUSTER,
          description="Information such as a \'pointer\' to the current CDC event in the database is stored by this processor, such that it can continue from the same location if restarted.")
 @WritesAttribute(attribute="cdc.sequence.id",description="A sequence identifier (i.e. strictly increasing integer value) specifying the order of the CDC event flow file relative to the other event flow file(s).") @WritesAttribute(attribute="cdc.event.type",description="A string indicating the type of CDC event that occurred, including (but not limited to) \'begin\', \'insert\', \'update\', \'delete\', \'ddl\' and \'commit\'.") @WritesAttribute(attribute="mime.type",description="The processor outputs flow file content in JSON format, and sets the mime.type attribute to application/json")
 @RequiresInstanceClassLoading
public class CaptureChangeMySQL
extends AbstractSessionFactoryProcessor

A processor to retrieve Change Data Capture (CDC) events and send them as flow files.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	CaptureChangeMySQL.DriverShim
private class	CaptureChangeMySQL.JDBCConnectionHolder

Field Summary

Fields

Modifier and Type	Field and Description
private BeginTransactionEventWriter	beginEventWriter
private com.github.shyiko.mysql.binlog.BinaryLogClient	binlogClient
private DistributedMapCacheClient	cacheClient
private Serializer<TableInfoCacheKey>	cacheKeySerializer
private Deserializer<TableInfo>	cacheValueDeserializer
private Serializer<TableInfo>	cacheValueSerializer
private CommitTransactionEventWriter	commitEventWriter
static PropertyDescriptor	CONNECT_TIMEOUT
private String	currentBinlogFile
private long	currentBinlogPosition
private String	currentDatabase
private BinlogEventInfo	currentEventInfo
private AbstractBinlogEventWriter<? extends BinlogEventInfo>	currentEventWriter
private String	currentGtidSet
private int	currentHost
private AtomicLong	currentSequenceId
private ProcessSession	currentSession
private TableInfo	currentTable
static PropertyDescriptor	DATABASE_NAME_PATTERN
private Pattern	databaseNamePattern
private DDLEventWriter	ddlEventWriter
private DeleteRowsWriter	deleteRowsWriter
static PropertyDescriptor	DIST_CACHE_CLIENT
private static int	DO_NOT_SET
static PropertyDescriptor	DRIVER_LOCATION
static PropertyDescriptor	DRIVER_NAME
private BinlogEventListener	eventListener
static PropertyDescriptor	EVENTS_PER_FLOWFILE_STRATEGY
private EventWriterConfiguration	eventWriterConfiguration
private com.github.shyiko.mysql.binlog.GtidSet	gtidSet
private AtomicBoolean	hasRun
static PropertyDescriptor	HOSTS
static PropertyDescriptor	INCLUDE_BEGIN_COMMIT
static PropertyDescriptor	INCLUDE_DDL_EVENTS
private boolean	includeBeginCommit
private boolean	includeDDLEvents
static PropertyDescriptor	INIT_BINLOG_FILENAME
static PropertyDescriptor	INIT_BINLOG_GTID
static PropertyDescriptor	INIT_BINLOG_POSITION
static PropertyDescriptor	INIT_SEQUENCE_ID
private InsertRowsWriter	insertRowsWriter
private boolean	inTransaction
private CaptureChangeMySQL.JDBCConnectionHolder	jdbcConnectionHolder
private BinlogLifecycleListener	lifecycleListener
private static Pattern	MULTI_COMMENT_PATTERN
static PropertyDescriptor	NUMBER_OF_EVENTS_PER_FLOWFILE
static PropertyDescriptor	PASSWORD
private static List<PropertyDescriptor>	propDescriptors
private BlockingQueue<RawBinlogEvent>	queue
static Relationship	REL_SUCCESS
protected static Set<Relationship>	relationships
static PropertyDescriptor	RETRIEVE_ALL_RECORDS
static PropertyDescriptor	SERVER_ID
private boolean	skipTable
static PropertyDescriptor	SSL_CONTEXT_SERVICE
static PropertyDescriptor	SSL_MODE
private static AllowableValue	SSL_MODE_DISABLED
private static AllowableValue	SSL_MODE_PREFERRED
private static AllowableValue	SSL_MODE_REQUIRED
private static AllowableValue	SSL_MODE_VERIFY_IDENTITY
static PropertyDescriptor	STATE_UPDATE_INTERVAL
static PropertyDescriptor	TABLE_NAME_PATTERN
private Pattern	tableNamePattern
private String	transitUri
private UpdateRowsWriter	updateRowsWriter
static PropertyDescriptor	USE_BINLOG_GTID
private boolean	useGtid
static PropertyDescriptor	USERNAME
private String	xactBinlogFile
private long	xactBinlogPosition
private String	xactGtidSet
private long	xactSequenceId

Constructor Summary

Constructors

Constructor and Description
CaptureChangeMySQL()

Method Summary

Modifier and Type	Method and Description
protected void	clearState()
protected void	connect(List<InetSocketAddress> hosts, String username, String password, Long serverId, boolean createEnrichmentConnection, String driverLocation, String driverName, long connectTimeout, SSLContextService sslContextService, com.github.shyiko.mysql.binlog.network.SSLMode sslMode)
protected com.github.shyiko.mysql.binlog.BinaryLogClient	createBinlogClient(String hostname, int port, String username, String password)
(package private) BinlogEventListener	createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client, BlockingQueue<RawBinlogEvent> q) Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.
(package private) BinlogLifecycleListener	createBinlogLifecycleListener() Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.
protected Collection<ValidationResult>	customValidate(ValidationContext validationContext)
private List<InetSocketAddress>	getHosts(String hostsString) Get a list of hosts from a NiFi property, e.g.
protected Connection	getJdbcConnection()
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
protected TableInfo	loadTableInfo(TableInfoCacheKey key) Retrieves the column information for the specified database and table.
protected String	normalizeQuery(String sql)
void	onPrimaryNodeChange(PrimaryNodeState state)
void	onStopped(ProcessContext context)
void	onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory)
void	outputEvents(ProcessSession session, ComponentLog log)
protected void	registerDriver(String locationString, String drvName) using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.
void	setup(ProcessContext context)
protected void	stop()
private void	updateState(ProcessSession session)
private void	updateState(ProcessSession session, String binlogFile, long binlogPosition, long sequenceId, String gtidSet, boolean inTransaction)

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.processor.Processor

isStateful

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

DO_NOT_SET

private static final int DO_NOT_SET

See Also:

Constant Field Values

MULTI_COMMENT_PATTERN

private static final Pattern MULTI_COMMENT_PATTERN

REL_SUCCESS

public static final Relationship REL_SUCCESS

relationships

protected static Set<Relationship> relationships

SSL_MODE_DISABLED

private static final AllowableValue SSL_MODE_DISABLED

SSL_MODE_PREFERRED

private static final AllowableValue SSL_MODE_PREFERRED

SSL_MODE_REQUIRED

private static final AllowableValue SSL_MODE_REQUIRED

SSL_MODE_VERIFY_IDENTITY

private static final AllowableValue SSL_MODE_VERIFY_IDENTITY

DATABASE_NAME_PATTERN

public static final PropertyDescriptor DATABASE_NAME_PATTERN

TABLE_NAME_PATTERN

public static final PropertyDescriptor TABLE_NAME_PATTERN

CONNECT_TIMEOUT

public static final PropertyDescriptor CONNECT_TIMEOUT

HOSTS

public static final PropertyDescriptor HOSTS

DRIVER_NAME

public static final PropertyDescriptor DRIVER_NAME

DRIVER_LOCATION

public static final PropertyDescriptor DRIVER_LOCATION

USERNAME

public static final PropertyDescriptor USERNAME

PASSWORD

public static final PropertyDescriptor PASSWORD

EVENTS_PER_FLOWFILE_STRATEGY

public static final PropertyDescriptor EVENTS_PER_FLOWFILE_STRATEGY

NUMBER_OF_EVENTS_PER_FLOWFILE

public static final PropertyDescriptor NUMBER_OF_EVENTS_PER_FLOWFILE

SERVER_ID

public static final PropertyDescriptor SERVER_ID

DIST_CACHE_CLIENT

public static final PropertyDescriptor DIST_CACHE_CLIENT

RETRIEVE_ALL_RECORDS

public static final PropertyDescriptor RETRIEVE_ALL_RECORDS

INCLUDE_BEGIN_COMMIT

public static final PropertyDescriptor INCLUDE_BEGIN_COMMIT

INCLUDE_DDL_EVENTS

public static final PropertyDescriptor INCLUDE_DDL_EVENTS

STATE_UPDATE_INTERVAL

public static final PropertyDescriptor STATE_UPDATE_INTERVAL

INIT_SEQUENCE_ID

public static final PropertyDescriptor INIT_SEQUENCE_ID

INIT_BINLOG_FILENAME

public static final PropertyDescriptor INIT_BINLOG_FILENAME

INIT_BINLOG_POSITION

public static final PropertyDescriptor INIT_BINLOG_POSITION

USE_BINLOG_GTID

public static final PropertyDescriptor USE_BINLOG_GTID

INIT_BINLOG_GTID

public static final PropertyDescriptor INIT_BINLOG_GTID

SSL_MODE

public static final PropertyDescriptor SSL_MODE

SSL_CONTEXT_SERVICE

public static final PropertyDescriptor SSL_CONTEXT_SERVICE

propDescriptors

private static final List<PropertyDescriptor> propDescriptors

currentSession

private volatile ProcessSession currentSession

binlogClient

private com.github.shyiko.mysql.binlog.BinaryLogClient binlogClient

eventListener

private BinlogEventListener eventListener

lifecycleListener

private BinlogLifecycleListener lifecycleListener

gtidSet

private com.github.shyiko.mysql.binlog.GtidSet gtidSet

queue

private final BlockingQueue<RawBinlogEvent> queue

currentBinlogFile

private volatile String currentBinlogFile

currentBinlogPosition

private volatile long currentBinlogPosition

currentGtidSet

private volatile String currentGtidSet

xactBinlogFile

private volatile String xactBinlogFile

xactBinlogPosition

private volatile long xactBinlogPosition

xactSequenceId

private volatile long xactSequenceId

xactGtidSet

private volatile String xactGtidSet

currentTable

private volatile TableInfo currentTable

currentDatabase

private volatile String currentDatabase

databaseNamePattern

private volatile Pattern databaseNamePattern

tableNamePattern

private volatile Pattern tableNamePattern

includeBeginCommit

private volatile boolean includeBeginCommit

includeDDLEvents

private volatile boolean includeDDLEvents

useGtid

private volatile boolean useGtid

inTransaction

private volatile boolean inTransaction

skipTable

private volatile boolean skipTable

hasRun

private final AtomicBoolean hasRun

currentHost

private int currentHost

transitUri

private String transitUri

currentSequenceId

private final AtomicLong currentSequenceId

cacheClient

private volatile DistributedMapCacheClient cacheClient

cacheKeySerializer

private final Serializer<TableInfoCacheKey> cacheKeySerializer

cacheValueSerializer

private final Serializer<TableInfo> cacheValueSerializer

cacheValueDeserializer

private final Deserializer<TableInfo> cacheValueDeserializer

jdbcConnectionHolder

private CaptureChangeMySQL.JDBCConnectionHolder jdbcConnectionHolder

beginEventWriter

private final BeginTransactionEventWriter beginEventWriter

commitEventWriter

private final CommitTransactionEventWriter commitEventWriter

ddlEventWriter

private final DDLEventWriter ddlEventWriter

insertRowsWriter

private final InsertRowsWriter insertRowsWriter

deleteRowsWriter

private final DeleteRowsWriter deleteRowsWriter

updateRowsWriter

private final UpdateRowsWriter updateRowsWriter

eventWriterConfiguration

private volatile EventWriterConfiguration eventWriterConfiguration

currentEventInfo

private volatile BinlogEventInfo currentEventInfo

currentEventWriter

private AbstractBinlogEventWriter<? extends BinlogEventInfo> currentEventWriter

Constructor Detail

CaptureChangeMySQL

public CaptureChangeMySQL()

Method Detail

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

customValidate

protected Collection<ValidationResult> customValidate(ValidationContext validationContext)

Overrides:

customValidate in class AbstractConfigurableComponent

onPrimaryNodeChange

@OnPrimaryNodeStateChange
public void onPrimaryNodeChange(PrimaryNodeState state)
                                                   throws CDCException

Throws:

CDCException

setup

public void setup(ProcessContext context)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSessionFactory sessionFactory)
               throws ProcessException

Throws:

ProcessException

onStopped

@OnStopped
 @OnShutdown
public void onStopped(ProcessContext context)

getHosts

private List<InetSocketAddress> getHosts(String hostsString)

Get a list of hosts from a NiFi property, e.g.

Parameters:

hostsString - A comma-separated list of hosts (host:port,host2:port2, etc.)

Returns:

List of InetSocketAddresses for the hosts

connect

protected void connect(List<InetSocketAddress> hosts,
                       String username,
                       String password,
                       Long serverId,
                       boolean createEnrichmentConnection,
                       String driverLocation,
                       String driverName,
                       long connectTimeout,
                       SSLContextService sslContextService,
                       com.github.shyiko.mysql.binlog.network.SSLMode sslMode)
                throws IOException

Throws:

IOException

outputEvents

public void outputEvents(ProcessSession session,
                         ComponentLog log)
                  throws IOException

Throws:

IOException

clearState

protected void clearState()
                   throws IOException

Throws:

IOException

normalizeQuery

protected String normalizeQuery(String sql)

stop

protected void stop()
             throws CDCException

Throws:

CDCException

updateState

private void updateState(ProcessSession session)
                  throws IOException

Throws:

IOException

updateState

private void updateState(ProcessSession session,
                         String binlogFile,
                         long binlogPosition,
                         long sequenceId,
                         String gtidSet,
                         boolean inTransaction)
                  throws IOException

Throws:

IOException

createBinlogEventListener

BinlogEventListener createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client,
                                              BlockingQueue<RawBinlogEvent> q)

Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.

Parameters:

client - A reference to a BinaryLogClient. The listener is associated with the given client, such that the listener is notified when events are available to the given client.

q - A queue used to communicate events between the listener and the NiFi processor thread.

Returns:

A BinlogEventListener instance, which will be notified of events associated with the specified client

createBinlogLifecycleListener

BinlogLifecycleListener createBinlogLifecycleListener()

Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.

Returns:

A BinlogLifecycleListener instance, which will be notified of events associated with the specified client

createBinlogClient

protected com.github.shyiko.mysql.binlog.BinaryLogClient createBinlogClient(String hostname,
                                                                            int port,
                                                                            String username,
                                                                            String password)

loadTableInfo

protected TableInfo loadTableInfo(TableInfoCacheKey key)
                           throws SQLException

Retrieves the column information for the specified database and table. The column information can be used to enrich CDC events coming from the RDBMS.

Parameters:

key - A TableInfoCacheKey reference, which contains the database and table names

Returns:

A TableInfo instance with the ColumnDefinitions provided (if retrieved successfully from the database)

Throws:

SQLException

getJdbcConnection

protected Connection getJdbcConnection()
                                throws SQLException

Throws:

SQLException

registerDriver

protected void registerDriver(String locationString,
                              String drvName)
                       throws InitializationException

using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.

Throws:

InitializationException - if there is a problem obtaining the ClassLoader