org.apache.nifi.cdc.mysql.processors

Class CaptureChangeMySQL

java.lang.Object

org.apache.nifi.components.AbstractConfigurableComponent

org.apache.nifi.processor.AbstractSessionFactoryProcessor

org.apache.nifi.cdc.mysql.processors.CaptureChangeMySQL

All Implemented Interfaces:

ConfigurableComponent, Processor

@TriggerSerially
 @InputRequirement(value=INPUT_FORBIDDEN)
 @Tags(value={"sql","jdbc","cdc","mysql"})
 @CapabilityDescription(value="Retrieves Change Data Capture (CDC) events from a MySQL database. CDC Events include INSERT, UPDATE, DELETE operations. Events are output as individual flow files ordered by the time at which the operation occurred.")
 @Stateful(scopes=CLUSTER,
          description="Information such as a \'pointer\' to the current CDC event in the database is stored by this processor, such that it can continue from the same location if restarted.")
 @WritesAttribute(attribute="cdc.sequence.id",description="A sequence identifier (i.e. strictly increasing integer value) specifying the order of the CDC event flow file relative to the other event flow file(s).") @WritesAttribute(attribute="cdc.event.type",description="A string indicating the type of CDC event that occurred, including (but not limited to) \'begin\', \'insert\', \'update\', \'delete\', \'ddl\' and \'commit\'.") @WritesAttribute(attribute="mime.type",description="The processor outputs flow file content in JSON format, and sets the mime.type attribute to application/json")
public class CaptureChangeMySQL
extends AbstractSessionFactoryProcessor

A processor to retrieve Change Data Capture (CDC) events and send them as flow files.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
private static class	CaptureChangeMySQL.DriverShim

Field Summary

Fields

Modifier and Type	Field and Description
private BeginTransactionEventWriter	beginEventWriter
private com.github.shyiko.mysql.binlog.BinaryLogClient	binlogClient
private DistributedMapCacheClient	cacheClient
private Serializer<TableInfoCacheKey>	cacheKeySerializer
private Deserializer<TableInfo>	cacheValueDeserializer
private Serializer<TableInfo>	cacheValueSerializer
private CommitTransactionEventWriter	commitEventWriter
static PropertyDescriptor	CONNECT_TIMEOUT
private String	currentBinlogFile
private long	currentBinlogPosition
private String	currentDatabase
private int	currentHost
private AtomicLong	currentSequenceId
private ProcessSession	currentSession
private TableInfo	currentTable
static PropertyDescriptor	DATABASE_NAME_PATTERN
private Pattern	databaseNamePattern
private DDLEventWriter	ddlEventWriter
private DeleteRowsWriter	deleteRowsWriter
static PropertyDescriptor	DIST_CACHE_CLIENT
private static int	DO_NOT_SET
private AtomicBoolean	doStop
static PropertyDescriptor	DRIVER_LOCATION
static PropertyDescriptor	DRIVER_NAME
private BinlogEventListener	eventListener
private AtomicBoolean	hasRun
static PropertyDescriptor	HOSTS
static PropertyDescriptor	INCLUDE_BEGIN_COMMIT
static PropertyDescriptor	INCLUDE_DDL_EVENTS
private boolean	includeBeginCommit
private boolean	includeDDLEvents
static PropertyDescriptor	INIT_BINLOG_FILENAME
static PropertyDescriptor	INIT_BINLOG_POSITION
static PropertyDescriptor	INIT_SEQUENCE_ID
private InsertRowsWriter	insertRowsWriter
private boolean	inTransaction
private Connection	jdbcConnection
private long	lastStateUpdate
private BinlogLifecycleListener	lifecycleListener
static PropertyDescriptor	PASSWORD
private static List<PropertyDescriptor>	propDescriptors
private LinkedBlockingQueue<RawBinlogEvent>	queue
static Relationship	REL_SUCCESS
protected static Set<Relationship>	relationships
static PropertyDescriptor	RETRIEVE_ALL_RECORDS
static PropertyDescriptor	SERVER_ID
private boolean	skipTable
static PropertyDescriptor	STATE_UPDATE_INTERVAL
private long	stateUpdateInterval
static PropertyDescriptor	TABLE_NAME_PATTERN
private Pattern	tableNamePattern
private String	transitUri
private UpdateRowsWriter	updateRowsWriter
static PropertyDescriptor	USERNAME
private String	xactBinlogFile
private long	xactBinlogPosition
private long	xactSequenceId

Constructor Summary

Constructors

Constructor and Description
CaptureChangeMySQL()

Method Summary

Modifier and Type	Method and Description
protected void	connect(List<InetSocketAddress> hosts, String username, String password, Long serverId, boolean createEnrichmentConnection, String driverLocation, String driverName, long connectTimeout)
(package private) com.github.shyiko.mysql.binlog.BinaryLogClient	createBinlogClient(String hostname, int port, String username, String password)
(package private) BinlogEventListener	createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client, LinkedBlockingQueue<RawBinlogEvent> q) Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.
(package private) BinlogLifecycleListener	createBinlogLifecycleListener() Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.
private List<InetSocketAddress>	getHosts(String hostsString) Get a list of hosts from a NiFi property, e.g.
protected Connection	getJdbcConnection(String locationString, String drvName, InetSocketAddress host, String username, String password, Map<String,String> customProperties) using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.
Set<Relationship>	getRelationships()
protected List<PropertyDescriptor>	getSupportedPropertyDescriptors()
protected TableInfo	loadTableInfo(TableInfoCacheKey key) Retrieves the column information for the specified database and table.
void	onShutdown(ProcessContext context)
void	onStopped(ProcessContext context)
void	onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory)
void	outputEvents(ProcessSession session, StateManager stateManager, ComponentLog log)
protected void	registerDriver(String locationString, String drvName)
void	setup(ProcessContext context)
protected void	stop(StateManager stateManager)
private void	updateState(StateManager stateManager, String binlogFile, long binlogPosition, long sequenceId)

Methods inherited from class org.apache.nifi.processor.AbstractSessionFactoryProcessor

getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTrue

Methods inherited from class org.apache.nifi.components.AbstractConfigurableComponent

customValidate, equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validate

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.nifi.components.ConfigurableComponent

getPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validate

Field Detail

DO_NOT_SET

private static final int DO_NOT_SET

See Also:

Constant Field Values

REL_SUCCESS

public static final Relationship REL_SUCCESS

relationships

protected static Set<Relationship> relationships

DATABASE_NAME_PATTERN

public static final PropertyDescriptor DATABASE_NAME_PATTERN

TABLE_NAME_PATTERN

public static final PropertyDescriptor TABLE_NAME_PATTERN

CONNECT_TIMEOUT

public static final PropertyDescriptor CONNECT_TIMEOUT

HOSTS

public static final PropertyDescriptor HOSTS

DRIVER_NAME

public static final PropertyDescriptor DRIVER_NAME

DRIVER_LOCATION

public static final PropertyDescriptor DRIVER_LOCATION

USERNAME

public static final PropertyDescriptor USERNAME

PASSWORD

public static final PropertyDescriptor PASSWORD

SERVER_ID

public static final PropertyDescriptor SERVER_ID

DIST_CACHE_CLIENT

public static final PropertyDescriptor DIST_CACHE_CLIENT

RETRIEVE_ALL_RECORDS

public static final PropertyDescriptor RETRIEVE_ALL_RECORDS

INCLUDE_BEGIN_COMMIT

public static final PropertyDescriptor INCLUDE_BEGIN_COMMIT

INCLUDE_DDL_EVENTS

public static final PropertyDescriptor INCLUDE_DDL_EVENTS

STATE_UPDATE_INTERVAL

public static final PropertyDescriptor STATE_UPDATE_INTERVAL

INIT_SEQUENCE_ID

public static final PropertyDescriptor INIT_SEQUENCE_ID

INIT_BINLOG_FILENAME

public static final PropertyDescriptor INIT_BINLOG_FILENAME

INIT_BINLOG_POSITION

public static final PropertyDescriptor INIT_BINLOG_POSITION

propDescriptors

private static List<PropertyDescriptor> propDescriptors

currentSession

private volatile ProcessSession currentSession

binlogClient

private com.github.shyiko.mysql.binlog.BinaryLogClient binlogClient

eventListener

private BinlogEventListener eventListener

lifecycleListener

private BinlogLifecycleListener lifecycleListener

queue

private volatile LinkedBlockingQueue<RawBinlogEvent> queue

currentBinlogFile

private volatile String currentBinlogFile

currentBinlogPosition

private volatile long currentBinlogPosition

xactBinlogFile

private volatile String xactBinlogFile

xactBinlogPosition

private volatile long xactBinlogPosition

xactSequenceId

private volatile long xactSequenceId

currentTable

private volatile TableInfo currentTable

currentDatabase

private volatile String currentDatabase

databaseNamePattern

private volatile Pattern databaseNamePattern

tableNamePattern

private volatile Pattern tableNamePattern

includeBeginCommit

private volatile boolean includeBeginCommit

includeDDLEvents

private volatile boolean includeDDLEvents

inTransaction

private volatile boolean inTransaction

skipTable

private volatile boolean skipTable

doStop

private AtomicBoolean doStop

hasRun

private AtomicBoolean hasRun

currentHost

private int currentHost

transitUri

private String transitUri

lastStateUpdate

private volatile long lastStateUpdate

stateUpdateInterval

private volatile long stateUpdateInterval

currentSequenceId

private AtomicLong currentSequenceId

cacheClient

private volatile DistributedMapCacheClient cacheClient

cacheKeySerializer

private final Serializer<TableInfoCacheKey> cacheKeySerializer

cacheValueSerializer

private final Serializer<TableInfo> cacheValueSerializer

cacheValueDeserializer

private final Deserializer<TableInfo> cacheValueDeserializer

jdbcConnection

private Connection jdbcConnection

beginEventWriter

private final BeginTransactionEventWriter beginEventWriter

commitEventWriter

private final CommitTransactionEventWriter commitEventWriter

ddlEventWriter

private final DDLEventWriter ddlEventWriter

insertRowsWriter

private final InsertRowsWriter insertRowsWriter

deleteRowsWriter

private final DeleteRowsWriter deleteRowsWriter

updateRowsWriter

private final UpdateRowsWriter updateRowsWriter

Constructor Detail

CaptureChangeMySQL

public CaptureChangeMySQL()

Method Detail

getRelationships

public Set<Relationship> getRelationships()

Specified by:

getRelationships in interface Processor

Overrides:

getRelationships in class AbstractSessionFactoryProcessor

getSupportedPropertyDescriptors

protected List<PropertyDescriptor> getSupportedPropertyDescriptors()

Overrides:

getSupportedPropertyDescriptors in class AbstractConfigurableComponent

setup

public void setup(ProcessContext context)

onTrigger

public void onTrigger(ProcessContext context,
                      ProcessSessionFactory sessionFactory)
               throws ProcessException

Throws:

ProcessException

onStopped

@OnStopped
public void onStopped(ProcessContext context)

onShutdown

@OnShutdown
public void onShutdown(ProcessContext context)

getHosts

private List<InetSocketAddress> getHosts(String hostsString)

Get a list of hosts from a NiFi property, e.g.

Parameters:

hostsString - A comma-separated list of hosts (host:port,host2:port2, etc.)

Returns:

List of InetSocketAddresses for the hosts

connect

protected void connect(List<InetSocketAddress> hosts,
                       String username,
                       String password,
                       Long serverId,
                       boolean createEnrichmentConnection,
                       String driverLocation,
                       String driverName,
                       long connectTimeout)
                throws IOException

Throws:

IOException

outputEvents

public void outputEvents(ProcessSession session,
                         StateManager stateManager,
                         ComponentLog log)
                  throws IOException

Throws:

IOException

stop

protected void stop(StateManager stateManager)
             throws CDCException

Throws:

CDCException

updateState

private void updateState(StateManager stateManager,
                         String binlogFile,
                         long binlogPosition,
                         long sequenceId)
                  throws IOException

Throws:

IOException

createBinlogEventListener

BinlogEventListener createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client,
                                              LinkedBlockingQueue<RawBinlogEvent> q)

Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.

Parameters:

client - A reference to a BinaryLogClient. The listener is associated with the given client, such that the listener is notified when events are available to the given client.

q - A queue used to communicate events between the listener and the NiFi processor thread.

Returns:

A BinlogEventListener instance, which will be notified of events associated with the specified client

createBinlogLifecycleListener

BinlogLifecycleListener createBinlogLifecycleListener()

Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.

Returns:

A BinlogLifecycleListener instance, which will be notified of events associated with the specified client

createBinlogClient

com.github.shyiko.mysql.binlog.BinaryLogClient createBinlogClient(String hostname,
                                                                  int port,
                                                                  String username,
                                                                  String password)

loadTableInfo

protected TableInfo loadTableInfo(TableInfoCacheKey key)
                           throws SQLException

Retrieves the column information for the specified database and table. The column information can be used to enrich CDC events coming from the RDBMS.

Parameters:

key - A TableInfoCacheKey reference, which contains the database and table names

Returns:

A TableInfo instance with the ColumnDefinitions provided (if retrieved successfully from the database)

Throws:

SQLException

getJdbcConnection

protected Connection getJdbcConnection(String locationString,
                                       String drvName,
                                       InetSocketAddress host,
                                       String username,
                                       String password,
                                       Map<String,String> customProperties)
                                throws InitializationException,
                                       SQLException

using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.

Throws:

InitializationException - if there is a problem obtaining the ClassLoader

SQLException

registerDriver

protected void registerDriver(String locationString,
                              String drvName)
                       throws InitializationException

Throws:

InitializationException