@TriggerSerially @InputRequirement(value=INPUT_FORBIDDEN) @Tags(value={"sql","jdbc","cdc","mysql"}) @CapabilityDescription(value="Retrieves Change Data Capture (CDC) events from a MySQL database. CDC Events include INSERT, UPDATE, DELETE operations. Events are output as individual flow files ordered by the time at which the operation occurred.") @Stateful(scopes=CLUSTER, description="Information such as a \'pointer\' to the current CDC event in the database is stored by this processor, such that it can continue from the same location if restarted.") @WritesAttribute(attribute="cdc.sequence.id",description="A sequence identifier (i.e. strictly increasing integer value) specifying the order of the CDC event flow file relative to the other event flow file(s).") @WritesAttribute(attribute="cdc.event.type",description="A string indicating the type of CDC event that occurred, including (but not limited to) \'begin\', \'insert\', \'update\', \'delete\', \'ddl\' and \'commit\'.") @WritesAttribute(attribute="mime.type",description="The processor outputs flow file content in JSON format, and sets the mime.type attribute to application/json") public class CaptureChangeMySQL extends AbstractSessionFactoryProcessor
| Modifier and Type | Class and Description |
|---|---|
private static class |
CaptureChangeMySQL.DriverShim |
| Constructor and Description |
|---|
CaptureChangeMySQL() |
| Modifier and Type | Method and Description |
|---|---|
protected void |
connect(List<InetSocketAddress> hosts,
String username,
String password,
Long serverId,
boolean createEnrichmentConnection,
String driverLocation,
String driverName,
long connectTimeout) |
(package private) com.github.shyiko.mysql.binlog.BinaryLogClient |
createBinlogClient(String hostname,
int port,
String username,
String password) |
(package private) BinlogEventListener |
createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client,
LinkedBlockingQueue<RawBinlogEvent> q)
Creates and returns a BinlogEventListener instance, associated with the specified binlog client and event queue.
|
(package private) BinlogLifecycleListener |
createBinlogLifecycleListener()
Creates and returns a BinlogLifecycleListener instance, associated with the specified binlog client and event queue.
|
private List<InetSocketAddress> |
getHosts(String hostsString)
Get a list of hosts from a NiFi property, e.g.
|
protected Connection |
getJdbcConnection(String locationString,
String drvName,
InetSocketAddress host,
String username,
String password,
Map<String,String> customProperties)
using Thread.currentThread().getContextClassLoader(); will ensure that you are using the ClassLoader for you NAR.
|
Set<Relationship> |
getRelationships() |
protected List<PropertyDescriptor> |
getSupportedPropertyDescriptors() |
protected TableInfo |
loadTableInfo(TableInfoCacheKey key)
Retrieves the column information for the specified database and table.
|
void |
onShutdown(ProcessContext context) |
void |
onStopped(ProcessContext context) |
void |
onTrigger(ProcessContext context,
ProcessSessionFactory sessionFactory) |
void |
outputEvents(ProcessSession session,
StateManager stateManager,
ComponentLog log) |
protected void |
registerDriver(String locationString,
String drvName) |
void |
setup(ProcessContext context) |
protected void |
stop(StateManager stateManager) |
private void |
updateState(StateManager stateManager,
String binlogFile,
long binlogPosition,
long sequenceId) |
getControllerServiceLookup, getIdentifier, getLogger, getNodeTypeProvider, init, initialize, isConfigurationRestored, isScheduled, toString, updateConfiguredRestoredTrue, updateScheduledFalse, updateScheduledTruecustomValidate, equals, getPropertyDescriptor, getPropertyDescriptors, getSupportedDynamicPropertyDescriptor, hashCode, onPropertyModified, validateclone, finalize, getClass, notify, notifyAll, wait, wait, waitgetPropertyDescriptor, getPropertyDescriptors, onPropertyModified, validateprivate static final int DO_NOT_SET
public static final Relationship REL_SUCCESS
protected static Set<Relationship> relationships
public static final PropertyDescriptor DATABASE_NAME_PATTERN
public static final PropertyDescriptor TABLE_NAME_PATTERN
public static final PropertyDescriptor CONNECT_TIMEOUT
public static final PropertyDescriptor HOSTS
public static final PropertyDescriptor DRIVER_NAME
public static final PropertyDescriptor DRIVER_LOCATION
public static final PropertyDescriptor USERNAME
public static final PropertyDescriptor PASSWORD
public static final PropertyDescriptor SERVER_ID
public static final PropertyDescriptor DIST_CACHE_CLIENT
public static final PropertyDescriptor RETRIEVE_ALL_RECORDS
public static final PropertyDescriptor INCLUDE_BEGIN_COMMIT
public static final PropertyDescriptor INCLUDE_DDL_EVENTS
public static final PropertyDescriptor STATE_UPDATE_INTERVAL
public static final PropertyDescriptor INIT_SEQUENCE_ID
public static final PropertyDescriptor INIT_BINLOG_FILENAME
public static final PropertyDescriptor INIT_BINLOG_POSITION
private static List<PropertyDescriptor> propDescriptors
private volatile ProcessSession currentSession
private com.github.shyiko.mysql.binlog.BinaryLogClient binlogClient
private BinlogEventListener eventListener
private BinlogLifecycleListener lifecycleListener
private volatile LinkedBlockingQueue<RawBinlogEvent> queue
private volatile String currentBinlogFile
private volatile long currentBinlogPosition
private volatile String xactBinlogFile
private volatile long xactBinlogPosition
private volatile long xactSequenceId
private volatile TableInfo currentTable
private volatile String currentDatabase
private volatile Pattern databaseNamePattern
private volatile Pattern tableNamePattern
private volatile boolean includeBeginCommit
private volatile boolean includeDDLEvents
private volatile boolean inTransaction
private volatile boolean skipTable
private AtomicBoolean doStop
private AtomicBoolean hasRun
private int currentHost
private String transitUri
private volatile long lastStateUpdate
private volatile long stateUpdateInterval
private AtomicLong currentSequenceId
private volatile DistributedMapCacheClient cacheClient
private final Serializer<TableInfoCacheKey> cacheKeySerializer
private final Serializer<TableInfo> cacheValueSerializer
private final Deserializer<TableInfo> cacheValueDeserializer
private Connection jdbcConnection
private final BeginTransactionEventWriter beginEventWriter
private final CommitTransactionEventWriter commitEventWriter
private final DDLEventWriter ddlEventWriter
private final InsertRowsWriter insertRowsWriter
private final DeleteRowsWriter deleteRowsWriter
private final UpdateRowsWriter updateRowsWriter
public Set<Relationship> getRelationships()
getRelationships in interface ProcessorgetRelationships in class AbstractSessionFactoryProcessorprotected List<PropertyDescriptor> getSupportedPropertyDescriptors()
getSupportedPropertyDescriptors in class AbstractConfigurableComponentpublic void setup(ProcessContext context)
public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException
ProcessException@OnStopped public void onStopped(ProcessContext context)
@OnShutdown public void onShutdown(ProcessContext context)
private List<InetSocketAddress> getHosts(String hostsString)
hostsString - A comma-separated list of hosts (host:port,host2:port2, etc.)protected void connect(List<InetSocketAddress> hosts, String username, String password, Long serverId, boolean createEnrichmentConnection, String driverLocation, String driverName, long connectTimeout) throws IOException
IOExceptionpublic void outputEvents(ProcessSession session, StateManager stateManager, ComponentLog log) throws IOException
IOExceptionprotected void stop(StateManager stateManager) throws CDCException
CDCExceptionprivate void updateState(StateManager stateManager, String binlogFile, long binlogPosition, long sequenceId) throws IOException
IOExceptionBinlogEventListener createBinlogEventListener(com.github.shyiko.mysql.binlog.BinaryLogClient client, LinkedBlockingQueue<RawBinlogEvent> q)
client - A reference to a BinaryLogClient. The listener is associated with the given client, such that the listener is notified when
events are available to the given client.q - A queue used to communicate events between the listener and the NiFi processor thread.BinlogLifecycleListener createBinlogLifecycleListener()
com.github.shyiko.mysql.binlog.BinaryLogClient createBinlogClient(String hostname, int port, String username, String password)
protected TableInfo loadTableInfo(TableInfoCacheKey key) throws SQLException
key - A TableInfoCacheKey reference, which contains the database and table namesSQLExceptionprotected Connection getJdbcConnection(String locationString, String drvName, InetSocketAddress host, String username, String password, Map<String,String> customProperties) throws InitializationException, SQLException
InitializationException - if there is a problem obtaining the ClassLoaderSQLExceptionprotected void registerDriver(String locationString, String drvName) throws InitializationException
InitializationExceptionCopyright © 2018 Apache NiFi Project. All rights reserved.