org.apache.flink.streaming.connectors.pulsar

Class FlinkPulsarSink<T>

java.lang.Object

org.apache.flink.api.common.functions.AbstractRichFunction

org.apache.flink.streaming.api.functions.sink.RichSinkFunction<T>

org.apache.flink.streaming.connectors.pulsar.FlinkPulsarSink<T>

Type Parameters:

T - Type of the Pojo class.

All Implemented Interfaces:

Serializable, org.apache.flink.api.common.functions.Function, org.apache.flink.api.common.functions.RichFunction, org.apache.flink.streaming.api.checkpoint.CheckpointedFunction, org.apache.flink.streaming.api.functions.sink.SinkFunction<T>

public class FlinkPulsarSink<T>
extends org.apache.flink.streaming.api.functions.sink.RichSinkFunction<T>

Write Pojo class to Flink.

See Also:

Serialized Form

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.flink.streaming.api.functions.sink.SinkFunction

org.apache.flink.streaming.api.functions.sink.SinkFunction.Context<T>

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.pulsar.client.admin.PulsarAdmin	admin
protected String	adminUrl
protected Map<String,String>	caseInsensitiveParams
protected org.apache.pulsar.client.impl.conf.ClientConfigurationData	clientConfigurationData
protected String	defaultTopic
protected Throwable	failedWrite
protected boolean	failOnWrite
protected boolean	flushOnCheckpoint
protected boolean	forcedTopic
protected long	pendingRecords Number of unacknowledged records.
protected org.apache.flink.util.SerializableObject	pendingRecordsLock Lock for accessing the pending records.
protected Map<String,Object>	producerConf
protected Properties	properties
protected BiConsumer<org.apache.pulsar.client.api.MessageId,Throwable>	sendCallback
protected org.apache.pulsar.client.api.Producer<byte[]>	singleProducer
protected Map<String,org.apache.pulsar.client.api.Producer<byte[]>>	topic2Producer

Constructor Summary

Constructors

Constructor and Description
FlinkPulsarSink(String adminUrl, Optional<String> defaultTopicName, org.apache.pulsar.client.impl.conf.ClientConfigurationData clientConf, Properties properties, PulsarSerializationSchema serializationSchema)
FlinkPulsarSink(String serviceUrl, String adminUrl, Optional<String> defaultTopicName, Properties properties, PulsarSerializationSchema<T> serializationSchema)

Method Summary

Modifier and Type	Method and Description
protected void	checkErroneous()
void	close()
protected org.apache.pulsar.client.api.Producer<byte[]>	createProducer(org.apache.pulsar.client.impl.conf.ClientConfigurationData clientConf, Map<String,Object> producerConf, String topic) To determine how to generate schema for pulsar producer
protected org.apache.pulsar.client.api.Producer<byte[]>	getProducer(String topic)
protected void	initializeSendCallback()
void	initializeState(org.apache.flink.runtime.state.FunctionInitializationContext context)
void	invoke(T value, org.apache.flink.streaming.api.functions.sink.SinkFunction.Context context)
void	open(org.apache.flink.configuration.Configuration parameters)
protected void	producerClose()
void	producerFlush()
void	snapshotState(org.apache.flink.runtime.state.FunctionSnapshotContext context)

Methods inherited from class org.apache.flink.api.common.functions.AbstractRichFunction

getIterationRuntimeContext, getRuntimeContext, setRuntimeContext

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.flink.streaming.api.functions.sink.SinkFunction

invoke

Field Detail

adminUrl

protected String adminUrl

clientConfigurationData

protected org.apache.pulsar.client.impl.conf.ClientConfigurationData clientConfigurationData

caseInsensitiveParams

protected final Map<String,String> caseInsensitiveParams

producerConf

protected final Map<String,Object> producerConf

properties

protected final Properties properties

flushOnCheckpoint

protected boolean flushOnCheckpoint

failOnWrite

protected boolean failOnWrite

pendingRecordsLock

protected final org.apache.flink.util.SerializableObject pendingRecordsLock

Lock for accessing the pending records.

pendingRecords

protected long pendingRecords

Number of unacknowledged records.

forcedTopic

protected final boolean forcedTopic

defaultTopic

protected final String defaultTopic

failedWrite

protected transient volatile Throwable failedWrite

admin

protected transient org.apache.pulsar.client.admin.PulsarAdmin admin

sendCallback

protected transient BiConsumer<org.apache.pulsar.client.api.MessageId,Throwable> sendCallback

singleProducer

protected transient org.apache.pulsar.client.api.Producer<byte[]> singleProducer

topic2Producer

protected transient Map<String,org.apache.pulsar.client.api.Producer<byte[]>> topic2Producer

Constructor Detail

FlinkPulsarSink

public FlinkPulsarSink(String adminUrl,
                       Optional<String> defaultTopicName,
                       org.apache.pulsar.client.impl.conf.ClientConfigurationData clientConf,
                       Properties properties,
                       PulsarSerializationSchema serializationSchema)

FlinkPulsarSink

public FlinkPulsarSink(String serviceUrl,
                       String adminUrl,
                       Optional<String> defaultTopicName,
                       Properties properties,
                       PulsarSerializationSchema<T> serializationSchema)

Method Detail

invoke

public void invoke(T value,
                   org.apache.flink.streaming.api.functions.sink.SinkFunction.Context context)
            throws Exception

Throws:

Exception

snapshotState

public void snapshotState(org.apache.flink.runtime.state.FunctionSnapshotContext context)
                   throws Exception

Specified by:

snapshotState in interface org.apache.flink.streaming.api.checkpoint.CheckpointedFunction

Throws:

Exception

initializeState

public void initializeState(org.apache.flink.runtime.state.FunctionInitializationContext context)
                     throws Exception

Specified by:

initializeState in interface org.apache.flink.streaming.api.checkpoint.CheckpointedFunction

Throws:

Exception

open

public void open(org.apache.flink.configuration.Configuration parameters)
          throws Exception

Specified by:

open in interface org.apache.flink.api.common.functions.RichFunction

Overrides:

open in class org.apache.flink.api.common.functions.AbstractRichFunction

Throws:

Exception

initializeSendCallback

protected void initializeSendCallback()

close

public void close()
           throws Exception

Specified by:

close in interface org.apache.flink.api.common.functions.RichFunction

Overrides:

close in class org.apache.flink.api.common.functions.AbstractRichFunction

Throws:

Exception

getProducer

protected org.apache.pulsar.client.api.Producer<byte[]> getProducer(String topic)

createProducer

protected org.apache.pulsar.client.api.Producer<byte[]> createProducer(org.apache.pulsar.client.impl.conf.ClientConfigurationData clientConf,
                                                                       Map<String,Object> producerConf,
                                                                       String topic)

To determine how to generate schema for pulsar producer

Returns:

the schema that our producer use.

producerFlush

public void producerFlush()
                   throws Exception

Throws:

Exception

producerClose

protected void producerClose()
                      throws Exception

Throws:

Exception

checkErroneous

protected void checkErroneous()
                       throws Exception

Throws:

Exception