org.apache.druid.indexing.kafka.supervisor

Class KafkaSupervisor

java.lang.Object

org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor<Integer,Long>

org.apache.druid.indexing.kafka.supervisor.KafkaSupervisor

All Implemented Interfaces:

Supervisor

public class KafkaSupervisor
extends SeekableStreamSupervisor<Integer,Long>

Supervisor responsible for managing the KafkaIndexTasks for a single dataSource. At a high level, the class accepts a KafkaSupervisorSpec which includes the Kafka topic and configuration as well as an ingestion spec which will be used to generate the indexing tasks. The run loop periodically refreshes its view of the Kafka topic's partitions and the list of running indexing tasks and ensures that all partitions are being read from and that there are enough tasks to satisfy the desired number of replicas. As tasks complete, new tasks are queued to process the next range of Kafka offsets.

Nested Class Summary

Nested classes/interfaces inherited from class org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor

SeekableStreamSupervisor.CheckpointNotice

Field Summary

Fields

Modifier and Type	Field and Description
static com.fasterxml.jackson.core.type.TypeReference<TreeMap<Integer,Map<Integer,Long>>>	CHECKPOINTS_TYPE_REF

Fields inherited from class org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor

CHECKPOINTS_CTX_KEY, IS_INCREMENTAL_HANDOFF_SUPPORTED, partitionIds, sequenceLastUpdated, sortingMapper

Constructor Summary

Constructors

Constructor and Description
KafkaSupervisor(TaskStorage taskStorage, TaskMaster taskMaster, IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator, KafkaIndexTaskClientFactory taskClientFactory, com.fasterxml.jackson.databind.ObjectMapper mapper, KafkaSupervisorSpec spec, RowIngestionMetersFactory rowIngestionMetersFactory)

Method Summary

Modifier and Type	Method and Description
protected String	baseTaskName()
protected boolean	checkSourceMetadataMatch(DataSourceMetadata metadata)
protected KafkaDataSourceMetadata	createDataSourceMetaDataForReset(String topic, Map<Integer,Long> map)
protected List<SeekableStreamIndexTask<Integer,Long>>	createIndexTasks(int replicas, String baseSequenceName, com.fasterxml.jackson.databind.ObjectMapper sortingMapper, TreeMap<Integer,Map<Integer,Long>> sequenceOffsets, SeekableStreamIndexTaskIOConfig taskIoConfig, SeekableStreamIndexTaskTuningConfig taskTuningConfig, RowIngestionMetersFactory rowIngestionMetersFactory)
protected SeekableStreamSupervisorReportPayload<Integer,Long>	createReportPayload(int numPartitions, boolean includeOffsets)
protected SeekableStreamIndexTaskIOConfig	createTaskIoConfig(int groupId, Map<Integer,Long> startPartitions, Map<Integer,Long> endPartitions, String baseSequenceName, org.joda.time.DateTime minimumMessageTime, org.joda.time.DateTime maximumMessageTime, Set<Integer> exclusiveStartSequenceNumberPartitions, SeekableStreamSupervisorIOConfig ioConfig)
protected boolean	doesTaskTypeMatchSupervisor(Task task)
protected Long	getEndOfPartitionMarker()
KafkaSupervisorIOConfig	getIoConfig()
protected Map<Integer,Long>	getLagPerPartition(Map<Integer,Long> currentOffsets)
protected Long	getNotSetMarker()
protected int	getTaskGroupIdForPartition(Integer partition)
protected boolean	isEndOfShard(Long seqNum)
protected OrderedSequenceNumber<Long>	makeSequenceNumber(Long seq, boolean isExclusive)
protected void	scheduleReporting(ScheduledExecutorService reportingExec)
protected RecordSupplier<Integer,Long>	setupRecordSupplier()
protected void	updateLatestSequenceFromStream(RecordSupplier<Integer,Long> recordSupplier, Set<StreamPartition<Integer>> partitions)
protected boolean	useExclusiveStartSequenceNumberForNonFirstSequence()

Methods inherited from class org.apache.druid.indexing.seekablestream.supervisor.SeekableStreamSupervisor

addTaskGroupToActivelyReadingTaskGroup, addTaskGroupToPendingCompletionTaskGroup, checkpoint, createBaseTaskContexts, generateSequenceName, getHighestCurrentOffsets, getInitRetryCounter, getNoticesQueueSize, getStats, getStatus, gracefulShutdownInternal, isLifecycleStarted, isStarted, moveTaskGroupToPendingCompletion, reset, resetInternal, runInternal, start, stop, tryInit, updateCurrentAndLatestOffsets

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

CHECKPOINTS_TYPE_REF

public static final com.fasterxml.jackson.core.type.TypeReference<TreeMap<Integer,Map<Integer,Long>>> CHECKPOINTS_TYPE_REF

Constructor Detail

KafkaSupervisor

public KafkaSupervisor(TaskStorage taskStorage,
                       TaskMaster taskMaster,
                       IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator,
                       KafkaIndexTaskClientFactory taskClientFactory,
                       com.fasterxml.jackson.databind.ObjectMapper mapper,
                       KafkaSupervisorSpec spec,
                       RowIngestionMetersFactory rowIngestionMetersFactory)

Method Detail

setupRecordSupplier

protected RecordSupplier<Integer,Long> setupRecordSupplier()

Specified by:

setupRecordSupplier in class SeekableStreamSupervisor<Integer,Long>

scheduleReporting

protected void scheduleReporting(ScheduledExecutorService reportingExec)

Specified by:

scheduleReporting in class SeekableStreamSupervisor<Integer,Long>

getTaskGroupIdForPartition

protected int getTaskGroupIdForPartition(Integer partition)

Specified by:

getTaskGroupIdForPartition in class SeekableStreamSupervisor<Integer,Long>

checkSourceMetadataMatch

protected boolean checkSourceMetadataMatch(DataSourceMetadata metadata)

Specified by:

checkSourceMetadataMatch in class SeekableStreamSupervisor<Integer,Long>

doesTaskTypeMatchSupervisor

protected boolean doesTaskTypeMatchSupervisor(Task task)

Specified by:

doesTaskTypeMatchSupervisor in class SeekableStreamSupervisor<Integer,Long>

createReportPayload

protected SeekableStreamSupervisorReportPayload<Integer,Long> createReportPayload(int numPartitions,
                                                                                  boolean includeOffsets)

Specified by:

createReportPayload in class SeekableStreamSupervisor<Integer,Long>

createTaskIoConfig

protected SeekableStreamIndexTaskIOConfig createTaskIoConfig(int groupId,
                                                             Map<Integer,Long> startPartitions,
                                                             Map<Integer,Long> endPartitions,
                                                             String baseSequenceName,
                                                             org.joda.time.DateTime minimumMessageTime,
                                                             org.joda.time.DateTime maximumMessageTime,
                                                             Set<Integer> exclusiveStartSequenceNumberPartitions,
                                                             SeekableStreamSupervisorIOConfig ioConfig)

Specified by:

createTaskIoConfig in class SeekableStreamSupervisor<Integer,Long>

createIndexTasks

protected List<SeekableStreamIndexTask<Integer,Long>> createIndexTasks(int replicas,
                                                                       String baseSequenceName,
                                                                       com.fasterxml.jackson.databind.ObjectMapper sortingMapper,
                                                                       TreeMap<Integer,Map<Integer,Long>> sequenceOffsets,
                                                                       SeekableStreamIndexTaskIOConfig taskIoConfig,
                                                                       SeekableStreamIndexTaskTuningConfig taskTuningConfig,
                                                                       RowIngestionMetersFactory rowIngestionMetersFactory)
                                                                throws com.fasterxml.jackson.core.JsonProcessingException

Specified by:

createIndexTasks in class SeekableStreamSupervisor<Integer,Long>

Throws:

com.fasterxml.jackson.core.JsonProcessingException

getLagPerPartition

protected Map<Integer,Long> getLagPerPartition(Map<Integer,Long> currentOffsets)

Specified by:

getLagPerPartition in class SeekableStreamSupervisor<Integer,Long>

createDataSourceMetaDataForReset

protected KafkaDataSourceMetadata createDataSourceMetaDataForReset(String topic,
                                                                   Map<Integer,Long> map)

Specified by:

createDataSourceMetaDataForReset in class SeekableStreamSupervisor<Integer,Long>

makeSequenceNumber

protected OrderedSequenceNumber<Long> makeSequenceNumber(Long seq,
                                                         boolean isExclusive)

Specified by:

makeSequenceNumber in class SeekableStreamSupervisor<Integer,Long>

getNotSetMarker

protected Long getNotSetMarker()

Specified by:

getNotSetMarker in class SeekableStreamSupervisor<Integer,Long>

getEndOfPartitionMarker

protected Long getEndOfPartitionMarker()

Specified by:

getEndOfPartitionMarker in class SeekableStreamSupervisor<Integer,Long>

isEndOfShard

protected boolean isEndOfShard(Long seqNum)

Specified by:

isEndOfShard in class SeekableStreamSupervisor<Integer,Long>

useExclusiveStartSequenceNumberForNonFirstSequence

protected boolean useExclusiveStartSequenceNumberForNonFirstSequence()

Specified by:

useExclusiveStartSequenceNumberForNonFirstSequence in class SeekableStreamSupervisor<Integer,Long>

updateLatestSequenceFromStream

protected void updateLatestSequenceFromStream(RecordSupplier<Integer,Long> recordSupplier,
                                              Set<StreamPartition<Integer>> partitions)

Specified by:

updateLatestSequenceFromStream in class SeekableStreamSupervisor<Integer,Long>

baseTaskName

protected String baseTaskName()

Specified by:

baseTaskName in class SeekableStreamSupervisor<Integer,Long>

getIoConfig

public KafkaSupervisorIOConfig getIoConfig()

Overrides:

getIoConfig in class SeekableStreamSupervisor<Integer,Long>