org.apache.gobblin.service.monitoring

Class KafkaJobStatusMonitor

java.lang.Object

com.google.common.util.concurrent.AbstractIdleService

org.apache.gobblin.runtime.kafka.HighLevelConsumer<byte[],byte[]>

org.apache.gobblin.service.monitoring.KafkaJobStatusMonitor

All Implemented Interfaces:

com.google.common.util.concurrent.Service

Direct Known Subclasses:

KafkaAvroJobStatusMonitor

public abstract class KafkaJobStatusMonitor
extends HighLevelConsumer<byte[],byte[]>

A Kafka monitor that tracks GobblinTrackingEvents reporting statuses of running jobs. The job statuses are stored as State objects in a FileContextBasedFsStateStore.

Nested Class Summary

Nested classes/interfaces inherited from interface com.google.common.util.concurrent.Service

com.google.common.util.concurrent.Service.Listener, com.google.common.util.concurrent.Service.State

Field Summary

Fields

Modifier and Type	Field and Description
static String	GET_AND_SET_JOB_STATUS
static String	STATE_STORE_KEY_SEPARATION_CHARACTER
static String	STATE_STORE_TABLE_SUFFIX

Fields inherited from class org.apache.gobblin.runtime.kafka.HighLevelConsumer

config, CONSUMER_CLIENT_FACTORY_CLASS_KEY, DEFAULT_AUTO_COMMIT_VALUE, DEFAULT_OFFSET_COMMIT_NUM_RECORDS_THRESHOLD, DEFAULT_OFFSET_COMMIT_TIME_THRESHOLD_SECS, ENABLE_AUTO_COMMIT_KEY, GROUP_ID_KEY, OFFSET_COMMIT_NUM_RECORDS_THRESHOLD_KEY, OFFSET_COMMIT_TIME_THRESHOLD_SECS_KEY, topic

Constructor Summary

Constructors

Constructor and Description
KafkaJobStatusMonitor(String topic, com.typesafe.config.Config config, int numThreads)

Method Summary

Modifier and Type	Method and Description
protected void	createMetrics()
static long	getExecutionIdFromTableName(String tableName)
static String	jobStatusStoreName(String flowGroup, String flowName)
static String	jobStatusTableName(long flowExecutionId, String jobGroup, String jobName)
static String	jobStatusTableName(String flowExecutionId, String jobGroup, String jobName)
abstract State	parseJobStatus(byte[] message)
protected void	processMessage(org.apache.gobblin.kafka.client.DecodeableKafkaRecord<byte[],byte[]> message)
void	shutDown()
protected void	startUp()

Methods inherited from class org.apache.gobblin.runtime.kafka.HighLevelConsumer

buildMetricsContextAndMetrics, commitOffsets, createConsumerClient, getGobblinKafkaConsumerClient, getMetricContext, getTagsForMetrics, getTopic, shutdownMetrics

Methods inherited from class com.google.common.util.concurrent.AbstractIdleService

addListener, awaitRunning, awaitRunning, awaitTerminated, awaitTerminated, executor, failureCause, isRunning, serviceName, start, startAndWait, startAsync, state, stop, stopAndWait, stopAsync, toString

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Field Detail

STATE_STORE_TABLE_SUFFIX

public static final String STATE_STORE_TABLE_SUFFIX

See Also:

Constant Field Values

STATE_STORE_KEY_SEPARATION_CHARACTER

public static final String STATE_STORE_KEY_SEPARATION_CHARACTER

See Also:

Constant Field Values

GET_AND_SET_JOB_STATUS

public static final String GET_AND_SET_JOB_STATUS

Constructor Detail

KafkaJobStatusMonitor

public KafkaJobStatusMonitor(String topic,
                             com.typesafe.config.Config config,
                             int numThreads)
                      throws ReflectiveOperationException

Throws:

ReflectiveOperationException

Method Detail

startUp

protected void startUp()

Overrides:

startUp in class HighLevelConsumer<byte[],byte[]>

shutDown

public void shutDown()

Overrides:

shutDown in class HighLevelConsumer<byte[],byte[]>

createMetrics

protected void createMetrics()

Overrides:

createMetrics in class HighLevelConsumer<byte[],byte[]>

processMessage

protected void processMessage(org.apache.gobblin.kafka.client.DecodeableKafkaRecord<byte[],byte[]> message)

Specified by:

processMessage in class HighLevelConsumer<byte[],byte[]>

jobStatusTableName

public static String jobStatusTableName(String flowExecutionId,
                                        String jobGroup,
                                        String jobName)

jobStatusTableName

public static String jobStatusTableName(long flowExecutionId,
                                        String jobGroup,
                                        String jobName)

jobStatusStoreName

public static String jobStatusStoreName(String flowGroup,
                                        String flowName)

getExecutionIdFromTableName

public static long getExecutionIdFromTableName(String tableName)

parseJobStatus

public abstract State parseJobStatus(byte[] message)
                              throws IOException

Throws:

IOException