org.apache.tez.dag.app.dag.impl

Class VertexImpl

java.lang.Object

org.apache.tez.dag.app.dag.impl.VertexImpl

All Implemented Interfaces:

Comparable<Vertex>, org.apache.hadoop.yarn.event.EventHandler<VertexEvent>, Vertex

public class VertexImpl
extends Object
implements Vertex, org.apache.hadoop.yarn.event.EventHandler<VertexEvent>

Implementation of Vertex interface. Maintains the state machines of Vertex. The read and write calls use ReadWriteLock for concurrency.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
static class	VertexImpl.BufferDataRecoverTransition
static class	VertexImpl.IgnoreInitInInitedTransition
static class	VertexImpl.InitTransition
static class	VertexImpl.NullEdgeInitializedTransition
static class	VertexImpl.OneToOneSourceSplitTransition
static class	VertexImpl.RecoverTransition
static class	VertexImpl.RootInputInitializedTransition
static class	VertexImpl.SourceVertexStartedTransition
static class	VertexImpl.StartRecoverTransition
static class	VertexImpl.StartTransition
static class	VertexImpl.StartWhileInitializingTransition
static class	VertexImpl.TerminateDuringRecoverTransition
static class	VertexImpl.VertexInitializedTransition

Field Summary

Fields

Modifier and Type	Field and Description
protected StateChangeNotifier	stateChangeNotifier
protected static org.apache.hadoop.yarn.state.StateMachineFactory<VertexImpl,VertexState,VertexEventType,VertexEvent>	stateMachineFactory

Constructor Summary

Constructors

Constructor and Description

VertexImpl(org.apache.tez.dag.records.TezVertexID vertexId, org.apache.tez.dag.api.records.DAGProtos.VertexPlan vertexPlan, String vertexName, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.event.EventHandler eventHandler, TaskAttemptListener taskAttemptListener, org.apache.hadoop.yarn.util.Clock clock, TaskHeartbeatHandler thh, boolean commitVertexOutputs, AppContext appContext, org.apache.tez.dag.api.VertexLocationHint vertexLocationHint, Map<String,org.apache.tez.dag.app.dag.impl.DAGImpl.VertexGroupInfo> dagVertexGroups, TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption, StateChangeNotifier entityStatusTracker)

Method Summary

Methods

Modifier and Type	Method and Description
void	addSharedOutputs(Set<String> outputs)
protected void	addTask(Task task)
int	compareTo(Vertex other)
void	constructFinalFullcounters()
protected RootInputInitializerManager	createRootInputInitializerManager(String dagName, String vertexName, org.apache.tez.dag.records.TezVertexID vertexID, org.apache.hadoop.yarn.event.EventHandler eventHandler, int numTasks, int numNodes, org.apache.hadoop.yarn.api.records.Resource vertexTaskResource, org.apache.hadoop.yarn.api.records.Resource totalResource)
boolean	equals(Object obj)
Map<String,org.apache.tez.dag.api.RootInputLeafOutput<org.apache.tez.dag.api.InputDescriptor,org.apache.tez.dag.api.InputInitializerDescriptor>>	getAdditionalInputs()
Map<String,org.apache.tez.dag.api.RootInputLeafOutput<org.apache.tez.dag.api.OutputDescriptor,org.apache.tez.dag.api.OutputCommitterDescriptor>>	getAdditionalOutputs()
org.apache.tez.common.counters.TezCounters	getAllCounters() Get all the counters of this vertex.
AppContext	getAppContext()
int	getCompletedTasks()
DAG	getDAG()
List<String>	getDiagnostics()
int	getDistanceFromRoot()
List<org.apache.tez.runtime.api.impl.GroupInputSpec>	getGroupInputSpecList(int taskIndex)
List<org.apache.tez.runtime.api.impl.InputSpec>	getInputSpecList(int taskIndex)
Map<Vertex,Edge>	getInputVertices()
int	getInputVerticesCount()
String	getLogIdentifier()
String	getName()
org.apache.tez.runtime.api.OutputCommitter	getOutputCommitter(String outputName)
Map<String,org.apache.tez.runtime.api.OutputCommitter>	getOutputCommitters()
List<org.apache.tez.runtime.api.impl.OutputSpec>	getOutputSpecList(int taskIndex)
Map<Vertex,Edge>	getOutputVertices()
int	getOutputVerticesCount()
org.apache.tez.dag.api.ProcessorDescriptor	getProcessorDescriptor()
float	getProgress()
int	getRunningTasks()
Set<String>	getSharedOutputs()
VertexState	getState()
protected org.apache.hadoop.yarn.state.StateMachine<VertexState,VertexEventType,VertexEvent>	getStateMachine()
int	getSucceededTasks()
Task	getTask(int taskIndex)
Task	getTask(org.apache.tez.dag.records.TezTaskID taskID)
org.apache.tez.dag.api.TaskLocationHint	getTaskLocationHint(org.apache.tez.dag.records.TezTaskID taskId)
org.apache.hadoop.yarn.api.records.Resource	getTaskResource()
Map<org.apache.tez.dag.records.TezTaskID,Task>	getTasks()
VertexTerminationCause	getTerminationCause()
int	getTotalTasks()
org.apache.tez.dag.records.TezVertexID	getVertexId()
org.apache.tez.dag.api.records.DAGProtos.VertexPlan	getVertexPlan()
ProgressBuilder	getVertexProgress()
VertexStats	getVertexStats()
VertexStatusBuilder	getVertexStatus(Set<org.apache.tez.dag.api.client.StatusGetOpts> statusOptions)
void	handle(VertexEvent event)
int	hashCode()
static org.apache.tez.common.counters.TezCounters	incrTaskCounters(org.apache.tez.common.counters.TezCounters counters, Collection<Task> tasks)
VertexState	restoreFromEvent(HistoryEvent historyEvent)
void	scheduleTasks(List<org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint> tasksToSchedule)
void	setAdditionalInputs(List<org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto> inputs)
void	setAdditionalOutputs(List<org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto> outputs)
void	setInputVertices(Map<Vertex,Edge> inVertices)
void	setOutputVertices(Map<Vertex,Edge> outVertices)
boolean	setParallelism(int parallelism, org.apache.tez.dag.api.VertexLocationHint vertexLocationHint, Map<String,org.apache.tez.dag.api.EdgeManagerPluginDescriptor> sourceEdgeManagers, Map<String,org.apache.tez.runtime.api.InputSpecUpdate> rootInputSpecUpdates)
void	setVertexLocationHint(org.apache.tez.dag.api.VertexLocationHint vertexLocationHint)
static VertexStats	updateVertexStats(VertexStats stats, Collection<Task> tasks)

Methods inherited from class java.lang.Object

clone, finalize, getClass, notify, notifyAll, toString, wait, wait, wait

Field Detail

stateChangeNotifier

protected final StateChangeNotifier stateChangeNotifier

stateMachineFactory

protected static final org.apache.hadoop.yarn.state.StateMachineFactory<VertexImpl,VertexState,VertexEventType,VertexEvent> stateMachineFactory

Constructor Detail

VertexImpl

public VertexImpl(org.apache.tez.dag.records.TezVertexID vertexId,
          org.apache.tez.dag.api.records.DAGProtos.VertexPlan vertexPlan,
          String vertexName,
          org.apache.hadoop.conf.Configuration conf,
          org.apache.hadoop.yarn.event.EventHandler eventHandler,
          TaskAttemptListener taskAttemptListener,
          org.apache.hadoop.yarn.util.Clock clock,
          TaskHeartbeatHandler thh,
          boolean commitVertexOutputs,
          AppContext appContext,
          org.apache.tez.dag.api.VertexLocationHint vertexLocationHint,
          Map<String,org.apache.tez.dag.app.dag.impl.DAGImpl.VertexGroupInfo> dagVertexGroups,
          TaskSpecificLaunchCmdOption taskSpecificLaunchCmdOption,
          StateChangeNotifier entityStatusTracker)

Method Detail

getStateMachine

protected org.apache.hadoop.yarn.state.StateMachine<VertexState,VertexEventType,VertexEvent> getStateMachine()

getVertexId

public org.apache.tez.dag.records.TezVertexID getVertexId()

Specified by:

getVertexId in interface Vertex

getVertexPlan

public org.apache.tez.dag.api.records.DAGProtos.VertexPlan getVertexPlan()

Specified by:

getVertexPlan in interface Vertex

getDistanceFromRoot

public int getDistanceFromRoot()

Specified by:

getDistanceFromRoot in interface Vertex

getName

public String getName()

Specified by:

getName in interface Vertex

getTask

public Task getTask(org.apache.tez.dag.records.TezTaskID taskID)

Specified by:

getTask in interface Vertex

getTask

public Task getTask(int taskIndex)

Specified by:

getTask in interface Vertex

getTotalTasks

public int getTotalTasks()

Specified by:

getTotalTasks in interface Vertex

getCompletedTasks

public int getCompletedTasks()

Specified by:

getCompletedTasks in interface Vertex

getSucceededTasks

public int getSucceededTasks()

Specified by:

getSucceededTasks in interface Vertex

getRunningTasks

public int getRunningTasks()

Specified by:

getRunningTasks in interface Vertex

getAllCounters

public org.apache.tez.common.counters.TezCounters getAllCounters()

Description copied from interface: Vertex

Get all the counters of this vertex.

Specified by:

getAllCounters in interface Vertex

Returns:

aggregate task-counters

getVertexStats

public VertexStats getVertexStats()

incrTaskCounters

public static org.apache.tez.common.counters.TezCounters incrTaskCounters(org.apache.tez.common.counters.TezCounters counters,
                                                          Collection<Task> tasks)

updateVertexStats

public static VertexStats updateVertexStats(VertexStats stats,
                            Collection<Task> tasks)

getDiagnostics

public List<String> getDiagnostics()

Specified by:

getDiagnostics in interface Vertex

getProgress

public float getProgress()

Specified by:

getProgress in interface Vertex

getVertexProgress

public ProgressBuilder getVertexProgress()

Specified by:

getVertexProgress in interface Vertex

getVertexStatus

public VertexStatusBuilder getVertexStatus(Set<org.apache.tez.dag.api.client.StatusGetOpts> statusOptions)

Specified by:

getVertexStatus in interface Vertex

getTaskLocationHint

public org.apache.tez.dag.api.TaskLocationHint getTaskLocationHint(org.apache.tez.dag.records.TezTaskID taskId)

Specified by:

getTaskLocationHint in interface Vertex

getTasks

public Map<org.apache.tez.dag.records.TezTaskID,Task> getTasks()

Specified by:

getTasks in interface Vertex

getState

public VertexState getState()

Specified by:

getState in interface Vertex

getTerminationCause

public VertexTerminationCause getTerminationCause()

Specified by:

getTerminationCause in interface Vertex

getAppContext

public AppContext getAppContext()

Specified by:

getAppContext in interface Vertex

restoreFromEvent

public VertexState restoreFromEvent(HistoryEvent historyEvent)

Specified by:

restoreFromEvent in interface Vertex

getLogIdentifier

public String getLogIdentifier()

Specified by:

getLogIdentifier in interface Vertex

scheduleTasks

public void scheduleTasks(List<org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint> tasksToSchedule)

Specified by:

scheduleTasks in interface Vertex

setParallelism

public boolean setParallelism(int parallelism,
                     org.apache.tez.dag.api.VertexLocationHint vertexLocationHint,
                     Map<String,org.apache.tez.dag.api.EdgeManagerPluginDescriptor> sourceEdgeManagers,
                     Map<String,org.apache.tez.runtime.api.InputSpecUpdate> rootInputSpecUpdates)

Specified by:

setParallelism in interface Vertex

setVertexLocationHint

public void setVertexLocationHint(org.apache.tez.dag.api.VertexLocationHint vertexLocationHint)

Specified by:

setVertexLocationHint in interface Vertex

handle

public void handle(VertexEvent event)

Specified by:

handle in interface org.apache.hadoop.yarn.event.EventHandler<VertexEvent>

addTask

protected void addTask(Task task)

createRootInputInitializerManager

protected RootInputInitializerManager createRootInputInitializerManager(String dagName,
                                                            String vertexName,
                                                            org.apache.tez.dag.records.TezVertexID vertexID,
                                                            org.apache.hadoop.yarn.event.EventHandler eventHandler,
                                                            int numTasks,
                                                            int numNodes,
                                                            org.apache.hadoop.yarn.api.records.Resource vertexTaskResource,
                                                            org.apache.hadoop.yarn.api.records.Resource totalResource)

constructFinalFullcounters

@InterfaceAudience.Private
public void constructFinalFullcounters()

setInputVertices

public void setInputVertices(Map<Vertex,Edge> inVertices)

Specified by:

setInputVertices in interface Vertex

setOutputVertices

public void setOutputVertices(Map<Vertex,Edge> outVertices)

Specified by:

setOutputVertices in interface Vertex

setAdditionalInputs

public void setAdditionalInputs(List<org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto> inputs)

Specified by:

setAdditionalInputs in interface Vertex

getOutputCommitters

@Nullable
public Map<String,org.apache.tez.runtime.api.OutputCommitter> getOutputCommitters()

Specified by:

getOutputCommitters in interface Vertex

getOutputCommitter

@Nullable
@InterfaceAudience.Private
public org.apache.tez.runtime.api.OutputCommitter getOutputCommitter(String outputName)

setAdditionalOutputs

public void setAdditionalOutputs(List<org.apache.tez.dag.api.records.DAGProtos.RootInputLeafOutputProto> outputs)

Specified by:

setAdditionalOutputs in interface Vertex

getAdditionalInputs

@Nullable
public Map<String,org.apache.tez.dag.api.RootInputLeafOutput<org.apache.tez.dag.api.InputDescriptor,org.apache.tez.dag.api.InputInitializerDescriptor>> getAdditionalInputs()

Specified by:

getAdditionalInputs in interface Vertex

getAdditionalOutputs

@Nullable
public Map<String,org.apache.tez.dag.api.RootInputLeafOutput<org.apache.tez.dag.api.OutputDescriptor,org.apache.tez.dag.api.OutputCommitterDescriptor>> getAdditionalOutputs()

Specified by:

getAdditionalOutputs in interface Vertex

compareTo

public int compareTo(Vertex other)

Specified by:

compareTo in interface Comparable<Vertex>

equals

public boolean equals(Object obj)

Overrides:

equals in class Object

hashCode

public int hashCode()

Overrides:

hashCode in class Object

getInputVertices

public Map<Vertex,Edge> getInputVertices()

Specified by:

getInputVertices in interface Vertex

getOutputVertices

public Map<Vertex,Edge> getOutputVertices()

Specified by:

getOutputVertices in interface Vertex

getInputVerticesCount

public int getInputVerticesCount()

Specified by:

getInputVerticesCount in interface Vertex

getOutputVerticesCount

public int getOutputVerticesCount()

Specified by:

getOutputVerticesCount in interface Vertex

getProcessorDescriptor

public org.apache.tez.dag.api.ProcessorDescriptor getProcessorDescriptor()

Specified by:

getProcessorDescriptor in interface Vertex

getDAG

public DAG getDAG()

Specified by:

getDAG in interface Vertex

getTaskResource

public org.apache.hadoop.yarn.api.records.Resource getTaskResource()

Specified by:

getTaskResource in interface Vertex

getInputSpecList

public List<org.apache.tez.runtime.api.impl.InputSpec> getInputSpecList(int taskIndex)

Specified by:

getInputSpecList in interface Vertex

getOutputSpecList

public List<org.apache.tez.runtime.api.impl.OutputSpec> getOutputSpecList(int taskIndex)

Specified by:

getOutputSpecList in interface Vertex

getGroupInputSpecList

public List<org.apache.tez.runtime.api.impl.GroupInputSpec> getGroupInputSpecList(int taskIndex)

Specified by:

getGroupInputSpecList in interface Vertex

addSharedOutputs

public void addSharedOutputs(Set<String> outputs)

Specified by:

addSharedOutputs in interface Vertex

getSharedOutputs

public Set<String> getSharedOutputs()

Specified by:

getSharedOutputs in interface Vertex