Packages

case class FinalStageResourceManager(session: SparkSession) extends Rule[SparkPlan] with FinalRebalanceStageHelper with Product with Serializable

This rule assumes the final write stage has less cores requirement than previous, otherwise this rule would take no effect.

It provide a feature: 1. Kill redundant executors before running final write stage

Linear Supertypes
Serializable, Serializable, Product, Equals, FinalRebalanceStageHelper, AdaptiveSparkPlanHelper, Rule[SparkPlan], Logging, SQLConfHelper, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. FinalStageResourceManager
  2. Serializable
  3. Serializable
  4. Product
  5. Equals
  6. FinalRebalanceStageHelper
  7. AdaptiveSparkPlanHelper
  8. Rule
  9. Logging
  10. SQLConfHelper
  11. AnyRef
  12. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new FinalStageResourceManager(session: SparkSession)

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. def apply(plan: SparkPlan): SparkPlan
    Definition Classes
    FinalStageResourceManager → Rule
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  7. def collect[B](p: SparkPlan)(pf: PartialFunction[SparkPlan, B]): Seq[B]
    Definition Classes
    AdaptiveSparkPlanHelper
  8. def collectFirst[B](p: SparkPlan)(pf: PartialFunction[SparkPlan, B]): Option[B]
    Definition Classes
    AdaptiveSparkPlanHelper
  9. def collectLeaves(p: SparkPlan): Seq[SparkPlan]
    Definition Classes
    AdaptiveSparkPlanHelper
  10. def collectWithSubqueries[B](p: SparkPlan)(f: PartialFunction[SparkPlan, B]): Seq[B]
    Definition Classes
    AdaptiveSparkPlanHelper
  11. def conf: SQLConf
    Definition Classes
    SQLConfHelper
  12. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  13. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  14. def find(p: SparkPlan)(f: (SparkPlan) ⇒ Boolean): Option[SparkPlan]
    Definition Classes
    AdaptiveSparkPlanHelper
  15. final def findFinalRebalanceStage(plan: SparkPlan): Option[ShuffleQueryStageExec]
    Attributes
    protected
    Definition Classes
    FinalRebalanceStageHelper
    Annotations
    @tailrec()
  16. def flatMap[A](p: SparkPlan)(f: (SparkPlan) ⇒ TraversableOnce[A]): Seq[A]
    Definition Classes
    AdaptiveSparkPlanHelper
  17. def foreach(p: SparkPlan)(f: (SparkPlan) ⇒ Unit): Unit
    Definition Classes
    AdaptiveSparkPlanHelper
  18. def foreachUp(p: SparkPlan)(f: (SparkPlan) ⇒ Unit): Unit
    Definition Classes
    AdaptiveSparkPlanHelper
  19. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  20. final def hasTableCache(plan: SparkPlan): Boolean
    Attributes
    protected
    Definition Classes
    FinalRebalanceStageHelper
  21. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  22. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  23. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  24. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  25. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  26. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  27. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  28. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  29. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  30. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  31. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  32. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  33. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  34. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  35. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  36. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  37. def mapPlans[A](p: SparkPlan)(f: (SparkPlan) ⇒ A): Seq[A]
    Definition Classes
    AdaptiveSparkPlanHelper
  38. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  39. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  40. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  41. lazy val ruleId: RuleId
    Attributes
    protected
    Definition Classes
    Rule
  42. val ruleName: String
    Definition Classes
    Rule
  43. val session: SparkSession
  44. def stripAQEPlan(p: SparkPlan): SparkPlan
    Definition Classes
    AdaptiveSparkPlanHelper
  45. def subqueriesAll(p: SparkPlan): Seq[SparkPlan]
    Definition Classes
    AdaptiveSparkPlanHelper
  46. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  47. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  48. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  49. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from Serializable

Inherited from Serializable

Inherited from Product

Inherited from Equals

Inherited from FinalRebalanceStageHelper

Inherited from AdaptiveSparkPlanHelper

Inherited from Rule[SparkPlan]

Inherited from Logging

Inherited from SQLConfHelper

Inherited from AnyRef

Inherited from Any

Ungrouped