Packages

t

cloudflow.spark

SparkStreamlet

trait SparkStreamlet extends Streamlet with Serializable

The base class for defining Spark streamlets. Derived classes need to override createLogic to provide the custom implementation for the behavior of the streamlet.

Here's an example:

// new custom `SparkStreamlet`
object MySparkProcessor extends SparkStreamlet {
  // Step 1: Define inlets and outlets. Note for the outlet you need to specify
  //         the partitioner function explicitly
  val in = AvroInlet[Data]("in")
  val out = AvroOutlet[Simple]("out", _.name)

  // Step 2: Define the shape of the streamlet. In this example the streamlet
  //         has 1 inlet and 1 outlet
  val shape = StreamletShape(in, out)

  // Step 3: Provide custom implementation of `SparkStreamletLogic` that defines
  //         the behavior of the streamlet
  override def createLogic() = new SparkStreamletLogic {
    override def buildStreamingQueries = {
      val dataset = readStream(in)
      val outStream = dataset.select($"name").as[Simple]
      val query = writeStream(outStream, out, OutputMode.Append)
      Seq(query)
    }
  }
}
Linear Supertypes
Serializable, Serializable, Streamlet, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. SparkStreamlet
  2. Serializable
  3. Serializable
  4. Streamlet
  5. AnyRef
  6. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Type Members

  1. final class SparkStreamletContextException extends Exception

Abstract Value Members

  1. abstract def createLogic(): SparkStreamletLogic
    Attributes
    protected
  2. abstract def shape(): StreamletShape
    Definition Classes
    Streamlet

Concrete Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. val applicationName: String
  5. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  6. def attributes: Set[StreamletAttribute]
    Attributes
    protected[cloudflow]
    Definition Classes
    Streamlet
  7. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  8. def configParameters: IndexedSeq[ConfigParameter]
    Definition Classes
    Streamlet
  9. final def configuredValue(context: SparkStreamletContext, configKey: String): String
  10. implicit final def context: SparkStreamletContext
    Attributes
    protected
  11. final def createContext(config: Config): SparkStreamletContext
    Attributes
    protected
    Definition Classes
    SparkStreamlet → Streamlet
  12. def customAttributes: Set[StreamletAttribute]
    Definition Classes
    Streamlet
  13. def defineConfigParameters(): Array[ConfigParameter]
    Definition Classes
    Streamlet
  14. def defineCustomAttributes(): Array[StreamletAttribute]
    Definition Classes
    Streamlet
  15. def defineVolumeMounts(): Array[VolumeMount]
    Definition Classes
    Streamlet
  16. def description: String
    Definition Classes
    Streamlet
  17. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  18. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  19. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  20. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  21. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  22. final def inlets: IndexedSeq[Inlet]
    Definition Classes
    Streamlet
  23. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  24. final def jsonDescriptor: String
    Definition Classes
    Streamlet
  25. def labels: IndexedSeq[String]
    Definition Classes
    Streamlet
  26. lazy val log: Logger
    Annotations
    @transient()
  27. def logStartRunnerMessage(buildInfo: String): Unit
    Definition Classes
    SparkStreamlet → Streamlet
  28. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  29. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  30. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  31. final def outlets: IndexedSeq[Outlet]
    Definition Classes
    Streamlet
  32. final def run(config: Config): StreamletExecution
    Definition Classes
    SparkStreamlet → Streamlet
  33. final val runtime: SparkStreamletRuntime.type
    Definition Classes
    SparkStreamlet → Streamlet
  34. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  35. def toString(): String
    Definition Classes
    AnyRef → Any
  36. def volumeMounts: IndexedSeq[VolumeMount]
    Definition Classes
    Streamlet
  37. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  38. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  39. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()

Inherited from Serializable

Inherited from Serializable

Inherited from Streamlet

Inherited from AnyRef

Inherited from Any

Ungrouped