object BigDiffy extends Command with Serializable

Big diff between two data sets given a primary key.

Linear Supertypes
Serializable, Command, AnyRef, Any
Ordering
  1. Alphabetic
  2. By Inheritance
Inherited
  1. BigDiffy
  2. Serializable
  3. Command
  4. AnyRef
  5. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. Protected

Type Members

  1. case class DeltaBigQuery(field: String, left: String, right: String, delta: DeltaValueBigQuery) extends Product with Serializable
  2. type DeltaSCollection = SCollection[(MultiKey, (Seq[Delta], DiffType.Value))]
  3. case class DeltaStatsBigQuery(deltaType: String, min: Double, max: Double, count: Long, mean: Double, variance: Double, stddev: Double, skewness: Double, kurtosis: Double) extends Product with Serializable
  4. case class DeltaValueBigQuery(deltaType: String, deltaValue: Option[Double]) extends Product with Serializable
  5. case class FieldStatsBigQuery(field: String @com.spotify.scio.bigquery.types.BigQueryTag, count: Long @com.spotify.scio.bigquery.types.BigQueryTag, fraction: Double @com.spotify.scio.bigquery.types.BigQueryTag, deltaStats: Option[DeltaStatsBigQuery]) extends HasAnnotation with Product with Serializable
    Annotations
    @BigQueryTag()
  6. case class GlobalStatsBigQuery(numTotal: Long @com.spotify.scio.bigquery.types.BigQueryTag, numSame: Long @com.spotify.scio.bigquery.types.BigQueryTag, numDiff: Long @com.spotify.scio.bigquery.types.BigQueryTag, numMissingLhs: Long @com.spotify.scio.bigquery.types.BigQueryTag, numMissingRhs: Long @com.spotify.scio.bigquery.types.BigQueryTag) extends HasAnnotation with Product with Serializable
    Annotations
    @BigQueryTag()
  7. case class KeyStatsBigQuery(key: String @com.spotify.scio.bigquery.types.BigQueryTag, diffType: String @com.spotify.scio.bigquery.types.BigQueryTag, delta: Option[DeltaBigQuery]) extends HasAnnotation with Product with Serializable
    Annotations
    @BigQueryTag()
  8. implicit class TextFileHeader extends AnyRef

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##: Int
    Definition Classes
    AnyRef → Any
  3. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  4. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  5. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.CloneNotSupportedException]) @native() @HotSpotIntrinsicCandidate()
  6. val command: String
    Definition Classes
    BigDiffy → Command
  7. def diff[T](lhs: SCollection[T], rhs: SCollection[T], d: Diffy[T], keyFn: (T) => MultiKey, ignoreNan: Boolean = false)(implicit arg0: ClassTag[T], arg1: Coder[T]): BigDiffy[T]

    Diff two data sets.

  8. def diffAvro[T <: SpecificRecordBase](sc: ScioContext, lhs: String, rhs: String, keyFn: (T) => MultiKey, diffy: AvroDiffy[T], ignoreNan: Boolean = false)(implicit arg0: ClassTag[T], arg1: Coder[T]): BigDiffy[T]

    Diff two Avro data sets.

  9. def diffParquet(sc: ScioContext, lhs: String, rhs: String, keyFn: (GenericRecord) => MultiKey, diffy: AvroDiffy[GenericRecord]): BigDiffy[GenericRecord]

    Diff two Parquet data sets.

    Diff two Parquet data sets. Note that both typed-parquet and avro-parquet inputs are supported. However, in either case the diff will be written in Parquet format as Avro GenericRecords.

  10. def diffProtoBuf[T <: AbstractMessage](sc: ScioContext, lhs: String, rhs: String, keyFn: (T) => MultiKey, diffy: ProtoBufDiffy[T])(implicit arg0: ClassTag[T]): BigDiffy[T]

    Diff two ProtoBuf data sets.

  11. def diffTableRow(sc: ScioContext, lhs: String, rhs: String, rowRestriction: Option[String], keyFn: (TableRow) => MultiKey, diffy: TableRowDiffy, ignoreNan: Boolean = false): BigDiffy[TableRow]

    Diff two TableRow data sets.

  12. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  13. def equals(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef → Any
  14. final def getClass(): Class[_ <: AnyRef]
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  15. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  16. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  17. def main(cmdlineArgs: Array[String]): Unit

    for easier running via sbt

  18. def mergeTableSchema(x: TableSchema, y: TableSchema): TableSchema

    Merge two BigQuery TableSchemas.

  19. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  20. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  21. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native() @HotSpotIntrinsicCandidate()
  22. def pathWithShards(path: String): String
  23. def run(cmdlineArgs: Array[String]): Unit

    Scio pipeline for BigDiffy.

  24. def saveStats[T](bigDiffy: BigDiffy[T], output: String, withHeader: Boolean = false, outputMode: OutputMode = GCS): Unit

    saves stats to either GCS as text, or BigQuery

  25. def stripQuoteWrap(input: String): String

    Remove quotes wrapping string argument.

    Remove quotes wrapping string argument. *

  26. final def synchronized[T0](arg0: => T0): T0
    Definition Classes
    AnyRef
  27. def toString(): String
    Definition Classes
    AnyRef → Any
  28. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  29. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException]) @native()
  30. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.InterruptedException])
  31. object FieldStatsBigQuery extends HasSchema[FieldStatsBigQuery] with (String, Long, Double, Option[DeltaStatsBigQuery]) => FieldStatsBigQuery with Serializable
  32. object GlobalStatsBigQuery extends HasSchema[GlobalStatsBigQuery] with (Long, Long, Long, Long, Long) => GlobalStatsBigQuery with Serializable
  33. object KeyStatsBigQuery extends HasSchema[KeyStatsBigQuery] with (String, String, Option[DeltaBigQuery]) => KeyStatsBigQuery with Serializable

Deprecated Value Members

  1. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws(classOf[java.lang.Throwable]) @Deprecated
    Deprecated

Inherited from Serializable

Inherited from Command

Inherited from AnyRef

Inherited from Any

Ungrouped