com.twitter.scalding.commons

source

package source

Visibility
  1. Public
  2. All

Type Members

  1. class CodecPailStructure[T] extends PailStructure[T]

    It is quite unlikely for client code to make a CodecPailStructure CodecPailStructure is constructed by PailSource's factory methods.

    It is quite unlikely for client code to make a CodecPailStructure CodecPailStructure is constructed by PailSource's factory methods.

    targetFn takes an instance of T and returns a list "path components". Pail joins these components with File.separator and sinks the instance of T into the pail at that location.

    Usual implementations of "validator" will check that the length of the supplied list is >= the length f the list returned by targetFn.

    CodecPailStructure has a default constructor because it is instantiated via reflection This unfortunately means params must be set via setParams to make it usefuls

  2. abstract class DailyPrefixSuffixLzoThrift[T <: TBase[_, _]] extends DailyPrefixSuffixSource with LzoThrift[T]

  3. case class DailyPrefixSuffixLzoTsv(prefix: String, suffix: String, fs: Fields = Fields.ALL)(implicit dateRange: DateRange) extends DailyPrefixSuffixSource with LzoTsv with Product with Serializable

  4. abstract class DailySuffixLongThriftSequenceFile[V <: TBase[_, _]] extends TimePathedLongThriftSequenceFile[V]

  5. abstract class DailySuffixLzoCodec[T] extends DailySuffixSource with LzoCodec[T]

  6. abstract class DailySuffixLzoProtobuf[T <: Message] extends DailySuffixSource with LzoProtobuf[T]

  7. abstract class DailySuffixLzoThrift[T <: TBase[_, _]] extends DailySuffixSource with LzoThrift[T]

  8. case class DailySuffixLzoTsv(prefix: String, fs: Fields = Fields.ALL)(implicit dateRange: DateRange) extends DailySuffixSource with LzoTsv with Product with Serializable

  9. abstract class DailySuffixMostRecentLzoProtobuf[T <: Message] extends DailySuffixMostRecentSource with LzoProtobuf[T]

  10. trait ErrorHandlingLzoCodec[T] extends FileSource with LzoCodec[T]

  11. trait ErrorThresholdLzoCodec[T] extends FileSource with ErrorHandlingLzoCodec[T]

  12. abstract class FixedPathLzoProtobuf[T <: Message] extends FixedPathSource with LzoProtobuf[T]

  13. abstract class FixedPathLzoThrift[T <: TBase[_, _]] extends FixedPathSource with LzoThrift[T]

  14. abstract class HourlySuffixLzoCodec[T] extends HourlySuffixSource with LzoCodec[T]

  15. abstract class HourlySuffixLzoProtobuf[T <: Message] extends HourlySuffixSource with LzoProtobuf[T]

  16. abstract class HourlySuffixLzoText extends HourlySuffixSource with LzoText

  17. abstract class HourlySuffixLzoThrift[T <: TBase[_, _]] extends HourlySuffixSource with LzoThrift[T]

  18. case class HourlySuffixLzoTsv(prefix: String, fs: Fields = Fields.ALL)(implicit dateRange: DateRange) extends HourlySuffixSource with LzoTsv with Product with Serializable

  19. trait LongThriftTransformer[V <: TBase[_, _]] extends Source

  20. trait LzoCodec[T] extends FileSource with SingleMappable[T] with typed.TypedSink[T] with LocalTapSource

  21. trait LzoProtobuf[T <: Message] extends SchemedSource with LocalTapSource with SingleMappable[T] with typed.TypedSink[T]

  22. trait LzoText extends SchemedSource with LocalTapSource with SingleMappable[String] with typed.TypedSink[String]

  23. trait LzoThrift[T <: TBase[_, _]] extends SchemedSource with LocalTapSource with SingleMappable[T] with typed.TypedSink[T]

  24. trait LzoTsv extends SchemedSource with DelimitedScheme with LocalTapSource

  25. trait LzoTypedTsv[T] extends SchemedSource with DelimitedScheme with Mappable[T] with typed.TypedSink[T] with LocalTapSource

  26. trait LzoTypedTsv1[A] extends SchemedSource with LzoTypedTsv[(A)] with Mappable1[A]

  27. trait LzoTypedTsv10[A, B, C, D, E, F, G, H, I, J] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J)] with Mappable10[A, B, C, D, E, F, G, H, I, J]

  28. trait LzoTypedTsv11[A, B, C, D, E, F, G, H, I, J, K] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K)] with Mappable11[A, B, C, D, E, F, G, H, I, J, K]

  29. trait LzoTypedTsv12[A, B, C, D, E, F, G, H, I, J, K, L] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L)] with Mappable12[A, B, C, D, E, F, G, H, I, J, K, L]

  30. trait LzoTypedTsv13[A, B, C, D, E, F, G, H, I, J, K, L, M] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M)] with Mappable13[A, B, C, D, E, F, G, H, I, J, K, L, M]

  31. trait LzoTypedTsv14[A, B, C, D, E, F, G, H, I, J, K, L, M, N] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N)] with Mappable14[A, B, C, D, E, F, G, H, I, J, K, L, M, N]

  32. trait LzoTypedTsv15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O)] with Mappable15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O]

  33. trait LzoTypedTsv16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P)] with Mappable16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P]

  34. trait LzoTypedTsv17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q)] with Mappable17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q]

  35. trait LzoTypedTsv18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R)] with Mappable18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R]

  36. trait LzoTypedTsv19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S)] with Mappable19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S]

  37. trait LzoTypedTsv2[A, B] extends SchemedSource with LzoTypedTsv[(A, B)] with Mappable2[A, B]

  38. trait LzoTypedTsv20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T)] with Mappable20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T]

  39. trait LzoTypedTsv21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U)] with Mappable21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U]

  40. trait LzoTypedTsv22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V)] with Mappable22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V]

  41. trait LzoTypedTsv3[A, B, C] extends SchemedSource with LzoTypedTsv[(A, B, C)] with Mappable3[A, B, C]

  42. trait LzoTypedTsv4[A, B, C, D] extends SchemedSource with LzoTypedTsv[(A, B, C, D)] with Mappable4[A, B, C, D]

  43. trait LzoTypedTsv5[A, B, C, D, E] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E)] with Mappable5[A, B, C, D, E]

  44. trait LzoTypedTsv6[A, B, C, D, E, F] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F)] with Mappable6[A, B, C, D, E, F]

  45. trait LzoTypedTsv7[A, B, C, D, E, F, G] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G)] with Mappable7[A, B, C, D, E, F, G]

  46. trait LzoTypedTsv8[A, B, C, D, E, F, G, H] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H)] with Mappable8[A, B, C, D, E, F, G, H]

  47. trait LzoTypedTsv9[A, B, C, D, E, F, G, H, I] extends SchemedSource with LzoTypedTsv[(A, B, C, D, E, F, G, H, I)] with Mappable9[A, B, C, D, E, F, G, H, I]

  48. abstract class MostRecentGoodLongThriftSequenceFile[V <: TBase[_, _]] extends MostRecentGoodSource with WritableSequenceFileScheme with Serializable with Mappable[(Long, V)] with LongThriftTransformer[V]

  49. class PailSource[T] extends Source with Mappable[T]

  50. class RichPipeEx extends Serializable

  51. abstract class TimePathedLongThriftSequenceFile[V <: TBase[_, _]] extends TimePathedSource with WritableSequenceFileScheme with Serializable with Mappable[(Long, V)] with LongThriftTransformer[V]

  52. class TsvWithHeader extends FixedPathSource with DelimitedScheme with FieldConversions

    A tsv source with the column name header info.

    A tsv source with the column name header info.

    Header file format: tab separated column names.

  53. class TypedRichPipeEx[K, V] extends Serializable

  54. class VersionedKeyValSource[K, V] extends Source with Mappable[(K, V)] with typed.TypedSink[(K, V)]

Value Members

  1. object LzoCodecSource

    Source used to write some type T into an LZO-compressed SequenceFile using a codec on T for serialization.

  2. object PailSource extends Serializable

    The PailSource enables scalding integration with the Pail class in the dfs-datastores library.

    The PailSource enables scalding integration with the Pail class in the dfs-datastores library. PailSource allows scalding to sink 1-tuples to subdirectories of a root folder by applying a routing function to each tuple.

    SEE EXAMPLE : https://gist.github.com/krishnanraman/5224937

  3. object RichPipeEx extends Serializable

  4. object VersionedKeyValSource extends Serializable

    Source used to write key-value pairs as byte arrays into a versioned store.

    Source used to write key-value pairs as byte arrays into a versioned store. Supports incremental updates via the monoid on V.

Ungrouped