org.apache.crunch.impl.spark
Class SparkRuntime

java.lang.Object
  com.google.common.util.concurrent.AbstractFuture<PipelineResult>
      org.apache.crunch.impl.spark.SparkRuntime

All Implemented Interfaces:

com.google.common.util.concurrent.ListenableFuture<PipelineResult>, Future<PipelineResult>, PipelineExecution

public class SparkRuntime
extends com.google.common.util.concurrent.AbstractFuture<PipelineResult>
implements PipelineExecution

Nested Class Summary

Nested classes/interfaces inherited from interface org.apache.crunch.PipelineExecution

PipelineExecution.Status

Constructor Summary

SparkRuntime(SparkPipeline pipeline, org.apache.spark.api.java.JavaSparkContext sparkContext, org.apache.hadoop.conf.Configuration conf, Map<PCollectionImpl<?>,Set<Target>> outputTargets, Map<PCollectionImpl<?>,org.apache.crunch.materialize.MaterializableIterable> toMaterialize, Map<PCollection<?>,org.apache.spark.storage.StorageLevel> toCache)

Method Summary

SparkRuntime	execute()
PipelineResult	get()
PipelineResult	get(long timeout, TimeUnit unit)
CombineFn	getCombineFn()
org.apache.hadoop.conf.Configuration	getConfiguration()
String	getPlanDotFile()
PipelineResult	getResult()
SparkRuntimeContext	getRuntimeContext()
org.apache.spark.api.java.JavaSparkContext	getSparkContext()
PipelineExecution.Status	getStatus()
org.apache.spark.storage.StorageLevel	getStorageLevel(PCollection<?> pcollection)
boolean	isValid(org.apache.spark.api.java.JavaRDDLike<?,?> rdd)
void	kill()
void	setCombineFn(CombineFn combineFn)
void	waitFor(long timeout, TimeUnit timeUnit)
void	waitUntilDone()

Methods inherited from class com.google.common.util.concurrent.AbstractFuture

addListener, cancel, isCancelled, isDone

Methods inherited from class java.lang.Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface com.google.common.util.concurrent.ListenableFuture

addListener

Methods inherited from interface java.util.concurrent.Future

cancel, isCancelled, isDone

Constructor Detail

SparkRuntime

public SparkRuntime(SparkPipeline pipeline,
                    org.apache.spark.api.java.JavaSparkContext sparkContext,
                    org.apache.hadoop.conf.Configuration conf,
                    Map<PCollectionImpl<?>,Set<Target>> outputTargets,
                    Map<PCollectionImpl<?>,org.apache.crunch.materialize.MaterializableIterable> toMaterialize,
                    Map<PCollection<?>,org.apache.spark.storage.StorageLevel> toCache)

Method Detail

setCombineFn

public void setCombineFn(CombineFn combineFn)

getCombineFn

public CombineFn getCombineFn()

execute

public SparkRuntime execute()

getSparkContext

public org.apache.spark.api.java.JavaSparkContext getSparkContext()

getRuntimeContext

public SparkRuntimeContext getRuntimeContext()

getConfiguration

public org.apache.hadoop.conf.Configuration getConfiguration()

isValid

public boolean isValid(org.apache.spark.api.java.JavaRDDLike<?,?> rdd)

getStorageLevel

public org.apache.spark.storage.StorageLevel getStorageLevel(PCollection<?> pcollection)

getPlanDotFile

public String getPlanDotFile()

Specified by:

getPlanDotFile in interface PipelineExecution

waitFor

public void waitFor(long timeout,
                    TimeUnit timeUnit)
             throws InterruptedException

Specified by:

waitFor in interface PipelineExecution

Throws:

InterruptedException

waitUntilDone

public void waitUntilDone()
                   throws InterruptedException

Specified by:

waitUntilDone in interface PipelineExecution

Throws:

InterruptedException

get

public PipelineResult get()
                   throws InterruptedException,
                          ExecutionException

Specified by:

get in interface Future<PipelineResult>

Overrides:

get in class com.google.common.util.concurrent.AbstractFuture<PipelineResult>

Throws:

InterruptedException

ExecutionException

get

public PipelineResult get(long timeout,
                          TimeUnit unit)
                   throws InterruptedException,
                          TimeoutException,
                          ExecutionException

Specified by:

get in interface Future<PipelineResult>

Overrides:

get in class com.google.common.util.concurrent.AbstractFuture<PipelineResult>

Throws:

InterruptedException

TimeoutException

ExecutionException

getStatus

public PipelineExecution.Status getStatus()

Specified by:

getStatus in interface PipelineExecution

getResult

public PipelineResult getResult()

Specified by:

getResult in interface PipelineExecution

kill

public void kill()
          throws InterruptedException

Specified by:

kill in interface PipelineExecution

Throws:

InterruptedException