org.apache.hudi.common.engine

Class HoodieEngineContext

java.lang.Object

org.apache.hudi.common.engine.HoodieEngineContext

Direct Known Subclasses:

HoodieLocalEngineContext

public abstract class HoodieEngineContext
extends Object

Base class contains the context information needed by the engine at runtime. It will be extended by different engine implementation if needed.

Field Summary

Fields

Modifier and Type	Field and Description
protected TaskContextSupplier	taskContextSupplier

Constructor Summary

Constructors

Constructor and Description
HoodieEngineContext(StorageConfiguration<?> storageConf, TaskContextSupplier taskContextSupplier)

Method Summary

Modifier and Type	Method and Description
abstract <I,O> O	aggregate(HoodieData<I> data, O zeroValue, Functions.Function2<O,I,O> seqOp, Functions.Function2<O,O,O> combOp) Aggregate the elements of each partition, and then the results for all the partitions, using given combine functions and a neutral "zero value".
abstract void	cancelAllJobs()
abstract void	cancelJob(String jobId)
abstract <T> HoodieData<T>	emptyHoodieData()
abstract <I,O> List<O>	flatMap(List<I> data, SerializableFunction<I,Stream<O>> func, int parallelism)
abstract <I> void	foreach(List<I> data, SerializableConsumer<I> consumer, int parallelism)
abstract List<Integer>	getCachedDataIds(HoodieData.HoodieDataCacheKey cacheKey)
abstract Option<String>	getProperty(EngineProperty key)
StorageConfiguration<?>	getStorageConf()
TaskContextSupplier	getTaskContextSupplier()
abstract <I,O> List<O>	map(List<I> data, SerializableFunction<I,O> func, int parallelism)
abstract <I,K,V> Stream<ImmutablePair<K,V>>	mapPartitionsToPairAndReduceByKey(Stream<I> data, SerializablePairFlatMapFunction<Iterator<I>,K,V> flatMapToPairFunc, SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
abstract <I,K,V> Map<K,V>	mapToPair(List<I> data, SerializablePairFunction<I,K,V> func, Integer parallelism)
abstract <I,K,V> List<V>	mapToPairAndReduceByKey(List<I> data, SerializablePairFunction<I,K,V> mapToPairFunc, SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
abstract HoodieAccumulator	newAccumulator()
<T> HoodieData<T>	parallelize(List<T> data)
abstract <T> HoodieData<T>	parallelize(List<T> data, int parallelism)
abstract void	putCachedDataIds(HoodieData.HoodieDataCacheKey cacheKey, int... ids)
abstract <I,K,V> List<V>	reduceByKey(List<Pair<K,V>> data, SerializableBiFunction<V,V,V> reduceFunc, int parallelism)
abstract List<Integer>	removeCachedDataIds(HoodieData.HoodieDataCacheKey cacheKey)
abstract void	setJobStatus(String activeModule, String activityDescription)
abstract void	setProperty(EngineProperty key, String value)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

taskContextSupplier

protected TaskContextSupplier taskContextSupplier

Constructor Detail

HoodieEngineContext

public HoodieEngineContext(StorageConfiguration<?> storageConf,
                           TaskContextSupplier taskContextSupplier)

Method Detail

getStorageConf

public StorageConfiguration<?> getStorageConf()

getTaskContextSupplier

public TaskContextSupplier getTaskContextSupplier()

newAccumulator

public abstract HoodieAccumulator newAccumulator()

emptyHoodieData

public abstract <T> HoodieData<T> emptyHoodieData()

parallelize

public <T> HoodieData<T> parallelize(List<T> data)

parallelize

public abstract <T> HoodieData<T> parallelize(List<T> data,
                                              int parallelism)

map

public abstract <I,O> List<O> map(List<I> data,
                                  SerializableFunction<I,O> func,
                                  int parallelism)

mapToPairAndReduceByKey

public abstract <I,K,V> List<V> mapToPairAndReduceByKey(List<I> data,
                                                        SerializablePairFunction<I,K,V> mapToPairFunc,
                                                        SerializableBiFunction<V,V,V> reduceFunc,
                                                        int parallelism)

mapPartitionsToPairAndReduceByKey

public abstract <I,K,V> Stream<ImmutablePair<K,V>> mapPartitionsToPairAndReduceByKey(Stream<I> data,
                                                                                     SerializablePairFlatMapFunction<Iterator<I>,K,V> flatMapToPairFunc,
                                                                                     SerializableBiFunction<V,V,V> reduceFunc,
                                                                                     int parallelism)

reduceByKey

public abstract <I,K,V> List<V> reduceByKey(List<Pair<K,V>> data,
                                            SerializableBiFunction<V,V,V> reduceFunc,
                                            int parallelism)

flatMap

public abstract <I,O> List<O> flatMap(List<I> data,
                                      SerializableFunction<I,Stream<O>> func,
                                      int parallelism)

foreach

public abstract <I> void foreach(List<I> data,
                                 SerializableConsumer<I> consumer,
                                 int parallelism)

mapToPair

public abstract <I,K,V> Map<K,V> mapToPair(List<I> data,
                                           SerializablePairFunction<I,K,V> func,
                                           Integer parallelism)

setProperty

public abstract void setProperty(EngineProperty key,
                                 String value)

getProperty

public abstract Option<String> getProperty(EngineProperty key)

setJobStatus

public abstract void setJobStatus(String activeModule,
                                  String activityDescription)

putCachedDataIds

public abstract void putCachedDataIds(HoodieData.HoodieDataCacheKey cacheKey,
                                      int... ids)

getCachedDataIds

public abstract List<Integer> getCachedDataIds(HoodieData.HoodieDataCacheKey cacheKey)

removeCachedDataIds

public abstract List<Integer> removeCachedDataIds(HoodieData.HoodieDataCacheKey cacheKey)

cancelJob

public abstract void cancelJob(String jobId)

cancelAllJobs

public abstract void cancelAllJobs()

aggregate

public abstract <I,O> O aggregate(HoodieData<I> data,
                                  O zeroValue,
                                  Functions.Function2<O,I,O> seqOp,
                                  Functions.Function2<O,O,O> combOp)

Aggregate the elements of each partition, and then the results for all the partitions, using given combine functions and a neutral "zero value".

Type Parameters:

I - input object type

O - output object type

Parameters:

data -

zeroValue - the initial value for the accumulated result of each partition for the seqOp operator

seqOp - function to aggregate the elements of each partition

combOp - function to combine results from different partitions

Returns:

the result of the aggregation