de.l3s.icrawl.crawler.yarn

Class BaseAppMaster

java.lang.Object

de.l3s.icrawl.crawler.yarn.BaseAppMaster

Direct Known Subclasses:

AppMaster, EvaluationAppMaster

public abstract class BaseAppMaster
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.conf.Configuration	conf
protected com.google.common.util.concurrent.AtomicDouble	progress

Constructor Summary

Constructors

Constructor and Description
BaseAppMaster()

Method Summary

Modifier and Type	Method and Description
protected void	addDefaultProps(com.google.common.collect.ImmutableMap.Builder<String,Object> defaultProps)
protected abstract void	crawl(org.springframework.context.ApplicationContext context, Crawler crawler)
protected long	getMaxUrls()
protected Set<String>	getProfiles()
void	run(String[] args)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

conf

protected final org.apache.hadoop.conf.Configuration conf

progress

protected final com.google.common.util.concurrent.AtomicDouble progress

Constructor Detail

BaseAppMaster

public BaseAppMaster()

Method Detail

run

public void run(String[] args)
         throws org.apache.hadoop.yarn.exceptions.YarnException,
                IOException

Throws:

org.apache.hadoop.yarn.exceptions.YarnException

IOException

getProfiles

protected Set<String> getProfiles()

crawl

protected abstract void crawl(org.springframework.context.ApplicationContext context,
                              Crawler crawler)
                       throws IOException

Throws:

IOException

addDefaultProps

protected void addDefaultProps(com.google.common.collect.ImmutableMap.Builder<String,Object> defaultProps)

getMaxUrls

protected long getMaxUrls()