de.l3s.icrawl.crawler.yarn

Class BaseClient

java.lang.Object

org.apache.hadoop.conf.Configured

de.l3s.icrawl.crawler.yarn.BaseClient

All Implemented Interfaces:

org.apache.hadoop.conf.Configurable, org.apache.hadoop.util.Tool

Direct Known Subclasses:

Client, EvaluationClient

public abstract class BaseClient
extends org.apache.hadoop.conf.Configured
implements org.apache.hadoop.util.Tool

Constructor Summary

Constructors

Constructor and Description
BaseClient()

Method Summary

Modifier and Type	Method and Description
protected void	addParameters(com.google.common.collect.ImmutableMap.Builder<String,String> env)
protected void	addResources(com.google.common.collect.ImmutableMap.Builder<String,org.apache.hadoop.yarn.api.records.LocalResource> resources)
protected abstract boolean	checkUsage(String[] args)
protected void	cleanup()
protected void	configure(String[] args)
protected abstract String	getAppMasterClass()
protected abstract String	getJarPath()
protected static org.apache.hadoop.yarn.api.records.LocalResource	getResource(org.apache.hadoop.fs.Path path, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.yarn.api.records.LocalResourceType resourceType)
protected void	prepare()
int	run(String[] args)
protected static org.apache.hadoop.fs.Path	uploadTempFile(org.apache.hadoop.fs.Path localPath, org.apache.hadoop.conf.Configuration conf, String prefix, String extension)

Methods inherited from class org.apache.hadoop.conf.Configured

getConf, setConf

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Methods inherited from interface org.apache.hadoop.conf.Configurable

getConf, setConf

Constructor Detail

BaseClient

public BaseClient()

Method Detail

run

public int run(String[] args)
        throws org.apache.hadoop.yarn.exceptions.YarnException,
               IOException,
               InterruptedException

Specified by:

run in interface org.apache.hadoop.util.Tool

Throws:

org.apache.hadoop.yarn.exceptions.YarnException

IOException

InterruptedException

cleanup

protected void cleanup()
                throws IOException

Throws:

IOException

prepare

protected void prepare()
                throws IOException

Throws:

IOException

checkUsage

protected abstract boolean checkUsage(String[] args)

addParameters

protected void addParameters(com.google.common.collect.ImmutableMap.Builder<String,String> env)

addResources

protected void addResources(com.google.common.collect.ImmutableMap.Builder<String,org.apache.hadoop.yarn.api.records.LocalResource> resources)
                     throws IOException

Throws:

IOException

getAppMasterClass

protected abstract String getAppMasterClass()

uploadTempFile

protected static org.apache.hadoop.fs.Path uploadTempFile(org.apache.hadoop.fs.Path localPath,
                                                          org.apache.hadoop.conf.Configuration conf,
                                                          String prefix,
                                                          String extension)
                                                   throws IOException

Throws:

IOException

getResource

protected static org.apache.hadoop.yarn.api.records.LocalResource getResource(org.apache.hadoop.fs.Path path,
                                                                              org.apache.hadoop.conf.Configuration conf,
                                                                              org.apache.hadoop.yarn.api.records.LocalResourceType resourceType)
                                                                       throws IOException

Throws:

IOException

getJarPath

protected abstract String getJarPath()

configure

protected void configure(String[] args)