de.l3s.icrawl.crawler

Class Crawler

java.lang.Object

de.l3s.icrawl.crawler.Crawler

public class Crawler
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static String	NUM_URLS
static String	RELEVANCE_THRESHOLD
static String	SNAPSHOTS_TO_ANALYZE
static String	WEIGHTING_METHOD

Constructor Summary

Constructors

Constructor and Description
Crawler(org.apache.hadoop.conf.Configuration conf, String indexPath, String dataPath, ResourceAnalyserFactory analyserFactory, ResultStorer.Factory storerFactory, com.codahale.metrics.MetricRegistry metrics, int numThreads)

Method Summary

Modifier and Type	Method and Description
void	crawlContinuously(ArchiveCrawlSpecification spec, StoppingCriterion stoppingCriterion, ResourceAnalyser.WeightingMethod method, double relevanceThreshold, int snapshotsToAnalyze)
Optional<ArchiveCrawlSpecification>	getCurrentSpec()
void	shutdown()
void	stop(boolean interruptRunningFetches)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

NUM_URLS

public static final String NUM_URLS

See Also:

Constant Field Values

WEIGHTING_METHOD

public static final String WEIGHTING_METHOD

See Also:

Constant Field Values

RELEVANCE_THRESHOLD

public static final String RELEVANCE_THRESHOLD

See Also:

Constant Field Values

SNAPSHOTS_TO_ANALYZE

public static final String SNAPSHOTS_TO_ANALYZE

See Also:

Constant Field Values

Constructor Detail

Crawler

public Crawler(org.apache.hadoop.conf.Configuration conf,
               String indexPath,
               String dataPath,
               ResourceAnalyserFactory analyserFactory,
               ResultStorer.Factory storerFactory,
               com.codahale.metrics.MetricRegistry metrics,
               int numThreads)
        throws IOException

Throws:

IOException

Method Detail

crawlContinuously

public void crawlContinuously(ArchiveCrawlSpecification spec,
                              StoppingCriterion stoppingCriterion,
                              ResourceAnalyser.WeightingMethod method,
                              double relevanceThreshold,
                              int snapshotsToAnalyze)
                       throws IOException

Throws:

IOException

stop

public void stop(boolean interruptRunningFetches)

shutdown

public void shutdown()

getCurrentSpec

public Optional<ArchiveCrawlSpecification> getCurrentSpec()