com.bytegriffin.get4j.core

类 Globals

java.lang.Object

com.bytegriffin.get4j.core.Globals

public final class Globals
extends Object

全局变量：缓存供全局访问的变量

字段概要

字段

限定符和类型	字段和说明
static Map<String,Chain>	CHAIN_CACHE 全局chain工作流缓存 key:seed_name value: site
static Map<String,DataSource>	DATASOURCE_CACHE key : seedName value: datasource
static Map<String,String>	DOWNLOAD_DISK_DIR_CACHE 全局download dir缓存 key:seed_name value: download dir
static Map<String,String>	DOWNLOAD_HDFS_DIR_CACHE key:seed_name value: hdfs://address:port/seedname/
static Map<String,Map<String,String>>	DYNAMIC_FIELDS_CACHE key：seedName value:field_map (key: name value: selector)
static EmailSender	emailSender 全局Email发送器，如果配置了email.recipient就表示当系统出现异常发送邮件提醒
static Map<String,String>	FETCH_DETAIL_SELECT_CACHE LIST_DETAIL模式下的详情页面url选择
static Map<String,PageMode>	FETCH_PAGE_MODE_CACHE 全局PageMode缓存 key:seed_name value: PageMode
static Map<String,PageChangeProber>	FETCH_PROBE_CACHE 抓取页面变化探测器缓存 key:seed_name value: PageChangeProber
static Map<String,FetchResourceSelector>	FETCH_RESOURCE_SELECTOR_CACHE 全局FetchFilter缓存 key:seed_name value: FetchResourceUrl
static Map<String,Integer>	FETCH_SLEEP_CACHE 全局http请求间隔缓存 key:seed_name value: fetch.sleep
static Map<String,SleepRandomSelector>	FETCH_SLEEP_RANGE_CACHE 全局sleep_selector缓存 key:seed_name value: SleepRangeSelector
static Map<String,HttpEngine>	HTTP_ENGINE_CACHE 全局Http探针缓存 key:seed_name value: HttpEngine
static Map<String,HttpProxySelector>	HTTP_PROXY_CACHE 全局http_proxy缓存 key:seed_name value: HttpProxySelector
static Map<String,org.apache.lucene.index.IndexWriter>	INDEX_WRITER_CACHE key : seedName value: IndexWriter
static Map<String,Launcher>	LAUNCHER_CACHE 当PageMode存放list列表url key：seed_name value：该seed对应的launcher
static Map<String,List<String>>	LIST_URLS_CACHE 当PageMode存放list列表url key：seed_name value：该seed下所有的列表url
static Map<String,String>	LUCENE_INDEX_DIR_CACHE 全局lucene index dir缓存 key:seed_name value: lucene index dir
static Map<String,com.mongodb.client.MongoCollection<org.bson.Document>>	MONGO_COLLECTION_CACHE key : seedName value: MongoCollection
static Map<String,okhttp3.OkHttpClient.Builder>	OK_HTTP_CLIENT_BUILDER_CACHE 全局okhttpclientbuilder缓存 key:seed_name value: HttpClientBuilder
static Map<String,PageParser>	PAGE_PARSER_CACHE 自定义的page_parser解析器缓存
static Map<String,String>	PER_START_TIME_CACHE key:seed_name | value:每次开始抓取时间
static Map<String,Seed>	SEED_CACHE 种子seed缓存，key : seed name key value : seed
static Map<String,UserAgentSelector>	USER_AGENT_CACHE 全局user_agent缓存 key:seed_name value: UserAgentSelector
static Map<String,org.openqa.selenium.WebDriver>	WEBDRIVER_CACHE 全局WebDriver缓存 key:seed_name value: WebDriver

构造器概要

构造器

构造器和说明
Globals()

方法概要

从类继承的方法 java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

字段详细资料

SEED_CACHE

public static Map<String,Seed> SEED_CACHE

种子seed缓存，key : seed name key value : seed

LIST_URLS_CACHE

public static Map<String,List<String>> LIST_URLS_CACHE

当PageMode存放list列表url key：seed_name value：该seed下所有的列表url

CHAIN_CACHE

public static final Map<String,Chain> CHAIN_CACHE

全局chain工作流缓存 key:seed_name value: site

HTTP_PROXY_CACHE

public static final Map<String,HttpProxySelector> HTTP_PROXY_CACHE

全局http_proxy缓存 key:seed_name value: HttpProxySelector

USER_AGENT_CACHE

public static final Map<String,UserAgentSelector> USER_AGENT_CACHE

全局user_agent缓存 key:seed_name value: UserAgentSelector

FETCH_SLEEP_CACHE

public static final Map<String,Integer> FETCH_SLEEP_CACHE

全局http请求间隔缓存 key:seed_name value: fetch.sleep

FETCH_PROBE_CACHE

public static final Map<String,PageChangeProber> FETCH_PROBE_CACHE

抓取页面变化探测器缓存 key:seed_name value: PageChangeProber

FETCH_SLEEP_RANGE_CACHE

public static final Map<String,SleepRandomSelector> FETCH_SLEEP_RANGE_CACHE

全局sleep_selector缓存 key:seed_name value: SleepRangeSelector

OK_HTTP_CLIENT_BUILDER_CACHE

public static final Map<String,okhttp3.OkHttpClient.Builder> OK_HTTP_CLIENT_BUILDER_CACHE

全局okhttpclientbuilder缓存 key:seed_name value: HttpClientBuilder

DOWNLOAD_DISK_DIR_CACHE

public static final Map<String,String> DOWNLOAD_DISK_DIR_CACHE

全局download dir缓存 key:seed_name value: download dir

DOWNLOAD_HDFS_DIR_CACHE

public static final Map<String,String> DOWNLOAD_HDFS_DIR_CACHE

key:seed_name value: hdfs://address:port/seedname/

LUCENE_INDEX_DIR_CACHE

public static final Map<String,String> LUCENE_INDEX_DIR_CACHE

全局lucene index dir缓存 key:seed_name value: lucene index dir

FETCH_PAGE_MODE_CACHE

public static final Map<String,PageMode> FETCH_PAGE_MODE_CACHE

全局PageMode缓存 key:seed_name value: PageMode

FETCH_DETAIL_SELECT_CACHE

public static final Map<String,String> FETCH_DETAIL_SELECT_CACHE

LIST_DETAIL模式下的详情页面url选择

FETCH_RESOURCE_SELECTOR_CACHE

public static final Map<String,FetchResourceSelector> FETCH_RESOURCE_SELECTOR_CACHE

全局FetchFilter缓存 key:seed_name value: FetchResourceUrl

PAGE_PARSER_CACHE

public static final Map<String,PageParser> PAGE_PARSER_CACHE

自定义的page_parser解析器缓存

DATASOURCE_CACHE

public static Map<String,DataSource> DATASOURCE_CACHE

key : seedName value: datasource

MONGO_COLLECTION_CACHE

public static Map<String,com.mongodb.client.MongoCollection<org.bson.Document>> MONGO_COLLECTION_CACHE

key : seedName value: MongoCollection

INDEX_WRITER_CACHE

public static Map<String,org.apache.lucene.index.IndexWriter> INDEX_WRITER_CACHE

key : seedName value: IndexWriter

WEBDRIVER_CACHE

public static final Map<String,org.openqa.selenium.WebDriver> WEBDRIVER_CACHE

全局WebDriver缓存 key:seed_name value: WebDriver

HTTP_ENGINE_CACHE

public static final Map<String,HttpEngine> HTTP_ENGINE_CACHE

全局Http探针缓存 key:seed_name value: HttpEngine

emailSender

public static EmailSender emailSender

全局Email发送器，如果配置了email.recipient就表示当系统出现异常发送邮件提醒

PER_START_TIME_CACHE

public static Map<String,String> PER_START_TIME_CACHE

key:seed_name | value:每次开始抓取时间

LAUNCHER_CACHE

public static Map<String,Launcher> LAUNCHER_CACHE

当PageMode存放list列表url key：seed_name value：该seed对应的launcher

DYNAMIC_FIELDS_CACHE

public static Map<String,Map<String,String>> DYNAMIC_FIELDS_CACHE

key：seedName value:field_map (key: name value: selector)

构造器详细资料

Globals

public Globals()