Versions in this module Expand all Collapse all v1 v1.0.0 Oct 4, 2022 Changes in this version + func AllowedDomains(domains []string) func(*Spider) + func BasicAuthHeader(username string, password string) string + func BytesToString(b []byte) string + func CloseFunc(fc func(*Spider)) func(*Spider) + func ConcurrentItems(items int) func(*Setting) + func ConcurrentRequests(requests int) func(*Setting) + func ConcurrentRequestsPerDomain(requests int) func(*Setting) + func CopyNew(old interface{}) interface + func DefaultParseFunc(fc ParseFunc) func(*Spider) + func DefaultRequestHeaders(headers http.Header) func(*Setting) + func DepthLimit(depth int) func(*Setting) + func DepthPriority(depth int) func(*Setting) + func DepthStatsVerbose(depth bool) func(*Setting) + func DownloadDelay(t time.Duration) func(*Setting) + func DownloadMaxSize(size int) func(*Setting) + func DownloadStats(stats bool) func(*Setting) + func DownloadTimeout(t time.Duration) func(*Setting) + func DownloadWarnSize(size int) func(*Setting) + func DownloaderClass(c Downloader) func(*Setting) + func DownloaderMiddlewares(d map[int]DownloaderMiddlewarer) func(*Setting) + func DownloaderMiddlewaresBase(d map[int]DownloaderMiddlewarer) func(*Setting) + func DropItem(reason string) error + func ErrFile(file string) func(*Setting) + func FilterClass(f DupeFilter) func(*Setting) + func FilterEnabled(filter bool) func(*Setting) + func FindScriptVar(b []byte, v string) string + func GetHeaderSize(header http.Header) int + func GetRequestSize(request *Request) int + func GetResponseSize(response *Response) int + func HttpAuthDomain(h []string) func(*Setting) + func HttpError(reason string) error + func HttpErrorAllowAll(http bool) func(*Setting) + func HttpErrorAllowedCodes(http []int) func(*Setting) + func HttpPass(h string) func(*Setting) + func HttpUser(h string) func(*Setting) + func IgnoreRequest(reason string) error + func InitLog(logPath, errPath string, logLevel zapcore.Level) *zap.SugaredLogger + func ItemPipelines(i map[int]ItemPipeliner) func(*Setting) + func ItemPipelinesBase(i map[int]ItemPipeliner) func(*Setting) + func LogEnabled(log bool) func(*Setting) + func LogFile(log string) func(*Setting) + func LogLevel(log zapcore.Level) func(*Setting) + func MaxRetryTimes(retry int) func(*Setting) + func Name(name string) func(*Spider) + func NewRequestLogger(log *zap.SugaredLogger, req *Request) *zap.SugaredLogger + func NewResponseLogger(log *zap.SugaredLogger, response *Response) *zap.SugaredLogger + func RandomizeDownloadDelay(delay bool) func(*Setting) + func ReadRequestBody(request *Request) []byte + func ResponseMaxActiveSize(size int) func(*Setting) + func RetryEnabled(retry bool) func(*Setting) + func RetryHttpCodes(retry []int) func(*Setting) + func RetryPriorityAdjust(retry int) func(*Setting) + func SanitizeFileName(fileName string) string + func SchedulerClass(c Scheduler) func(*Setting) + func SchedulerPriorityQueue(pq PriorityQueuer) func(*Setting) + func Settings(settings *Setting) func(*Spider) + func SpiderMiddlewares(m map[int]SpiderMiddlewarer) func(*Setting) + func SpiderMiddlewaresBase(m map[int]SpiderMiddlewarer) func(*Setting) + func StartRequestsFunc(fc func(*Spider) []*Request) func(*Spider) + func StartUrls(urls []string) func(*Spider) + func StringToBytes(s string) []byte + func UrlLengthLimit(limits int) func(*Setting) + func UrlSafeBase64Decode(source string) []byte + func UrlSafeBase64Encode(source []byte) string + func UserAgent(ua string) func(*Setting) + type BaseItemPipeline struct + Logger *zap.SugaredLogger + ModuleName string + Stats StatsCollector + func (i *BaseItemPipeline) CloseSpider(spider *Spider) + func (i *BaseItemPipeline) FromSpider(spider *Spider) + func (i *BaseItemPipeline) GetModuleName() string + func (i *BaseItemPipeline) OpenSpider(spider *Spider) + func (i *BaseItemPipeline) ProcessItem(item *Item, spider *Spider) + type BaseMiddleware struct + Logger *zap.SugaredLogger + ModuleName string + Stats StatsCollector + func (mw *BaseMiddleware) FromSpider(spider *Spider) + func (mw *BaseMiddleware) GetModuleName() string + func (mw *BaseMiddleware) ProcessException(request *Request, err error, spider *Spider) RequestResponse + func (mw *BaseMiddleware) ProcessRequest(request *Request, spider *Spider) RequestResponse + func (mw *BaseMiddleware) ProcessResponse(request *Request, response *Response, spider *Spider) RequestResponse + func (mw *BaseMiddleware) ProcessSpiderException(response *Response, err error, spider *Spider) RequestItems + func (mw *BaseMiddleware) ProcessSpiderInput(response *Response, spider *Spider) + func (mw *BaseMiddleware) ProcessSpiderOutput(response *Response, result RequestItems, spider *Spider) RequestItems + func (mw *BaseMiddleware) ProcessStartRequests(result []*Request, spider *Spider) []*Request + type CloseSpiderFunc func(*Spider) + type Context struct + func NewContext() *Context + func (c *Context) ForEach(fn func(k string, v interface{}) interface{}) []interface{} + func (c *Context) Get(key string) interface{} + func (c *Context) GetBool(key string) bool + func (c *Context) GetBoolWithDefault(key string, dft bool) bool + func (c *Context) GetFloat64(key string) float64 + func (c *Context) GetFloat64WithDefault(key string, dft float64) float64 + func (c *Context) GetInt(key string) int + func (c *Context) GetIntWithDefault(key string, dft int) int + func (c *Context) GetString(key string) string + func (c *Context) GetStringWithDefault(key string, dft string) string + func (c *Context) GetWithDefault(key string, dft interface{}) interface{} + func (c *Context) Has(key string) bool + func (c *Context) Put(key string, value interface{}) + type DefaultDownloader struct + func (d *DefaultDownloader) Close() + func (d *DefaultDownloader) Fetch(request *Request) (*Response, error) + func (d *DefaultDownloader) FromSpider(spider *Spider) + func (d *DefaultDownloader) IsEmpty() bool + func (d *DefaultDownloader) IsFree() bool + func (d *DefaultDownloader) NextRequestCircle(heartbeat time.Duration) + func (d *DefaultDownloader) ProcessDownloader(signal *Signal, spider *Spider) + type DefaultDupeFilter struct + func (df *DefaultDupeFilter) Close() + func (df *DefaultDupeFilter) FromSpider(spider *Spider) + func (df *DefaultDupeFilter) Log(request *Request) + func (df *DefaultDupeFilter) RequestFingerprint(request *Request) string + func (df *DefaultDupeFilter) RequestSeen(request *Request) bool + type DefaultScheduler struct + func (s *DefaultScheduler) Close() + func (s *DefaultScheduler) EnqueueRequest(request *Request) bool + func (s *DefaultScheduler) FromSpider(spider *Spider) + func (s *DefaultScheduler) HasPendingRequests() bool + func (s *DefaultScheduler) NextRequest() *Request + type DepthMiddleware struct + func (mw *DepthMiddleware) FromSpider(spider *Spider) + func (mw *DepthMiddleware) ProcessSpiderOutput(response *Response, result RequestItems, spider *Spider) RequestItems + type DownloadStatsMiddleware struct + func (mw *DownloadStatsMiddleware) FromSpider(spider *Spider) + func (mw *DownloadStatsMiddleware) ProcessException(request *Request, err error, spider *Spider) RequestResponse + func (mw *DownloadStatsMiddleware) ProcessRequest(request *Request, spider *Spider) RequestResponse + func (mw *DownloadStatsMiddleware) ProcessResponse(request *Request, response *Response, spider *Spider) RequestResponse + type DownloadTimeoutMiddleware struct + func (mw *DownloadTimeoutMiddleware) FromSpider(spider *Spider) + func (mw *DownloadTimeoutMiddleware) ProcessRequest(request *Request, spider *Spider) RequestResponse + type Downloader interface + Close func() + Fetch func(request *Request) (*Response, error) + FromSpider func(spider *Spider) + IsEmpty func() bool + IsFree func() bool + NextRequestCircle func(heartbeat time.Duration) + ProcessDownloader func(signal *Signal, spider *Spider) + type DownloaderMiddlewareManager struct + ModuleName string + func (d *DownloaderMiddlewareManager) FromSpider(spider *Spider) + func (d *DownloaderMiddlewareManager) ProcessException(signal *Signal, spider *Spider) (index int) + func (d *DownloaderMiddlewareManager) ProcessRequest(signal *Signal, spider *Spider) (index int) + func (d *DownloaderMiddlewareManager) ProcessResponse(signal *Signal, spider *Spider) (index int) + type DownloaderMiddlewarer interface + FromSpider func(spider *Spider) + GetModuleName func() string + ProcessException func(request *Request, err error, spider *Spider) RequestResponse + ProcessRequest func(request *Request, spider *Spider) RequestResponse + ProcessResponse func(request *Request, response *Response, spider *Spider) RequestResponse + type DownloaderSlot struct + func (s *DownloaderSlot) DownloadDelay() int64 + func (s *DownloaderSlot) IsFree() bool + func (s *DownloaderSlot) Len() int + type DupeFilter interface + Close func() + FromSpider func(spider *Spider) + Log func(request *Request) + RequestFingerprint func(request *Request) string + RequestSeen func(request *Request) bool + type ErrBase struct + Reason string + func (err *ErrBase) Error() string + type ErrDropItem struct + type ErrHttpError struct + type ErrIgnoreRequest struct + type ErrorbackFunc func(*Failure) RequestItems + type Failure struct + Error error + Request *Request + Response *Response + Spider *Spider + type HttpAuthMiddleware struct + func (mw *HttpAuthMiddleware) FromSpider(spider *Spider) + func (mw *HttpAuthMiddleware) ProcessRequest(request *Request, spider *Spider) RequestResponse + type HttpErrorMiddleware struct + func (mw *HttpErrorMiddleware) FromSpider(spider *Spider) + func (mw *HttpErrorMiddleware) ProcessSpiderException(response *Response, err error, spider *Spider) RequestItems + func (mw *HttpErrorMiddleware) ProcessSpiderInput(response *Response, spider *Spider) + type Item map[string]interface + type ItemPipelineManager struct + ModuleName string + func (i *ItemPipelineManager) CloseSpider(spider *Spider) (index int) + func (i *ItemPipelineManager) FromSpider(spider *Spider) + func (i *ItemPipelineManager) OpenSpider(spider *Spider) (index int) + func (i *ItemPipelineManager) ProcessItem(item Item, spider *Spider) (index int) + type ItemPipeliner interface + CloseSpider func(*Spider) + GetModuleName func() string + OpenSpider func(*Spider) + ProcessItem func(Item, *Spider) Item + type ItemSlot struct + func (s *ItemSlot) AddItem(item Item) + func (s *ItemSlot) FinishItem(item Item) + func (s *ItemSlot) FromSpider(spider *Spider) + func (s *ItemSlot) IsFree() bool + type LifoPriorityQueue struct + func (q *LifoPriorityQueue) Close() + func (q *LifoPriorityQueue) FromSpider(spider *Spider) + func (q *LifoPriorityQueue) IsEmpty() bool + func (q *LifoPriorityQueue) Peek() interface{} + func (q *LifoPriorityQueue) Pop() interface{} + func (q *LifoPriorityQueue) Push(v interface{}, i int) + func (q *LifoPriorityQueue) Size() int + type OpenSpiderFunc func(*Spider) + type ParseFunc func(*Response, *Spider) RequestItems + type PriorityQueuer interface + Close func() + FromSpider func(*Spider) + IsEmpty func() bool + Peek func() interface{} + Pop func() interface{} + Push func(item interface{}, priority int) + Size func() int + type ProcessExceptionFunc func(request *Request, err error, spider *Spider) RequestResponse + type ProcessItemFunc func(Item, *Spider) Item + type ProcessRequestFunc func(request *Request, spider *Spider) RequestResponse + type ProcessResponseFunc func(request *Request, response *Response, spider *Spider) RequestResponse + type ProcessSpiderExceptionFunc func(response *Response, err error, spider *Spider) RequestItems + type ProcessSpiderInputFunc func(response *Response, spider *Spider) + type ProcessSpiderOutputFunc func(response *Response, result RequestItems, spider *Spider) RequestItems + type ProcessStartRequestsFunc func(result []*Request, spider *Spider) []*Request + type Request struct + Body io.Reader + Callback ParseFunc + Cookies []http.Cookie + Ctx *Context + DontFilter bool + Encoding string + Errback ErrorbackFunc + Headers *http.Header + Method string + Priority int + Url *url.URL + func DefaultStartRequests(s *Spider) []*Request + func NewRequest(method, URL string, body io.Reader) (*Request, error) + func (r *Request) Domain() string + type RequestItems []interface + type RequestResponse interface + type Response struct + Body []byte + Ctx *Context + Headers *http.Header + Request *Request + StatusCode int + func NewResponse(response *http.Response, request *Request) (*Response, error) + func (r *Response) FileName() string + func (r *Response) Save(fileName string) error + type ResponseSlot struct + func (s *ResponseSlot) AddResponse(response *Response) + func (s *ResponseSlot) FinishResponse(response *Response) + func (s *ResponseSlot) FromSpider(spider *Spider) + func (s *ResponseSlot) IsFree() bool + type RetryMiddleware struct + func (mw *RetryMiddleware) FromSpider(spider *Spider) + func (mw *RetryMiddleware) ProcessException(request *Request, err error, spider *Spider) RequestResponse + func (mw *RetryMiddleware) ProcessResponse(request *Request, response *Response, spider *Spider) RequestResponse + type Scheduler interface + Close func() + EnqueueRequest func(request *Request) bool + FromSpider func(spider *Spider) + HasPendingRequests func() bool + NextRequest func() *Request + type Setter interface + Add func(elems ...interface{}) + Clear func() + Discard func(elems ...interface{}) + Has func(elems ...interface{}) bool + Len func() int + Pop func() interface{} + Remove func(elems ...interface{}) error + func NewSet(size int) Setter + type Setting struct + ConcurrentItems int + ConcurrentRequests int + ConcurrentRequestsPerDomain int + DefaultRequestHeaders http.Header + DepthLimit int + DepthPriority int + DepthStatsVerbose bool + DownloadDelay time.Duration + DownloadMaxSize int + DownloadStats bool + DownloadTimeout time.Duration + DownloadWarnSize int + DownloaderClass Downloader + DownloaderMiddlewares map[int]DownloaderMiddlewarer + DownloaderMiddlewaresBase map[int]DownloaderMiddlewarer + ErrFile string + ExtensionSettings *Context + FilterClass DupeFilter + FilterEnabled bool + HttpAuthDomain []string + HttpErrorAllowAll bool + HttpErrorAllowedCodes []int + HttpPass string + HttpUser string + ItemPipelines map[int]ItemPipeliner + ItemPipelinesBase map[int]ItemPipeliner + LogEnabled bool + LogFile string + LogLevel zapcore.Level + MaxRetryTimes int + RandomizeDownloadDelay bool + ResponseMaxActiveSize int + RetryEnabled bool + RetryHttpCodes []int + RetryPriorityAdjust int + SchedulerClass Scheduler + SchedulerPriorityQueue PriorityQueuer + SpiderMiddlewares map[int]SpiderMiddlewarer + SpiderMiddlewaresBase map[int]SpiderMiddlewarer + UrlLengthLimit int + UserAgent string + func NewSetting(options ...func(*Setting)) *Setting + func (s *Setting) Init() + type Signal struct + Body interface{} + From int + Index int + To int + type Spider struct + AllowedDomains []string + CloseFunc func(*Spider) + DefaultParseFunc ParseFunc + Log *zap.SugaredLogger + Name string + Settings *Setting + StartRequestsFunc func(*Spider) []*Request + StartUrls []string + Stats StatsCollector + func NewSpider(options ...func(*Spider)) *Spider + func (s *Spider) Init() + func (s *Spider) Run() + type SpiderMiddlewareManager struct + ModuleName string + func (s *SpiderMiddlewareManager) FromSpider(spider *Spider) + func (s *SpiderMiddlewareManager) ProcessSpiderException(signal *Signal, spider *Spider) (index int) + func (s *SpiderMiddlewareManager) ProcessSpiderInput(signal *Signal, spider *Spider) (index int) + func (s *SpiderMiddlewareManager) ProcessSpiderOutput(signal *Signal, spider *Spider) (index int) + func (s *SpiderMiddlewareManager) ProcessStartRequests(signal *Signal, spider *Spider) (index int) + type SpiderMiddlewarer interface + FromSpider func(spider *Spider) + GetModuleName func() string + ProcessSpiderException func(response *Response, err error, spider *Spider) RequestItems + ProcessSpiderInput func(response *Response, spider *Spider) + ProcessSpiderOutput func(response *Response, result RequestItems, spider *Spider) RequestItems + ProcessStartRequests func(result []*Request, spider *Spider) []*Request + type SpiderOutputData struct + Response *Response + Result RequestItems + type StatsCollect struct + func (s *StatsCollect) ClearStats() + func (s *StatsCollect) GetStats() StatsMap + func (s *StatsCollect) GetValue(key string, dft int) int + func (s *StatsCollect) IncValue(key string, count int, start int) + func (s *StatsCollect) MaxValue(key string, value int) + func (s *StatsCollect) MinValue(key string, value int) + func (s *StatsCollect) SetStats(stats StatsMap) + func (s *StatsCollect) SetValue(key string, value int) + type StatsCollector interface + ClearStats func() + GetStats func() StatsMap + GetValue func(key string, dft int) int + IncValue func(key string, count int, start int) + MaxValue func(key string, value int) + MinValue func(key string, value int) + SetStats func(stats StatsMap) + SetValue func(key string, value int) + func NewStatsCollector() StatsCollector + type StatsMap map[string]int + type UrlLengthMiddleware struct + func (mw *UrlLengthMiddleware) FromSpider(spider *Spider) + func (mw *UrlLengthMiddleware) ProcessSpiderOutput(response *Response, result RequestItems, spider *Spider) RequestItems + type UserAgentMiddleware struct + func (mw *UserAgentMiddleware) FromSpider(spider *Spider) + func (mw *UserAgentMiddleware) ProcessRequest(request *Request, spider *Spider) RequestResponse
Go to main page