arachne

README

arachne

Documentation

Overview

Package arachne is a generated GoMock package.

Package arachne is a generated GoMock package.

Package arachne is a generated GoMock package.

Index

• type HTTPClient
• type LogLevel
• type Logger
• type MockHTTPClient
  • func NewMockHTTPClient(ctrl *gomock.Controller) *MockHTTPClient
  • func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error)
  • func (m *MockHTTPClient) EXPECT() *MockHTTPClientMockRecorder
• type MockHTTPClientMockRecorder
  • func (mr *MockHTTPClientMockRecorder) Do(req interface{}) *gomock.Call
• type MockLogger
  • func NewMockLogger(ctrl *gomock.Controller) *MockLogger
  • func (m *MockLogger) Criticalf(format string, v ...interface{})
  • func (m *MockLogger) Debugf(format string, v ...interface{})
  • func (m *MockLogger) EXPECT() *MockLoggerMockRecorder
  • func (m *MockLogger) Errorf(format string, v ...interface{})
  • func (m *MockLogger) Infof(format string, v ...interface{})
  • func (m *MockLogger) Warnf(format string, v ...interface{})
• type MockLoggerMockRecorder
  • func (mr *MockLoggerMockRecorder) Criticalf(format interface{}, v ...interface{}) *gomock.Call
  • func (mr *MockLoggerMockRecorder) Debugf(format interface{}, v ...interface{}) *gomock.Call
  • func (mr *MockLoggerMockRecorder) Errorf(format interface{}, v ...interface{}) *gomock.Call
  • func (mr *MockLoggerMockRecorder) Infof(format interface{}, v ...interface{}) *gomock.Call
  • func (mr *MockLoggerMockRecorder) Warnf(format interface{}, v ...interface{}) *gomock.Call
• type MockWorkerQueue
  • func NewMockWorkerQueue(ctrl *gomock.Controller) *MockWorkerQueue
  • func (m *MockWorkerQueue) EXPECT() *MockWorkerQueueMockRecorder
  • func (m *MockWorkerQueue) PublishRequest(request *Request) error
  • func (m *MockWorkerQueue) RetryRequest(request *Request) error
  • func (m *MockWorkerQueue) SubscribeRequests(ctx context.Context) (<-chan *Request, error)
• type MockWorkerQueueMockRecorder
  • func (mr *MockWorkerQueueMockRecorder) PublishRequest(request interface{}) *gomock.Call
  • func (mr *MockWorkerQueueMockRecorder) RetryRequest(request interface{}) *gomock.Call
  • func (mr *MockWorkerQueueMockRecorder) SubscribeRequests(ctx interface{}) *gomock.Call
• type Request
  • func NewGetRequest(urlStr string) (*Request, error)
  • func NewRequestFromHTTPRequest(request *http.Request) (*Request, error)
  • func (r *Request) BodyReader() io.Reader
  • func (r *Request) HTTPRequest() (*http.Request, error)
  • func (r *Request) URLHost() string
• type Response
  • func NewResponseFromHTTPResponse(response *http.Response) (*Response, error)
  • func (r *Response) ContentType() string
  • func (r *Response) Follow(urlString string) (string, error)
  • func (r *Response) FollowRequest(urlString string) (*Request, error)
  • func (r *Response) Text() string
• type Worker
  • func (w *Worker) RetryMiddleware(request *Request)
  • func (w *Worker) Start(ctx context.Context) error
  • func (w *Worker) StartWithFirstRequest(ctx context.Context, URL string) error
• type WorkerQueue

Types

type HTTPClient

type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

HTTPClient is like http.Client

type LogLevel

type LogLevel int

LogLevel : DebugLevel, InfoLevel, WarnLevel, ErrorLevel, CriticalLevel

const (
	// DebugLevel : debug or higher
	DebugLevel LogLevel = iota
	// InfoLevel : info or higher
	InfoLevel
	// WarnLevel : warn or higher
	WarnLevel
	// ErrorLevel : error or higher
	ErrorLevel
	// CriticalLevel : critical
	CriticalLevel
)

type Logger

type Logger interface {
	Debugf(format string, v ...interface{})
	Infof(format string, v ...interface{})
	Warnf(format string, v ...interface{})
	Errorf(format string, v ...interface{})
	Criticalf(format string, v ...interface{})
}

Logger is interface for logging in arachne crawler.

type MockHTTPClient

type MockHTTPClient struct {
	// contains filtered or unexported fields
}

MockHTTPClient is a mock of HTTPClient interface

func NewMockHTTPClient

func NewMockHTTPClient(ctrl *gomock.Controller) *MockHTTPClient

NewMockHTTPClient creates a new mock instance

func (*MockHTTPClient) Do

func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error)

Do mocks base method

func (*MockHTTPClient) EXPECT

func (m *MockHTTPClient) EXPECT() *MockHTTPClientMockRecorder

EXPECT returns an object that allows the caller to indicate expected use

type MockHTTPClientMockRecorder

type MockHTTPClientMockRecorder struct {
	// contains filtered or unexported fields
}

MockHTTPClientMockRecorder is the mock recorder for MockHTTPClient

func (*MockHTTPClientMockRecorder) Do

func (mr *MockHTTPClientMockRecorder) Do(req interface{}) *gomock.Call

Do indicates an expected call of Do

type MockLogger

type MockLogger struct {
	// contains filtered or unexported fields
}

MockLogger is a mock of Logger interface

func NewMockLogger

func NewMockLogger(ctrl *gomock.Controller) *MockLogger

NewMockLogger creates a new mock instance

func (*MockLogger) Criticalf

func (m *MockLogger) Criticalf(format string, v ...interface{})

Criticalf mocks base method

func (*MockLogger) Debugf

func (m *MockLogger) Debugf(format string, v ...interface{})

Debugf mocks base method

func (*MockLogger) EXPECT

func (m *MockLogger) EXPECT() *MockLoggerMockRecorder

EXPECT returns an object that allows the caller to indicate expected use

func (*MockLogger) Errorf

func (m *MockLogger) Errorf(format string, v ...interface{})

Errorf mocks base method

func (*MockLogger) Infof

func (m *MockLogger) Infof(format string, v ...interface{})

Infof mocks base method

func (*MockLogger) Warnf

func (m *MockLogger) Warnf(format string, v ...interface{})

Warnf mocks base method

type MockLoggerMockRecorder

type MockLoggerMockRecorder struct {
	// contains filtered or unexported fields
}

MockLoggerMockRecorder is the mock recorder for MockLogger

func (*MockLoggerMockRecorder) Criticalf

func (mr *MockLoggerMockRecorder) Criticalf(format interface{}, v ...interface{}) *gomock.Call

Criticalf indicates an expected call of Criticalf

func (*MockLoggerMockRecorder) Debugf

func (mr *MockLoggerMockRecorder) Debugf(format interface{}, v ...interface{}) *gomock.Call

Debugf indicates an expected call of Debugf

func (*MockLoggerMockRecorder) Errorf

func (mr *MockLoggerMockRecorder) Errorf(format interface{}, v ...interface{}) *gomock.Call

Errorf indicates an expected call of Errorf

func (*MockLoggerMockRecorder) Infof

func (mr *MockLoggerMockRecorder) Infof(format interface{}, v ...interface{}) *gomock.Call

Infof indicates an expected call of Infof

func (*MockLoggerMockRecorder) Warnf

func (mr *MockLoggerMockRecorder) Warnf(format interface{}, v ...interface{}) *gomock.Call

Warnf indicates an expected call of Warnf

type MockWorkerQueue

type MockWorkerQueue struct {
	// contains filtered or unexported fields
}

MockWorkerQueue is a mock of WorkerQueue interface

func NewMockWorkerQueue

func NewMockWorkerQueue(ctrl *gomock.Controller) *MockWorkerQueue

NewMockWorkerQueue creates a new mock instance

func (*MockWorkerQueue) EXPECT

func (m *MockWorkerQueue) EXPECT() *MockWorkerQueueMockRecorder

EXPECT returns an object that allows the caller to indicate expected use

func (*MockWorkerQueue) PublishRequest

func (m *MockWorkerQueue) PublishRequest(request *Request) error

PublishRequest mocks base method

func (*MockWorkerQueue) RetryRequest

func (m *MockWorkerQueue) RetryRequest(request *Request) error

RetryRequest mocks base method

func (*MockWorkerQueue) SubscribeRequests

func (m *MockWorkerQueue) SubscribeRequests(ctx context.Context) (<-chan *Request, error)

SubscribeRequests mocks base method

type MockWorkerQueueMockRecorder

type MockWorkerQueueMockRecorder struct {
	// contains filtered or unexported fields
}

MockWorkerQueueMockRecorder is the mock recorder for MockWorkerQueue

func (*MockWorkerQueueMockRecorder) PublishRequest

func (mr *MockWorkerQueueMockRecorder) PublishRequest(request interface{}) *gomock.Call

PublishRequest indicates an expected call of PublishRequest

func (*MockWorkerQueueMockRecorder) RetryRequest

func (mr *MockWorkerQueueMockRecorder) RetryRequest(request interface{}) *gomock.Call

RetryRequest indicates an expected call of RetryRequest

func (*MockWorkerQueueMockRecorder) SubscribeRequests

func (mr *MockWorkerQueueMockRecorder) SubscribeRequests(ctx interface{}) *gomock.Call

SubscribeRequests indicates an expected call of SubscribeRequests

type Request

type Request struct {
	URL       string
	Method    string
	Header    http.Header
	Body      []byte
	Priority  int64
	QueueName string
	Meta      map[string]interface{}
	// contains filtered or unexported fields
}

Request is a domain model that represents http request.

func NewGetRequest

func NewGetRequest(urlStr string) (*Request, error)

NewGetRequest creates simple GET request.

func NewRequestFromHTTPRequest

func NewRequestFromHTTPRequest(request *http.Request) (*Request, error)

NewRequestFromHTTPRequest constructs Request from http.Request

func (*Request) BodyReader

func (r *Request) BodyReader() io.Reader

BodyReader returns io.Reader of Body

func (*Request) HTTPRequest

func (r *Request) HTTPRequest() (*http.Request, error)

HTTPRequest constructs http.Request from Request

func (*Request) URLHost

func (r *Request) URLHost() string

URLHost returns the host of the request url.

type Response

type Response struct {
	StatusCode int
	Headers    http.Header
	Body       []byte
	Request    *Request
}

Response is a domain model that represents http response.

func NewResponseFromHTTPResponse

func NewResponseFromHTTPResponse(response *http.Response) (*Response, error)

NewResponseFromHTTPResponse constructs Response from http.Response

func (*Response) ContentType

func (r *Response) ContentType() string

ContentType returns detected Content-Type of the response body. If it cannot determine a more specific one, it returns "application/octet-stream".

func (*Response) Follow

func (r *Response) Follow(urlString string) (string, error)

Follow creates a url whose url schema and host is the same as those of response.

func (*Response) FollowRequest

func (r *Response) FollowRequest(urlString string) (*Request, error)

FollowRequest creates a simple GET request whose the schema and the host of the url is the same as those of response.

func (*Response) Text

func (r *Response) Text() string

Text returns string(Response.Body) Note that this method does not decode body. To ensure that the Response.Body is decoded, use http.Client with DefaultTransport.

type Worker

type Worker struct {
	WorkerQueue         WorkerQueue
	HTTPClient          HTTPClient
	Logger              Logger
	RequestMiddlewares  []func(request *Request)
	ResponseMiddlewares []func(response *Response)
	Spider              func(response *Response) ([]*Request, error)
}

Worker handles scheduled requests.

func (*Worker) RetryMiddleware

func (w *Worker) RetryMiddleware(request *Request)

RetryMiddleware is request middleware that remove request in worker pipeline and send request to worker queue if Request.Meta['retry'] flas is true.

func (*Worker) Start

func (w *Worker) Start(ctx context.Context) error

Start kicks worker off.

func (*Worker) StartWithFirstRequest

func (w *Worker) StartWithFirstRequest(ctx context.Context, URL string) error

StartWithFirstRequest kicks worker off with first request.

type WorkerQueue

type WorkerQueue interface {
	SubscribeRequests(ctx context.Context) (<-chan *Request, error)
	RetryRequest(request *Request) error
	PublishRequest(request *Request) error
}

WorkerQueue is queue operations for scheduled requests.