colly_protofiles

package
v0.0.0-...-6c943e3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 14, 2019 License: Apache-2.0 Imports: 4 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Collector 

type Collector struct {
	ID                     int64    `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
	UA                     string   `protobuf:"bytes,2,opt,name=UA,proto3" json:"UA,omitempty"`
	MaxDepth               int32    `protobuf:"varint,14,opt,name=MaxDepth,proto3" json:"MaxDepth,omitempty"`
	AllowedDomains         []string `protobuf:"bytes,17,rep,name=AllowedDomains,proto3" json:"AllowedDomains,omitempty"`
	DisallowedURLFilters   *any.Any `protobuf:"bytes,18,opt,name=DisallowedURLFilters,proto3" json:"DisallowedURLFilters,omitempty"`
	AllowURLRevisit        bool     `protobuf:"varint,19,opt,name=AllowURLRevisit,proto3" json:"AllowURLRevisit,omitempty"`
	MaxBodySize            int32    `protobuf:"varint,20,opt,name=MaxBodySize,proto3" json:"MaxBodySize,omitempty"`
	CacheDir               string   `protobuf:"bytes,21,opt,name=CacheDir,proto3" json:"CacheDir,omitempty"`
	IgnoreRobotsTxt        bool     `protobuf:"varint,22,opt,name=IgnoreRobotsTxt,proto3" json:"IgnoreRobotsTxt,omitempty"`
	Async                  bool     `protobuf:"varint,3,opt,name=Async,proto3" json:"Async,omitempty"`
	ParseHTTPErrorResponse bool     `protobuf:"varint,4,opt,name=ParseHTTPErrorResponse,proto3" json:"ParseHTTPErrorResponse,omitempty"`
	DetectCharset          bool     `protobuf:"varint,5,opt,name=DetectCharset,proto3" json:"DetectCharset,omitempty"`
	RedirectHandler        *any.Any `protobuf:"bytes,6,opt,name=redirectHandler,proto3" json:"redirectHandler,omitempty"`
	CheckHead              bool     `protobuf:"varint,23,opt,name=CheckHead,proto3" json:"CheckHead,omitempty"`
	// Probably override with Storage
	Store *any.Any `protobuf:"bytes,7,opt,name=store,proto3" json:"store,omitempty"`
	// Probably override with Debug
	Debugger             *any.Any            `protobuf:"bytes,8,opt,name=debugger,proto3" json:"debugger,omitempty"`
	RobotsMap            map[string]*any.Any `` /* 160-byte string literal not displayed */
	HTMLCallbacks        *any.Any            `protobuf:"bytes,25,opt,name=HTMLCallbacks,proto3" json:"HTMLCallbacks,omitempty"`
	XMLCallbacks         *any.Any            `protobuf:"bytes,26,opt,name=XMLCallbacks,proto3" json:"XMLCallbacks,omitempty"`
	RequestCallbacks     *any.Any            `protobuf:"bytes,27,opt,name=RequestCallbacks,proto3" json:"RequestCallbacks,omitempty"`
	ResponseCallbacks    *any.Any            `protobuf:"bytes,28,opt,name=ResponseCallbacks,proto3" json:"ResponseCallbacks,omitempty"`
	ErrorCallback        *any.Any            `protobuf:"bytes,29,opt,name=ErrorCallback,proto3" json:"ErrorCallback,omitempty"`
	ScrapedCallback      *any.Any            `protobuf:"bytes,30,opt,name=ScrapedCallback,proto3" json:"ScrapedCallback,omitempty"`
	RequestCount         uint32              `protobuf:"varint,9,opt,name=requestCount,proto3" json:"requestCount,omitempty"`
	ResponseCount        uint32              `protobuf:"varint,10,opt,name=responseCount,proto3" json:"responseCount,omitempty"`
	HttpBackend          *any.Any            `protobuf:"bytes,11,opt,name=httpBackend,proto3" json:"httpBackend,omitempty"`
	Wg                   *any.Any            `protobuf:"bytes,12,opt,name=wg,proto3" json:"wg,omitempty"`
	Lock                 *any.Any            `protobuf:"bytes,13,opt,name=lock,proto3" json:"lock,omitempty"`
	XXX_NoUnkeyedLiteral struct{}            `json:"-"`
	XXX_unrecognized     []byte              `json:"-"`
	XXX_sizecache        int32               `json:"-"`
}

func (*Collector) Descriptor 

func (*Collector) Descriptor() ([]byte, []int)

func (*Collector) GetAllowURLRevisit 

func (m *Collector) GetAllowURLRevisit() bool

func (*Collector) GetAllowedDomains 

func (m *Collector) GetAllowedDomains() []string

func (*Collector) GetAsync 

func (m *Collector) GetAsync() bool

func (*Collector) GetCacheDir 

func (m *Collector) GetCacheDir() string

func (*Collector) GetCheckHead 

func (m *Collector) GetCheckHead() bool

func (*Collector) GetDebugger 

func (m *Collector) GetDebugger() *any.Any

func (*Collector) GetDetectCharset 

func (m *Collector) GetDetectCharset() bool

func (*Collector) GetDisallowedURLFilters 

func (m *Collector) GetDisallowedURLFilters() *any.Any

func (*Collector) GetErrorCallback 

func (m *Collector) GetErrorCallback() *any.Any

func (*Collector) GetHTMLCallbacks 

func (m *Collector) GetHTMLCallbacks() *any.Any

func (*Collector) GetHttpBackend 

func (m *Collector) GetHttpBackend() *any.Any

func (*Collector) GetID 

func (m *Collector) GetID() int64

func (*Collector) GetIgnoreRobotsTxt 

func (m *Collector) GetIgnoreRobotsTxt() bool

func (*Collector) GetLock 

func (m *Collector) GetLock() *any.Any

func (*Collector) GetMaxBodySize 

func (m *Collector) GetMaxBodySize() int32

func (*Collector) GetMaxDepth 

func (m *Collector) GetMaxDepth() int32

func (*Collector) GetParseHTTPErrorResponse 

func (m *Collector) GetParseHTTPErrorResponse() bool

func (*Collector) GetRedirectHandler 

func (m *Collector) GetRedirectHandler() *any.Any

func (*Collector) GetRequestCallbacks 

func (m *Collector) GetRequestCallbacks() *any.Any

func (*Collector) GetRequestCount 

func (m *Collector) GetRequestCount() uint32

func (*Collector) GetResponseCallbacks 

func (m *Collector) GetResponseCallbacks() *any.Any

func (*Collector) GetResponseCount 

func (m *Collector) GetResponseCount() uint32

func (*Collector) GetRobotsMap 

func (m *Collector) GetRobotsMap() map[string]*any.Any

func (*Collector) GetScrapedCallback 

func (m *Collector) GetScrapedCallback() *any.Any

func (*Collector) GetStore 

func (m *Collector) GetStore() *any.Any

func (*Collector) GetUA 

func (m *Collector) GetUA() string

func (*Collector) GetWg 

func (m *Collector) GetWg() *any.Any

func (*Collector) GetXMLCallbacks 

func (m *Collector) GetXMLCallbacks() *any.Any

func (*Collector) ProtoMessage 

func (*Collector) ProtoMessage()

func (*Collector) Reset 

func (m *Collector) Reset()

func (*Collector) String 

func (m *Collector) String() string

func (*Collector) XXX_DiscardUnknown 

func (m *Collector) XXX_DiscardUnknown()

func (*Collector) XXX_Marshal 

func (m *Collector) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Collector) XXX_Merge 

func (m *Collector) XXX_Merge(src proto.Message)

func (*Collector) XXX_Size 

func (m *Collector) XXX_Size() int

func (*Collector) XXX_Unmarshal 

func (m *Collector) XXX_Unmarshal(b []byte) error

type Context 

type Context struct {
	Ctx                  map[string]string `` /* 147-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
	XXX_unrecognized     []byte            `json:"-"`
	XXX_sizecache        int32             `json:"-"`
}

func (*Context) Descriptor 

func (*Context) Descriptor() ([]byte, []int)

func (*Context) GetCtx 

func (m *Context) GetCtx() map[string]string

func (*Context) ProtoMessage 

func (*Context) ProtoMessage()

func (*Context) Reset 

func (m *Context) Reset()

func (*Context) String 

func (m *Context) String() string

func (*Context) XXX_DiscardUnknown 

func (m *Context) XXX_DiscardUnknown()

func (*Context) XXX_Marshal 

func (m *Context) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Context) XXX_Merge 

func (m *Context) XXX_Merge(src proto.Message)

func (*Context) XXX_Size 

func (m *Context) XXX_Size() int

func (*Context) XXX_Unmarshal 

func (m *Context) XXX_Unmarshal(b []byte) error
type Header struct {
	Header               map[string]string `` /* 153-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
	XXX_unrecognized     []byte            `json:"-"`
	XXX_sizecache        int32             `json:"-"`
}

func (*Header) Descriptor 

func (*Header) Descriptor() ([]byte, []int)

func (*Header) GetHeader 

func (m *Header) GetHeader() map[string]string

func (*Header) ProtoMessage 

func (*Header) ProtoMessage()

func (*Header) Reset 

func (m *Header) Reset()

func (*Header) String 

func (m *Header) String() string

func (*Header) XXX_DiscardUnknown 

func (m *Header) XXX_DiscardUnknown()

func (*Header) XXX_Marshal 

func (m *Header) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Header) XXX_Merge 

func (m *Header) XXX_Merge(src proto.Message)

func (*Header) XXX_Size 

func (m *Header) XXX_Size() int

func (*Header) XXX_Unmarshal 

func (m *Header) XXX_Unmarshal(b []byte) error

type Request 

type Request struct {
	URL                       *any.Any `protobuf:"bytes,1,opt,name=URL,proto3" json:"URL,omitempty"`
	Header                    *Header  `protobuf:"bytes,2,opt,name=header,proto3" json:"header,omitempty"`
	Ctx                       *Context `protobuf:"bytes,3,opt,name=Ctx,proto3" json:"Ctx,omitempty"`
	Depth                     int32    `protobuf:"varint,4,opt,name=Depth,proto3" json:"Depth,omitempty"`
	Method                    string   `protobuf:"bytes,5,opt,name=method,proto3" json:"method,omitempty"`
	Body                      *any.Any `protobuf:"bytes,6,opt,name=Body,proto3" json:"Body,omitempty"`
	ResponseCharacterEncoding string   `protobuf:"bytes,7,opt,name=ResponseCharacterEncoding,proto3" json:"ResponseCharacterEncoding,omitempty"`
	ID                        int32    `protobuf:"varint,8,opt,name=ID,proto3" json:"ID,omitempty"`
	Collector                 *any.Any `protobuf:"bytes,9,opt,name=collector,proto3" json:"collector,omitempty"`
	Abort                     bool     `protobuf:"varint,10,opt,name=abort,proto3" json:"abort,omitempty"`
	BaseURL                   *any.Any `protobuf:"bytes,11,opt,name=baseURL,proto3" json:"baseURL,omitempty"`
	ProxyURL                  string   `protobuf:"bytes,12,opt,name=proxyURL,proto3" json:"proxyURL,omitempty"`
	XXX_NoUnkeyedLiteral      struct{} `json:"-"`
	XXX_unrecognized          []byte   `json:"-"`
	XXX_sizecache             int32    `json:"-"`
}

func (*Request) Descriptor 

func (*Request) Descriptor() ([]byte, []int)

func (*Request) GetAbort 

func (m *Request) GetAbort() bool

func (*Request) GetBaseURL 

func (m *Request) GetBaseURL() *any.Any

func (*Request) GetBody 

func (m *Request) GetBody() *any.Any

func (*Request) GetCollector 

func (m *Request) GetCollector() *any.Any

func (*Request) GetCtx 

func (m *Request) GetCtx() *Context

func (*Request) GetDepth 

func (m *Request) GetDepth() int32

func (*Request) GetHeader 

func (m *Request) GetHeader() *Header

func (*Request) GetID 

func (m *Request) GetID() int32

func (*Request) GetMethod 

func (m *Request) GetMethod() string

func (*Request) GetProxyURL 

func (m *Request) GetProxyURL() string

func (*Request) GetResponseCharacterEncoding 

func (m *Request) GetResponseCharacterEncoding() string

func (*Request) GetURL 

func (m *Request) GetURL() *any.Any

func (*Request) ProtoMessage 

func (*Request) ProtoMessage()

func (*Request) Reset 

func (m *Request) Reset()

func (*Request) String 

func (m *Request) String() string

func (*Request) XXX_DiscardUnknown 

func (m *Request) XXX_DiscardUnknown()

func (*Request) XXX_Marshal 

func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Request) XXX_Merge 

func (m *Request) XXX_Merge(src proto.Message)

func (*Request) XXX_Size 

func (m *Request) XXX_Size() int

func (*Request) XXX_Unmarshal 

func (m *Request) XXX_Unmarshal(b []byte) error

type Response 

type Response struct {
	StatusCode           int32             `protobuf:"varint,1,opt,name=StatusCode,proto3" json:"StatusCode,omitempty"`
	Body                 [][]byte          `protobuf:"bytes,2,rep,name=Body,proto3" json:"Body,omitempty"`
	Ctx                  map[string]string `` /* 147-byte string literal not displayed */
	Request              *any.Any          `protobuf:"bytes,4,opt,name=Request,proto3" json:"Request,omitempty"`
	Header               map[string]string `` /* 153-byte string literal not displayed */
	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
	XXX_unrecognized     []byte            `json:"-"`
	XXX_sizecache        int32             `json:"-"`
}

func (*Response) Descriptor 

func (*Response) Descriptor() ([]byte, []int)

func (*Response) GetBody 

func (m *Response) GetBody() [][]byte

func (*Response) GetCtx 

func (m *Response) GetCtx() map[string]string

func (*Response) GetHeader 

func (m *Response) GetHeader() map[string]string

func (*Response) GetRequest 

func (m *Response) GetRequest() *any.Any

func (*Response) GetStatusCode 

func (m *Response) GetStatusCode() int32

func (*Response) ProtoMessage 

func (*Response) ProtoMessage()

func (*Response) Reset 

func (m *Response) Reset()

func (*Response) String 

func (m *Response) String() string

func (*Response) XXX_DiscardUnknown 

func (m *Response) XXX_DiscardUnknown()

func (*Response) XXX_Marshal 

func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Response) XXX_Merge 

func (m *Response) XXX_Merge(src proto.Message)

func (*Response) XXX_Size 

func (m *Response) XXX_Size() int

func (*Response) XXX_Unmarshal 

func (m *Response) XXX_Unmarshal(b []byte) error

type SerializableRequest 

type SerializableRequest struct {
	URL                  string   `protobuf:"bytes,1,opt,name=URL,proto3" json:"URL,omitempty"`
	Method               string   `protobuf:"bytes,2,opt,name=method,proto3" json:"method,omitempty"`
	Body                 [][]byte `protobuf:"bytes,3,rep,name=body,proto3" json:"body,omitempty"`
	ID                   int32    `protobuf:"varint,4,opt,name=ID,proto3" json:"ID,omitempty"`
	Ctx                  *Context `protobuf:"bytes,5,opt,name=ctx,proto3" json:"ctx,omitempty"`
	Headers              *Header  `protobuf:"bytes,6,opt,name=headers,proto3" json:"headers,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

func (*SerializableRequest) Descriptor 

func (*SerializableRequest) Descriptor() ([]byte, []int)

func (*SerializableRequest) GetBody 

func (m *SerializableRequest) GetBody() [][]byte

func (*SerializableRequest) GetCtx 

func (m *SerializableRequest) GetCtx() *Context

func (*SerializableRequest) GetHeaders 

func (m *SerializableRequest) GetHeaders() *Header

func (*SerializableRequest) GetID 

func (m *SerializableRequest) GetID() int32

func (*SerializableRequest) GetMethod 

func (m *SerializableRequest) GetMethod() string

func (*SerializableRequest) GetURL 

func (m *SerializableRequest) GetURL() string

func (*SerializableRequest) ProtoMessage 

func (*SerializableRequest) ProtoMessage()

func (*SerializableRequest) Reset 

func (m *SerializableRequest) Reset()

func (*SerializableRequest) String 

func (m *SerializableRequest) String() string

func (*SerializableRequest) XXX_DiscardUnknown 

func (m *SerializableRequest) XXX_DiscardUnknown()

func (*SerializableRequest) XXX_Marshal 

func (m *SerializableRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*SerializableRequest) XXX_Merge 

func (m *SerializableRequest) XXX_Merge(src proto.Message)

func (*SerializableRequest) XXX_Size 

func (m *SerializableRequest) XXX_Size() int

func (*SerializableRequest) XXX_Unmarshal 

func (m *SerializableRequest) XXX_Unmarshal(b []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.