config

Documentation

Index

• func INIT_CONFIG_ERR()
• func InitConfig(name string, dpptrMap map[int]DownloadPartPtr, ...)
• type CommonConfig
  • func (this *CommonConfig) Get(key string) *ConfigValue
  • func (this *CommonConfig) GetName() string
• type ConfigType
• type ConfigValue
  • func Get(name string, confType ConfigType, key string) *ConfigValue
  • func NewConfigValue(name string, data interface{}) *ConfigValue
  • func (this *ConfigValue) Bool() bool
  • func (this *ConfigValue) DPPtr(depth int) DownloadPartPtr
  • func (this *ConfigValue) Int() int
  • func (this *ConfigValue) Int64() int64
  • func (this *ConfigValue) RPPtr(depth int) ResolverPartPtr
  • func (this *ConfigValue) String() string
• type DownloadPartPtr
• type HttpConfig
  • func (this *HttpConfig) Get(key string) *ConfigValue
  • func (this *HttpConfig) GetName() string
• type IDownloader
• type IDownloaderPart
• type IResolver
• type IResolverPart
• type Link
  • func NewLink(addr string) *Link
  • func (this *Link) AddCookie(cookie *http.Cookie)
  • func (this *Link) AddHeader(key, value string)
  • func (this *Link) AddReqParam(key, value string)
• type LogConfig
  • func (this *LogConfig) Get(key string) *ConfigValue
  • func (this *LogConfig) GetName() string
• type Method
• type PartConfig
  • func (this *PartConfig) Get(key string) *ConfigValue
  • func (this *PartConfig) GetName() string
  • func (this *PartConfig) SetDPPtrMap(dpptrMap map[int]DownloadPartPtr)
  • func (this *PartConfig) SetRPPtrMap(rpptrMap map[int]ResolverPartPtr)
• type RawData
  • func NewRawData(link *Link, data string) *RawData
• type ResolverPartPtr
• type Scheme
• type Text
  • func NewText(link *Link, data string) *Text

Functions

func INIT_CONFIG_ERR

func INIT_CONFIG_ERR()

func InitConfig

func InitConfig(name string, dpptrMap map[int]DownloadPartPtr, rpptrMap map[int]ResolverPartPtr, args ...interface{})

Types

type CommonConfig

type CommonConfig struct {
	MaxDepth int `json:"maxDepth, int"`

	// a Group composite of {NumUnit} Units.
	NumUnit int `json:"numUnit, int"`

	// storage data in cachedText when len(cachedText) is equal to {NumCachedText}
	NumCachedText int `json:"numCachedText, int"`
	// contains filtered or unexported fields
}

func (*CommonConfig) Get

func (this *CommonConfig) Get(key string) *ConfigValue

maybe should use reflect to make a common Get() method, but not good for efficience.

func (*CommonConfig) GetName

func (this *CommonConfig) GetName() string

type ConfigType

type ConfigType int

const (
	COMMON_CONF ConfigType = iota
	LOG_CONF
	HTTP_CONF
	PART_CONF
)

type ConfigValue

type ConfigValue struct {
	// contains filtered or unexported fields
}

func Get

func Get(name string, confType ConfigType, key string) *ConfigValue

func NewConfigValue

func NewConfigValue(name string, data interface{}) *ConfigValue

func (*ConfigValue) Bool

func (this *ConfigValue) Bool() bool

func (*ConfigValue) DPPtr

func (this *ConfigValue) DPPtr(depth int) DownloadPartPtr

func (*ConfigValue) Int

func (this *ConfigValue) Int() int

func (*ConfigValue) Int64

func (this *ConfigValue) Int64() int64

func (*ConfigValue) RPPtr

func (this *ConfigValue) RPPtr(depth int) ResolverPartPtr

func (*ConfigValue) String

func (this *ConfigValue) String() string

type DownloadPartPtr

type DownloadPartPtr func(string, *Link) IDownloaderPart

type HttpConfig

type HttpConfig struct {
	// contains filtered or unexported fields
}

func (*HttpConfig) Get

func (this *HttpConfig) Get(key string) *ConfigValue

func (*HttpConfig) GetName

func (this *HttpConfig) GetName() string

type IDownloader

type IDownloader interface {
	GenerateRawData() (*RawData, error)
}

type IDownloaderPart

type IDownloaderPart interface {
	layer.IPart
	IDownloader
	SetDownloader(downloader IDownloader)
}

type IResolver

type IResolver interface {
	GenerateLinks() ([]*Link, error)
	GenerateText() (*Text, error)
}

type IResolverPart

type IResolverPart interface {
	layer.IPart
	IResolver
	SetResolver(resolver IResolver)
}

type Link

type Link struct {
	Protocol Scheme

	URL *url.URL

	// this link retry times for http request
	RetryTimes int

	// "GET" or "POST"
	Type Method

	// form data for "POST" request
	FormData url.Values

	Header  http.Header
	Cookies []*http.Cookie

	// Dialer timeout and client timeout.
	Timeout time.Duration

	// whether this Link is available. such as http://javascript_xxx is not available.
	Available       bool
	UnavailableAddr string
}

TODO: setup proxy service

func NewLink

func NewLink(addr string) *Link

note: only handle http or https, others will be setup unavailable. such as mailto:xxx.com

func (*Link) AddCookie

func (this *Link) AddCookie(cookie *http.Cookie)

func (*Link) AddHeader

func (this *Link) AddHeader(key, value string)

func (*Link) AddReqParam

func (this *Link) AddReqParam(key, value string)

type LogConfig

type LogConfig struct {
	LogRootDir  string `json:"logRootDir, string"`
	LogFileSize int64  `json:"logFileSize, int64"`
	// contains filtered or unexported fields
}

func (*LogConfig) Get

func (this *LogConfig) Get(key string) *ConfigValue

func (*LogConfig) GetName

func (this *LogConfig) GetName() string

type Method

type Method string

const (
	POST Method = "POST"
	GET  Method = "GET"
)

type PartConfig

type PartConfig struct {
	NewDownloadPartMap map[int]DownloadPartPtr
	NewResolverPartMap map[int]ResolverPartPtr
	// contains filtered or unexported fields
}

func (*PartConfig) Get

func (this *PartConfig) Get(key string) *ConfigValue

func (*PartConfig) GetName

func (this *PartConfig) GetName() string

func (*PartConfig) SetDPPtrMap

func (this *PartConfig) SetDPPtrMap(dpptrMap map[int]DownloadPartPtr)

func (*PartConfig) SetRPPtrMap

func (this *PartConfig) SetRPPtrMap(rpptrMap map[int]ResolverPartPtr)

type RawData

type RawData struct {
	LINK *Link
	Data string

	// if LINK is not available, the RawData should be the same.
	Available bool
}

func NewRawData

func NewRawData(link *Link, data string) *RawData

type ResolverPartPtr

type ResolverPartPtr func(string, *RawData) IResolverPart

type Scheme

type Scheme string

const (
	HTTP  Scheme = "http://"
	HTTPS Scheme = "https://"
)

type Text

type Text struct {
	LINK *Link
	Data string

	// if LINK is not available, the Text should be the same.
	Available bool
}

func NewText

func NewText(link *Link, data string) *Text