internal

Documentation

Index

• Constants
• type Config
  • func (c *Config) IsSTDIN() bool
  • func (c *Config) MustParse() error
• type Input
  • func (i *Input) Next() (req *Request, err error)
  • func (i *Input) ParseStream()
  • func (i *Input) ParserStreamLine(inputLine []byte) (*Request, error)
  • func (i *Input) RegistPaserFn(name string, fn InputParserFn)
• type InputFormatJSONRequest
  • func (jr *InputFormatJSONRequest) String() string
• type InputParserFn
• type Request
• type Response
  • func (r *Response) Bytes() ([]byte, error)
• type Worker
  • func NewWorker(id int, workerPool *WorkerPool) *Worker
  • func (w *Worker) Close()
  • func (w *Worker) Log(v ...interface{})
  • func (w *Worker) Logf(format string, v ...interface{})
  • func (w *Worker) LogfBase(format string, v ...interface{})
  • func (w *Worker) Talk(req *Request) error
  • func (w *Worker) TalkWithHTTP(req *Request) error
• type WorkerPool
  • func NewWorkerPool() *WorkerPool
  • func NewWorkerPoolWithConfig(config *Config, input *Input) *WorkerPool
  • func (wp *WorkerPool) Close()
  • func (wp *WorkerPool) Start()

Constants

const (
	// InputFormatURLListGet  输入格式为url列表，一行一个url
	InputFormatURLListGet string = "url_list_get"

	// InputFormatJSON 输入格式为json，一行一条json数据
	InputFormatJSON string = "json"
)

const Version string = "v1.0.2020040414"

Version 版本信息

Types

type Config

type Config struct {
	// 数据源
	Input string

	// 数据输入格式
	InputFormat string

	// 发送并发数，worker的数量
	Conc uint

	// 待发送队列长度
	RequestQueueSize uint32

	// 调试模式
	Trace bool

	// 超时时间
	TimeoutMs uint

	// log前缀
	LogFileName string

	Retry uint

	// 内容输出文件
	OutFileName string
}

Config 程序配置

func (*Config) IsSTDIN

func (c *Config) IsSTDIN() bool

IsSTDIN 是否从标准输入读取信息

func (*Config) MustParse

func (c *Config) MustParse() error

MustParse 解析、判断配置是否正确

type Input

type Input struct {
	// contains filtered or unexported fields
}

func (*Input) Next

func (i *Input) Next() (req *Request, err error)

Next 获取一个请求

func (*Input) ParseStream

func (i *Input) ParseStream()

ParseStream 解析数据流

func (*Input) ParserStreamLine

func (i *Input) ParserStreamLine(inputLine []byte) (*Request, error)

func (*Input) RegistPaserFn

func (i *Input) RegistPaserFn(name string, fn InputParserFn)

type InputFormatJSONRequest

type InputFormatJSONRequest struct {
	ID     string            `json:"id"`
	Method string            `json:"method"`
	URL    string            `json:"url"`
	Header map[string]string `json:"header"`
	Body   string            `json:"body"`
}

func (*InputFormatJSONRequest) String

func (jr *InputFormatJSONRequest) String() string

type InputParserFn

type InputParserFn func(config *Config, inputLine []byte) (*Request, error)

type Request

type Request struct {
	LineNo      uint64
	URL         string
	ID          string
	HTTPRequest *http.Request
}

type Response

type Response struct {
	ID  string `json:"id"`
	URL string `json:"url"`

	// http status code
	StatusCode int    `json:"status"`
	Error      string `json:"error"`
	RespBody   string `json:"body"`

	// 耗时
	Cost int64 `json:"cost_ms"`

	// 请求所在文件行数
	LineNo uint64 `json:"line_no"`
}

func (*Response) Bytes

func (r *Response) Bytes() ([]byte, error)

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

Worker 一个worker(实际进行网络操作的worker)

func NewWorker

func NewWorker(id int, workerPool *WorkerPool) *Worker

NewWorker 创建新worker

func (*Worker) Close

func (w *Worker) Close()

Close 回收资源

func (*Worker) Log

func (w *Worker) Log(v ...interface{})

Log 打印日志

func (*Worker) Logf

func (w *Worker) Logf(format string, v ...interface{})

Logf 打印日志

func (*Worker) LogfBase

func (w *Worker) LogfBase(format string, v ...interface{})

LogfBase 打印日志

func (*Worker) Talk

func (w *Worker) Talk(req *Request) error

Talk 和远端进行网络交互-同步

func (*Worker) TalkWithHTTP

func (w *Worker) TalkWithHTTP(req *Request) error

TalkWithHTTP HTTP协议的交互

type WorkerPool

type WorkerPool struct {
	// contains filtered or unexported fields
}

WorkerPool worker工作池

func NewWorkerPool

func NewWorkerPool() *WorkerPool

NewWorkerPool 创建wokrer工作池对象

func NewWorkerPoolWithConfig

func NewWorkerPoolWithConfig(config *Config, input *Input) *WorkerPool

NewWorkerPoolWithConfig 创建worker工作池对象

func (*WorkerPool) Close

func (wp *WorkerPool) Close()

Close 资源回收

func (*WorkerPool) Start

func (wp *WorkerPool) Start()

Start 启动进程