utils

Documentation

Index

• func CheckDirectory(path string) error
• func CheckFile(path string) error
• func CheckOutputDir(path string) error
• func DownloadFile(url string, targetPath string) (int, error)
• func DownloadFileWithOptions(ctx context.Context, downloadURL, targetPath string, opt DownloadOptions) (int, error)
• func GetCacheDir() (string, error)
• func UnzipFile(zipPath, targetPath string, includeNestedZip ...bool) error
• type CSVWriter
  • func NewCSVWriter[T any](filename string) (*CSVWriter[T], error)
  • func (cw *CSVWriter[T]) Close() error
  • func (cw *CSVWriter[T]) Write(data []T) error
• type Download
• type DownloadOptions
• type Pipeline
  • func NewPipeline[I, O any](opts ...PipelineOption) *Pipeline[I, O]
  • func (p *Pipeline[I, O]) Run(ctx context.Context, inputs []I, ...) (*PipelineResult, error)
  • func (p *Pipeline[I, O]) RunWithWriter(ctx context.Context, inputs []I, ...) (*PipelineResult, error)
• type PipelineOption
  • func WithBufferSize(n int) PipelineOption
  • func WithConcurrency(n int) PipelineOption
• type PipelineResult
  • func (r *PipelineResult) ErrorSummary() string
  • func (r *PipelineResult) FirstError() error
  • func (r *PipelineResult) HasErrors() bool

Functions

func CheckDirectory

func CheckDirectory(path string) error

func CheckFile

func CheckFile(path string) error

func CheckOutputDir

func CheckOutputDir(path string) error

func DownloadFile

func DownloadFile(url string, targetPath string) (int, error)

DownloadFile 下载文件并返回 HTTP 状态码 (向后兼容入口)。 走默认 options: 浏览器 UA + 自动 Referer + 3 次重试。 若状态码为 404 或 200，error 为 nil。

func DownloadFileWithOptions

func DownloadFileWithOptions(ctx context.Context, downloadURL, targetPath string, opt DownloadOptions) (int, error)

DownloadFileWithOptions 带请求头 / 重试 / 超时的下载。整体下载 (HEAD + 分段或单线程) 作为一次尝试, 失败按 RetryBackoff 递增退避重试, 直到成功或耗尽 MaxAttempts; 全程响应 ctx 取消。404 视为确定性结果, 立即返回不重试。

func GetCacheDir

func GetCacheDir() (string, error)

func UnzipFile

func UnzipFile(zipPath, targetPath string, includeNestedZip ...bool) error

UnzipFile 解压 zip。默认跳过内嵌的 .zip 文件（用于 day4/tic4 等包含嵌套归档的场景）。 传 includeNestedZip=true 时会一并解压嵌套的 .zip 文件。

Types

type CSVWriter

type CSVWriter[T any] struct {
	// contains filtered or unexported fields
}

CSVWriter 通用 CSV 写入器

func NewCSVWriter

func NewCSVWriter[T any](filename string) (*CSVWriter[T], error)

NewCSVWriter 初始化

func (*CSVWriter[T]) Close

func (cw *CSVWriter[T]) Close() error

func (*CSVWriter[T]) Write

func (cw *CSVWriter[T]) Write(data []T) error

Write 写入数据

type Download

type Download struct {
	Url           string
	Target        string
	TotalSections int
	// contains filtered or unexported fields
}

Download 封装下载任务

type DownloadOptions

type DownloadOptions struct {
	Headers      map[string]string
	MaxAttempts  int
	RetryBackoff time.Duration
	Timeout      time.Duration // 每次尝试的 http.Client 超时; 0 = 不限
}

DownloadOptions 控制单次下载的请求头 / 重试 / 超时。零值字段走默认: MaxAttempts<=0 → 3, RetryBackoff<=0 → 3s, Timeout==0 → 不限 (沿用大文件慢下载), Headers 始终在默认 UA + 自动 Referer 之上叠加 (同名覆盖)。

type Pipeline

type Pipeline[I, O any] struct {
	// contains filtered or unexported fields
}

Pipeline 通用并发处理管道

func NewPipeline

func NewPipeline[I, O any](opts ...PipelineOption) *Pipeline[I, O]

func (*Pipeline[I, O]) Run

func (p *Pipeline[I, O]) Run(
	ctx context.Context,
	inputs []I,
	process func(ctx context.Context, input I) ([]O, error),
	consume func(rows []O) error,
) (*PipelineResult, error)

func (*Pipeline[I, O]) RunWithWriter

func (p *Pipeline[I, O]) RunWithWriter(
	ctx context.Context,
	inputs []I,
	process func(ctx context.Context, input I) ([]O, error),
	writer *CSVWriter[O],
) (*PipelineResult, error)

type PipelineOption

type PipelineOption func(*pipelineConfig)

func WithBufferSize

func WithBufferSize(n int) PipelineOption

func WithConcurrency

func WithConcurrency(n int) PipelineOption

type PipelineResult

type PipelineResult struct {
	TotalItems     int
	ProcessedItems int64
	OutputRows     int64
	Errors         []error
	Duration       time.Duration
}

PipelineResult 执行结果统计

func (*PipelineResult) ErrorSummary

func (r *PipelineResult) ErrorSummary() string

func (*PipelineResult) FirstError

func (r *PipelineResult) FirstError() error

func (*PipelineResult) HasErrors

func (r *PipelineResult) HasErrors() bool