engine

Documentation

Index

• Constants
• Variables
• func BuildCacherInputFromCrawlerDownloaded(d *crawler.Downloaded) *cacher.Input
• type Config
  • func ParseConfig(arg0 string, otherArgs []string, output io.Writer) (*Config, error)
• type Engine
  • func FromConfig(fs cacher.Fs, config *Config) Engine
  • func New(fs cacher.Fs, httpClient *http.Client, logger *logrus.Logger) Engine

Constants

const (
	// ConfigEnvVarPrefix the environment variable prefix
	ConfigEnvVarPrefix = "SITEMIRROR"
	// ConfigDefaultLoggerLevel default value for .LoggerLevel
	ConfigDefaultLoggerLevel = logrus.InfoLevel
	// ConfigDefaultBumpTTL default value for .BumpTTL
	ConfigDefaultBumpTTL = time.Minute
	// ConfigDefaultAutoEnqueueInterval default value for .AutoEnqueueInterval
	ConfigDefaultAutoEnqueueInterval = time.Duration(0)
	// ConfigDefaultHttpTimeout default value for .HttpTimeout
	ConfigDefaultHttpTimeout = 10 * time.Second
	// ConfigDefaultCacherDefaultTTL default value for .Cacher.DefaultTTL
	ConfigDefaultCacherDefaultTTL = 10 * time.Minute
	// ConfigDefaultCrawlerAutoDownloadDepth default value for .Crawler.AutoDownloadDepth
	ConfigDefaultCrawlerAutoDownloadDepth = uint64(1)
	// ConfigDefaultCrawlerNoCrossHost default value for .Crawler.NoCrossHost
	ConfigDefaultCrawlerNoCrossHost = false
	// ConfigDefaultCrawlerWorkerCount default value for .Crawler.WorkerCount
	ConfigDefaultCrawlerWorkerCount = uint64(4)
	// ConfigDefaultPort default value for .Port
	ConfigDefaultPort = int64(-1)
)

Variables

var (
	// ResponseBodyMethodNotAllowed the text to respond when user request method is not allowed
	ResponseBodyMethodNotAllowed = "Sorry, your request is not supported and cannot be processed."
)

Functions

func BuildCacherInputFromCrawlerDownloaded

func BuildCacherInputFromCrawlerDownloaded(d *crawler.Downloaded) *cacher.Input

BuildCacherInputFromCrawlerDownloaded returns a cacher.Input with data copied from the specified crawler.Downloaded

Types

type Config

type Config struct {
	LoggerLevel configLoggerLevel

	HostRewrites        configStringMap
	HostsWhitelist      configStringSlice
	BumpTTL             time.Duration
	AutoEnqueueInterval time.Duration
	HttpTimeout         time.Duration

	Cacher  configCacher
	Crawler configCrawler

	Port        int64
	MirrorURLs  configURLSlice
	MirrorPorts configIntSlice
}

Config represents an engine configuration

func ParseConfig

func ParseConfig(arg0 string, otherArgs []string, output io.Writer) (*Config, error)

ParseConfig returns configuration derived from command line arguments or environment variables

type Engine

type Engine interface {
	GetCacher() cacher.Cacher
	GetCrawler() crawler.Crawler
	GetServer() web.Server

	AddHostRewrite(string, string)
	GetHostRewrites() map[string]string
	AddHostWhitelisted(string)
	GetHostsWhitelist() []string
	SetBumpTTL(time.Duration)
	GetBumpTTL() time.Duration
	SetAutoEnqueueInterval(time.Duration)
	GetAutoEnqueueInterval() time.Duration

	Mirror(*url.URL, int) error
	Stop()
	// contains filtered or unexported methods
}

Engine represents an object that can mirror urls

func FromConfig

func FromConfig(fs cacher.Fs, config *Config) Engine

FromConfig return an Engine instance with all configuration applied

func New

func New(fs cacher.Fs, httpClient *http.Client, logger *logrus.Logger) Engine

New returns a new Engine instance