pipe

Documentation

Index

• Constants
• func GetKey(args ...string) []byte
• func NewTaskJoint(task *model.Task) Joint
• type EmptyJoint
  • func (this EmptyJoint) Name() string
  • func (this EmptyJoint) Process(s *Context) error
• type FetchJoint
  • func (this FetchJoint) Name() string
  • func (this FetchJoint) Process(context *Context) error
• type HashJoint
  • func (this HashJoint) Name() string
  • func (this HashJoint) Process(context *Context) error
• type HtmlToTextJoint
  • func (this HtmlToTextJoint) Name() string
  • func (this HtmlToTextJoint) Process(context *Context) error
• type IgnoreTimeoutJoint
  • func (this IgnoreTimeoutJoint) Name() string
  • func (this IgnoreTimeoutJoint) Process(context *Context) error
• type InitTaskJoint
  • func (this InitTaskJoint) Name() string
  • func (this InitTaskJoint) Process(context *Context) error
• type LoadMetadataJoint
  • func (this LoadMetadataJoint) Name() string
  • func (this LoadMetadataJoint) Process(context *Context) error
• type ParsePageJoint
  • func (this ParsePageJoint) Name() string
  • func (this ParsePageJoint) Process(context *Context) error
• type PublishJoint
  • func (this PublishJoint) Name() string
  • func (this PublishJoint) Process(c *Context) error
• type SaveSnapshotToDBJoint
  • func (this SaveSnapshotToDBJoint) Name() string
  • func (this SaveSnapshotToDBJoint) Process(c *Context) error
• type SaveSnapshotToFileSystemJoint
  • func (this SaveSnapshotToFileSystemJoint) Name() string
  • func (this SaveSnapshotToFileSystemJoint) Process(c *Context) error
• type SaveTaskJoint
  • func (this SaveTaskJoint) Name() string
  • func (this SaveTaskJoint) Process(context *Context) error
• type UrlCheckFilterJoint
  • func (this UrlCheckFilterJoint) Name() string
  • func (this UrlCheckFilterJoint) Process(context *Context) error
• type UrlExtFilterJoint
  • func (this UrlExtFilterJoint) Name() string
  • func (this UrlExtFilterJoint) Process(context *Context) error
• type UrlNormalizationJoint
  • func (this UrlNormalizationJoint) Name() string
  • func (this UrlNormalizationJoint) Process(context *Context) error

Constants

const (
	CONTEXT_CRAWLER_DOMAIN   ParaKey = "CRAWLER_DOMAIN"
	CONTEXT_CRAWLER_TASK     ParaKey = "CRAWLER_TASK"
	CONTEXT_CRAWLER_SNAPSHOT ParaKey = "CRAWLER_SNAPSHOT"

	CONTEXT_PAGE_LINKS ParaKey = "PAGE_LINKS"
)

const Cookie ParaKey = "cookie"

const Empty JointKey = "empty"

const Fetch JointKey = "fetch"

const Hash JointKey = "hash"

const HtmlToText JointKey = "html2text"

const IgnoreTimeout JointKey = "ignore_timeout"

const InitTask JointKey = "init_task"

const KeyDelimiter string = "/"

const LoadMetadata JointKey = "load_metadata"

const ParsePage JointKey = "parse"

const Proxy ParaKey = "proxy"

const Publish JointKey = "publish"

const SaveSnapshotToDB JointKey = "save_snapshot_db"

const SaveSnapshotToFileSystem JointKey = "save_snapshot_fs"

const SaveTask JointKey = "save_task"

const TaskID ParaKey = "TASK_ID"

const UrlCheckFilter JointKey = "url_check_filter"

const UrlExtFilter JointKey = "url_ext_filter"

const UrlNormalization JointKey = "url_normalization"

Functions

func GetKey

func GetKey(args ...string) []byte

func NewTaskJoint

func NewTaskJoint(task *model.Task) Joint

Types

type EmptyJoint

type EmptyJoint struct {
}

func (EmptyJoint) Name

func (this EmptyJoint) Name() string

func (EmptyJoint) Process

func (this EmptyJoint) Process(s *Context) error

type FetchJoint

type FetchJoint struct {
	Parameters
	// contains filtered or unexported fields
}

func (FetchJoint) Name

func (this FetchJoint) Name() string

func (FetchJoint) Process

func (this FetchJoint) Process(context *Context) error

type HashJoint

type HashJoint struct {
	DictRoot string
	Simhash  bool
}

func (HashJoint) Name

func (this HashJoint) Name() string

func (HashJoint) Process

func (this HashJoint) Process(context *Context) error

type HtmlToTextJoint

type HtmlToTextJoint struct {
	MergeWhitespace bool //merge whitespace and \n
}

func (HtmlToTextJoint) Name

func (this HtmlToTextJoint) Name() string

func (HtmlToTextJoint) Process

func (this HtmlToTextJoint) Process(context *Context) error

type IgnoreTimeoutJoint

type IgnoreTimeoutJoint struct {
	IgnoreTimeoutAfterCount int64
}

func (IgnoreTimeoutJoint) Name

func (this IgnoreTimeoutJoint) Name() string

func (IgnoreTimeoutJoint) Process

func (this IgnoreTimeoutJoint) Process(context *Context) error

type InitTaskJoint

type InitTaskJoint struct {
	Parameters
	Task *model.Task
}

func (InitTaskJoint) Name

func (this InitTaskJoint) Name() string

func (InitTaskJoint) Process

func (this InitTaskJoint) Process(context *Context) error

type LoadMetadataJoint

type LoadMetadataJoint struct {
}

load metadata from db

func (LoadMetadataJoint) Name

func (this LoadMetadataJoint) Name() string

func (LoadMetadataJoint) Process

func (this LoadMetadataJoint) Process(context *Context) error

type ParsePageJoint

type ParsePageJoint struct {
	DispatchLinks    bool
	MaxDepth         int         //max depth of page to follow
	MaxBreadth       int         //max breadth of the domain to follow
	MaxPageOfBreadth map[int]int //max page to fetch in each level's breadth, eg: 1:100;2:50;3:5;4:1

}

func (ParsePageJoint) Name

func (this ParsePageJoint) Name() string

func (ParsePageJoint) Process

func (this ParsePageJoint) Process(context *Context) error

type PublishJoint

type PublishJoint struct {
}

func (PublishJoint) Name

func (this PublishJoint) Name() string

func (PublishJoint) Process

func (this PublishJoint) Process(c *Context) error

type SaveSnapshotToDBJoint

type SaveSnapshotToDBJoint struct {
	CompressBody bool
	Bucket       string
}

func (SaveSnapshotToDBJoint) Name

func (this SaveSnapshotToDBJoint) Name() string

func (SaveSnapshotToDBJoint) Process

func (this SaveSnapshotToDBJoint) Process(c *Context) error

type SaveSnapshotToFileSystemJoint

type SaveSnapshotToFileSystemJoint struct {
	// contains filtered or unexported fields
}

func (SaveSnapshotToFileSystemJoint) Name

func (this SaveSnapshotToFileSystemJoint) Name() string

func (SaveSnapshotToFileSystemJoint) Process

func (this SaveSnapshotToFileSystemJoint) Process(c *Context) error

type SaveTaskJoint

type SaveTaskJoint struct {
	IsCreate bool
}

func (SaveTaskJoint) Name

func (this SaveTaskJoint) Name() string

func (SaveTaskJoint) Process

func (this SaveTaskJoint) Process(context *Context) error

type UrlCheckFilterJoint

type UrlCheckFilterJoint struct {
	Parameters
	//ignore files end with js,css,apk,zip
	SkipPageParsePattern *regexp.Regexp
}

func (UrlCheckFilterJoint) Name

func (this UrlCheckFilterJoint) Name() string

func (UrlCheckFilterJoint) Process

func (this UrlCheckFilterJoint) Process(context *Context) error

type UrlExtFilterJoint

type UrlExtFilterJoint struct {
	//ignore files end with js,css,apk,zip
	SkipPageParsePattern *regexp.Regexp
}

func (UrlExtFilterJoint) Name

func (this UrlExtFilterJoint) Name() string

func (UrlExtFilterJoint) Process

func (this UrlExtFilterJoint) Process(context *Context) error

type UrlNormalizationJoint

type UrlNormalizationJoint struct {
	FollowAllDomain      bool
	FollowDomainSettings bool
	FollowSubDomain      bool
	// contains filtered or unexported fields
}

func (UrlNormalizationJoint) Name

func (this UrlNormalizationJoint) Name() string

func (UrlNormalizationJoint) Process

func (this UrlNormalizationJoint) Process(context *Context) error