Versions in this module Expand all Collapse all v1 v1.2.1 Jun 18, 2022 Changes in this version + const LANGEMPTY + var B2S = regexp.MustCompile("\\<[\\S\\s]+?\\>") + var Blue = color.New(color.FgBlue).SprintFunc() + var BlueBack = color.New(color.BgBlue, color.FgHiWhite, color.Bold).SprintFunc() + var Bold = color.New(color.Bold).SprintFunc() + var CacheStatus = false + var ConfigDocument = ... + var DEFAULT_BREAKPOINT_FILE = "default-skip.txt" + var DateMatcher = map[*regexp.Regexp]string + var DeafultHeaders = map[string]string + var DefaultLoger = &defaultloger + var DefaultProxyDialer NewDialler + var DefaultProxyPool ProxyDiallerPool + var DelHtmlTag = regexp.MustCompile("\\<[\\S\\s]+?\\>") + var DelScript = regexp.MustCompile("\\<script[\\S\\s]+?\\</script\\>") + var DelSpaceContinuesly = regexp.MustCompile("\\s{2,}") + var DelStyle = regexp.MustCompile("\\<style[\\S\\s]+?\\</style\\>") + var FileTp = regexp.MustCompile(`\.[a-zA-Z0-9]+`) + var FlowMode = 0 + var Green = color.New(color.FgGreen).SprintFunc() + var GreenBack = color.New(color.BgGreen, color.Bold).SprintFunc() + var Hblue = color.New(color.FgHiBlue).SprintFunc() + var Hgreen = color.New(color.FgHiGreen).SprintFunc() + var HostParse = regexp.MustCompile(`Host: .+`) + var Hyello = color.New(color.FgHiYellow).SprintFunc() + var KeyEx = regexp.MustCompile(`\{\w+\}`) + var LogLevl = 0 + var MAX_POOL = 100 + var Magenta = color.New(color.FgMagenta).SprintFunc() + var MsgCacheChan = make(chan string) + var NW = regexp.MustCompile(`\W`) + var RandomMode = 1 + var Red = color.New(color.FgRed).SprintFunc() + var STOP = "[STOP]" + var UA = []string + var Underline = color.New(color.Underline).SprintFunc() + var Yello = color.New(color.FgYellow).SprintFunc() + func AndSimHash(a, b string) float32 + func DecodeHTMLBody(body io.Reader, charset string) (io.Reader, error) + func DefaultProxyMode(mode int) + func Failed(args ...interface{}) + func FindMostMayDate(title, raw string) (t time.Time) + func GetMD5(c []byte) string + func Info(args ...interface{}) + func L(args ...interface{}) + func ProgressLog(now int, all int, msg string) + func SetProxyGenerater(proxyCreator NewDialler) + func ShowDemo() + func Skip(u string, us ...string) bool + func Socks5Dialer(addr string) proxy.Dialer + func Success(args ...interface{}) + func UrlJoin(f ...string) string + type ArrayFilter []string + func (array ArrayFilter) Add(o string) ArrayFilter + func (array ArrayFilter) Every(handler func(no int, every string) string) ArrayFilter + func (array ArrayFilter) Filter(reStrOrFunc_str_bool interface{}) (newArray ArrayFilter) + func (array ArrayFilter) FilterFunc(reStrOrFunc_str_bool func(no int, every string) bool) (newArray ArrayFilter) + func (array ArrayFilter) In(o string) int + func (array ArrayFilter) Sort() ArrayFilter + type Article struct + Author string + Date time.Time + Link string + Text string + Title string + func NewArticle(doc *goquery.Document) (article *Article) + func (article *Article) WaitToFile() + type Async struct + Done []string + NeedRestart bool + func (async *Async) Async(url string, proxy ...string) *Async + func (async *Async) Each(do func(out *AsyncOut)) *Async + func (async *Async) EndAsync() *Session + func (async *Async) LoadCache(name string) *Async + func (async *Async) Restart() *Async + func (async *Async) State() *Async + type AsyncOut struct + End bool + Res *WithOper + Url string + type ConsoleBar struct + All int64 + Interval int + Last time.Time + LastBar string + LastMsg string + LastWrite string + Now int64 + NowWidth int + Width int + func NewConsoleBar(all int64) (bar *ConsoleBar, err error) + func (pro *ConsoleBar) Add(i int) int + func (pro *ConsoleBar) Error(err error) + func (pro *ConsoleBar) Finished() + func (pro *ConsoleBar) GetPercent() float32 + func (pro *ConsoleBar) Increment() int + func (pro *ConsoleBar) Println(args ...interface{}) + func (pro *ConsoleBar) Reset() + func (pro *ConsoleBar) SetAll(all int) + func (pro *ConsoleBar) SetMsg(msg string) + func (pro *ConsoleBar) Update() + func (pro *ConsoleBar) Write(args ...interface{}) + type Dict map[string]string + type DictBool map[string]bool + func (dict DictBool) Keys() (newArray ArrayFilter) + type EnumeConfig struct + Domain string + EndId int + IdFile string + Names []string + Output string + Proxy string + Proxy2 string + Proxy3 string + StartId int + Template map[string]string + func ReadConf(f string) (config *EnumeConfig) + func (conf *EnumeConfig) Marshal() string + type FilterOption struct + Distance float32 + Proxy interface{} + Rank int + type G map[string]interface + type Gfunc map[string]NextValue + type Links [][]*UrlSim + func (link Links) AsString(rank int) (a [][]string) + type Loger interface + Error func(error) + Println func(args ...interface{}) + SetMsg func(msg string) + type NewDialler func(proxy interface{}) proxy.Dialer + type NextValue func(v Value) Value + type Payloader string + func (pay Payloader) AsFile(howHandle func(f *os.File, err error)) + func (pay Payloader) Format(args ...interface{}) Payloader + func (pay Payloader) FormatMap(args map[string]interface{}) Payloader + func (pay Payloader) Lines() (a ArrayFilter) + func (pay Payloader) Render(name string, v Value) string + func (pay Payloader) String() string + type ProxyDiallerPool interface + Add func(url string) + GetDialer func() proxy.Dialer + SetMode func(int) + type Result struct + Res interface{} + Url string + type Resulter struct + Errs []error + Oks []string + func (r *Resulter) Json() string + func (r *Resulter) PutErr(err error) + func (r *Resulter) PutOk(m string) + type RunnerPool struct + After func(res Result, loger Loger) + Bar *ConsoleBar + ErrDo func(error, int, Result, Loger) + Handle func(arg string, tryTime int) interface{} + LogLevl int + RetryTime int + Thread int + func NewAwaitPool(thread int) (pool *RunnerPool) + func (pool *RunnerPool) Loop(args []string, showBar bool) + func (pool *RunnerPool) LoopByFunc(generate func() (string, bool)) + func (pool *RunnerPool) Tick(sec int) + type Selection struct + type Session struct + Document *goquery.Document + Header map[string]string + MultiGetRetryTime int + Proxy string + RandomeUA bool + Timeout int + Transprot httplib.Transport + func NewSession() (sess *Session) + func (sess *Session) CheckAlive(urls []string, showBar bool, after func(res *SmartResponse) bool, ...) (alived []string) + func (sess *Session) GetsWith(urltemp string, mapFuncs Gfunc, ...) + func (sess *Session) MultiGet(urls []string, handleRes func(loger Loger, res *SmartResponse, err error), ...) + func (session *Session) Asyncs(work int, loadCache bool, showState bool, do func(each *AsyncOut), ...) *Session + func (session *Session) Copy() *Session + func (session *Session) Get(url string, proxy ...interface{}) (resp *SmartResponse, err error) + func (session *Session) Json(url string, data map[string]interface{}, proxy ...interface{}) (resp *SmartResponse, err error) + func (session *Session) Post(httpurl string, data map[string]string, proxy ...interface{}) (resp *SmartResponse, err error) + func (session *Session) Send(raw string, proxy ...interface{}) (resp *SmartResponse, err error) + func (session *Session) SetHeader(key string, value string) + func (session *Session) SetProxy(proxy interface{}) + func (session *Session) SetProxyDialer(dialer proxy.Dialer) + func (session *Session) SetSocks5Proxy(proxyAddr string) (err error) + func (session *Session) SetTimeout(t int) + func (session *Session) StartAsync(i int) *Async + func (session *Session) TestErrorPage(url string, proxy ...interface{}) (string, string, string) + func (session *Session) Upload(url string, filePath string, fileKey string, data map[string]string, ...) (resp *SmartResponse, err error) + func (session *Session) UrlJoin(f ...string) string + func (session *Session) With(urlstr string, proxy ...interface{}) (with *WithOper) + type SmartResponse struct + Code int + func ParseRawData(buf []byte, url string) (r *SmartResponse, err error) + func (res *SmartResponse) Base64() string + func (res *SmartResponse) Base64Mime() []byte + func (res *SmartResponse) CssExtract(cssSelctors Dict) (out G) + func (res *SmartResponse) CssSelect(css string, each func(i int, s *Selection)) + func (res *SmartResponse) FastCheckLineByLine(found func(line string) bool) (string, bool) + func (res *SmartResponse) FileLinks(includeouter ...bool) (s []string) + func (res *SmartResponse) HashMMH3() int32 + func (res *SmartResponse) HashMMH3Base64() int32 + func (res *SmartResponse) Links(includeouter ...bool) (s []string) + func (res *SmartResponse) PageTextHash() string + func (resp *SmartResponse) Text() string + func (smartres *SmartResponse) HeaderJson() string + func (smartres *SmartResponse) HeaderString() (d string) + func (smartres *SmartResponse) Html() []byte + func (smartres *SmartResponse) Json(obj ...interface{}) (jdata map[string]interface{}) + func (smartres *SmartResponse) Md5() string + func (smartres *SmartResponse) ReExtractString(re string) []string + func (smartres *SmartResponse) RequestURL() *url.URL + func (smartres *SmartResponse) Search(key string, toLower bool) bool + func (smartres *SmartResponse) Soup() (m *goquery.Document) + func (smartres *SmartResponse) String() string + func (smartres *SmartResponse) Title() string + type UrlSim struct + func AsUrlSim(urlstr string, title ...string) (u *UrlSim) + func (u *UrlSim) GetTitle() string + func (u *UrlSim) GetUrl() string + func (u *UrlSim) SetTitle(title string) + func (u *UrlSim) Sub(other interface{}) (score float32) + type Value struct + func NewValue(i interface{}) Value + func (v Value) Add(one int) Value + func (v Value) AsInt() (int, error) + func (v Value) Empty() bool + func (v Value) Increase() Value + func (v Value) String() string + type WithOper struct + Article *Article + Document *goquery.Document + Err error + LastSelections []*goquery.Selection + Links Links + URL *url.URL + func (with *WithOper) AsArticle() *WithOper + func (with *WithOper) AsSiteMap(do func(out *AsyncOut), breakpointContinue bool, showState bool, ...) *WithOper + func (with *WithOper) Each(css string, do ...func(i int, s *Selection)) *WithOper + func (with *WithOper) EndCache() *WithOper + func (with *WithOper) Entry(url string) *WithOper + func (with *WithOper) For(do func(i int, s *Selection)) *WithOper + func (with *WithOper) News(filters ...FilterOption) *WithOper + func (with *WithOper) PreTestSkip(name string, urls ...string) (o []string) + func (with *WithOper) SimpleNews() *WithOper + func (with *WithOper) StartCache(name string) *WithOper v1.1.1 Mar 28, 2021