idun

Apr 7, 2021

Changes in this version

type S

+ Cache *memcache.CacheType

+ UserAgent string

Apr 7, 2021

Changes in this version

+ const APIRetryMax

+ const ConsulAdvertisedPort

+ const ConsulAdvertisedService

+ const CrawlFilterRetry

+ const CrawlerExtra

+ const CrawlerMaxRunTime

+ const Environment

+ const ErrMsg

+ const GetDomainsRetry

+ const HalfGig

+ const HeadCheckTimeout

+ const HostsURL

+ const IdleTimeout

+ const KillSleep

+ const MaxDomainsInMap

+ const MaxPerCore

+ const MaxPerGig

+ const OneGig

+ const OneK

+ const OneMeg

+ const Parallelism

+ const PeerURL

+ const RandomDelay

+ const ReadTimeout

+ const RobotsTimeout

+ const TickEvery

+ const TwoGigs

+ const WriteTimeout

+ const YacyTimeout

+ var APIBase = "https://api.domainsproject.org/api/vo"

+ var BannedExtensions = []string

+ var BannedLocalRedirects = map[string]string

+ var Build = "unset"

+ var BuildDate = "unset"

+ var FreyaKey = os.Getenv("FREYA")

+ var GoVersion = "unset"

+ var IgnoreNoFollow = map[string]string

+ var Version = "unset"

+ func CalculateMaxWorkers() (int64, error)

+ func CrawlURL(client *Client, targetURL string, debugMode bool, serverAddr string)

+ func CrawlYacyHosts(apiHost string, address string, debugMode bool, s *S)

+ func DeduplicateSlice(incoming []string) (outgoing []string)

+ func FilterAndSubmit(domainMap map[string]bool, client *Client, serverAddr string)

+ func GetAllRemoteHosts(remoteURLs []string, domainCh chan string)

+ func GetHostNames(target string) ([]string, error)

+ func GetHostURLs(target string) ([]string, error)

+ func GetUA(reqURL string, logger *log.Logger) (string, error)

+ func HeadCheck(domain string, ua string) bool

+ func HeadCheckDomains(domains []string, ua string) map[string]bool

+ func KillPid(pid int)

+ func PIDWatcher(pid int)

+ func ParseXML(target string, outgoing interface{}) error

+ func PrepareClient(logger *log.Logger) *retryablehttp.Client

+ func RunAgent(consulURL string, logger *log.Logger, agentPort int)

+ func RunCrawl(target, serverAddr string, debugMode bool)

+ func RunWithAPI(client *Client, address string, debugMode bool, srvr *S)

+ func SubmitOutgoingDomains(client *Client, domains []string, serverAddr string)

+ func WaitAndKill(sleepTime time.Duration, pid int)

+ type Client struct

+ APIBase string

+ CustomDomainsURL string

+ Key string

+ Logger *log.Logger

+ func (c *Client) FilterDomains(incoming []string) (outgoing []string, err error)

+ func (c *Client) GetDomains() ([]string, error)

+ func (c *Client) GetUA() (string, error)

+ type ConsulClient struct

+ func NewConsul(consulURL string, logger *log.Logger) *ConsulClient

+ func (cc *ConsulClient) Deregister()

+ func (cc *ConsulClient) GetServices()

+ func (cc *ConsulClient) Register()

+ func (cc *ConsulClient) SetAdvertisedPort(port int)

+ func (cc *ConsulClient) SetServiceName(serviceName string)

+ type ConsulRegistration struct

+ Address string

+ ID string

+ Name string

+ Port int

+ Tags []string

+ type DomainsResponse struct

+ Domains []string

+ type Host struct

+ Name string

+ XMLName xml.Name

+ type HostBrowserResponse struct

+ Hosts Hosts

+ XMLName xml.Name

+ type Hosts struct

+ Host []Host

+ XMLName xml.Name

+ type JSONResponse struct

+ Code int64

+ Message string

+ type Peer struct

+ Address string

+ XMLName xml.Name

+ type PeerResponse struct

+ Peers []Peer

+ XMLName xml.Name

+ type RoboTester struct

+ func NewRoboTester(fullURL, userAgent string) (*RoboTester, error)

+ func (rt *RoboTester) GetDelay() time.Duration

+ func (rt *RoboTester) GetRobots(path string) (robots *robotstxt.RobotsData, err error)

+ func (rt *RoboTester) Test(path string) bool

+ type S struct

+ func (s *S) Pop() string

+ func (s *S) UA(w http.ResponseWriter, r *http.Request)

+ func (s *S) UploadDomains(w http.ResponseWriter, r *http.Request)

+ type WorkerNode struct

+ func (w WorkerNode) GetItem(ctx context.Context) (interface{}, error)

+ func (w WorkerNode) Process(ctx context.Context, item interface{}) (interface{}, error)

+ func (w WorkerNode) SubmitResult(ctx context.Context, result interface{}) error