utils

Documentation

Index

• Variables
• func DedupeStrings(input []string) []string
• func DedupeURLs(URLs []*url.URL) []*url.URL
• func FileExists(filename string) bool
• func GetHostname() string
• func GetOutboundIP() net.IP
• func GetSHA1(str string) string
• func GetSHA1FromFile(filePath string) string
• func MakeAbsolute(base *url.URL, URLs []*url.URL) []*url.URL
• func RemoveFragments(URLs []*url.URL) []*url.URL
• func SetupLogging(jobPath string, liveStats bool, esURL string) (logInfo, logWarning, logError *logrus.Logger)
• func StringContainsSliceElements(target string, slice []string) bool
• func StringInSlice(a string, list []string) bool
• func StringSliceToURLSlice(rawURLs []string) (URLs []*url.URL)
• func URLToString(u *url.URL) string
• func ValidateURL(u *url.URL) error
• type DiskStatus
  • func GetFreeDiskSpace(path string) (disk DiskStatus)
• type TAtomBool
  • func (b *TAtomBool) Get() bool
  • func (b *TAtomBool) Set(value bool)
• type Version
  • func GetVersion() (version Version)

Variables

var LogInfo, LogWarning, LogError *logrus.Logger

Functions

func DedupeStrings

func DedupeStrings(input []string) []string

DedupeStrings take a slice of string and dedupe it

func DedupeURLs

func DedupeURLs(URLs []*url.URL) []*url.URL

DedupeURLs take a slice of *url.URL and dedupe it

func FileExists

func FileExists(filename string) bool

FileExists checks if a file exists and is not a directory before we try using it to prevent further errors

func GetHostname

func GetHostname() string

func GetOutboundIP

func GetOutboundIP() net.IP

Note: GetOutboundIP does not establish any connection and the destination does not need to exist for this function to work.

func GetSHA1

func GetSHA1(str string) string

GetSHA1 take a string and return the SHA1 hash of the string, as a string

func GetSHA1FromFile

func GetSHA1FromFile(filePath string) string

GetSHA1FromFile calculate the SHA1 of a file

func MakeAbsolute

func MakeAbsolute(base *url.URL, URLs []*url.URL) []*url.URL

MakeAbsolute turn all URLs in a slice of url.URL into absolute URLs, based on a given base *url.URL

func RemoveFragments

func RemoveFragments(URLs []*url.URL) []*url.URL

func SetupLogging

func SetupLogging(jobPath string, liveStats bool, esURL string) (logInfo, logWarning, logError *logrus.Logger)

SetupLogging setup the logger for the crawl

func StringContainsSliceElements

func StringContainsSliceElements(target string, slice []string) bool

StringContainsSliceElements if the string contains one of the elements of a slice

func StringInSlice

func StringInSlice(a string, list []string) bool

StringInSlice return true if the string is in the slice

func StringSliceToURLSlice

func StringSliceToURLSlice(rawURLs []string) (URLs []*url.URL)

StringSliceToURLSlice takes a slice of string and return a slice of url.URL

func URLToString

func URLToString(u *url.URL) string

func ValidateURL

func ValidateURL(u *url.URL) error

ValidateURL validates a *url.URL

Types

type DiskStatus

type DiskStatus struct {
	All   uint64 `json:"all"`
	Used  uint64 `json:"used"`
	Free  uint64 `json:"free"`
	Avail uint64 `json:"avail"`
}

func GetFreeDiskSpace

func GetFreeDiskSpace(path string) (disk DiskStatus)

type TAtomBool

type TAtomBool struct {
	// contains filtered or unexported fields
}

TAtomBool define an atomic boolean

func (*TAtomBool) Get

func (b *TAtomBool) Get() bool

Get return the value of an atomic boolean

func (*TAtomBool) Set

func (b *TAtomBool) Set(value bool)

Set set the value of an atomic boolean

type Version

type Version struct {
	Version     string
	GoVersion   string
	WarcVersion string
	ZenoVersion string
}

func GetVersion

func GetVersion() (version Version)