gg_html

Documentation

Index

• Constants
• Variables
• type HTMLHelper
  • func (instance *HTMLHelper) Html2TextFromString(input string) (string, error)
  • func (instance *HTMLHelper) LoadCrawlerSettings(filename string) (*HtmlCrawlerSettings, error)
  • func (instance *HTMLHelper) NewCrawler(settings *HtmlCrawlerSettings) *HtmlCrawler
  • func (instance *HTMLHelper) NewParser(input interface{}) (*HtmlParser, error)
  • func (instance *HTMLHelper) UrlMatch(url string, list []string) bool
• type HtmlCrawler
  • func NewHtmlCrawler(settings *HtmlCrawlerSettings) *HtmlCrawler
  • func (instance *HtmlCrawler) Crawl(path string)
  • func (instance *HtmlCrawler) IsWorking() bool
  • func (instance *HtmlCrawler) Join()
  • func (instance *HtmlCrawler) OnContent(callback func(event *HtmlCrawlerContend))
  • func (instance *HtmlCrawler) Start()
  • func (instance *HtmlCrawler) Stop()
  • func (instance *HtmlCrawler) String() string
• type HtmlCrawlerContend
• type HtmlCrawlerSettings
  • func LoadHtmlCrawlerSettings(filename string) (*HtmlCrawlerSettings, error)
  • func (instance *HtmlCrawlerSettings) String() string
• type HtmlParser
  • func NewHtmlParser(input interface{}) (*HtmlParser, error)
  • func (instance *HtmlParser) BaseUrl() string
  • func (instance *HtmlParser) Document() *html.Node
  • func (instance *HtmlParser) FileName() string
  • func (instance *HtmlParser) ForEach(callback func(node *html.Node) bool)
  • func (instance *HtmlParser) GeNodeAttributes(nodes []*html.Node) []map[string]string
  • func (instance *HtmlParser) GelLinks() []*html.Node
  • func (instance *HtmlParser) GetLinkURLs() []string
  • func (instance *HtmlParser) GetMetaContent(name string) string
  • func (instance *HtmlParser) GetText() []string
  • func (instance *HtmlParser) InnerHtml(n *html.Node) string
  • func (instance *HtmlParser) IsURL() bool
  • func (instance *HtmlParser) Lang() string
  • func (instance *HtmlParser) MetaAuthor() string
  • func (instance *HtmlParser) MetaDescription() string
  • func (instance *HtmlParser) MetaKeywords() []string
  • func (instance *HtmlParser) MetaTitle() string
  • func (instance *HtmlParser) OuterHtml(n *html.Node) string
  • func (instance *HtmlParser) Path() string
  • func (instance *HtmlParser) RootUrl() string
  • func (instance *HtmlParser) Select(selector string) []*html.Node
  • func (instance *HtmlParser) SemanticBlocks(node *html.Node) []*SemanticBlock
  • func (instance *HtmlParser) SemanticBlocksAll() []*SemanticBlock
  • func (instance *HtmlParser) String() string
  • func (instance *HtmlParser) Text(node *html.Node) string
  • func (instance *HtmlParser) TextAll() string
  • func (instance *HtmlParser) Title() string
• type SemanticBlock
  • func (instance *SemanticBlock) GetBody() string
  • func (instance *SemanticBlock) GetText() string
  • func (instance *SemanticBlock) Json() string

Constants

const (
	EventOnContent = "on_content"
)

Variables

var (
	DefaultBlackList = []string{
		"http*//*facebook.*",
		"http*//*github.*",
		"http*//*linkedin.*",
		"http*//*bitbucket.*",
		"http*//*pinterest.*",
		"http*//*instagram.*",
		"http*//*twitter.*",
		"http*//*telegram.*",
		"http*//*google.*",
		"http*//*repubblica.*",
		"http*//*akismet.*",
		"http*//*jetpack.*",
	}
)

Types

type HTMLHelper

type HTMLHelper struct {
}

var HTML *HTMLHelper

func (*HTMLHelper) Html2TextFromString

func (instance *HTMLHelper) Html2TextFromString(input string) (string, error)

func (*HTMLHelper) LoadCrawlerSettings

func (instance *HTMLHelper) LoadCrawlerSettings(filename string) (*HtmlCrawlerSettings, error)

func (*HTMLHelper) NewCrawler

func (instance *HTMLHelper) NewCrawler(settings *HtmlCrawlerSettings) *HtmlCrawler

func (*HTMLHelper) NewParser

func (instance *HTMLHelper) NewParser(input interface{}) (*HtmlParser, error)

func (*HTMLHelper) UrlMatch

func (instance *HTMLHelper) UrlMatch(url string, list []string) bool

type HtmlCrawler

type HtmlCrawler struct {
	Settings        *HtmlCrawlerSettings
	HistoryCallback func(path string) bool
	// contains filtered or unexported fields
}

func NewHtmlCrawler

func NewHtmlCrawler(settings *HtmlCrawlerSettings) *HtmlCrawler

func (*HtmlCrawler) Crawl

func (instance *HtmlCrawler) Crawl(path string)

func (*HtmlCrawler) IsWorking

func (instance *HtmlCrawler) IsWorking() bool

func (*HtmlCrawler) Join

func (instance *HtmlCrawler) Join()

func (*HtmlCrawler) OnContent

func (instance *HtmlCrawler) OnContent(callback func(event *HtmlCrawlerContend))

func (*HtmlCrawler) Start

func (instance *HtmlCrawler) Start()

func (*HtmlCrawler) Stop

func (instance *HtmlCrawler) Stop()

func (*HtmlCrawler) String

func (instance *HtmlCrawler) String() string

type HtmlCrawlerContend

type HtmlCrawlerContend struct {
	Url    string           `json:"url"`
	Blocks []*SemanticBlock `json:"blocks"`
	Error  string           `json:"error"`
	Links  []string         `json:"links"`
}

type HtmlCrawlerSettings

type HtmlCrawlerSettings struct {
	StartPoints             []string `json:"start_points"`
	MaxThreads              int      `json:"max_threads"`
	AllowExternals          bool     `json:"allow_externals"` // are allowed external links
	WhiteList               []string `json:"while_list"`      // always allowed
	BlackList               []string `json:"black_list"`      // never allowed
	ExcludeDefaultBlackList bool     `json:"exclude_default_black_list"`
}

func LoadHtmlCrawlerSettings

func LoadHtmlCrawlerSettings(filename string) (*HtmlCrawlerSettings, error)

func (*HtmlCrawlerSettings) String

func (instance *HtmlCrawlerSettings) String() string

type HtmlParser

type HtmlParser struct {
	// contains filtered or unexported fields
}

func NewHtmlParser

func NewHtmlParser(input interface{}) (*HtmlParser, error)

func (*HtmlParser) BaseUrl

func (instance *HtmlParser) BaseUrl() string

func (*HtmlParser) Document

func (instance *HtmlParser) Document() *html.Node

func (*HtmlParser) FileName

func (instance *HtmlParser) FileName() string

func (*HtmlParser) ForEach

func (instance *HtmlParser) ForEach(callback func(node *html.Node) bool)

func (*HtmlParser) GeNodeAttributes

func (instance *HtmlParser) GeNodeAttributes(nodes []*html.Node) []map[string]string

func (*HtmlParser) GelLinks

func (instance *HtmlParser) GelLinks() []*html.Node

func (*HtmlParser) GetLinkURLs

func (instance *HtmlParser) GetLinkURLs() []string

func (*HtmlParser) GetMetaContent

func (instance *HtmlParser) GetMetaContent(name string) string

func (*HtmlParser) GetText

func (instance *HtmlParser) GetText() []string

func (*HtmlParser) InnerHtml

func (instance *HtmlParser) InnerHtml(n *html.Node) string

func (*HtmlParser) IsURL

func (instance *HtmlParser) IsURL() bool

func (*HtmlParser) Lang

func (instance *HtmlParser) Lang() string

func (*HtmlParser) MetaAuthor

func (instance *HtmlParser) MetaAuthor() string

func (*HtmlParser) MetaDescription

func (instance *HtmlParser) MetaDescription() string

func (*HtmlParser) MetaKeywords

func (instance *HtmlParser) MetaKeywords() []string

func (*HtmlParser) MetaTitle

func (instance *HtmlParser) MetaTitle() string

func (*HtmlParser) OuterHtml

func (instance *HtmlParser) OuterHtml(n *html.Node) string

func (*HtmlParser) Path

func (instance *HtmlParser) Path() string

func (*HtmlParser) RootUrl

func (instance *HtmlParser) RootUrl() string

func (*HtmlParser) Select

func (instance *HtmlParser) Select(selector string) []*html.Node

func (*HtmlParser) SemanticBlocks

func (instance *HtmlParser) SemanticBlocks(node *html.Node) []*SemanticBlock

func (*HtmlParser) SemanticBlocksAll

func (instance *HtmlParser) SemanticBlocksAll() []*SemanticBlock

func (*HtmlParser) String

func (instance *HtmlParser) String() string

func (*HtmlParser) Text

func (instance *HtmlParser) Text(node *html.Node) string

func (*HtmlParser) TextAll

func (instance *HtmlParser) TextAll() string

func (*HtmlParser) Title

func (instance *HtmlParser) Title() string

type SemanticBlock

type SemanticBlock struct {
	Lang     string   `json:"lang"`  // detected language (maybe different from page lang)
	Level    int      `json:"level"` // title level. 0 is when a block is free text with no title
	Title    string   `json:"title"`
	Body     string   `json:"body"`
	Keywords []string `json:"keywords"`
	// contains filtered or unexported fields
}

func (*SemanticBlock) GetBody

func (instance *SemanticBlock) GetBody() string

func (*SemanticBlock) GetText

func (instance *SemanticBlock) GetText() string

func (*SemanticBlock) Json

func (instance *SemanticBlock) Json() string