tokenizer

Documentation

Index

• Constants
• Variables
• func AtomLookupString(tagName string) atom.Atom
• func NewTokenizerFile(filename string) (*html.Tokenizer, error)
• func NextStartToken(z *html.Tokenizer, skipErrors bool, htmlAtoms ...atom.Atom) (html.Token, error)
• func NextTextToken(z *html.Tokenizer, skipErrors bool, htmlAtoms ...atom.Atom) (html.Token, error)
• func NextTokenMatch(z *html.Tokenizer, skipErrors, includeChain, includeMatch bool, ...) ([]html.Token, error)
• func ParseLink(tokens ...html.Token) (href string, desc string, err error)
• func TokenAttribute(token html.Token, attrName string) (string, error)
• func TokenMap(t html.Token) map[string]string
• func TokensBetween(z *html.Tokenizer, skipErrors, inclusive bool, begin, end TokenFilters) ([]html.Token, error)
• func TokensBetweenAtom(z *html.Tokenizer, skipErrors, inclusive bool, htmlAtom atom.Atom) ([]html.Token, error)
• func TokensSubset(startFilter, endFilter *TokenFilter, inclusive, greedy bool, toks []html.Token) []html.Token
• type AtomSet
  • func NewAtomSet(htmlAtoms ...atom.Atom) AtomSet
  • func NewAtomSetString(tagNames ...string) (AtomSet, error)
  • func NewAtomSetStringMust(tagNames ...string) AtomSet
  • func (set AtomSet) Add(htmlAtoms ...atom.Atom)
  • func (set AtomSet) Exists(htmlAtom atom.Atom) bool
  • func (set AtomSet) Len() int
  • func (set AtomSet) Names() []string
• type Attributes
  • func (attrs Attributes) GetOne(attributeKey string) (html.Attribute, error)
• type Description
  • func (d *Description) DescriptionString() string
  • func (d *Description) Empty() bool
  • func (d *Description) Strings() []string
  • func (d *Description) TermString() string
• type DescriptionList
  • func ParseDescriptionListTokens(toks ...html.Token) DescriptionList
  • func TokenizerDescriptionListNext(z *html.Tokenizer) (DescriptionList, error)
  • func (dl DescriptionList) Strings() [][]string
• type DescriptionLists
  • func TokenizerDescriptionLists(z *html.Tokenizer) (DescriptionLists, error)
  • func (dls DescriptionLists) Strings() [][][]string
• type TokenFilter
  • func NewTokenFilter(tokenType html.TokenType, atoms ...atom.Atom) *TokenFilter
  • func (tf *TokenFilter) Match(t html.Token) bool
• type TokenFilters
  • func (filters TokenFilters) ByTokenType(tt html.TokenType) []TokenFilter
• type Tokens
  • func (tokens Tokens) Maps() []map[string]string
  • func (tokens Tokens) String() string

Constants

const (
	AttrHref = "href"
)

Variables

var ErrTokenNotFound = errors.New("token(s) not found")

Functions

func AtomLookupString

func AtomLookupString(tagName string) atom.Atom

func NewTokenizerFile

func NewTokenizerFile(filename string) (*html.Tokenizer, error)

func NextStartToken

func NextStartToken(z *html.Tokenizer, skipErrors bool, htmlAtoms ...atom.Atom) (html.Token, error)

func NextTextToken

func NextTextToken(z *html.Tokenizer, skipErrors bool, htmlAtoms ...atom.Atom) (html.Token, error)

func NextTokenMatch

func NextTokenMatch(z *html.Tokenizer, skipErrors, includeChain, includeMatch bool, filters ...TokenFilter) ([]html.Token, error)

NextTokenMatch returns a string of matches. `includeMatch` is only used when `includeChain` is included.

func ParseLink

func ParseLink(tokens ...html.Token) (href string, desc string, err error)

func TokenAttribute

func TokenAttribute(token html.Token, attrName string) (string, error)

func TokenMap

func TokenMap(t html.Token) map[string]string

func TokensBetween

func TokensBetween(z *html.Tokenizer, skipErrors, inclusive bool, begin, end TokenFilters) ([]html.Token, error)

func TokensBetweenAtom

func TokensBetweenAtom(z *html.Tokenizer, skipErrors, inclusive bool, htmlAtom atom.Atom) ([]html.Token, error)

TokensBetweenAtom returns the tokens that represent the `innerHtml` between a start and end tag token.

func TokensSubset

func TokensSubset(startFilter, endFilter *TokenFilter, inclusive, greedy bool, toks []html.Token) []html.Token

Types

type AtomSet

type AtomSet struct {
	Atoms map[string]atom.Atom
}

func NewAtomSet

func NewAtomSet(htmlAtoms ...atom.Atom) AtomSet

func NewAtomSetString

func NewAtomSetString(tagNames ...string) (AtomSet, error)

func NewAtomSetStringMust

func NewAtomSetStringMust(tagNames ...string) AtomSet

func (AtomSet) Add

func (set AtomSet) Add(htmlAtoms ...atom.Atom)

func (AtomSet) Exists

func (set AtomSet) Exists(htmlAtom atom.Atom) bool

func (AtomSet) Len

func (set AtomSet) Len() int

func (AtomSet) Names

func (set AtomSet) Names() []string

type Attributes

type Attributes []html.Attribute

func (Attributes) GetOne

func (attrs Attributes) GetOne(attributeKey string) (html.Attribute, error)

type Description

type Description struct {
	Term        []html.Token
	Description []html.Token
}

func (*Description) DescriptionString

func (d *Description) DescriptionString() string

func (*Description) Empty

func (d *Description) Empty() bool

func (*Description) Strings

func (d *Description) Strings() []string

func (*Description) TermString

func (d *Description) TermString() string

type DescriptionList

type DescriptionList []Description

func ParseDescriptionListTokens

func ParseDescriptionListTokens(toks ...html.Token) DescriptionList

func TokenizerDescriptionListNext

func TokenizerDescriptionListNext(z *html.Tokenizer) (DescriptionList, error)

func (DescriptionList) Strings

func (dl DescriptionList) Strings() [][]string

type DescriptionLists

type DescriptionLists []DescriptionList

func TokenizerDescriptionLists

func TokenizerDescriptionLists(z *html.Tokenizer) (DescriptionLists, error)

func (DescriptionLists) Strings

func (dls DescriptionLists) Strings() [][][]string

type TokenFilter

type TokenFilter struct {
	TokenType html.TokenType
	AtomSet   AtomSet
}

find next <tr> or </table>

func NewTokenFilter

func NewTokenFilter(tokenType html.TokenType, atoms ...atom.Atom) *TokenFilter

func (*TokenFilter) Match

func (tf *TokenFilter) Match(t html.Token) bool

type TokenFilters

type TokenFilters []TokenFilter

func (TokenFilters) ByTokenType

func (filters TokenFilters) ByTokenType(tt html.TokenType) []TokenFilter

type Tokens

type Tokens []html.Token

func (Tokens) Maps

func (tokens Tokens) Maps() []map[string]string

func (Tokens) String

func (tokens Tokens) String() string