gowiki

README

gowiki

Gowiki is a golang library to parse mediawiki markup as found in Wikipedia pages.

Documentation

Index

• Constants
• Variables
• type Article
  • func NewArticle(title, text string) (*Article, error)
  • func ParseArticle(title, text string, g PageGetter) (*Article, error)
  • func (a *Article) CheckDisambiguation() bool
  • func (a *Article) CheckRedirect() (bool, *WikiLink)
  • func (a *Article) GenText() error
  • func (a *Article) GetAbstract() string
  • func (a *Article) GetExternalLinks() []string
  • func (a *Article) GetLinks() []WikiLink
  • func (a *Article) GetMedia() []WikiLink
  • func (a *Article) GetText() string
  • func (a *Article) GetTextLinks() []FullWikiLink
  • func (a *Article) ParseSimple() error
  • func (a *Article) PrintParseTree()
  • func (a *Article) Tokenize(mw string, g PageGetter) ([]*Token, error)
  • func (a *Article) TranscludeTemplatesRecursive(mw string, params map[string]string, g PageGetter, depth int) string
• type DummyPageGetter
  • func (g *DummyPageGetter) Get(wl WikiLink) (string, error)
• type FullWikiLink
• type Namespaces
  • func (namespaces Namespaces) WikiCanonicalFormNamespaceEsc(l string, defaultNamespace string, unescape bool) WikiLink
• type PageGetter
• type ParseNode
• type Prefixes
• type Template
• type TemplateRenderer
• type Token
• type WikiLink
  • func WikiCanonicalForm(l string) WikiLink
  • func WikiCanonicalFormEsc(l string, unescape bool) WikiLink
  • func WikiCanonicalFormNamespace(l string, defaultNamespace string) WikiLink
  • func (wl *WikiLink) FullPagename() string
  • func (wl *WikiLink) FullPagenameAnchor() string
  • func (wl *WikiLink) GetAnchor() string
  • func (wl *WikiLink) HasAnchor() bool
  • func (wl *WikiLink) IsImplicitSelfLink() bool

Constants

const (
	QS_none int = iota
	QS_i
	QS_b
	QS_ib
	QS_bi
)

const (
	TClosed int = 1 << iota
)

Variables

var DebugLevel int = 0

var MagicMap map[string]TemplateRenderer = map[string]TemplateRenderer{
	"DISPLAYTITLE": nil,
}

var NoHashFunctionsMap map[string]bool = map[string]bool{
	"displaytitle":     true,
	"formatdate":       true,
	"int":              true,
	"namespace":        true,
	"pagesinnamespace": true,
	"speciale":         true,
	"special":          true,
	"tag":              true,
	"anchorencode":     true, "basepagenamee": true, "basepagename": true, "canonicalurle": true,
	"canonicalurl": true, "cascadingsources": true, "defaultsort": true, "filepath": true,
	"formatnum": true, "fullpagenamee": true, "fullpagename": true, "fullurle": true,
	"fullurl": true, "gender": true, "grammar": true, "language": true,
	"lcfirst": true, "lc": true, "localurle": true, "localurl": true,
	"namespacee": true, "namespacenumber": true, "nse": true, "ns": true,
	"numberingroup": true, "numberofactiveusers": true, "numberofadmins": true, "numberofarticles": true,
	"numberofedits": true, "numberoffiles": true, "numberofpages": true, "numberofusers": true,
	"numberofviews": true, "padleft": true, "padright": true, "pageid": true,
	"pagenamee": true, "pagename": true, "pagesincategory": true, "pagesize": true,
	"plural": true, "protectionlevel": true, "revisionday2": true, "revisionday": true,
	"revisionid": true, "revisionmonth1": true, "revisionmonth": true, "revisiontimestamp": true,
	"revisionuser": true, "revisionyear": true, "rootpagenamee": true, "rootpagename": true,
	"subjectpagenamee": true, "subjectpagename": true, "subjectspacee": true, "subjectspace": true,
	"subpagenamee": true, "subpagename": true, "talkpagenamee": true, "talkpagename": true,
	"talkspacee": true, "talkspace": true, "ucfirst": true, "uc": true,
	"urlencode": true,
}

var VariablesMap map[string]bool = map[string]bool{
	"articlepath":         true,
	"basepagenamee":       true,
	"basepagename":        true,
	"cascadingsources":    true,
	"contentlanguage":     true,
	"currentday2":         true,
	"currentdayname":      true,
	"currentday":          true,
	"currentdow":          true,
	"currenthour":         true,
	"currentmonth1":       true,
	"currentmonthabbrev":  true,
	"currentmonthnamegen": true,
	"currentmonthname":    true,
	"currentmonth":        true,
	"currenttimestamp":    true,
	"currenttime":         true,
	"currentversion":      true,
	"currentweek":         true,
	"currentyear":         true,
	"directionmark":       true,
	"fullpagenamee":       true,
	"fullpagename":        true,
	"localday2":           true,
	"localdayname":        true,
	"localday":            true,
	"localdow":            true,
	"localhour":           true,
	"localmonth1":         true,
	"localmonthabbrev":    true,
	"localmonthnamegen":   true,
	"localmonthname":      true,
	"localmonth":          true,
	"localtimestamp":      true,
	"localtime":           true,
	"localweek":           true,
	"localyear":           true,
	"namespacee":          true,
	"namespacenumber":     true,
	"namespace":           true,
	"numberofactiveusers": true,
	"numberofadmins":      true,
	"numberofarticles":    true,
	"numberofedits":       true,
	"numberoffiles":       true,
	"numberofpages":       true,
	"numberofusers":       true,
	"numberofviews":       true,
	"pageid":              true,
	"pagenamee":           true,
	"pagename":            true,
	"revisionday2":        true,
	"revisionday":         true,
	"revisionid":          true,
	"revisionmonth1":      true,
	"revisionmonth":       true,
	"revisionsize":        true,
	"revisiontimestamp":   true,
	"revisionuser":        true,
	"revisionyear":        true,
	"rootpagenamee":       true,
	"rootpagename":        true,
	"scriptpath":          true,
	"servername":          true,
	"server":              true,
	"sitename":            true,
	"stylepath":           true,
	"subjectpagenamee":    true,
	"subjectpagename":     true,
	"subjectspacee":       true,
	"subjectspace":        true,
	"subpagenamee":        true,
	"subpagename":         true,
	"talkpagenamee":       true,
	"talkpagename":        true,
	"talkspacee":          true,
	"talkspace":           true,
}

Types

type Article

type Article struct {
	MediaWiki    string
	Title        string
	Links        []WikiLink
	ExtLinks     []string
	Type         string
	AbstractText string
	Media        []WikiLink
	Tokens       []*Token
	Root         *ParseNode
	Parsed       bool
	Text         string
	TextLinks    []FullWikiLink
	Templates    []*Template
	// contains filtered or unexported fields
}

func NewArticle

func NewArticle(title, text string) (*Article, error)

func ParseArticle

func ParseArticle(title, text string, g PageGetter) (*Article, error)

func (*Article) CheckDisambiguation

func (a *Article) CheckDisambiguation() bool

func (*Article) CheckRedirect

func (a *Article) CheckRedirect() (bool, *WikiLink)

func (*Article) GenText

func (a *Article) GenText() error

func (*Article) GetAbstract

func (a *Article) GetAbstract() string

func (*Article) GetExternalLinks

func (a *Article) GetExternalLinks() []string

func (*Article) GetLinks

func (a *Article) GetLinks() []WikiLink

func (*Article) GetMedia

func (a *Article) GetMedia() []WikiLink

func (*Article) GetText

func (a *Article) GetText() string

func (*Article) GetTextLinks

func (a *Article) GetTextLinks() []FullWikiLink

func (*Article) ParseSimple

func (a *Article) ParseSimple() error

func (*Article) PrintParseTree

func (a *Article) PrintParseTree()

func (*Article) Tokenize

func (a *Article) Tokenize(mw string, g PageGetter) ([]*Token, error)

func (*Article) TranscludeTemplatesRecursive

func (a *Article) TranscludeTemplatesRecursive(mw string, params map[string]string, g PageGetter, depth int) string

type DummyPageGetter

type DummyPageGetter struct{}

func (*DummyPageGetter) Get

func (g *DummyPageGetter) Get(wl WikiLink) (string, error)

type FullWikiLink

type FullWikiLink struct {
	Link  WikiLink
	Text  string
	Start int // rune offset of beginning
	End   int // rune offset of end (index of the char after the last)
}

type Namespaces

type Namespaces map[string]string

var StandardNamespaces Namespaces = map[string]string{
	"media":                  "Media",
	"special":                "Special",
	"talk":                   "Talk",
	"user":                   "User",
	"user talk":              "User talk",
	"wikipedia":              "Wikipedia",
	"wikipedia talk":         "Wikipedia talk",
	"file":                   "File",
	"file talk":              "File talk",
	"mediawiki":              "MediaWiki",
	"mediawiki talk":         "MediaWiki talk",
	"template":               "Template",
	"template talk":          "Template talk",
	"help":                   "Help",
	"help talk":              "Help talk",
	"category":               "Category",
	"category talk":          "Category talk",
	"portal":                 "Portal",
	"portal talk":            "Portal talk",
	"book":                   "Book",
	"book talk":              "Book talk",
	"draft":                  "Draft",
	"draft talk":             "Draft talk",
	"education program":      "Education Program",
	"education program talk": "Education Program talk",
	"timedtext":              "TimedText",
	"timedtext talk":         "TimedText talk",
	"module":                 "Module",
	"module talk":            "Module talk",
	"topic":                  "Topic",
}

func (Namespaces) WikiCanonicalFormNamespaceEsc

func (namespaces Namespaces) WikiCanonicalFormNamespaceEsc(l string, defaultNamespace string, unescape bool) WikiLink

type PageGetter

type PageGetter interface {
	Get(page WikiLink) (string, error)
}

type ParseNode

type ParseNode struct {
	NType    string
	NSubType string
	Link     WikiLink
	Contents string
	Flags    int
	Nodes    []*ParseNode
}

type Prefixes

type Prefixes []string

var ExtLinkPrefixes Prefixes = []string{"https://", "http://", "ftp://", "//"}

var FileLinkPrefixes Prefixes = []string{"[[image:", "[[media:", "[[file:"}

type Template

type Template struct {
	Typ        string            `json:"type"` //magic,normal,ext,param
	Name       string            `json:"name"`
	Attr       string            `json:"attr"` //text after the ':' in magic templates
	Parameters map[string]string `json:"parameters"`
}

type TemplateRenderer

type TemplateRenderer func(name string, params map[string]string) string

type Token

type Token struct {
	TText   string   `json:"tText,omitempty"`
	TType   string   `json:"tType,omitempty"`
	TAttr   string   `json:"tAttr,omitempty"`
	TLink   WikiLink `json:"tLink,omitempty"`
	TClosed bool     `json:"tClosed,omitempty"`
	TPipes  []string `json:"tPipes,omitempty"`
}

type WikiLink

type WikiLink struct {
	Namespace string
	PageName  string
	Anchor    string
}

func WikiCanonicalForm

func WikiCanonicalForm(l string) WikiLink

func WikiCanonicalFormEsc

func WikiCanonicalFormEsc(l string, unescape bool) WikiLink

func WikiCanonicalFormNamespace

func WikiCanonicalFormNamespace(l string, defaultNamespace string) WikiLink

func (*WikiLink) FullPagename

func (wl *WikiLink) FullPagename() string

func (*WikiLink) FullPagenameAnchor

func (wl *WikiLink) FullPagenameAnchor() string

func (*WikiLink) GetAnchor

func (wl *WikiLink) GetAnchor() string

func (*WikiLink) HasAnchor

func (wl *WikiLink) HasAnchor() bool

func (*WikiLink) IsImplicitSelfLink

func (wl *WikiLink) IsImplicitSelfLink() bool