wikidump

Documentation

Index

• func New(dir string) (*dump, error)
• type Contributer
• type Page
  • func Find(pages []*Page, pageIDs map[int64]struct{}) []*Page
  • func ParseStream(stream []byte) ([]*Page, error)
• type Redirect
• type Revision
• type Stream

Functions

func New

func New(dir string) (*dump, error)

Types

type Contributer

type Contributer struct {
	Username string `xml:"username"`
	ID       int64  `xml:"id"`
}

type Page

type Page struct {
	Title    string   `xml:"title"`
	Redirect Redirect `xml:"redirect"`
	Revision Revision `xml:"revision"`
	NS       int64    `xml:"ns"`
	ID       int64    `xml:"id"`
}

func Find

func Find(pages []*Page, pageIDs map[int64]struct{}) []*Page

Only returns the pages with IDs that are in the pageIDs map.

func ParseStream

func ParseStream(stream []byte) ([]*Page, error)

type Redirect

type Redirect struct {
	Title string `xml:"title,attr"`
}

type Revision

type Revision struct {
	Timestamp   time.Time   `xml:"timestamp"`
	Format      string      `xml:"format"`
	Text        string      `xml:"text"`
	Comment     string      `xml:"comment"`
	Model       string      `xml:"model"`
	SHA1        string      `xml:"sha1"`
	Contributer Contributer `xml:"contributer"`
	ID          int64       `xml:"id"`
	ParentID    int64       `xml:"parentid"`
}

type Stream

type Stream struct {
	XMLName xml.Name `xml:"stream"`
	Pages   []*Page  `xml:"page"`
}

TODO: convert wikitext to plain text by expanding wikipedia templates.