store

Documentation

Index

• Variables
• func GetInsertQuery(table string, fields ...string) string
• func MimeAny(string) bool
• func MimeIsText(mime string) bool
• func NewURLStore(db *sql.DB, opts ...URLStoreOpt) (*urlStore, error)
• func OnlyTLD(ending string) func(*url.URL) bool
• type ActionStore
  • func NewActionStore(db *sql.DB, fs *FileStore) (*ActionStore, error)
  • func (as *ActionStore) Save(tx *sql.Tx, id int64, actions []*kraaler.CrawlAction) error
• type BodyStore
  • func NewBodyStore(db *sql.DB, fs *FileStore) (*BodyStore, error)
  • func (ss *BodyStore) Save(tx *sql.Tx, id int64, body kraaler.ResponseBody, mime string) error
• type Compressor
• type ConsoleStore
  • func NewConsoleStore(db *sql.DB) (*ConsoleStore, error)
  • func (cs *ConsoleStore) Save(tx *sql.Tx, id int64, console []*kraaler.JavaScriptConsole) error
• type FileStore
  • func NewFileStore(root string, opts ...FileStoreOpt) (*FileStore, error)
  • func (fs *FileStore) Store(raw []byte) (StoredFile, error)
• type FileStoreOpt
  • func WithCompression(c Compressor) FileStoreOpt
  • func WithMimeTypes(types ...MimeValidator) FileStoreOpt
• type Hasher
• type HeaderStore
  • func NewHeaderStore(db *sql.DB) (*HeaderStore, error)
  • func (hs *HeaderStore) SaveRequest(tx *sql.Tx, id int64, key, value string) error
  • func (hs *HeaderStore) SaveResponse(tx *sql.Tx, id int64, key, value string) error
• type IDStore
  • func NewIDStore(table string, cache *cache.Cache, fields ...string) *IDStore
  • func (is *IDStore) Get(tx *sql.Tx, items ...interface{}) (int64, error)
• type InitiatorStackStore
  • func NewInitiatorStackStore(db *sql.DB) (*InitiatorStackStore, error)
  • func (is *InitiatorStackStore) Save(tx *sql.Tx, id int64, cf kraaler.CallFrame) error
• type MimeValidator
• type PostDataStore
  • func NewPostDataStore(db *sql.DB) (*PostDataStore, error)
  • func (ps *PostDataStore) Save(tx *sql.Tx, id int64, data string) error
• type Sampler
  • func PairSampler(pw int) Sampler
  • func UniformSampler() Sampler
• type ScreenStore
  • func NewScreenStore(db *sql.DB, ss *ScreenshotStore) (*ScreenStore, error)
  • func (ss *ScreenStore) Save(tx *sql.Tx, id int64, urlstr string, screenshots []*kraaler.BrowserScreenshot) error
• type ScreenshotStore
  • func NewScreenshotStore(dir string) *ScreenshotStore
  • func (ss *ScreenshotStore) Store(s *kraaler.BrowserScreenshot, domain string) (string, error)
• type SecurityStore
  • func NewSecurityStore(db *sql.DB) (*SecurityStore, error)
  • func (ss *SecurityStore) Save(tx *sql.Tx, id int64, sd *network.SecurityDetails) error
• type SessionStore
  • func NewSessionStore(db *sql.DB) (*SessionStore, error)
  • func (ss *SessionStore) Save(tx *sql.Tx, sess *kraaler.Page) (int64, error)
• type Store
  • func NewStore(db *sql.DB, bodyPath, screenPath string) (*Store, error)
  • func (s *Store) SaveSession(cs kraaler.Page) error
• type StoredFile
• type URLFilter
• type URLStoreOpt
  • func WithNoResampling() URLStoreOpt
  • func WithSampler(s Sampler) URLStoreOpt
  • func WithURLFilters(f ...URLFilter) URLStoreOpt
• type UrlStore
  • func NewUrlStore(db *sql.DB) (*UrlStore, error)
  • func (us *UrlStore) Save(tx *sql.Tx, id int64, urlstr string) error
• type WarehouseInserter
  • func (m WarehouseInserter) Add(s string, i interface{})
  • func (m WarehouseInserter) Store(tx *sql.Tx, table string) (int64, error)

Variables

var GzipCompression = gzipComp{}

var NoCompression = noComp{}

var (
	NotAllowedMimeErr = errors.New("mime type is not allowed to be stored")
)

var Sha256Hasher = &hasher{sum: func(d []byte) string { return fmt.Sprintf("%x", sha256.Sum256(d)) }, name: "sha256"}

var (
	StoreIsEmptyErr = errors.New("store is empty")
)

Functions

func GetInsertQuery

func GetInsertQuery(table string, fields ...string) string

func MimeAny

func MimeAny(string) bool

func MimeIsText

func MimeIsText(mime string) bool

func NewURLStore

func NewURLStore(db *sql.DB, opts ...URLStoreOpt) (*urlStore, error)

func OnlyTLD

func OnlyTLD(ending string) func(*url.URL) bool

Types

type ActionStore

type ActionStore struct {
	// contains filtered or unexported fields
}

func NewActionStore

func NewActionStore(db *sql.DB, fs *FileStore) (*ActionStore, error)

func (*ActionStore) Save

func (as *ActionStore) Save(tx *sql.Tx, id int64, actions []*kraaler.CrawlAction) error

type BodyStore

type BodyStore struct {
	// contains filtered or unexported fields
}

func NewBodyStore

func NewBodyStore(db *sql.DB, fs *FileStore) (*BodyStore, error)

func (*BodyStore) Save

func (ss *BodyStore) Save(tx *sql.Tx, id int64, body kraaler.ResponseBody, mime string) error

type Compressor

type Compressor interface {
	NewWriter(io.Writer) (io.Writer, error)
	Ext() string
}

type ConsoleStore

type ConsoleStore struct {
	// contains filtered or unexported fields
}

func NewConsoleStore

func NewConsoleStore(db *sql.DB) (*ConsoleStore, error)

func (*ConsoleStore) Save

func (cs *ConsoleStore) Save(tx *sql.Tx, id int64, console []*kraaler.JavaScriptConsole) error

type FileStore

type FileStore struct {
	// contains filtered or unexported fields
}

func NewFileStore

func NewFileStore(root string, opts ...FileStoreOpt) (*FileStore, error)

func (*FileStore) Store

func (fs *FileStore) Store(raw []byte) (StoredFile, error)

type FileStoreOpt

type FileStoreOpt func(fs *FileStore)

func WithCompression

func WithCompression(c Compressor) FileStoreOpt

func WithMimeTypes

func WithMimeTypes(types ...MimeValidator) FileStoreOpt

type Hasher

type Hasher interface {
	Sum([]byte) string
	Name() string
}

type HeaderStore

type HeaderStore struct {
	// contains filtered or unexported fields
}

func NewHeaderStore

func NewHeaderStore(db *sql.DB) (*HeaderStore, error)

func (*HeaderStore) SaveRequest

func (hs *HeaderStore) SaveRequest(tx *sql.Tx, id int64, key, value string) error

func (*HeaderStore) SaveResponse

func (hs *HeaderStore) SaveResponse(tx *sql.Tx, id int64, key, value string) error

type IDStore

type IDStore struct {
	// contains filtered or unexported fields
}

func NewIDStore

func NewIDStore(table string, cache *cache.Cache, fields ...string) *IDStore

func (*IDStore) Get

func (is *IDStore) Get(tx *sql.Tx, items ...interface{}) (int64, error)

type InitiatorStackStore

type InitiatorStackStore struct{}

func NewInitiatorStackStore

func NewInitiatorStackStore(db *sql.DB) (*InitiatorStackStore, error)

func (*InitiatorStackStore) Save

func (is *InitiatorStackStore) Save(tx *sql.Tx, id int64, cf kraaler.CallFrame) error

type MimeValidator

type MimeValidator func(string) bool

type PostDataStore

type PostDataStore struct{}

func NewPostDataStore

func NewPostDataStore(db *sql.DB) (*PostDataStore, error)

func (*PostDataStore) Save

func (ps *PostDataStore) Save(tx *sql.Tx, id int64, data string) error

type Sampler

type Sampler func(map[*url.URL]*time.Time) *url.URL

func PairSampler

func PairSampler(pw int) Sampler

func UniformSampler

func UniformSampler() Sampler

type ScreenStore

type ScreenStore struct {
	// contains filtered or unexported fields
}

func NewScreenStore

func NewScreenStore(db *sql.DB, ss *ScreenshotStore) (*ScreenStore, error)

func (*ScreenStore) Save

func (ss *ScreenStore) Save(tx *sql.Tx, id int64, urlstr string, screenshots []*kraaler.BrowserScreenshot) error

type ScreenshotStore

type ScreenshotStore struct {
	// contains filtered or unexported fields
}

func NewScreenshotStore

func NewScreenshotStore(dir string) *ScreenshotStore

func (*ScreenshotStore) Store

func (ss *ScreenshotStore) Store(s *kraaler.BrowserScreenshot, domain string) (string, error)

type SecurityStore

type SecurityStore struct {
	// contains filtered or unexported fields
}

func NewSecurityStore

func NewSecurityStore(db *sql.DB) (*SecurityStore, error)

func (*SecurityStore) Save

func (ss *SecurityStore) Save(tx *sql.Tx, id int64, sd *network.SecurityDetails) error

type SessionStore

type SessionStore struct {
	// contains filtered or unexported fields
}

func NewSessionStore

func NewSessionStore(db *sql.DB) (*SessionStore, error)

func (*SessionStore) Save

func (ss *SessionStore) Save(tx *sql.Tx, sess *kraaler.Page) (int64, error)

type Store

type Store struct {
	// contains filtered or unexported fields
}

func NewStore

func NewStore(db *sql.DB, bodyPath, screenPath string) (*Store, error)

func (*Store) SaveSession

func (s *Store) SaveSession(cs kraaler.Page) error

type StoredFile

type StoredFile struct {
	HashType string
	Hash     string
	Path     string
	OrgSize  int
	CompSize int
	MimeType string
}

type URLFilter

type URLFilter func(*url.URL) bool

type URLStoreOpt

type URLStoreOpt func(*urlStore)

func WithNoResampling

func WithNoResampling() URLStoreOpt

func WithSampler

func WithSampler(s Sampler) URLStoreOpt

func WithURLFilters

func WithURLFilters(f ...URLFilter) URLStoreOpt

type UrlStore

type UrlStore struct {
	// contains filtered or unexported fields
}

func NewUrlStore

func NewUrlStore(db *sql.DB) (*UrlStore, error)

func (*UrlStore) Save

func (us *UrlStore) Save(tx *sql.Tx, id int64, urlstr string) error

type WarehouseInserter

type WarehouseInserter map[string]func(tx *sql.Tx) (interface{}, error)

func (WarehouseInserter) Add

func (m WarehouseInserter) Add(s string, i interface{})

func (WarehouseInserter) Store

func (m WarehouseInserter) Store(tx *sql.Tx, table string) (int64, error)