redisstorage

README

Redis Storage for Colly

This is a redis based storage backend for Colly collectors.

Install

go get -u github.com/gocolly/redisstorage

Usage

import (
	"github.com/gocolly/colly"
	"github.com/gocolly/redisstorage"
)

c := colly.NewCollector()

storage := &redisstorage.Storage{
    Address:  "127.0.0.1:6379",
    Password: "",
    DB:       0,
    Prefix:   "job01",
}

err := c.SetStorage(storage)
if err != nil {
    panic(err)
}

Bugs

Bugs or suggestions? Visit the issue tracker or join #colly on freenode

Documentation

Index

• type Storage
  • func (s *Storage) AddRequest(r []byte) error
  • func (s *Storage) Clear() error
  • func (s *Storage) Cookies(u *url.URL) string
  • func (s *Storage) GetRequest() ([]byte, error)
  • func (s *Storage) Init() error
  • func (s *Storage) IsVisited(requestID uint64) (bool, error)
  • func (s *Storage) QueueSize() (int, error)
  • func (s *Storage) SetCookies(u *url.URL, cookies string)
  • func (s *Storage) Visited(requestID uint64) error

Types

type Storage

type Storage struct {
	// Address is the redis server address
	Address string
	// Password is the password for the redis server
	Password string
	// DB is the redis database. Default is 0
	DB int
	// Prefix is an optional string in the keys. It can be used
	// to use one redis database for independent scraping tasks.
	Prefix string
	// Client is the redis connection
	Client *redis.Client

	// Expiration time for Visited keys. After expiration pages
	// are to be visited again.
	Expires time.Duration
	// contains filtered or unexported fields
}

Storage implements the redis storage backend for Colly

func (*Storage) AddRequest

func (s *Storage) AddRequest(r []byte) error

AddRequest implements queue.Storage.AddRequest() function

func (*Storage) Clear

func (s *Storage) Clear() error

Clear removes all entries from the storage

func (*Storage) Cookies

func (s *Storage) Cookies(u *url.URL) string

Cookies implements colly/storage.Cookies()

func (*Storage) GetRequest

func (s *Storage) GetRequest() ([]byte, error)

GetRequest implements queue.Storage.GetRequest() function

func (*Storage) Init

func (s *Storage) Init() error

Init initializes the redis storage

func (*Storage) IsVisited

func (s *Storage) IsVisited(requestID uint64) (bool, error)

IsVisited implements colly/storage.IsVisited()

func (*Storage) QueueSize

func (s *Storage) QueueSize() (int, error)

QueueSize implements queue.Storage.QueueSize() function

func (*Storage) SetCookies

func (s *Storage) SetCookies(u *url.URL, cookies string)

SetCookies implements colly/storage..SetCookies()

func (*Storage) Visited

func (s *Storage) Visited(requestID uint64) error

Visited implements colly/storage.Visited()