leveldb

package
v0.8.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 24, 2017 License: Apache-2.0 Imports: 5 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type LeveldbStore

type LeveldbStore struct {
	WalkPrefix         string
	FetchPrefix        string
	ParsePrefix        string
	PendingFetchPrefix string
	OffsetPrefix       string

	PersistFileName string
	Leveldb         *leveldb.DB
}

func (*LeveldbStore) Add

func (filter *LeveldbStore) Add(key []byte) error

func (*LeveldbStore) AddFetchFailedUrl

func (this *LeveldbStore) AddFetchFailedUrl(url []byte)

func (*LeveldbStore) AddFetchedUrl

func (this *LeveldbStore) AddFetchedUrl(url []byte)

func (*LeveldbStore) AddParsedFile

func (this *LeveldbStore) AddParsedFile(url []byte)

func (*LeveldbStore) AddPendingFetchUrl

func (this *LeveldbStore) AddPendingFetchUrl(url []byte)

func (*LeveldbStore) AddSavedUrl

func (this *LeveldbStore) AddSavedUrl(url []byte)

func (*LeveldbStore) AddValue

func (filter *LeveldbStore) AddValue(key []byte, value []byte) error

func (*LeveldbStore) AddWalkedUrl

func (this *LeveldbStore) AddWalkedUrl(url []byte)

func (*LeveldbStore) Close

func (this *LeveldbStore) Close() error

func (*LeveldbStore) FileHasParsed

func (this *LeveldbStore) FileHasParsed(url []byte) bool

func (*LeveldbStore) FileHasSaved

func (this *LeveldbStore) FileHasSaved(file string) bool

func (*LeveldbStore) Get

func (this *LeveldbStore) Get(key string) []byte

get webpage file

func (*LeveldbStore) GetValue

func (filter *LeveldbStore) GetValue(key []byte) []byte

func (*LeveldbStore) InitPendingFetchBloomFilter

func (this *LeveldbStore) InitPendingFetchBloomFilter(fileName string)

func (*LeveldbStore) List

func (this *LeveldbStore) List(from int, size int) [][]byte

func (*LeveldbStore) LoadOffset

func (this *LeveldbStore) LoadOffset(fileName string) int64

func (*LeveldbStore) LogFetchFailedUrl

func (this *LeveldbStore) LogFetchFailedUrl(path string, content string)

func (*LeveldbStore) LogPendingFetchUrl

func (this *LeveldbStore) LogPendingFetchUrl(path string, content string)

func (*LeveldbStore) LogSavedFile

func (this *LeveldbStore) LogSavedFile(path string, content string)

func (*LeveldbStore) Lookup

func (filter *LeveldbStore) Lookup(key []byte) bool

func (*LeveldbStore) Open

func (this *LeveldbStore) Open() error

func (*LeveldbStore) PendingFetchUrlHasAdded

func (this *LeveldbStore) PendingFetchUrlHasAdded(url []byte) bool

func (*LeveldbStore) Persist

func (filter *LeveldbStore) Persist() error

TODO REMOVE

func (*LeveldbStore) PersistBloomFilter

func (this *LeveldbStore) PersistBloomFilter()

func (*LeveldbStore) PersistOffset

func (this *LeveldbStore) PersistOffset(fileName string, offset int64)

func (*LeveldbStore) Store

func (this *LeveldbStore) Store(url string, data []byte)

store webpage file

func (*LeveldbStore) TaskEnqueue

func (this *LeveldbStore) TaskEnqueue(url []byte)

func (*LeveldbStore) UrlHasFetched

func (this *LeveldbStore) UrlHasFetched(url []byte) bool

func (*LeveldbStore) UrlHasWalked

func (this *LeveldbStore) UrlHasWalked(url []byte) bool

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL