datastore

Documentation

Index

• func GetPgDataSource(host string, port string, user string, pwd string, dbname string, ...) string
• func NewPostgresModelList(ctx context.Context, config configpackage.Configuration, logger logger.Logger) (mapPgModel map[string]*PostgresDB, err error)
• type Datastore
• type PostgresDB
  • func GetListPgModel(ctx context.Context, config configpackage.Configuration, logger logger.Logger) (listPostgresModel []*PostgresDB)
  • func (d *PostgresDB) CheckUrlIsExist(link string) (contains bool, err error)
  • func (m *PostgresDB) Connect() (err error)
  • func (d *PostgresDB) ContainsD(link string) (contains bool, err error)
  • func (d *PostgresDB) CrawlpageList(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListData, err error)
  • func (d *PostgresDB) CrawlpageListParsed(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListParsedData, err error)
  • func (d *PostgresDB) GetExistingQueue(task string) (dataList []string, err error)
  • func (d *PostgresDB) GetLatestSeedUrl(task string, seedurl string) (latest_seedurl string, err error)
  • func (d *PostgresDB) StoreD(pagesource string, link string, task string, documenttype string, ...) (err error)
  • func (d *PostgresDB) StoreDocument(link string, task string, documenttype string, document []byte, ...) (err error)
  • func (d *PostgresDB) StoreE(link string, href string, task string) (err error)

Functions

func GetPgDataSource

func GetPgDataSource(host string, port string, user string, pwd string, dbname string, sslmode string, poolmaxcon string) string

func NewPostgresModelList

func NewPostgresModelList(ctx context.Context, config configpackage.Configuration, logger logger.Logger) (mapPgModel map[string]*PostgresDB, err error)

Types

type Datastore

type Datastore interface {
	StoreD(pagesource string, link string, task string, documenttype string, mimetype string) (err error)
	StoreDocument(link string, task string, filename string, document []byte, documentcontenttype string) (err error)
	StoreE(link string, href string, task string) (err error)
	ContainsD(url string) (contains bool, err error)
	GetExistingQueue(task string) (queue []string, err error)
	GetLatestSeedUrl(task string, seedurl string) (res_seedurl string, err error)
	CrawlpageList(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListData, err error)
	CrawlpageListParsed(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListParsedData, err error)
	CheckUrlIsExist(link string) (isexist bool, err error)
}

type PostgresDB

type PostgresDB struct {
	// contains filtered or unexported fields
}

func GetListPgModel

func GetListPgModel(ctx context.Context, config configpackage.Configuration, logger logger.Logger) (listPostgresModel []*PostgresDB)

func (*PostgresDB) CheckUrlIsExist

func (d *PostgresDB) CheckUrlIsExist(link string) (contains bool, err error)

func (*PostgresDB) Connect

func (m *PostgresDB) Connect() (err error)

func (*PostgresDB) ContainsD

func (d *PostgresDB) ContainsD(link string) (contains bool, err error)

func (*PostgresDB) CrawlpageList

func (d *PostgresDB) CrawlpageList(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListData, err error)

func (*PostgresDB) CrawlpageListParsed

func (d *PostgresDB) CrawlpageListParsed(param entity.CrawlpageListParam) (dataList []entity.CrawlpageListParsedData, err error)

func (*PostgresDB) GetExistingQueue

func (d *PostgresDB) GetExistingQueue(task string) (dataList []string, err error)

func (*PostgresDB) GetLatestSeedUrl

func (d *PostgresDB) GetLatestSeedUrl(task string, seedurl string) (latest_seedurl string, err error)

func (*PostgresDB) StoreD

func (d *PostgresDB) StoreD(pagesource string, link string, task string, documenttype string, mimetype string) (err error)

func (*PostgresDB) StoreDocument

func (d *PostgresDB) StoreDocument(link string, task string, documenttype string, document []byte, documentcontenttype string) (err error)

func (*PostgresDB) StoreE

func (d *PostgresDB) StoreE(link string, href string, task string) (err error)