data

Documentation

Index

• Variables
• func BatchInsertItems(repo Repository) func(ctx context.Context, i interface{}) (interface{}, error)
• func DispatchChecksumFetch() func(context.Context, interface{}) (interface{}, error)
• func FetchFiles(k *koanf.Koanf, dataRepo Repository) (map[string]string, error)
• func NotNilFilter() func(i interface{}) bool
• func ParseDumpModel(ctx context.Context) func(item rxgo.Item) rxgo.Observable
• func PopulateFromUri() func(ctx context.Context, i interface{}) (interface{}, error)
• func SetChecksumValues(m map[time.Time]map[string]string) func(ctx context.Context, i interface{}) (interface{}, error)
• func UpdateData(ctx context.Context, repo Repository) (int, error)
• func ValidUriFilter() func(i interface{}) bool
• type Data
  • func (d *Data) Dump() *opendiscogsmodel.DiscogsDump
• type ImportPlan
  • func FetchImportPlan(k *koanf.Koanf, dataRepo Repository) (*ImportPlan, error)
• type Repository
  • func NewDataRepository(db *gorm.DB) Repository

Variables

var DiscogsDataBaseURL = "https://data.discogs.com/"

var DiscogsS3BaseUrl = "https://discogs-data-dumps.s3.us-west-2.amazonaws.com/"

Functions

func BatchInsertItems

func BatchInsertItems(repo Repository) func(ctx context.Context, i interface{}) (interface{}, error)

func DispatchChecksumFetch

func DispatchChecksumFetch() func(context.Context, interface{}) (interface{}, error)

func FetchFiles

func FetchFiles(k *koanf.Koanf, dataRepo Repository) (map[string]string, error)

func NotNilFilter

func NotNilFilter() func(i interface{}) bool

func ParseDumpModel

func ParseDumpModel(ctx context.Context) func(item rxgo.Item) rxgo.Observable

func PopulateFromUri

func PopulateFromUri() func(ctx context.Context, i interface{}) (interface{}, error)

func SetChecksumValues

func SetChecksumValues(m map[time.Time]map[string]string) func(ctx context.Context, i interface{}) (interface{}, error)

func UpdateData

func UpdateData(ctx context.Context, repo Repository) (int, error)

func ValidUriFilter

func ValidUriFilter() func(i interface{}) bool

ValidUriFilter filter items by validating URI, judged by date, types and uri pattern.

Types

type Data

type Data struct {
	ETag        string `xml:"ETag" gorm:"column:etag"`
	GeneratedAt time.Time
	Checksum    string
	TargetType  string `gorm:"column:target_type"`
	Uri         string `xml:"Key"`
	SizeBytes   int64  `xml:"Size"`
}

func (*Data) Dump

func (d *Data) Dump() *opendiscogsmodel.DiscogsDump

type ImportPlan

type ImportPlan struct {
	Resources map[string]string
	Dumps     []*opendiscogsmodel.DiscogsDump
}

func FetchImportPlan

func FetchImportPlan(k *koanf.Koanf, dataRepo Repository) (*ImportPlan, error)

type Repository

type Repository interface {
	BatchInsert([]*Data) (int, error)
	FindByYearMonthType(year, month, typ string) (*opendiscogsmodel.DiscogsDump, error)
}

func NewDataRepository

func NewDataRepository(db *gorm.DB) Repository