redditmongo

README

Reddit Mongo

Description

Scraps a particular subreddit and saves its posts to a mongodb

Install

go get github.com/jarivas/redditmongo

Usage

.env

REDDIT_USERNAME=reddit_bot
REDDIT_PASSWORD=snoo
REDDIT_CLIENT_ID=p-jcoLKBynTLew
REDDIT_APP_SECRET=gko_LXELoV07ZBNUXrvWZfzE3aI
REDDIT_TIME_SLEEP=30s
MONGO_PORT=27017
MONGO_PORT_UI=8081
MONGO_DB_NAME=reddit
MONGO_INITDB_ROOT_USERNAME=root
MONGO_INITDB_ROOT_PASSWORD=example

demo.go

package demo

import (
	"github.com/jarivas/redditmongo"
    "log"
)

func main() {
	rm, err := RedditMongo{}.FromEnv("redditdev")

	if err != nil {
		log.Fatal(err)
	}

	e := make(chan error)

	go rm.Scrape(e)

	for err = range(e) {
		log.Fatal(err)
	}
}

Documentation

Index

• type MongoStorage
  • func (m *MongoStorage) CreateCollection(name string) error
  • func (m MongoStorage) FromEnv() (*MongoStorage, error)
  • func (m *MongoStorage) GetCollection(name string) *mgm.Collection
  • func (m MongoStorage) New(url, dbName string) (*MongoStorage, error)
  • func (m *MongoStorage) ResetColection(name string) error
• type Post
  • func (p *Post) CheckExists(m *MongoStorage) (bool, error)
  • func (p Post) FromScraped(post *redditscraper.Post, subreddit string) *Post
  • func (p Post) GetLast(m *MongoStorage, subreddit string) (*Post, error)
  • func (p *Post) Save(m *MongoStorage) error
  • func (p *Post) Validate() bool
• type RedditMongo
  • func (rm RedditMongo) FromEnv(subreddit string) (*RedditMongo, error)
  • func (rm RedditMongo) New(ms *MongoStorage, rs *redditscraper.RedditScraper, subreddit string) *RedditMongo
  • func (rm *RedditMongo) Scrape(e chan<- error)

Types

type MongoStorage

type MongoStorage struct {
	// contains filtered or unexported fields
}

func (*MongoStorage) CreateCollection

func (m *MongoStorage) CreateCollection(name string) error

func (MongoStorage) FromEnv

func (m MongoStorage) FromEnv() (*MongoStorage, error)

func (*MongoStorage) GetCollection

func (m *MongoStorage) GetCollection(name string) *mgm.Collection

func (MongoStorage) New

func (m MongoStorage) New(url, dbName string) (*MongoStorage, error)

func (*MongoStorage) ResetColection

func (m *MongoStorage) ResetColection(name string) error

type Post

type Post struct {
	mgm.DefaultModel `bson:",inline"`
	Id               string `json:"id" bson:"id"`
	Title            string `json:"title" bson:"title"`
	Body             string `json:"body" bson:"body"`
	// contains filtered or unexported fields
}

func (*Post) CheckExists

func (p *Post) CheckExists(m *MongoStorage) (bool, error)

func (Post) FromScraped

func (p Post) FromScraped(post *redditscraper.Post, subreddit string) *Post

func (Post) GetLast

func (p Post) GetLast(m *MongoStorage, subreddit string) (*Post, error)

func (*Post) Save

func (p *Post) Save(m *MongoStorage) error

func (*Post) Validate

func (p *Post) Validate() bool

type RedditMongo

type RedditMongo struct {
	Subreddit string
	// contains filtered or unexported fields
}

func (RedditMongo) FromEnv

func (rm RedditMongo) FromEnv(subreddit string) (*RedditMongo, error)

func (RedditMongo) New

func (rm RedditMongo) New(ms *MongoStorage, rs *redditscraper.RedditScraper, subreddit string) *RedditMongo

func (*RedditMongo) Scrape

func (rm *RedditMongo) Scrape(e chan<- error)