manager

Documentation

Index

• func InitFeedManager(db store.Store)
• type ApplyResult
• type DeleteResult
• type FeedManager
  • func NewFeedManager(db store.Store) *FeedManager
  • func (m *FeedManager) ApplyFeeds(configs []models.FeedConfig, defaults *models.Selectors, dryRun bool) (ApplyResult, error)
  • func (m *FeedManager) DeleteFeeds(feedIds []string, dryRun bool) (DeleteResult, error)
  • func (m *FeedManager) DiagnoseFeed(id string) (models.FeedDiagnosis, error)
  • func (m *FeedManager) FetchFeed(feed models.Feed, from *time.Time, to *time.Time, overwrite bool, restore bool, ...) (models.FetchResult, error)
  • func (m *FeedManager) GetFeed(id string) (models.Feed, error)
  • func (m *FeedManager) GetScraper(feedType string) (scrapers.Scraper, error)
  • func (m *FeedManager) InspectArticle(rawURL, mode string, headers map[string]string, sel *models.Selectors, ...) models.ArticleInspection
  • func (m *FeedManager) InspectFeedURL(feedURL string, headers map[string]string, maxLinks int) scrapers.FeedInspection
  • func (m *FeedManager) ListFeeds() ([]models.Feed, error)
  • func (m *FeedManager) RefreshAllFeeds(wg *sync.WaitGroup)
  • func (m *FeedManager) RefreshFeed(id string) (models.FetchResult, error)
  • func (m *FeedManager) RefreshFeedWithTimeWindow(id string, from *time.Time, to *time.Time, overwrite bool, restore bool, ...) (models.FetchResult, error)
  • func (m *FeedManager) RegisterFeed(config models.FeedConfig) error
  • func (m *FeedManager) RegisterScraper(feedType string, scraper scrapers.Scraper)
  • func (m *FeedManager) RemoveFeed(feedId string) error
  • func (m *FeedManager) SetSolimenAddr(addr string)
  • func (m *FeedManager) SuggestSelectors(rawURL, mode string, headers map[string]string, max int) ([]models.SelectorCandidate, error)
  • func (m *FeedManager) UpdateFeedEnabled(id string, enabled bool) error

Functions

func InitFeedManager

func InitFeedManager(db store.Store)

Types

type ApplyResult

type ApplyResult struct {
	Created  []string
	Updated  []string
	Disabled []string
}

ApplyResult reports the outcome of an ApplyFeeds reconciliation. Each list holds human-readable feed labels ("title — url"), not ids.

type DeleteResult

type DeleteResult struct {
	Deleted  []string
	NotFound []string
}

DeleteResult reports the outcome of a DeleteFeeds call. Deleted holds human-readable feed labels; NotFound holds the requested ids with no match.

type FeedManager

type FeedManager struct {
	// contains filtered or unexported fields
}

FeedManager manages feeds and articles

var (
	Manager *FeedManager
)

func NewFeedManager

func NewFeedManager(db store.Store) *FeedManager

NewFeedManager creates a new FeedManager instance

func (*FeedManager) ApplyFeeds

func (m *FeedManager) ApplyFeeds(configs []models.FeedConfig, defaults *models.Selectors, dryRun bool) (ApplyResult, error)

ApplyFeeds reconciles the stored feeds against the desired set: feeds in configs are created or updated, feeds with enabled:false in configs are disabled, and feeds absent from configs that are still enabled get disabled (articles preserved). When dryRun is true the plan is computed but nothing is written.

func (*FeedManager) DeleteFeeds

func (m *FeedManager) DeleteFeeds(feedIds []string, dryRun bool) (DeleteResult, error)

DeleteFeeds removes the given feeds (by id), cascading article deletion. When dryRun is true the targets are reported but nothing is deleted.

func (*FeedManager) DiagnoseFeed

func (m *FeedManager) DiagnoseFeed(id string) (models.FeedDiagnosis, error)

DiagnoseFeed fetches a feed's raw HTTP response and returns a structured diagnosis of what came back. It is read-only: nothing is stored and the feed's last-fetch time is left untouched. This backs the `feeds diagnose` command.

func (*FeedManager) FetchFeed

func (m *FeedManager) FetchFeed(feed models.Feed, from *time.Time, to *time.Time, overwrite bool, restore bool, lastN int) (models.FetchResult, error)

FetchFeed fetches a feed and stores its articles Optional time window filtering: from and to can be nil to disable filtering If provided, only articles with PublishedAt between from and to (inclusive) will be stored If overwrite is true, existing articles will be overwritten instead of skipped If restore is true, only existing articles with no content will be overwritten

func (*FeedManager) GetFeed

func (m *FeedManager) GetFeed(id string) (models.Feed, error)

GetFeed returns a feed by Id

func (*FeedManager) GetScraper

func (m *FeedManager) GetScraper(feedType string) (scrapers.Scraper, error)

GetScraper returns the scraper for a feed type

func (*FeedManager) InspectArticle

func (m *FeedManager) InspectArticle(rawURL, mode string, headers map[string]string, sel *models.Selectors, htmlLimit int) models.ArticleInspection

InspectArticle scrapes a single article URL in the requested mode and, when an article selector is supplied, returns the extracted content. It is read-only and stores nothing — it backs the feed-config builder's selector testing and mode probing. Modes, in escalating cost: "" / static, dynamic, full_browser.

func (*FeedManager) InspectFeedURL

func (m *FeedManager) InspectFeedURL(feedURL string, headers map[string]string, maxLinks int) scrapers.FeedInspection

InspectFeedURL fetches and parses a feed URL (read-only), returning a diagnosis plus sample article links and the detected title. It is the pre-registration counterpart to DiagnoseFeed, working on a raw URL rather than a stored feed.

func (*FeedManager) ListFeeds

func (m *FeedManager) ListFeeds() ([]models.Feed, error)

ListFeeds returns all registered feeds

func (*FeedManager) RefreshAllFeeds

func (m *FeedManager) RefreshAllFeeds(wg *sync.WaitGroup)

RefreshAllFeeds refreshes all enabled feeds

func (*FeedManager) RefreshFeed

func (m *FeedManager) RefreshFeed(id string) (models.FetchResult, error)

RefreshFeed refreshes a feed by Id

func (*FeedManager) RefreshFeedWithTimeWindow

func (m *FeedManager) RefreshFeedWithTimeWindow(id string, from *time.Time, to *time.Time, overwrite bool, restore bool, lastN int) (models.FetchResult, error)

RefreshFeedWithTimeWindow refreshes a feed by Id with optional time window filtering from and to can be nil to disable filtering

func (*FeedManager) RegisterFeed

func (m *FeedManager) RegisterFeed(config models.FeedConfig) error

RegisterFeed registers a feed

func (*FeedManager) RegisterScraper

func (m *FeedManager) RegisterScraper(feedType string, scraper scrapers.Scraper)

RegisterScraper registers a scraper for a feed type

func (*FeedManager) RemoveFeed

func (m *FeedManager) RemoveFeed(feedId string) error

RemoveFeed removes a feed and its data from the system

func (*FeedManager) SetSolimenAddr

func (m *FeedManager) SetSolimenAddr(addr string)

SetSolimenAddr sets the solimen service address used for full_browser feeds.

func (*FeedManager) SuggestSelectors

func (m *FeedManager) SuggestSelectors(rawURL, mode string, headers map[string]string, max int) ([]models.SelectorCandidate, error)

SuggestSelectors scrapes an article page in the given mode and returns ranked candidate article-content selectors (by text length, penalized by link density). Read-only; backs the autobuild agent's selector discovery.

func (*FeedManager) UpdateFeedEnabled

func (m *FeedManager) UpdateFeedEnabled(id string, enabled bool) error

UpdateFeedEnabled updates the enabled status of a feed