scraper

Documentation

Index

• Constants
• type Scraper
  • func New(headless bool, debugPauseAfterScrape bool) *Scraper
  • func (s *Scraper) ScrapeForYou(ctx context.Context, cookies []*network.Cookie, count int) ([]types.Post, error)

Constants

const (
	// Feed selectors
	FeedContainer = `[data-testid="primaryColumn"]`
	TweetArticle  = `article[data-testid="tweet"]`

	// Tweet content selectors
	TweetText      = `[data-testid="tweetText"]`
	TweetShowMore  = `button[data-testid="tweet-text-show-more-link"]`
	TweetAuthor    = `[data-testid="User-Name"]`
	TweetTimestamp = `time`
	TweetLink      = `a[href*="/status/"]`
	TweetMedia     = `[data-testid="tweetPhoto"], [data-testid="videoPlayer"]`

	// Engagement selectors
	ReplyCount   = `[data-testid="reply"]`
	RetweetCount = `[data-testid="retweet"]`
	LikeCount    = `[data-testid="like"]`

	// Tweet type indicators
	RetweetIndicator = `[data-testid="socialContext"]`
	QuoteIndicator   = `[data-testid="quoteTweet"]`
	ReplyIndicator   = `[data-testid="tweet"] a[href*="/status/"][dir="ltr"]`

	// Login page indicators (for detecting auth state)
	HomeIndicator = `[data-testid="SideNav_NewTweet_Button"]`
	LoginForm     = `[data-testid="loginButton"]`
)

const (
	WaitForTweets = TweetArticle
)

Common wait conditions

Types

type Scraper

type Scraper struct {
	// contains filtered or unexported fields
}

Scraper handles extracting posts from X.com

func New

func New(headless bool, debugPauseAfterScrape bool) *Scraper

New creates a new scraper

func (*Scraper) ScrapeForYou

func (s *Scraper) ScrapeForYou(ctx context.Context, cookies []*network.Cookie, count int) ([]types.Post, error)

ScrapeForYou fetches posts from the For You feed