Documentation
¶
Overview ¶
Package crawler provides a simple web crawler with concurrent support
Example (BasicUsage) ¶
package main
import (
"context"
"fmt"
"time"
"github.com/IceWhaleTech/ZimaOS-Blue/server/internal/crawler"
)
func main() {
// Create a crawler with default configuration
config := crawler.DefaultConfig()
config.MaxDepth = 1
config.MaxConcurrency = 3
c := crawler.New(config)
// Crawl URLs
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
urls := []string{
"https://example.com",
}
results := c.CrawlSync(ctx, urls)
for _, result := range results {
if result.Error != "" {
fmt.Printf("Error crawling %s: %s\n", result.URL, result.Error)
continue
}
fmt.Printf("Title: %s\n", result.Title)
fmt.Printf("URL: %s\n", result.URL)
fmt.Printf("Links found: %d\n", len(result.Links))
}
}
Output:
Example (DomainRestricted) ¶
package main
import (
"context"
"fmt"
"time"
"github.com/IceWhaleTech/ZimaOS-Blue/server/internal/crawler"
)
func main() {
config := crawler.DefaultConfig()
config.MaxDepth = 3
config.AllowedDomains = []string{"example.com"} // Only crawl example.com
c := crawler.New(config)
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
results := c.CrawlSync(ctx, []string{"https://example.com"})
fmt.Printf("Crawled %d pages from example.com\n", len(results))
}
Output:
Example (StreamingResults) ¶
package main
import (
"context"
"fmt"
"time"
"github.com/IceWhaleTech/ZimaOS-Blue/server/internal/crawler"
)
func main() {
config := crawler.DefaultConfig()
c := crawler.New(config)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
// Use streaming storage for large crawls
storage, err := crawler.NewStreamingJSONStorage("./output/stream.jsonl")
if err != nil {
fmt.Printf("Failed to create storage: %v\n", err)
return
}
defer storage.Close()
// Process results as they come in
resultChan := c.Crawl(ctx, []string{"https://example.com"})
for result := range resultChan {
// Save each result immediately
if err := storage.SaveOne(result); err != nil {
fmt.Printf("Failed to save result: %v\n", err)
}
fmt.Printf("Crawled: %s\n", result.URL)
}
}
Output:
Example (WithStorage) ¶
package main
import (
"context"
"fmt"
"time"
"github.com/IceWhaleTech/ZimaOS-Blue/server/internal/crawler"
)
func main() {
config := crawler.DefaultConfig()
c := crawler.New(config)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
results := c.CrawlSync(ctx, []string{"https://example.com"})
// Save to JSON
jsonStorage := crawler.NewJSONStorage("./output/results.json")
if err := jsonStorage.Save(results); err != nil {
fmt.Printf("Failed to save JSON: %v\n", err)
}
// Save to CSV
csvStorage := crawler.NewCSVStorage("./output/results.csv")
if err := csvStorage.Save(results); err != nil {
fmt.Printf("Failed to save CSV: %v\n", err)
}
}
Output:
Index ¶
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func GenerateFilename ¶
GenerateFilename generates a filename with timestamp
Types ¶
type CSVStorage ¶
type CSVStorage struct {
// contains filtered or unexported fields
}
CSVStorage saves results to a CSV file
func NewCSVStorage ¶
func NewCSVStorage(filePath string) *CSVStorage
NewCSVStorage creates a new CSV storage
func (*CSVStorage) Save ¶
func (s *CSVStorage) Save(results []Result) error
Save writes results to a CSV file
type Config ¶
type Config struct {
// MaxDepth is the maximum depth to crawl (0 = only initial URLs)
MaxDepth int
// MaxConcurrency is the maximum number of concurrent requests
MaxConcurrency int
// RequestTimeout is the timeout for each HTTP request
RequestTimeout time.Duration
// UserAgent is the User-Agent header to use
UserAgent string
// RateLimit is the minimum delay between requests to the same domain
RateLimit time.Duration
// AllowedDomains restricts crawling to specific domains (empty = allow all)
AllowedDomains []string
}
Config holds crawler configuration
func DefaultConfig ¶
func DefaultConfig() Config
DefaultConfig returns a default crawler configuration
type Crawler ¶
type Crawler struct {
// contains filtered or unexported fields
}
Crawler is a concurrent web crawler
type JSONStorage ¶
type JSONStorage struct {
// contains filtered or unexported fields
}
JSONStorage saves results to a JSON file
func NewJSONStorage ¶
func NewJSONStorage(filePath string) *JSONStorage
NewJSONStorage creates a new JSON storage
func (*JSONStorage) Save ¶
func (s *JSONStorage) Save(results []Result) error
Save writes results to a JSON file
type Result ¶
type Result struct {
URL string `json:"url"`
Title string `json:"title"`
Links []string `json:"links"`
Text string `json:"text"`
Headers map[string]string `json:"headers"`
Status int `json:"status"`
Error string `json:"error,omitempty"`
CrawledAt time.Time `json:"crawled_at"`
}
Result represents a crawled page result
type StreamingJSONStorage ¶
type StreamingJSONStorage struct {
// contains filtered or unexported fields
}
StreamingJSONStorage saves results incrementally to a JSON Lines file
func NewStreamingJSONStorage ¶
func NewStreamingJSONStorage(filePath string) (*StreamingJSONStorage, error)
NewStreamingJSONStorage creates a new streaming JSON storage
func (*StreamingJSONStorage) Close ¶
func (s *StreamingJSONStorage) Close() error
Close closes the file
func (*StreamingJSONStorage) Save ¶
func (s *StreamingJSONStorage) Save(results []Result) error
Save writes multiple results to the file
func (*StreamingJSONStorage) SaveOne ¶
func (s *StreamingJSONStorage) SaveOne(result Result) error
SaveOne writes a single result to the file
Source Files
Click to show internal directories.
Click to hide internal directories.