queue

Documentation

Index

• Constants
• Variables
• type PriorityURLQueue
  • func NewPriorityURLQueue(storage domain.Storage) *PriorityURLQueue
  • func (q *PriorityURLQueue) Close() error
  • func (q *PriorityURLQueue) GetMemoryUsageMB() float64
  • func (q *PriorityURLQueue) IsEmpty() bool
  • func (q *PriorityURLQueue) IsFull() bool
  • func (q *PriorityURLQueue) Pop() (domain.URLTask, error)
  • func (q *PriorityURLQueue) Push(task domain.URLTask) error
  • func (q *PriorityURLQueue) Size() int
• type QueueError
  • func (e *QueueError) Error() string

Constants

const (
	MaxQueueSize    = 100000 // Increased from 50k for better throughput - roughly 80mb for normal urls
	RefillThreshold = 0.2    // Refill when queue is <20% full (more aggressive)
)

Variables

var (
	ErrQueueFull  = &QueueError{Message: "queue is full"}
	ErrQueueEmpty = &QueueError{Message: "queue is empty"}
)

Custom errors

Types

type PriorityURLQueue

type PriorityURLQueue struct {
	// contains filtered or unexported fields
}

func NewPriorityURLQueue

func NewPriorityURLQueue(storage domain.Storage) *PriorityURLQueue

NewPriorityURLQueue creates a new priority URL queue

func (*PriorityURLQueue) Close

func (q *PriorityURLQueue) Close() error

Close closes the queue

func (*PriorityURLQueue) GetMemoryUsageMB

func (q *PriorityURLQueue) GetMemoryUsageMB() float64

GetMemoryUsageMB estimated memory usage

func (*PriorityURLQueue) IsEmpty

func (q *PriorityURLQueue) IsEmpty() bool

IsEmpty checks if the queue is empty

func (*PriorityURLQueue) IsFull

func (q *PriorityURLQueue) IsFull() bool

IsFull checks if the queue is full

func (*PriorityURLQueue) Pop

func (q *PriorityURLQueue) Pop() (domain.URLTask, error)

remove and returns the highest priority URL task

func (*PriorityURLQueue) Push

func (q *PriorityURLQueue) Push(task domain.URLTask) error

Push adds a URL task to the queue

func (*PriorityURLQueue) Size

func (q *PriorityURLQueue) Size() int

Size returns the current size of the queue

type QueueError

type QueueError struct {
	Message string
}

func (*QueueError) Error

func (e *QueueError) Error() string