Documentation

Overview

    Package pipeline is the persistent and offline process part of crawler.

    Index

    Constants

    This section is empty.

    Variables

    This section is empty.

    Functions

    This section is empty.

    Types

    type CollectPipeline

    type CollectPipeline interface {
    	Pipeline
    
    	// The GetCollected returns result saved in in process's memory temporarily.
    	GetCollected() []*page_items.PageItems
    }

      The interface CollectPipeline recommend result in process's memory temporarily.

      type CollectPipelinePageItems

      type CollectPipelinePageItems struct {
      	// contains filtered or unexported fields
      }

      func NewCollectPipelinePageItems

      func NewCollectPipelinePageItems() *CollectPipelinePageItems

      func (*CollectPipelinePageItems) GetCollected

      func (this *CollectPipelinePageItems) GetCollected() []*page_items.PageItems

      func (*CollectPipelinePageItems) Process

      type Pipeline

      type Pipeline interface {
      	// The Process implements result persistent.
      	// The items has the result be crawled.
      	// The t has informations of this crawl task.
      	Process(items *page_items.PageItems, t com_interfaces.Task)
      }

        The interface Pipeline can be implemented to customize ways of persistent.

        type PipelineConsole

        type PipelineConsole struct {
        }

        func NewPipelineConsole

        func NewPipelineConsole() *PipelineConsole

        func (*PipelineConsole) Process

        func (this *PipelineConsole) Process(items *page_items.PageItems, t com_interfaces.Task)

        type PipelineFile

        type PipelineFile struct {
        	// contains filtered or unexported fields
        }

        func NewPipelineFile

        func NewPipelineFile(path string) *PipelineFile

        func (*PipelineFile) Process

        func (this *PipelineFile) Process(items *page_items.PageItems, t com_interfaces.Task)