constants

Documentation

Index

• Constants
• Variables
• type TaskSignal

Constants

const (
	ActionTypeVisit          = "visit"
	ActionTypeInstallDep     = "install_dep"
	ActionTypeInstallLang    = "install_lang"
	ActionTypeViewDisclaimer = "view_disclaimer"
)

const (
	AnchorStartStage = "START_STAGE"
	AnchorStartUrl   = "START_URL"
	AnchorItems      = "ITEMS"
	AnchorParsers    = "PARSERS"
)

const (
	OwnerTypeAll    = "all"
	OwnerTypeMe     = "me"
	OwnerTypePublic = "public"
)

const (
	CacheColName             = "cache"
	CacheColKey              = "k"
	CacheColValue            = "v"
	CacheColTime             = "t"
	CacheKeyPublicPlugins    = "public_plugins"
	CacheKeyPublicPluginInfo = "public_plugin_info"
)

const (
	ChannelAllNode = "nodes:public"

	ChannelWorkerNode = "nodes:"

	ChannelMasterNode = "nodes:master"
)

const (
	ASCENDING  = "asc"
	DESCENDING = "dsc"
)

const (
	EngineScrapy = "scrapy"
	EngineColly  = "colly"
)

const (
	DataFieldTypeGeneral  = "general"
	DataFieldTypeNumeric  = "numeric"
	DataFieldTypeDate     = "date"
	DataFieldTypeCurrency = "currency"
	DataFieldTypeUrl      = "url"
	DataFieldTypeImage    = "image"
	DataFieldTypeAudio    = "audio"
	DataFieldTypeVideo    = "video"
)

const (
	DataSourceTypeMongo         = "mongo"
	DataSourceTypeMysql         = "mysql"
	DataSourceTypePostgresql    = "postgresql"
	DataSourceTypeMssql         = "mssql"
	DataSourceTypeSqlite        = "sqlite"
	DataSourceTypeCockroachdb   = "cockroachdb"
	DataSourceTypeElasticSearch = "elasticsearch"
	DataSourceTypeKafka         = "kafka"
)

const (
	DefaultMongoPort         = "27017"
	DefaultMysqlPort         = "3306"
	DefaultPostgresqlPort    = "5432"
	DefaultMssqlPort         = "1433"
	DefaultCockroachdbPort   = "26257"
	DefaultElasticsearchPort = "9200"
	DefaultKafkaPort         = "9092"
)

const (
	DataSourceStatusOnline  = "on"
	DataSourceStatusOffline = "off"
)

const (
	GrpcEventServiceTypeRegister = "register"
	GrpcEventServiceTypeSend     = "send"
)

const (
	ExportTypeCsv  = "csv"
	ExportTypeJson = "json"
)

const (
	FilterQueryFieldConditions = "conditions"
	FilterQueryFieldAll        = "all"
)

const (
	FilterObjectTypeString  = "string"
	FilterObjectTypeNumber  = "number"
	FilterObjectTypeBoolean = "boolean"
)

const (
	FilterOpNotSet           = "ns"
	FilterOpContains         = "c"
	FilterOpNotContains      = "nc"
	FilterOpRegex            = "r"
	FilterOpEqual            = "eq"
	FilterOpNotEqual         = "ne"
	FilterOpIn               = "in"
	FilterOpNotIn            = "nin"
	FilterOpGreaterThan      = "gt"
	FilterOpLessThan         = "lt"
	FilterOpGreaterThanEqual = "gte"
	FilterOpLessThanEqual    = "lte"
	FilterOpSearch           = "s"
)

const (
	GitAuthTypeHttp = "http"
	GitAuthTypeSsh  = "ssh"
)

const (
	GitRemoteNameUpstream = "upstream"
	GitRemoteNameOrigin   = "origin"
)

const (
	GitBranchMaster = "master"
	GitBranchMain   = "main"
)

const (
	DefaultGrpcServerHost       = ""
	DefaultGrpcServerPort       = "9666"
	DefaultGrpcClientRemoteHost = "localhost"
	DefaultGrpcClientRemotePort = DefaultGrpcServerPort
	DefaultGrpcAuthKey          = "Crawlab2021!"
)

const (
	GrpcSubscribeTypeNode   = "node"
	GrpcSubscribeTypePlugin = "plugin"
)

const (
	HttpResponseStatusOk       = "ok"
	HttpResponseMessageSuccess = "success"
	HttpResponseMessageError   = "error"
)

const (
	MsgTypeGetLog        = "get-log"
	MsgTypeGetSystemInfo = "get-sys-info"
	MsgTypeCancelTask    = "cancel-task"
	MsgTypeRemoveLog     = "remove-log"
	MsgTypeRemoveSpider  = "remove-spider"
)

const (
	NodeStatusUnregistered = "u"
	NodeStatusRegistered   = "r"
	NodeStatusOnline       = "on"
	NodeStatusOffline      = "off"
)

const (
	NotificationTriggerOnTaskEnd   = "notification_trigger_on_task_end"
	NotificationTriggerOnTaskError = "notification_trigger_on_task_error"
	NotificationTriggerNever       = "notification_trigger_never"
)

const (
	NotificationTypeMail     = "notification_type_mail"
	NotificationTypeDingTalk = "notification_type_ding_talk"
	NotificationTypeWechat   = "notification_type_wechat"
)

const (
	PluginTypeApi  = "api"
	PluginTypeWeb  = "web"
	PluginTypeTask = "task"
)

const (
	PluginProtoGrpc = "grpc"
	PluginProtoHttp = "http"
)

const (
	PluginUIComponentTypeView = "view"
	PluginUIComponentTypeTab  = "tab"
)

const (
	PluginDeployModeMaster = "master"
	PluginDeployModeAll    = "all"
)

const (
	PluginInstallTypePublic = "public"
	PluginInstallTypeGit    = "git"
	PluginInstallTypeLocal  = "local"
)

const (
	PluginStatusInstalling   = "installing"
	PluginStatusInstallError = "install_error"
	PluginStatusStopped      = "stopped"
	PluginStatusRunning      = "running"
	PluginStatusError        = "error"
)

const (
	RegisterTypeMac        = "mac"
	RegisterTypeIp         = "ip"
	RegisterTypeHostname   = "hostname"
	RegisterTypeCustomName = "customName"
)

const (
	DedupTypeIgnore    = "ignore"
	DedupTypeOverwrite = "overwrite"
)

const (
	RpcInstallLang          = "install_lang"
	RpcInstallDep           = "install_dep"
	RpcUninstallDep         = "uninstall_dep"
	RpcGetInstalledDepList  = "get_installed_dep_list"
	RpcGetLang              = "get_lang"
	RpcCancelTask           = "cancel_task"
	RpcGetSystemInfoService = "get_system_info"
	RpcRemoveSpider         = "remove_spider"
)

const (
	ScheduleStatusStop    = "stopped"
	ScheduleStatusRunning = "running"
	ScheduleStatusError   = "error"

	ScheduleStatusErrorNotFoundNode   = "Not Found Node"
	ScheduleStatusErrorNotFoundSpider = "Not Found Spider"
)

const (
	SettingPlugin                = "plugin"
	SettingPluginBaseUrl         = "plugin_base_url"
	SettingPluginGithubPublicOrg = "github_public_org"
	SettingPluginRepoPrefix      = "repo_prefix"
)

const (
	DefaultSettingPluginBaseUrl         = "https://plugins.crawlab.cn"
	DefaultSettingPluginBaseUrlDocker   = "/app/plugins"
	DefaultSettingPluginBaseUrlGitHub   = "https://github.com"
	DefaultSettingPluginGithubPublicOrg = "crawlab-team"
	DefaultSettingPluginRepoPrefix      = "plugin-"
)

const (
	Customized   = "customized"
	Configurable = "configurable"
	Plugin       = "plugin"
)

const (
	Windows = "windows"
	Linux   = "linux"
	Darwin  = "darwin"
)

const (
	Python = "python"
	Nodejs = "node"
	Java   = "java"
)

const (
	InstallStatusNotInstalled    = "not-installed"
	InstallStatusInstalling      = "installing"
	InstallStatusInstallingOther = "installing-other"
	InstallStatusInstalled       = "installed"
)

const (
	LangTypeLang      = "lang"
	LangTypeWebDriver = "webdriver"
)

const (
	TaskStatusPending   = "pending"
	TaskStatusRunning   = "running"
	TaskStatusFinished  = "finished"
	TaskStatusError     = "error"
	TaskStatusCancelled = "cancelled"
	TaskStatusAbnormal  = "abnormal"
)

const (
	RunTypeAllNodes      = "all-nodes"
	RunTypeRandom        = "random"
	RunTypeSelectedNodes = "selected-nodes"
)

const (
	TaskTypeSpider = "spider"
	TaskTypeSystem = "system"
)

const (
	TaskListQueuePrefixPublic = "tasks:public"
	TaskListQueuePrefixNodes  = "tasks:nodes"
)

const (
	RoleAdmin  = "admin"
	RoleNormal = "normal"
)

const (
	DefaultAdminUsername = "admin"
	DefaultAdminPassword = "admin"
)

const (
	String  = "string"
	Number  = "number"
	Boolean = "boolean"
	Array   = "array"
	Object  = "object"
)

const (
	ColJob = "jobs"
)

const (
	DataCollectionKey = "_col"
)

const (
	DefaultEncryptServerKey = "0123456789abcdef"
)

const (
	DefaultFilerAuthKey = "Crawlab2021!"
)

const (
	DefaultHost = "localhost"
)

const EmptyFileData = " "

const (
	ErrorRegexPattern = "(?:[ :,.]|^)((?:error|exception|traceback)s?)(?:[ :,.]|$)"
)

const FsKeepFileName = ".gitkeep"

const (
	GrpcHeaderAuthorization = "authorization"
)

const (
	HashKey = "_h"
)

const (
	HttpContentTypeApplicationJson = "application/json"
)

const ScrapyProtectedFieldNames = "_id,task_id,ts"

const ScrapyProtectedStageNames = ""

const (
	SignalQuit = iota
)

const (
	SortQueryField = "sort"
)

const (
	TaskKey = "_tid"
)

const (
	UserContextKey = "user"
)

Variables

var (
	//ErrorMongoError                = e.NewSystemOPError(1001, "system error:[mongo]%s", http.StatusInternalServerError)
	//ErrorUserNotFound              = e.NewBusinessError(10001, "user not found.", http.StatusUnauthorized)
	//ErrorUsernameOrPasswordInvalid = e.NewBusinessError(11001, "username or password invalid", http.StatusUnauthorized)
	ErrAlreadyExists    = errors.New("already exists")
	ErrNotExists        = errors.New("not exists")
	ErrForbidden        = errors.New("forbidden")
	ErrInvalidOperation = errors.New("invalid operation")
	ErrInvalidOptions   = errors.New("invalid options")
	ErrNoTasksAvailable = errors.New("no tasks available")
	ErrInvalidType      = errors.New("invalid type")
	ErrInvalidSignal    = errors.New("invalid signal")
	ErrEmptyValue       = errors.New("empty value")
	ErrTaskError        = errors.New("task error")
	ErrTaskLost         = errors.New("task lost")
	ErrTaskCancelled    = errors.New("task cancelled")
	ErrUnableToCancel   = errors.New("unable to cancel")
	ErrUnableToDispose  = errors.New("unable to dispose")
	ErrAlreadyDisposed  = errors.New("already disposed")
	ErrStopped          = errors.New("stopped")
	ErrMissingCol       = errors.New("missing col")
	ErrInvalidValue     = errors.New("invalid value")
	ErrInvalidCronSpec  = errors.New("invalid cron spec")
)

var PaginationDefaultPage = 1

var PaginationDefaultSize = 10

Types

type TaskSignal

type TaskSignal int

const (
	TaskSignalFinish TaskSignal = iota
	TaskSignalCancel
	TaskSignalError
	TaskSignalLost
)