interfaces

package
v0.6.0-beta.20210726.1120 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 26, 2021 License: BSD-3-Clause Imports: 12 Imported by: 6

Documentation

Index

Constants

View Source
const (
	ModelIdArtifact = iota
	ModelIdTag
	ModelIdNode
	ModelIdProject
	ModelIdSpider
	ModelIdTask
	ModelIdJob
	ModelIdSchedule
	ModelIdUser
	ModelIdSetting
	ModelIdToken
	ModelIdVariable
	ModelIdTaskQueue
	ModelIdTaskStat
	ModelIdPlugin
	ModelIdSpiderStat
	ModelIdDataSource
	ModelIdDataCollection
	ModelIdResult
	ModelIdPassword
)
View Source
const (
	ModelColNameArtifact       = "artifacts"
	ModelColNameTag            = "tags"
	ModelColNameNode           = "nodes"
	ModelColNameProject        = "projects"
	ModelColNameSpider         = "spiders"
	ModelColNameTask           = "tasks"
	ModelColNameJob            = "jobs"
	ModelColNameSchedule       = "schedules"
	ModelColNameUser           = "users"
	ModelColNameSetting        = "settings"
	ModelColNameToken          = "tokens"
	ModelColNameVariable       = "variables"
	ModelColNameTaskQueue      = "task_queue"
	ModelColNameTaskStat       = "task_stats"
	ModelColNamePlugin         = "plugins"
	ModelColNameSpiderStat     = "spider_stats"
	ModelColNameDataSource     = "data_sources"
	ModelColNameDataCollection = "data_collections"
	ModelColNamePasswords      = "passwords"
)
View Source
const (
	ModelDelegateMethodAdd = iota
	ModelDelegateMethodSave
	ModelDelegateMethodDelete
	ModelDelegateMethodGetArtifact
	ModelDelegateMethodRefresh
)

Variables

This section is empty.

Functions

This section is empty.

Types

type Address

type Address interface {
	Entity
	String() string
	IsEmpty() bool
}

type Color

type Color interface {
	Entity
	GetHex() string
	GetName() string
}

type ColorService

type ColorService interface {
	Injectable
	GetByName(name string) (res Color, err error)
	GetRandom() (res Color, err error)
}

type Entity

type Entity interface {
	Value() interface{}
}

type FsFileInfo

type FsFileInfo interface {
	GetName() string
	GetPath() string
	GetFullPath() string
	GetExtension() string
	GetMd5() string
	GetIsDir() bool
	GetFileSize() int64
	GetChildren() []FsFileInfo
}

type FsService

type FsService interface {
	WithConfigPath
	List(path string, opts ...ServiceCrudOption) (files []FsFileInfo, err error)
	GetFile(path string, opts ...ServiceCrudOption) (data []byte, err error)
	GetFileInfo(path string, opts ...ServiceCrudOption) (file FsFileInfo, err error)
	Save(path string, data []byte, opts ...ServiceCrudOption) (err error)
	Rename(path, newPath string, opts ...ServiceCrudOption) (err error)
	Delete(path string, opts ...ServiceCrudOption) (err error)
	Copy(path, newPath string, opts ...ServiceCrudOption) (err error)
	Commit(msg string) (err error)
	SyncToFs(opts ...ServiceCrudOption) (err error)
	SyncToWorkspace() (err error)
	GetFsPath() (path string)
	SetFsPath(path string)
	GetWorkspacePath() (path string)
	SetWorkspacePath(path string)
	GetRepoPath() (path string)
	SetRepoPath(path string)
	GetFs() (fs cfs.Manager)
}

type GrpcBase

type GrpcBase interface {
	WithConfigPath
	Init() (err error)
	Start() (err error)
	Stop() (err error)
	Register() (err error)
}

type GrpcBaseServiceParams

type GrpcBaseServiceParams interface {
	Entity
}

type GrpcClient

type GrpcClient interface {
	GrpcBase
	WithConfigPath
	GetModelDelegateClient() grpc.ModelDelegateClient
	GetModelBaseServiceClient() grpc.ModelBaseServiceClient
	GetNodeClient() grpc.NodeServiceClient
	GetTaskClient() grpc.TaskServiceClient
	GetPluginClient() grpc.PluginServiceClient
	SetAddress(Address)
	SetTimeout(time.Duration)
	Context() (context.Context, context.CancelFunc)
	NewRequest(interface{}) *grpc.Request
	GetMessageChannel() chan *grpc.StreamMessage
	Restart() error
	NewModelBaseServiceRequest(ModelId, GrpcBaseServiceParams) (*grpc.Request, error)
	IsStarted() bool
	IsClosed() bool
	Err() error
}

type GrpcClientModelBaseService

type GrpcClientModelBaseService interface {
	WithModelId
	WithConfigPath
	ModelBaseService
}

type GrpcClientModelDelegate

type GrpcClientModelDelegate interface {
	ModelDelegate
	WithConfigPath
	Close() error
}

type GrpcClientModelNodeService

type GrpcClientModelNodeService interface {
	ModelBaseService
	GetNodeById(id primitive.ObjectID) (n Node, err error)
	GetNode(query bson.M, opts *mongo.FindOptions) (n Node, err error)
	GetNodeByKey(key string) (n Node, err error)
	GetNodeList(query bson.M, opts *mongo.FindOptions) (res []Node, err error)
}

type GrpcClientModelService

type GrpcClientModelService interface {
	WithConfigPath
	NewBaseServiceDelegate(id ModelId) (GrpcClientModelBaseService, error)
}

type GrpcClientModelSpiderService

type GrpcClientModelSpiderService interface {
	ModelBaseService
	GetSpiderById(id primitive.ObjectID) (s Spider, err error)
	GetSpider(query bson.M, opts *mongo.FindOptions) (s Spider, err error)
	GetSpiderList(query bson.M, opts *mongo.FindOptions) (res []Spider, err error)
}

type GrpcClientModelTaskService

type GrpcClientModelTaskService interface {
	ModelBaseService
	GetTaskById(id primitive.ObjectID) (s Task, err error)
	GetTask(query bson.M, opts *mongo.FindOptions) (s Task, err error)
	GetTaskList(query bson.M, opts *mongo.FindOptions) (res []Task, err error)
}

type GrpcClientModelTaskStatService

type GrpcClientModelTaskStatService interface {
	ModelBaseService
	GetTaskStatById(id primitive.ObjectID) (s TaskStat, err error)
	GetTaskStat(query bson.M, opts *mongo.FindOptions) (s TaskStat, err error)
	GetTaskStatList(query bson.M, opts *mongo.FindOptions) (res []TaskStat, err error)
}

type GrpcClientPool

type GrpcClientPool interface {
	WithConfigPath
	Init() error
	NewClient() error
	GetClient() (GrpcClient, error)
	SetSize(int)
}

type GrpcModelBaseServiceMessage

type GrpcModelBaseServiceMessage interface {
	GetModelId() ModelId
	GetData() []byte
	ToBytes() (data []byte)
}

type GrpcModelBinder

type GrpcModelBinder interface {
	// contains filtered or unexported methods
}

type GrpcModelDelegateMessage

type GrpcModelDelegateMessage interface {
	GetModelId() ModelId
	GetMethod() ModelDelegateMethod
	GetData() []byte
	ToBytes() (data []byte)
}

type GrpcModelListBinder

type GrpcModelListBinder interface {
	// contains filtered or unexported methods
}

type GrpcServer

type GrpcServer interface {
	GrpcBase
	SetAddress(Address)
	GetSubscribe(key string) (sub GrpcSubscribe, err error)
	SetSubscribe(key string, sub GrpcSubscribe)
	DeleteSubscribe(key string)
	SendStreamMessage(nodeKey string, code grpc.StreamMessageCode) (err error)
	SendStreamMessageWithData(nodeKey string, code grpc.StreamMessageCode, d interface{}) (err error)
	IsStopped() (res bool)
}

type GrpcSubscribe

type GrpcSubscribe interface {
	GetStream() grpc.NodeService_SubscribeServer
	GetFinished() chan bool
}

type Injectable

type Injectable interface {
	Inject() error
}

type List

type List interface {
	ToArray() interface{}
}

type Model

type Model interface {
	GetId() (id primitive.ObjectID)
	SetId(id primitive.ObjectID)
}

type ModelArtifact

type ModelArtifact interface {
	Model
	GetSys() (sys ModelArtifactSys)
	GetTagIds() (ids []primitive.ObjectID)
	SetTagIds(ids []primitive.ObjectID)
	SetObj(obj Model)
	SetDel(del bool)
}

type ModelArtifactSys

type ModelArtifactSys interface {
	GetCreateTs() time.Time
	SetCreateTs(ts time.Time)
	GetUpdateTs() time.Time
	SetUpdateTs(ts time.Time)
	GetDeleteTs() time.Time
	SetDeleteTs(ts time.Time)
	GetCreateUid() primitive.ObjectID
	SetCreateUid(id primitive.ObjectID)
	GetUpdateUid() primitive.ObjectID
	SetUpdateUid(id primitive.ObjectID)
	GetDeleteUid() primitive.ObjectID
	SetDeleteUid(id primitive.ObjectID)
}

type ModelBaseService

type ModelBaseService interface {
	GetModelId() (id ModelId)
	SetModelId(id ModelId)
	GetById(id primitive.ObjectID) (res Model, err error)
	Get(query bson.M, opts *mongo.FindOptions) (res Model, err error)
	GetList(query bson.M, opts *mongo.FindOptions) (res arraylist.List, err error)
	DeleteById(id primitive.ObjectID) (err error)
	Delete(query bson.M) (err error)
	DeleteList(query bson.M) (err error)
	ForceDeleteList(query bson.M) (err error)
	UpdateById(id primitive.ObjectID, update bson.M) (err error)
	Update(query bson.M, update bson.M, fields []string) (err error)
	UpdateDoc(query bson.M, doc Model, fields []string) (err error)
	Insert(docs ...interface{}) (err error)
	Count(query bson.M) (total int, err error)
}

type ModelBinder

type ModelBinder interface {
	AssignFields(d Model, fieldIds ...ModelId) (res Model, err error)
	// contains filtered or unexported methods
}

type ModelDelegate

type ModelDelegate interface {
	Add() error
	Save() error
	Delete() error
	GetArtifact() (ModelArtifact, error)
	GetModel() Model
	Refresh() error
}

type ModelDelegateMethod

type ModelDelegateMethod int

type ModelId

type ModelId int

type ModelListBinder

type ModelListBinder interface {
	MustBindListAsPtr() (res arraylist.List)
	MustBindListWithNoFields() (res arraylist.List)
	// contains filtered or unexported methods
}

type ModelNodeDelegate

type ModelNodeDelegate interface {
	ModelDelegate
	UpdateStatus(active bool, activeTs *time.Time, status string) (err error)
	UpdateStatusOnline() (err error)
	UpdateStatusOffline() (err error)
}

type ModelService

type ModelService interface {
	GetBaseService(id ModelId) (svc ModelBaseService)
}

type ModelWithTags

type ModelWithTags interface {
	Model
	SetTags(tags []Tag)
	GetTags() (tags []Tag)
}

type Module

type Module interface {
	Init() error
	Start()
	Wait()
	Stop()
}

type ModuleId

type ModuleId int

type Node

type Node interface {
	ModelWithTags
	GetKey() (key string)
	GetIsMaster() (ok bool)
	SetActive(active bool)
	SetActiveTs(activeTs time.Time)
	GetStatus() (status string)
	SetStatus(status string)
	GetEnabled() (enabled bool)
	SetEnabled(enabled bool)
	GetAvailableRunners() (runners int)
	SetAvailableRunners(runners int)
	GetMaxRunners() (runners int)
	SetMaxRunners(runners int)
	IncrementAvailableRunners()
	DecrementAvailableRunners()
}

type NodeConfigService

type NodeConfigService interface {
	WithConfigPath
	Init() error
	Reload() error
	GetBasicNodeInfo() Entity
	GetNodeKey() string
	IsMaster() bool
	GetAuthKey() string
	GetMaxRunners() int
}

type NodeMasterService

type NodeMasterService interface {
	NodeService
	Monitor()
	SetMonitorInterval(duration time.Duration)
	Register() error
	StopOnError()
	GetServer() GrpcServer
}

type NodeService

type NodeService interface {
	Module
	WithConfigPath
	WithAddress
	GetConfigService() NodeConfigService
}

type NodeServiceOption

type NodeServiceOption interface {
}

type NodeWorkerService

type NodeWorkerService interface {
	NodeService
	Register()
	Recv()
	ReportStatus()
	SetHeartbeatInterval(duration time.Duration)
}

type PluginService

type PluginService interface {
	Module
	SetDirPath(path string)
	SetFsPathBase(path string)
}

type Provide

type Provide func(env string)

type Result

type Result interface {
	Value() map[string]interface{}
	SetValue(key string, value interface{})
	GetValue(key string) (value interface{})
	GetTaskId() (id primitive.ObjectID)
	SetTaskId(id primitive.ObjectID)
}

type ResultService

type ResultService interface {
	GetId() (id primitive.ObjectID)
	SetId(id primitive.ObjectID)
	GetList(query bson.M, opts *mongo.FindOptions) (results []Result, err error)
	Count(query bson.M) (total int, err error)
	Insert(docs ...interface{}) (err error)
}

type Schedule

type Schedule interface {
	Model
	GetEnabled() (enabled bool)
	SetEnabled(enabled bool)
	GetEntryId() (id cron.EntryID)
	SetEntryId(id cron.EntryID)
	GetCron() (c string)
	SetCron(c string)
	GetSpiderId() (id primitive.ObjectID)
	SetSpiderId(id primitive.ObjectID)
	GetMode() (mode string)
	SetMode(mode string)
	GetNodeIds() (ids []primitive.ObjectID)
	SetNodeIds(ids []primitive.ObjectID)
	GetNodeTags() (tags []string)
	SetNodeTags(tags []string)
	GetCmd() (cmd string)
	SetCmd(cmd string)
	GetParam() (param string)
	SetParam(param string)
	GetPriority() (p int)
	SetPriority(p int)
}

type ScheduleService

type ScheduleService interface {
	WithConfigPath
	Module
	GetLocation() (loc *time.Location)
	SetLocation(loc *time.Location)
	GetDelay() (delay bool)
	SetDelay(delay bool)
	GetSkip() (skip bool)
	SetSkip(skip bool)
	GetUpdateInterval() (interval time.Duration)
	SetUpdateInterval(interval time.Duration)
	Enable(s Schedule) (err error)
	Disable(s Schedule) (err error)
	Update()
	GetCron() (c *cron.Cron)
}

type ServiceCrudOption

type ServiceCrudOption func(o *ServiceCrudOptions)

func WithNotSyncToWorkspace

func WithNotSyncToWorkspace() ServiceCrudOption

func WithOnlyFromWorkspace

func WithOnlyFromWorkspace() ServiceCrudOption

type ServiceCrudOptions

type ServiceCrudOptions struct {
	IsAbsolute         bool // whether the path is absolute
	OnlyFromWorkspace  bool // whether only sync from workspace
	NotSyncToWorkspace bool // whether not sync to workspace
}

type Spider

type Spider interface {
	Model
	GetType() (ty string)
	GetMode() (mode string)
	SetMode(mode string)
	GetNodeIds() (ids []primitive.ObjectID)
	SetNodeIds(ids []primitive.ObjectID)
	GetNodeTags() (tags []string)
	SetNodeTags(tags []string)
	GetCmd() (cmd string)
	SetCmd(cmd string)
	GetParam() (param string)
	SetParam(param string)
	GetPriority() (p int)
	SetPriority(p int)
}

type SpiderAdminService

type SpiderAdminService interface {
	WithConfigPath
	// Schedule a new task of the spider
	Schedule(id primitive.ObjectID, opts *SpiderRunOptions) (err error)
	// Clone the spider
	Clone(id primitive.ObjectID, opts *SpiderCloneOptions) (err error)
	// Delete the spider
	Delete(id primitive.ObjectID) (err error)
}

type SpiderCloneOptions

type SpiderCloneOptions struct {
	Name string
}

type SpiderFsService

type SpiderFsService interface {
	WithConfigPath
	Init() (err error)
	SetId(id primitive.ObjectID)
	List(path string) (files []FsFileInfo, err error)
	GetFile(path string) (data []byte, err error)
	GetFileInfo(path string) (file FsFileInfo, err error)
	Save(path string, data []byte) (err error)
	Rename(path, newPath string) (err error)
	Delete(path string) (err error)
	Copy(path, newPath string) (err error)
	Commit(msg string) (err error)
	GetFsPath() (res string)
	GetWorkspacePath() (res string)
	GetRepoPath() (res string)
	SetFsPathBase(path string)
	SetWorkspacePathBase(path string)
	SetRepoPathBase(path string)
	GetFsService() (fsSvc FsService)
}

type SpiderRunOptions

type SpiderRunOptions struct {
	Mode       string               `json:"mode"`
	NodeIds    []primitive.ObjectID `json:"node_ids"`
	NodeTags   []string             `json:"node_tags"`
	Cmd        string               `json:"cmd"`
	Param      string               `json:"param"`
	ScheduleId primitive.ObjectID   `json:"schedule_id"`
	Priority   int                  `json:"priority"`
}

type SpiderSyncService

type SpiderSyncService interface {
	WithConfigPath
	SetFsPathBase(path string)
	SetWorkspacePathBase(path string)
	SetRepoPathBase(path string)
	GetFsService(id primitive.ObjectID) (fsSvc SpiderFsService, err error)
	ForceGetFsService(id primitive.ObjectID) (fsSvc SpiderFsService, err error)
	SyncToFs(id primitive.ObjectID) (err error)
	SyncToWorkspace(id primitive.ObjectID) (err error)
}

type StatsService

type StatsService interface {
	GetOverviewStats(query bson.M) (data interface{}, err error)
	GetDailyStats(query bson.M) (data interface{}, err error)
	GetTaskStats(query bson.M) (data interface{}, err error)
}

type Tag

type Tag interface {
	Model
	GetName() string
	GetColor() string
	SetCol(string)
}

type Task

type Task interface {
	Model
	GetNodeId() (id primitive.ObjectID)
	SetNodeId(id primitive.ObjectID)
	GetNodeIds() (ids []primitive.ObjectID)
	GetNodeTags() (tags []string)
	GetStatus() (status string)
	SetStatus(status string)
	GetError() (error string)
	SetError(error string)
	GetSpiderId() (id primitive.ObjectID)
	GetType() (ty string)
	GetCmd() (cmd string)
	GetParam() (param string)
	GetPriority() (p int)
}

type TaskBaseService

type TaskBaseService interface {
	WithConfigPath
	Module
	SaveTask(t Task, status string) (err error)
	IsStopped() (res bool)
	GetQueue(nodeId primitive.ObjectID) (queue string)
}

type TaskHandlerService

type TaskHandlerService interface {
	TaskBaseService
	// Run task and execute locally
	Run(taskId primitive.ObjectID) (err error)
	// Cancel task locally
	Cancel(taskId primitive.ObjectID) (err error)
	// ReportStatus periodically report handler status to master
	ReportStatus()
	// Reset reset internals to default
	Reset()
	// IsSyncLocked whether the given spider is locked for files sync
	IsSyncLocked(spiderId primitive.ObjectID) (ok bool)
	// LockSync lock files sync for given spider
	LockSync(spiderId primitive.ObjectID)
	// UnlockSync unlock files sync for given spider
	UnlockSync(spiderId primitive.ObjectID)
	// GetExitWatchDuration get max runners
	GetExitWatchDuration() (duration time.Duration)
	// SetExitWatchDuration set max runners
	SetExitWatchDuration(duration time.Duration)
	// GetReportInterval get report interval
	GetReportInterval() (interval time.Duration)
	// SetReportInterval set report interval
	SetReportInterval(interval time.Duration)
	// GetModelService get model service
	GetModelService() (modelSvc GrpcClientModelService)
	// GetModelSpiderService get model spider service
	GetModelSpiderService() (modelSpiderSvc GrpcClientModelSpiderService)
	// GetModelTaskService get model task service
	GetModelTaskService() (modelTaskSvc GrpcClientModelTaskService)
	// GetModelTaskStatService get model task stat service
	GetModelTaskStatService() (modelTaskStatSvc GrpcClientModelTaskStatService)
	// GetNodeConfigService get node config service
	GetNodeConfigService() (cfgSvc NodeConfigService)
	// GetCurrentNode get node of the handler
	GetCurrentNode() (n Node, err error)
	// GetTaskById get task by id
	GetTaskById(id primitive.ObjectID) (t Task, err error)
	// GetSpiderById get task by id
	GetSpiderById(id primitive.ObjectID) (t Spider, err error)
}

type TaskRunner

type TaskRunner interface {
	Init() (err error)
	Run() (err error)
	Cancel() (err error)
	Dispose() (err error)
	SetLogDriverType(driverType string)
	SetSubscribeTimeout(timeout time.Duration)
	GetTaskId() (id primitive.ObjectID)
}

type TaskSchedulerService

type TaskSchedulerService interface {
	TaskBaseService
	// Enqueue task into the task queue
	Enqueue(t Task) (err error)
	// DequeueAndSchedule continuously dequeue task and schedule to corresponding node
	DequeueAndSchedule()
	// Dequeue task with node info from the task queue
	Dequeue() (tasks []Task, err error)
	// Schedule task to corresponding node
	Schedule(tasks []Task) (err error)
	// Cancel task to corresponding node
	Cancel(id primitive.ObjectID) (err error)
	// SetInterval set the interval or duration between two adjacent fetches
	SetInterval(interval time.Duration)
}

type TaskStat

type TaskStat interface {
	Model
	GetCreateTs() (ts time.Time)
	SetCreateTs(ts time.Time)
	GetStartTs() (ts time.Time)
	SetStartTs(ts time.Time)
	GetEndTs() (ts time.Time)
	SetEndTs(ts time.Time)
	GetWaitDuration() (d int64)
	SetWaitDuration(d int64)
	GetRuntimeDuration() (d int64)
	SetRuntimeDuration(d int64)
	GetTotalDuration() (d int64)
	SetTotalDuration(d int64)
	GetResultCount() (c int64)
	SetResultCount(c int64)
	GetErrorLogCount() (c int64)
	SetErrorLogCount(c int64)
}

type TaskStatsService

type TaskStatsService interface {
	TaskBaseService
	InsertData(id primitive.ObjectID, records ...interface{}) (err error)
	InsertLogs(id primitive.ObjectID, logs ...string) (err error)
}

type Test

type Test interface {
	Setup(*testing.T)
	Cleanup()
}

type User

type User interface {
	Model
	GetUsername() (name string)
	GetPassword() (p string)
	GetRole() (r string)
	GetEmail() (email string)
}

type UserCreateOptions

type UserCreateOptions struct {
	Username string
	Password string
	Email    string
	Role     string
}

type UserLoginOptions

type UserLoginOptions struct {
	Username string
	Password string
}

type UserService

type UserService interface {
	Init() (err error)
	SetJwtSecret(secret string)
	SetJwtSigningMethod(method jwt.SigningMethod)
	Create(opts *UserCreateOptions) (err error)
	Login(opts *UserLoginOptions) (token string, u User, err error)
	CheckToken(token string) (u User, err error)
	ChangePassword(id primitive.ObjectID, password string) (err error)
	MakeToken(user User) (tokenStr string, err error)
	GetCurrentUser(c *gin.Context) (u User, err error)
}

type WithAddress

type WithAddress interface {
	GetAddress() (address Address)
	SetAddress(address Address)
}

type WithConfigPath

type WithConfigPath interface {
	GetConfigPath() (path string)
	SetConfigPath(path string)
}

type WithModelId

type WithModelId interface {
	GetModelId() (id ModelId)
	SetModelId(id ModelId)
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL