core

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 20, 2025 License: Apache-2.0 Imports: 54 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CREATED string = "CREATED" //任务创建，但还没有开始执行
	STARTED string = "STARTED" //任务在执行中
	SUCCESS string = "SUCCESS" //任务执行完成，结果为SUCCESS
	FAILURE string = "FAILURE" //任务执行完成，结果为FAILURE

	TopicActive     = "active"
	TopicFinger     = "finger"
	TopicPassive    = "passive"
	TopicPocscan    = "pocscan"
	TopicCustom     = "custom"
	TopicStandalone = "standalone"

	TopicMQPrefix = "nemo_mq"
)
View Source 
const (
	EnvServiceHost = "SERVICE_HOST"
	EnvServicePort = "SERVICE_PORT"
	EnvServiceAuth = "SERVICE_AUTH"
)
View Source 
const (
	SliceNone              int = 0
	SliceByLine            int = 1
	SliceByIP              int = 2
	SliceByPort            int = 3
	DefaultIpSliceNumber   int = 64
	DefaultPortSliceNumber int = 1000

	TopPorts1000 = "" /* 3813-byte string literal not displayed */
	TopPorts100  = "" /* 393-byte string literal not displayed */
	TopPorts10   = "21-23,80,139,443,445,3306,3389,8080"
)

Variables

View Source 
var (
	RsaPrivateKeyText []byte
	RsaPublicKeyText  []byte
)
View Source 
var (
	TLSCertFile string
	TLSKeyFile  string
)
View Source 
var CustomTaskWorkspaceMap = make(map[string]struct{})

CustomTaskWorkspaceMap 自定义任务关联的工作空间GUID

View Source 
var WorkerName string

Functions

func CallImageServiceXClient 

func CallImageServiceXClient(serviceMethod string, args interface{}, reply interface{}) error

CallImageServiceXClient 调用图像服务的RPC远程调用

func CallXClient 

func CallXClient(serviceMethod string, args interface{}, reply interface{}) error

CallXClient RPC远程调用

func CheckRequiredResource 

func CheckRequiredResource(resourceList []RequiredResource) error

func CheckWorkerPerformance 

func CheckWorkerPerformance(workerPerformance int)

func CloseRedisClient 

func CloseRedisClient(client *redis.Client) error

func DaemonStartWorker 

func DaemonStartWorker() bool

DaemonStartWorker 启动worker进程

func DeleteWorkerStatusFromRedis 

func DeleteWorkerStatusFromRedis(client *redis.Client, workerID string) error

DeleteWorkerStatusFromRedis 从 Redis 中删除指定 workerID 的记录

func DoKeepAlive 

func DoKeepAlive(ws *WorkerStatus) bool

DoKeepAlive worker请求keepAlive

func GetRedisClient 

func GetRedisClient() (*redis.Client, error)

func GetRoutingKeyByTopic 

func GetRoutingKeyByTopic(topicName string) string

func GetServerTaskMQServer 

func GetServerTaskMQServer(topicName string) *machinery.Server

GetServerTaskMQServer 根据server配置文件，获取到消息中心的连接

func GetTopicByMQRoutingKey 

func GetTopicByMQRoutingKey(routingKey string) string

func GetTopicByTaskName 

func GetTopicByTaskName(taskName string, workspaceGUID string) string

func GetWorkerMQServer 

func GetWorkerMQServer(topicName string, prefetchCount int) *machinery.Server

GetWorkerMQServer 根据worker配置文件，获取到消息中心的连接

func GetWorkerNameByDaemon 

func GetWorkerNameByDaemon() string

func GetWorkerNameBySelf 

func GetWorkerNameBySelf() string

func IsCIDR 

func IsCIDR(ip string) bool

func KillWorker 

func KillWorker() bool

KillWorker 停止当前worker进程

func LoadCronTask 

func LoadCronTask() (err error)

func LoadWorkerStatusFromRedis 

func LoadWorkerStatusFromRedis(client *redis.Client) (map[string]*WorkerStatus, error)

LoadWorkerStatusFromRedis 从 Redis 中读取 WorkerAliveStatus

func Notify 

func Notify(docId []string, data NotifyData) (err error)

func PrepareWorkerConfig 

func PrepareWorkerConfig(opts *WorkerOption) bool

func PrepareWorkerRunEnv 

func PrepareWorkerRunEnv() bool

func PrepareWorkerServiceOptions 

func PrepareWorkerServiceOptions(opts *WorkerOption) bool

func ReloadCronTask 

func ReloadCronTask()

func ReloadWorkerRunEnv 

func ReloadWorkerRunEnv() bool

func RunOnceTaskCron 

func RunOnceTaskCron(taskId string) bool

RunOnceTaskCron 立即执行一次任务

func SetCronTaskUpdateFlag 

func SetCronTaskUpdateFlag(flag string) (err error)

func SetWorkerStatusToRedis 

func SetWorkerStatusToRedis(client *redis.Client, workerID string, status *WorkerStatus) error

SetWorkerStatusToRedis 将 WorkerAliveStatus 存储到 Redis 中

func SetupCloseHandler 

func SetupCloseHandler()

func StartCronTaskDamon 

func StartCronTaskDamon()

StartCronTaskDamon 启动定时任务守护和调度

func StartMainTaskDamon 

func StartMainTaskDamon()

StartMainTaskDamon MainTask任务的后台监控

func StartRedisReverseProxy 

func StartRedisReverseProxy()

func StartSaveRuntimeLog 

func StartSaveRuntimeLog(source string)

func StartServiceServer 

func StartServiceServer()

StartServiceServer 启动RPC server

func StartWorkerDaemon 

func StartWorkerDaemon()

StartWorkerDaemon 启动worker的daemon

func SyncTaskAsset 

func SyncTaskAsset(workspaceId string, taskId string) (result string)

func SyncTaskHistoryVul 

func SyncTaskHistoryVul(workspaceId string, taskId string, mongoClient *mongo.Client) (newVul, updateVul int)

func Test 

func Test(t *testing.T)

func WatchWorkerProcess 

func WatchWorkerProcess()

WatchWorkerProcess worker进程状态监控

Types

type AssetSaveResultResp 

type AssetSaveResultResp struct {
	AssetTotal  int `json:"assetTotal,omitempty"`
	AssetNew    int `json:"assetNew,omitempty"`
	AssetUpdate int `json:"assetUpdate,omitempty"`
	HostTotal   int `json:"hostTotal,omitempty"`
	HostNew     int `json:"hostNew,omitempty"`
	HostUpdate  int `json:"hostUpdate,omitempty"`
	ScreenShot  int `json:"screenshot,omitempty"`
	VulTotal    int `json:"vulTotal,omitempty"`
	VulNew      int `json:"vulNew,omitempty"`
	VulUpdate   int `json:"vulUpdate,omitempty"`
}

func (*AssetSaveResultResp) String 

func (r *AssetSaveResultResp) String() string

type Blacklist 

type Blacklist struct {
	// contains filtered or unexported fields
}

func NewBlacklist 

func NewBlacklist() *Blacklist

func (*Blacklist) AddDomain 

func (b *Blacklist) AddDomain(domain string)

func (*Blacklist) AddIP 

func (b *Blacklist) AddIP(ip string) error

func (*Blacklist) IsDomainBlocked 

func (b *Blacklist) IsDomainBlocked(domain string) bool

func (*Blacklist) IsHostBlocked 

func (b *Blacklist) IsHostBlocked(host string) bool

func (*Blacklist) IsIPBlocked 

func (b *Blacklist) IsIPBlocked(ip string) bool

func (*Blacklist) LoadBlacklist 

func (b *Blacklist) LoadBlacklist(workspaceId string) bool

type CheckTaskArgs 

type CheckTaskArgs struct {
	TaskID     string
	MainTaskID string
}

type CronTaskJob 

type CronTaskJob struct {
	TaskId string
}

func (CronTaskJob) Run 

func (j CronTaskJob) Run()

Run 当定时任务启动时，创建任务执行并发送到消息队列中

type DingTalkNotify 

type DingTalkNotify struct {
	Token string
}

func (*DingTalkNotify) Notify 

func (d *DingTalkNotify) Notify(message string) (err error)

type DingTalkResponseInfo 

type DingTalkResponseInfo struct {
	Code    int    `json:"errcode"`
	Message string `json:"errmsg"`
}

type FeishuNotify 

type FeishuNotify struct {
	Token string
}

func (*FeishuNotify) Notify 

func (f *FeishuNotify) Notify(message string) (err error)

type FeishuResponseInfo 

type FeishuResponseInfo struct {
	Code    int    `json:"StatusCode"`
	Message string `json:"StatusMessage"`
}

type Honeypot 

type Honeypot struct {
	// contains filtered or unexported fields
}

func NewHoneypot 

func NewHoneypot(workspaceId string) *Honeypot

func (*Honeypot) IsHoneypot 

func (h *Honeypot) IsHoneypot(ipOrDomain string) bool

type KeepAliveDaemonInfo 

type KeepAliveDaemonInfo struct {
	ManualReloadFlag       bool
	ManualFileSyncFlag     bool
	ManualUpdateOptionFlag bool
	WorkerRunOption        *WorkerOption
}

func DoDaemonKeepAlive 

func DoDaemonKeepAlive() (replay KeepAliveDaemonInfo, err error)

DoDaemonKeepAlive worker请求keepAlive

type NewTaskArgs 

type NewTaskArgs struct {
	TaskName      string
	ConfigJSON    string
	MainTaskID    string
	LastRunTaskId string
}

NewTaskArgs 新建任务请求与返回参数

type NotifyData 

type NotifyData struct {
	TaskName string
	Target   string
	Runtime  string
	Result   string
}

type NotifyExecutor 

type NotifyExecutor interface {
	Notify(message string) (err error)
}

func NewNotifyExecutor 

func NewNotifyExecutor(category string, token string) NotifyExecutor

type RedisLock 

type RedisLock struct {
	// contains filtered or unexported fields
}

func NewRedisLock 

func NewRedisLock(key string, expiration time.Duration, redisCli *redis.Client) *RedisLock

func (*RedisLock) TryLock 

func (rl *RedisLock) TryLock() (bool, error)

func (*RedisLock) Unlock 

func (rl *RedisLock) Unlock() error

type RedisProxyServer 

type RedisProxyServer struct {
	ReverseServerAddr string
	LocalListenAddr   string
	AuthPass          string
}

func NewRedisProxyServer 

func NewRedisProxyServer(reverseServerAddr string, localListenAddr string, authPass string) *RedisProxyServer

func (*RedisProxyServer) Start 

func (rp *RedisProxyServer) Start()

type RedisReverseServer 

type RedisReverseServer struct {
	RedisAddr  string
	ListenAddr string
	AuthPass   string
}

func NewRedisReverseServer 

func NewRedisReverseServer(redisAddr string, listenAddr string, authPass string) *RedisReverseServer

func (*RedisReverseServer) Start 

func (rr *RedisReverseServer) Start()

type RequestResourceArgs 

type RequestResourceArgs struct {
	Category string
	Name     string
}

type RequiredResource 

type RequiredResource struct {
	Category string
	Name     string
}

type ResourceResultArgs 

type ResourceResultArgs struct {
	Path  string
	Hash  string
	Bytes []byte
}

type RuntimeLogArgs 

type RuntimeLogArgs struct {
	Source     string
	LogMessage []byte
}

type ScreenShotResultArgs 

type ScreenShotResultArgs struct {
	WorkspaceId    string
	Scheme         string
	Host           string
	Port           string
	ScreenshotByte []byte
}

type ServerChan 

type ServerChan struct {
	Token string
}

func (*ServerChan) Notify 

func (s *ServerChan) Notify(message string) (err error)

type ServerChanResponseInfo 

type ServerChanResponseInfo struct {
	Code    int    `json:"code"`
	Message string `json:"message"`
	Info    string `json:"info"`
}

type ServerOption 

type ServerOption struct {
	Web         bool `long:"web" description:"web service"`
	Cron        bool `long:"cron" description:"cron service"`
	Service     bool `long:"service" description:"rpc service"`
	RedisTunnel bool `long:"redis-tunnel" description:"redis tunnel service"`
	WebOption   `group:"web-option"`
}

type Service 

type Service struct{}

Service RPC服务

func (*Service) CheckTask 

func (s *Service) CheckTask(ctx context.Context, args *CheckTaskArgs, replay *TaskStatusArgs) error

CheckTask 检查任务在数据库中的状态：任务是否存在、是否被取消，任务状态、结果

func (*Service) KeepAlive 

func (s *Service) KeepAlive(ctx context.Context, args *WorkerStatus, replay *string) error

KeepAlive worker通过RPC，保持与server的心跳与同步

func (*Service) KeepDaemonAlive 

func (s *Service) KeepDaemonAlive(ctx context.Context, args *string, replay *KeepAliveDaemonInfo) error

KeepDaemonAlive worker的daemon通过RPC，保持与server的心跳与同步

func (*Service) LoadWorkerConfig 

func (s *Service) LoadWorkerConfig(ctx context.Context, args *string, replay *conf.Worker) error

func (*Service) LookupQueryData 

func (s *Service) LookupQueryData(ctx context.Context, args *db.QueryDocument, replay *db.QueryDocument) error

func (*Service) NewTask 

func (s *Service) NewTask(ctx context.Context, args *execute.ExecutorTaskInfo, replay *bool) error

func (*Service) RequestResource 

func (s *Service) RequestResource(ctx context.Context, args *RequestResourceArgs, replay *ResourceResultArgs) error

func (*Service) SaveQueryData 

func (s *Service) SaveQueryData(ctx context.Context, args *[]db.QueryDocument, replay *string) error

func (*Service) SaveRuntimeLog 

func (s *Service) SaveRuntimeLog(ctx context.Context, args *RuntimeLogArgs, replay *string) error

SaveRuntimeLog 保存RuntimeLog

func (*Service) SaveScreenShotResult 

func (s *Service) SaveScreenShotResult(ctx context.Context, args *[]ScreenShotResultArgs, replay *string) error

SaveScreenShotResult 保存截图结果到本地文件 

保存截图结果到本地文件
  由于v3版本里service与web是可以分开部署的，所以截图结果保存到本地文件的方式需要修改；如果service与web部署在同一台机器，则可以直接保存到web目录下。
  如果service与web部署在不同的机器，则需要通过rpc调用web的接口，将截图结果保存到web目录下。
  保存逻辑判断为：
      如果在server.yml中定义了imageService，检查host，port和auth均不为空的话，则调用imageService的rpc接口，将截图结果保存到web目录下；
      否则，则保存到本地文件。

func (*Service) SaveTaskResult 

func (s *Service) SaveTaskResult(ctx context.Context, args *TaskAssetDocumentResultArgs, replay *string) error

func (*Service) SaveVulResult 

func (s *Service) SaveVulResult(ctx context.Context, args *VulResultArgs, replay *string) error

func (*Service) UpdateTask 

func (s *Service) UpdateTask(ctx context.Context, args *TaskStatusArgs, replay *bool) error

UpdateTask 更新任务状态到数据库中

func (*Service) UploadScreenShotResult 

func (s *Service) UploadScreenShotResult(ctx context.Context, args *[]ScreenShotResultArgs, replay *string) error

type ServiceOptions 

type ServiceOptions struct {
	ServiceHost string `long:"service" description:"Service host" json:"service_host" form:"service_host"`
	ServicePort int    `long:"port" description:"Service port" default:"5001" json:"service_port" form:"service_port"`
	ServiceAuth string `long:"auth" description:"Service auth" json:"service_auth" form:"service_auth"`
}

type TaskAssetDocumentResultArgs 

type TaskAssetDocumentResultArgs struct {
	WorkspaceId string
	MainTaskId  string
	Result      []db.AssetDocument
}

type TaskResult 

type TaskResult struct {
	Status string `json:"status"`
	Msg    string `json:"msg"`
}

type TaskSlice 

type TaskSlice struct {
	IpTarget        []string
	DomainTarget    []string
	Port            string
	TaskMode        int
	IpSliceNumber   int
	PortSliceNumber int
}

TaskSlice 任务切分

func NewTaskSlice 

func NewTaskSlice() *TaskSlice

NewTaskSlice 创建任务切分对象

func (*TaskSlice) DoDomainSlice 

func (t *TaskSlice) DoDomainSlice() (target []string)

DoDomainSlice 对域名任务目标进行切分 只支持0，1两种模式

func (*TaskSlice) DoIpSlice 

func (t *TaskSlice) DoIpSlice() (target []string, port []string)

DoIpSlice 对IP任务目标进行切分 支持0-4共5种切分模式

type TaskStatusArgs 

type TaskStatusArgs struct {
	WorkspaceId string
	TaskID      string
	IsExist     bool
	IsFinished  bool
	State       string
	Worker      string
	Result      string
}

TaskStatusArgs 任务状态请求与返回参数

type TrieNode 

type TrieNode struct {
	// contains filtered or unexported fields
}

type VulResultArgs 

type VulResultArgs struct {
	WorkspaceId string
	MainTaskId  string
	Result      []db.VulDocument
}

type WebOption 

type WebOption struct {
	TLSCertFile string `long:"tls_cert_file" description:"TLS certificate file" default:"server.crt" form:"tls_cert_file"`
	TLSKeyFile  string `long:"tls_key_file" description:"TLS key file" default:"server.key" form:"tls_key_file"`
}

type WorkerOption 

type WorkerOption struct {
	ServiceOptions   `group:"services"`
	WorkerTaskOption `group:"worker-tasks"`
	ConfigFile       string `short:"f" long:"config-file" description:"config file" json:"default_config_file" form:"default_config_file"`
	NoProxy          bool   `` /* 128-byte string literal not displayed */
	NoRedisProxy     bool   `long:"no-redis-proxy" description:"disable redis proxy configuration" json:"no_redis_proxy" form:"no_redis_proxy"`
	IpV6Support      bool   `long:"ipv6" description:"support ipv6 portscan" json:"ipv6" form:"ipv6"`
}
var DaemonRunOption *WorkerOption
var WorkerRunOption *WorkerOption

func PrepareWorkerOptions 

func PrepareWorkerOptions() *WorkerOption

type WorkerRunTaskMode 

type WorkerRunTaskMode int
const (
	TaskModeDefault WorkerRunTaskMode = iota
	TaskModeActive
	TaskModeFinger
	TaskModePassive
	TaskModePocscan
	TaskModeStandalone
)

type WorkerStatus 

type WorkerStatus struct {
	sync.Mutex `json:"-"`
	// worker's task status
	WorkerName         string    `json:"worker_name"`
	WorkerTopics       string    `json:"worker_topic"`
	CreateTime         time.Time `json:"create_time"`
	UpdateTime         time.Time `json:"update_time"`
	TaskExecutedNumber int       `json:"task_number"`
	TaskStartedNumber  int       `json:"started_number"`
	// worker's run status
	ManualReloadFlag       bool   `json:"manual_reload_flag"`
	ManualFileSyncFlag     bool   `json:"manual_file_sync_flag"`
	ManualUpdateOptionFlag bool   `json:"manual_update_daemon_option"`
	CPULoad                string `json:"cpu_load"`
	MemUsed                string `json:"mem_used"`
	// worker's option
	WorkerRunOption    []byte `json:"worker_run_option"`    //worker当前运行的启动参数
	WorkerUpdateOption []byte `json:"worker_update_option"` //worker需要更新的启动参数
	// daemon option
	IsDaemonProcess        bool      `json:"is_daemon_process"`
	WorkerDaemonUpdateTime time.Time `json:"worker_daemon_update_time"`
}

func GetWorkerStatusFromRedis 

func GetWorkerStatusFromRedis(client *redis.Client, workerID string) (*WorkerStatus, error)

GetWorkerStatusFromRedis 从 Redis 中读取指定 workerID 的 WorkerStatus

type WorkerTaskOption 

type WorkerTaskOption struct {
	Concurrency       int                 `short:"c" long:"concurrency" description:"Number of concurrent workers" default:"2" json:"concurrency" form:"concurrency"`
	WorkerPerformance int                 `` /* 187-byte string literal not displayed */
	WorkerRunTaskMode string              `` /* 238-byte string literal not displayed */
	WorkerTopic       map[string]struct{} `json:"-"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.