headmast

README

headmast is a light weight job scheduler used by rule chain master

Documentation

Overview

Licensed under the Apache License, Version 2.0 (the "License"); you may not use p file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use p file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use p file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• type Client
  • func NewClient(opts *ClientOptions) *Client
  • func (c *Client) ControlJob(jobID string, action string)
  • func (c *Client) CreateNewJob(job *Job) error
  • func (c *Client) DeleteJob(jobID string) error
  • func (c *Client) GetJob(jobID string) (*Job, error)
  • func (c *Client) GetJobs() ([]*Job, error)
  • func (c *Client) GetJobsWithDomain(domain string) ([]*Job, error)
  • func (c *Client) WatchJobPath(jobPath string, handler WatchPathHandler) error
• type ClientOptions
• type HeadmastService
  • func NewHeadmastService(servingOptions *ServingOptions) *HeadmastService
  • func (s *HeadmastService) Run() error
• type Job
  • func NewJob() *Job
  • func (job *Job) MarshalBinary() ([]byte, error)
  • func (job *Job) UnmarshalBinary(buf []byte) error
• type JobManager
  • func NewJobManager(servingOptions *ServingOptions) JobManager
• type JobObserver
• type JobScheduler
  • func NewJobScheduler(servingOptions *ServingOptions, jobManager JobManager, ...) JobScheduler
• type ScheduleContext
  • func NewScheduleContext() *ScheduleContext
• type SchedulePolicy
  • func NewSchedulePolicy(servingOptions *ServingOptions) SchedulePolicy
• type ServingOptions
  • func NewServingOptions() *ServingOptions
  • func (s *ServingOptions) AddFlags(fs *pflag.FlagSet)
• type WatchPathHandler
• type Worker
  • func NewWorker(ctx WorkerContext) *Worker
  • func (w Worker) KillerPath() string
  • func (w *Worker) MarshalBinary() ([]byte, error)
  • func (w *Worker) RetrieveJobs(jobCh chan *Job, errCh chan error)
  • func (w *Worker) UnmarshalBinary(buf []byte) error
  • func (w Worker) WorkingPath() string
• type WorkerContext
• type WorkerManager
  • func NewWorkerManager(servingOptions *ServingOptions) WorkerManager
• type WorkersObserver

Constants

const (
	JOB_STATUS_CREATED = "created"
	JOB_STATUS_KILLED  = "killed"
)

const (
	HEADMAST_JOBS_PATH    = "/headmast/jobs"
	HEADMAST_WORKERS_PATH = "/headmast/workers"
	HEADMAST_KILLER_PATH  = "/headmast/killers"
)

const (
	SCHEDULE_POLICY_ROUNDBIN = "roundbin"
	SCHEDULE_POLICY_BLANCE   = "balance"
)

const (
	HEADMAST_CHANGES_ADDED   = "added"
	HEADMAST_CHANGES_DELETED = "deleted"
)

const (
	HEADMAST_WORKER_PATH = "/headmast/workers"
)

Types

type Client

type Client struct {
	// contains filtered or unexported fields
}

Client is endpoint for headmast

func NewClient

func NewClient(opts *ClientOptions) *Client

NewClient create a client endpoint

func (*Client) ControlJob

func (c *Client) ControlJob(jobID string, action string)

ControlJob control a job to be killed or alived

func (*Client) CreateNewJob

func (c *Client) CreateNewJob(job *Job) error

CreateNewJob create a new job in headmast

func (*Client) DeleteJob

func (c *Client) DeleteJob(jobID string) error

DeleteJob remove a job from headmast

func (*Client) GetJob

func (c *Client) GetJob(jobID string) (*Job, error)

GetJob return job detail from headmast

func (*Client) GetJobs

func (c *Client) GetJobs() ([]*Job, error)

GetJobs return current all jobs

func (*Client) GetJobsWithDomain

func (c *Client) GetJobsWithDomain(domain string) ([]*Job, error)

GetJobsWithDomain return jobs within specific domain

func (*Client) WatchJobPath

func (c *Client) WatchJobPath(jobPath string, handler WatchPathHandler) error

WatchJobPath watch a specific job path and call handler when envent occured

type ClientOptions

type ClientOptions struct {
	ServerAddr string // header master server address
}

type HeadmastService

type HeadmastService struct {
	// contains filtered or unexported fields
}

func NewHeadmastService

func NewHeadmastService(servingOptions *ServingOptions) *HeadmastService

NewHeadmastService manage http rest api server to handle client's request JobManager and JobScheduler is backend for job's control

func (*HeadmastService) Run

func (s *HeadmastService) Run() error

type Job

type Job struct {
	ID      string `json:"id"`
	Domain  string `json:"domain"`  // job's domain
	Payload []byte `json:"payload"` // job's payload
	Status  string `json:"status"`  // job's  status
}

func NewJob

func NewJob() *Job

func (*Job) MarshalBinary

func (job *Job) MarshalBinary() ([]byte, error)

func (*Job) UnmarshalBinary

func (job *Job) UnmarshalBinary(buf []byte) error

type JobManager

type JobManager interface {
	AddJob(job *Job) error
	RemoveJob(jobID string) error
	KillJob(jobID string) error
	GetJob(jobID string) (*Job, error)
	GetJobs() []*Job
	UpdateJob(job *Job) error
	RegisterObserver(JobObserver)
}

JobManager is light weight scheduler service based on etcd backend

func NewJobManager

func NewJobManager(servingOptions *ServingOptions) JobManager

NewJobManager return manager instance

type JobObserver

type JobObserver func(job *Job, reason string)

type JobScheduler

type JobScheduler interface {
}

JobScheduler schedule job based on available workers

func NewJobScheduler

func NewJobScheduler(servingOptions *ServingOptions, jobManager JobManager, workerManager WorkerManager) JobScheduler

NewJobSchduler return instance of job scheduler that based on schedule policy created from serving options

type ScheduleContext

type ScheduleContext struct {
	LastScheduledWorkerIndex int
}

ScheduleContext encapsulate informations for job sechduling

func NewScheduleContext

func NewScheduleContext() *ScheduleContext

NewScheduleContex create default schedule context

type SchedulePolicy

type SchedulePolicy interface {

	// DeterminWithJobChange caculate the last result for scheduler when jobs
	// is added or deleted
	DeterminWithJobChanged(ctx *ScheduleContext, affectedJob *Job, allWorkers []*Worker, added bool) []*Worker

	// DeterminWithWorkerChange caculate the last result for scheduler when
	// worker is added or deleted
	DeterminWithWorkerChanged(ctx *ScheduleContext, affectedWorker *Worker, allWorkers []*Worker, added bool) []*Worker
}

SchedulePolicy determain scheduler policy that affect how the new job and worker should be scheduled SchedulePolicy doesn't update etcd directly, it just caculated the result

func NewSchedulePolicy

func NewSchedulePolicy(servingOptions *ServingOptions) SchedulePolicy

NewSchedulePolicy return scheduler policy based on serving options's schedule-policy

type ServingOptions

type ServingOptions struct {
	SecureServing  *genericoptions.SecureServingOptions
	EtcdEndpoints  string
	SchedulePolicy string
}

func NewServingOptions

func NewServingOptions() *ServingOptions

func (*ServingOptions) AddFlags

func (s *ServingOptions) AddFlags(fs *pflag.FlagSet)

type WatchPathHandler

type WatchPathHandler func(jobPath string, job *Job)

type Worker

type Worker struct {
	ID          string   `json:"id"`
	WorkingJobs []string `json:"workingJobs"`
	KillingJobs []string `json:"killingJobs"`
	// contains filtered or unexported fields
}

Worker represent worker node on which the job is run, worker monitor its working path and recive job that assigned to it. the worker ID is a UUID that created by worker, when worker watch it working path, it will post the worker ID to server.

func NewWorker

func NewWorker(ctx WorkerContext) *Worker

func (Worker) KillerPath

func (w Worker) KillerPath() string

func (*Worker) MarshalBinary

func (w *Worker) MarshalBinary() ([]byte, error)

func (*Worker) RetrieveJobs

func (w *Worker) RetrieveJobs(jobCh chan *Job, errCh chan error)

func (*Worker) UnmarshalBinary

func (w *Worker) UnmarshalBinary(buf []byte) error

func (Worker) WorkingPath

func (w Worker) WorkingPath() string

type WorkerContext

type WorkerContext struct {
	EtcdEndpoints []string
}

WorkerContext hold working context

type WorkerManager

type WorkerManager interface {
	GetWorker(wid string) (*Worker, error)
	UpdateWorkers([]*Worker)
	GetWorkers() []*Worker
	RemoveWorker(wid string)
	RegisterObserver(WorkersObserver)
}

WorkerManager monitors worker path on etcd and jobs to workers WorkerManager also manager all nodes info, if none node exist, the worker's job will be assign to other works

func NewWorkerManager

func NewWorkerManager(servingOptions *ServingOptions) WorkerManager

NewWorkerManager return worker manager instance, one service should only have one schduler

type WorkersObserver

type WorkersObserver func(worker *Worker, reason string)