README

je - Job Engine

A distributed job execution engine for the execution of batch jobs, workflows, remediations and more. You could also use je as a simple FaaS (Function as a Service) or "Serverless Computing" aka "Lambda".

Status: Early alpha stages, in heavy development.

Features

  • Simple HTTP API
  • Simple command-line client
  • UNIX friendly

Install

$ go install github.com/prologic/je/...

Usage

Run the je daemon/server:

$ je -d
INFO[0000] je 0.0.1-dev (HEAD) listening on 0.0.0.0:8000

Run a simple job:

$ job run -r echo -- 'hello world'
hello world

You should see something like this on the server side:

$ je -d
INFO[0000] je 0.0.1-dev (HEAD) listening on 0.0.0.0:8000
[je] 2018/05/20 20:33:40 ([::1]:50853) "POST /echo?args=hello+world HTTP/1.1" 302 0 10.342742ms
[je] 2018/05/20 20:33:40 ([::1]:50853) "GET /search/47 HTTP/1.1" 200 212 198.135µs

License

je is licensed under the MIT License

Documentation

Index

Constants

View Source
const (
	DefaultDataPath = "./data"
	DefaultBacklog  = 32
	DefaultThreads  = 16
)

Variables

View Source
var (
	// Version release version
	Version = "1.0.0"

	// Build will be overwritten automatically by the build system
	Build = "dev"

	// GitCommit will be overwritten automatically by the build system
	GitCommit = "HEAD"
)
View Source
var DefObjectives = map[float64]float64{
	0.50: 0.05,
	0.90: 0.01,
	0.95: 0.005,
	0.99: 0.001,
}

    DefObjectives ...

    View Source
    var (
    	ErrNotExist = errors.New("key does not exist")
    )

    Functions

    func FullVersion

    func FullVersion() string

      FullVersion returns the full version, build and commit hash

      func SafeParseInt

      func SafeParseInt(s string, d int) int

        SafeParseInt ...

        func SafeParseUint64

        func SafeParseUint64(s string, d uint64) uint64

          SafeParseUint64 ...

          Types

          type BoltStore

          type BoltStore struct {
          	// contains filtered or unexported fields
          }

          func (*BoltStore) All

          func (store *BoltStore) All() (jobs []*Job, err error)

          func (*BoltStore) Close

          func (store *BoltStore) Close() error

          func (*BoltStore) Find

          func (store *BoltStore) Find(ids ...ID) (jobs []*Job, err error)

          func (*BoltStore) Get

          func (store *BoltStore) Get(id ID) (*Job, error)

          func (*BoltStore) NextId

          func (store *BoltStore) NextId() ID

          func (*BoltStore) Save

          func (store *BoltStore) Save(job *Job) error

          func (*BoltStore) Search

          func (store *BoltStore) Search(q string) (jobs []*Job, err error)

          type Data

          type Data interface {
          	Read(id ID, dtype DataType) (io.ReadCloser, error)
          	Write(id ID, dtype DataType) (io.WriteCloser, error)
          	Tail(id ID, dtype DataType, ctx context.Context) (chan string, chan error)
          }

          func InitData

          func InitData(path string) (Data, error)

          func NewLocalData

          func NewLocalData(path string) (data Data, err error)

          type DataType

          type DataType int
          const (
          	DATA_INPUT DataType = iota
          	DATA_OUTPUT
          	DATA_LOGS
          )

          func (DataType) String

          func (dt DataType) String() string

          type ID

          type ID uint64

          func ParseId

          func ParseId(s string) ID

          func (ID) String

          func (id ID) String() string

          type IdGenerator

          type IdGenerator struct {
          	sync.Mutex
          	// contains filtered or unexported fields
          }

          func (*IdGenerator) Next

          func (id *IdGenerator) Next() ID

          type Job

          type Job struct {
          	sync.RWMutex
          
          	ID          ID        `json:"id"`
          	Name        string    `json:"name"`
          	Args        []string  `json:"args"`
          	Interactive bool      `json:"interactive"`
          	Worker      string    `json:"worker"`
          	State       State     `json:"state"`
          	Status      int       `json:"status"`
          	CreatedAt   time.Time `json:"created"`
          	StartedAt   time.Time `json:"started"`
          	StoppedAt   time.Time `json:"stopped"`
          	KilledAt    time.Time `json:"killed"`
          	ErroredAt   time.Time `json:"errored"`
          	// contains filtered or unexported fields
          }

            Job ...

            func NewJob

            func NewJob(name string, args []string, interactive bool) (job *Job, err error)

            func (*Job) Close

            func (j *Job) Close() error

            func (*Job) Enqueue

            func (j *Job) Enqueue() error

            func (*Job) Error

            func (j *Job) Error(err error) error

            func (*Job) Execute

            func (j *Job) Execute() (err error)

            func (*Job) Id

            func (j *Job) Id() ID

            func (*Job) Kill

            func (j *Job) Kill(force bool) (err error)

            func (*Job) Killed

            func (j *Job) Killed() bool

            func (*Job) Start

            func (j *Job) Start(worker string) error

            func (*Job) Stop

            func (j *Job) Stop() error

            func (*Job) Wait

            func (j *Job) Wait()

            func (*Job) Write

            func (j *Job) Write(input io.Reader) (int64, error)

            type KeyError

            type KeyError struct {
            	Key ID
            	Err error
            }

            func (*KeyError) Error

            func (e *KeyError) Error() string

            type LocalData

            type LocalData struct {
            	// contains filtered or unexported fields
            }

            func (*LocalData) Read

            func (d *LocalData) Read(id ID, dtype DataType) (io.ReadCloser, error)

            func (*LocalData) Tail

            func (d *LocalData) Tail(id ID, dtype DataType, ctx context.Context) (lines chan string, errors chan error)

            func (*LocalData) Write

            func (d *LocalData) Write(id ID, dtype DataType) (io.WriteCloser, error)

            type MemoryStore

            type MemoryStore struct {
            	sync.RWMutex
            	// contains filtered or unexported fields
            }

            func (*MemoryStore) All

            func (store *MemoryStore) All() (jobs []*Job, err error)

            func (*MemoryStore) Close

            func (store *MemoryStore) Close() error

            func (*MemoryStore) Find

            func (store *MemoryStore) Find(ids ...ID) (jobs []*Job, err error)

            func (*MemoryStore) Get

            func (store *MemoryStore) Get(id ID) (job *Job, err error)

            func (*MemoryStore) NextId

            func (store *MemoryStore) NextId() ID

            func (*MemoryStore) Save

            func (store *MemoryStore) Save(job *Job) error

            func (*MemoryStore) Search

            func (store *MemoryStore) Search(q string) (jobs []*Job, err error)

            type Metrics

            type Metrics struct {
            	sync.RWMutex
            	// contains filtered or unexported fields
            }

              Metrics ...

              func InitMetrics

              func InitMetrics(name string) *Metrics

              func NewMetrics

              func NewMetrics(namespace string) *Metrics

                NewMetrics ...

                func (*Metrics) Counter

                func (m *Metrics) Counter(subsystem, name string) prometheus.Counter

                  Counter ...

                  func (*Metrics) CounterVec

                  func (m *Metrics) CounterVec(subsystem, name string) *prometheus.CounterVec

                    CounterVec ...

                    func (*Metrics) Gauge

                    func (m *Metrics) Gauge(subsystem, name string) prometheus.Gauge

                      Gauge ...

                      func (*Metrics) GaugeVec

                      func (m *Metrics) GaugeVec(subsystem, name string) *prometheus.GaugeVec

                        GaugeVec ...

                        func (*Metrics) Handler

                        func (m *Metrics) Handler() http.Handler

                          Handler ...

                          func (*Metrics) NewCounter

                          func (m *Metrics) NewCounter(subsystem, name, help string) prometheus.Counter

                            NewCounter ...

                            func (*Metrics) NewCounterFunc

                            func (m *Metrics) NewCounterFunc(subsystem, name, help string, f func() float64) prometheus.CounterFunc

                              NewCounterFunc ...

                              func (*Metrics) NewCounterVec

                              func (m *Metrics) NewCounterVec(subsystem, name, help string, labels []string) *prometheus.CounterVec

                                NewCounterVec ...

                                func (*Metrics) NewGauge

                                func (m *Metrics) NewGauge(subsystem, name, help string) prometheus.Gauge

                                  NewGauge ...

                                  func (*Metrics) NewGaugeFunc

                                  func (m *Metrics) NewGaugeFunc(subsystem, name, help string, f func() float64) prometheus.GaugeFunc

                                    NewGaugeFunc ...

                                    func (*Metrics) NewGaugeVec

                                    func (m *Metrics) NewGaugeVec(subsystem, name, help string, labels []string) *prometheus.GaugeVec

                                      NewGaugeVec ...

                                      func (*Metrics) NewSummary

                                      func (m *Metrics) NewSummary(subsystem, name, help string) prometheus.Summary

                                        NewSummary ...

                                        func (*Metrics) NewSummaryVec

                                        func (m *Metrics) NewSummaryVec(subsystem, name, help string, labels []string) *prometheus.SummaryVec

                                          NewSummaryVec ...

                                          func (*Metrics) Run

                                          func (m *Metrics) Run(addr string)

                                            Run ...

                                            func (*Metrics) Summary

                                            func (m *Metrics) Summary(subsystem, name string) prometheus.Summary

                                              Summary ...

                                              func (*Metrics) SummaryVec

                                              func (m *Metrics) SummaryVec(subsystem, name string) *prometheus.SummaryVec

                                                SummaryVec ...

                                                type Options

                                                type Options struct {
                                                	Data    string
                                                	Backlog int
                                                	Threads int
                                                }

                                                  Options ...

                                                  type Server

                                                  type Server struct {
                                                  	// contains filtered or unexported fields
                                                  }

                                                    Server ...

                                                    func NewServer

                                                    func NewServer(bind string, options *Options) *Server

                                                      NewServer ...

                                                      func (*Server) AddRoute

                                                      func (s *Server) AddRoute(method, path string, handler http.Handler)

                                                      func (*Server) CloseHandler

                                                      func (s *Server) CloseHandler() httprouter.Handle

                                                        CloseHandler ...

                                                        func (*Server) CreateHandler

                                                        func (s *Server) CreateHandler() httprouter.Handle

                                                          CreateHandler ...

                                                          func (*Server) IndexHandler

                                                          func (s *Server) IndexHandler() httprouter.Handle

                                                            IndexHandler ...

                                                            func (*Server) KillHandler

                                                            func (s *Server) KillHandler() httprouter.Handle

                                                              KillHandler ...

                                                              func (*Server) ListenAndServe

                                                              func (s *Server) ListenAndServe()

                                                                ListenAndServe ...

                                                                func (*Server) LogsHandler

                                                                func (s *Server) LogsHandler() httprouter.Handle

                                                                  LogsHandler ...

                                                                  func (*Server) OutputHandler

                                                                  func (s *Server) OutputHandler() httprouter.Handle

                                                                    OutputHandler ...

                                                                    func (*Server) SearchHandler

                                                                    func (s *Server) SearchHandler() httprouter.Handle

                                                                      SearchHandler ...

                                                                      func (*Server) Shutdown

                                                                      func (s *Server) Shutdown()

                                                                      func (*Server) WriteHandler

                                                                      func (s *Server) WriteHandler() httprouter.Handle

                                                                        WriteHandler ...

                                                                        type State

                                                                        type State int

                                                                          State ...

                                                                          const (
                                                                          	STATE_CREATED State
                                                                          	STATE_WAITING
                                                                          	STATE_RUNNING
                                                                          	STATE_STOPPED
                                                                          	STATE_KILLED
                                                                          	STATE_ERRORED
                                                                          )

                                                                          func ParseState

                                                                          func ParseState(s string) State

                                                                          func (State) String

                                                                          func (s State) String() string

                                                                          type Store

                                                                          type Store interface {
                                                                          	Close() error
                                                                          	NextId() ID
                                                                          	Save(job *Job) error
                                                                          	Get(id ID) (*Job, error)
                                                                          	Find(id ...ID) ([]*Job, error)
                                                                          	All() ([]*Job, error)
                                                                          	Search(q string) ([]*Job, error)
                                                                          }

                                                                          func InitDB

                                                                          func InitDB(uri string) (Store, error)

                                                                          func NewBoltStore

                                                                          func NewBoltStore(dbpath string) (Store, error)

                                                                          func NewMemoryStore

                                                                          func NewMemoryStore() (Store, error)

                                                                          type URI

                                                                          type URI struct {
                                                                          	Type string
                                                                          	Path string
                                                                          }

                                                                          func ParseURI

                                                                          func ParseURI(uri string) (*URI, error)

                                                                          func (*URI) String

                                                                          func (u *URI) String() string

                                                                          Directories

                                                                          Path Synopsis
                                                                          cmd
                                                                          je
                                                                          job
                                                                          Package codec contains sub-packages with different codecs that can be used to encode and decode entities in Storm.
                                                                          Package codec contains sub-packages with different codecs that can be used to encode and decode entities in Storm.
                                                                          gob
                                                                          Package gob contains a codec to encode and decode entities in Gob format
                                                                          Package gob contains a codec to encode and decode entities in Gob format
                                                                          json
                                                                          Package json contains a codec to encode and decode entities in JSON format
                                                                          Package json contains a codec to encode and decode entities in JSON format
                                                                          msgpack
                                                                          Package msgpack contains a codec to encode and decode entities in msgpack format
                                                                          Package msgpack contains a codec to encode and decode entities in msgpack format