je

package module
Version: v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 13, 2018 License: MIT Imports: 27 Imported by: 0

README

je - Job Engine

A distributed job execution engine for the execution of batch jobs, workflows, remediations and more. You could also use je as a simple FaaS (Function as a Service) or "Serverless Computing" aka "Lambda".

Status: Early alpha stages, in heavy development.

Features

  • Simple HTTP API
  • Simple command-line client
  • UNIX friendly

Install

$ go install github.com/prologic/je/...

Usage

Run the je daemon/server:

$ je -d
INFO[0000] je 0.0.1-dev (HEAD) listening on 0.0.0.0:8000

Run a simple job:

$ job run -r echo -- 'hello world'
hello world

You should see something like this on the server side:

$ je -d
INFO[0000] je 0.0.1-dev (HEAD) listening on 0.0.0.0:8000
[je] 2018/05/20 20:33:40 ([::1]:50853) "POST /echo?args=hello+world HTTP/1.1" 302 0 10.342742ms
[je] 2018/05/20 20:33:40 ([::1]:50853) "GET /search/47 HTTP/1.1" 200 212 198.135µs

License

je is licensed under the MIT License

Documentation

Index

Constants

View Source
const (
	DefaultDataPath = "./data"
	DefaultBacklog  = 32
	DefaultThreads  = 16
)

Variables

View Source
var (
	// Version release version
	Version = "1.0.0"

	// Build will be overwritten automatically by the build system
	Build = "dev"

	// GitCommit will be overwritten automatically by the build system
	GitCommit = "HEAD"
)
View Source
var DefObjectives = map[float64]float64{
	0.50: 0.05,
	0.90: 0.01,
	0.95: 0.005,
	0.99: 0.001,
}

DefObjectives ...

View Source
var (
	ErrNotExist = errors.New("key does not exist")
)

Functions

func FullVersion

func FullVersion() string

FullVersion returns the full version, build and commit hash

func SafeParseInt

func SafeParseInt(s string, d int) int

SafeParseInt ...

func SafeParseUint64

func SafeParseUint64(s string, d uint64) uint64

SafeParseUint64 ...

Types

type BoltStore

type BoltStore struct {
	// contains filtered or unexported fields
}

func (*BoltStore) All

func (store *BoltStore) All() (jobs []*Job, err error)

func (*BoltStore) Close

func (store *BoltStore) Close() error

func (*BoltStore) Find

func (store *BoltStore) Find(ids ...ID) (jobs []*Job, err error)

func (*BoltStore) Get

func (store *BoltStore) Get(id ID) (*Job, error)

func (*BoltStore) NextId

func (store *BoltStore) NextId() ID

func (*BoltStore) Save

func (store *BoltStore) Save(job *Job) error

func (*BoltStore) Search

func (store *BoltStore) Search(q string) (jobs []*Job, err error)

type Data

type Data interface {
	Read(id ID, dtype DataType) (io.ReadCloser, error)
	Write(id ID, dtype DataType) (io.WriteCloser, error)
	Tail(id ID, dtype DataType, ctx context.Context) (chan string, chan error)
}

func InitData

func InitData(path string) (Data, error)

func NewLocalData

func NewLocalData(path string) (data Data, err error)

type DataType

type DataType int
const (
	DATA_INPUT DataType = iota
	DATA_OUTPUT
	DATA_LOGS
)

func (DataType) String

func (dt DataType) String() string

type ID

type ID uint64

func ParseId

func ParseId(s string) ID

func (ID) String

func (id ID) String() string

type IdGenerator

type IdGenerator struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func (*IdGenerator) Next

func (id *IdGenerator) Next() ID

type Job

type Job struct {
	sync.RWMutex

	ID          ID        `json:"id"`
	Name        string    `json:"name"`
	Args        []string  `json:"args"`
	Interactive bool      `json:"interactive"`
	Worker      string    `json:"worker"`
	State       State     `json:"state"`
	Status      int       `json:"status"`
	CreatedAt   time.Time `json:"created"`
	StartedAt   time.Time `json:"started"`
	StoppedAt   time.Time `json:"stopped"`
	KilledAt    time.Time `json:"killed"`
	ErroredAt   time.Time `json:"errored"`
	// contains filtered or unexported fields
}

Job ...

func NewJob

func NewJob(name string, args []string, interactive bool) (job *Job, err error)

func (*Job) Close

func (j *Job) Close() error

func (*Job) Enqueue

func (j *Job) Enqueue() error

func (*Job) Error

func (j *Job) Error(err error) error

func (*Job) Execute

func (j *Job) Execute() (err error)

func (*Job) Id

func (j *Job) Id() ID

func (*Job) Kill

func (j *Job) Kill(force bool) (err error)

func (*Job) Killed

func (j *Job) Killed() bool

func (*Job) Start

func (j *Job) Start(worker string) error

func (*Job) Stop

func (j *Job) Stop() error

func (*Job) Wait

func (j *Job) Wait()

func (*Job) Write

func (j *Job) Write(input io.Reader) (int64, error)

type KeyError

type KeyError struct {
	Key ID
	Err error
}

func (*KeyError) Error

func (e *KeyError) Error() string

type LocalData

type LocalData struct {
	// contains filtered or unexported fields
}

func (*LocalData) Read

func (d *LocalData) Read(id ID, dtype DataType) (io.ReadCloser, error)

func (*LocalData) Tail

func (d *LocalData) Tail(id ID, dtype DataType, ctx context.Context) (lines chan string, errors chan error)

func (*LocalData) Write

func (d *LocalData) Write(id ID, dtype DataType) (io.WriteCloser, error)

type MemoryStore

type MemoryStore struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func (*MemoryStore) All

func (store *MemoryStore) All() (jobs []*Job, err error)

func (*MemoryStore) Close

func (store *MemoryStore) Close() error

func (*MemoryStore) Find

func (store *MemoryStore) Find(ids ...ID) (jobs []*Job, err error)

func (*MemoryStore) Get

func (store *MemoryStore) Get(id ID) (job *Job, err error)

func (*MemoryStore) NextId

func (store *MemoryStore) NextId() ID

func (*MemoryStore) Save

func (store *MemoryStore) Save(job *Job) error

func (*MemoryStore) Search

func (store *MemoryStore) Search(q string) (jobs []*Job, err error)

type Metrics

type Metrics struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Metrics ...

func InitMetrics

func InitMetrics(name string) *Metrics

func NewMetrics

func NewMetrics(namespace string) *Metrics

NewMetrics ...

func (*Metrics) Counter

func (m *Metrics) Counter(subsystem, name string) prometheus.Counter

Counter ...

func (*Metrics) CounterVec

func (m *Metrics) CounterVec(subsystem, name string) *prometheus.CounterVec

CounterVec ...

func (*Metrics) Gauge

func (m *Metrics) Gauge(subsystem, name string) prometheus.Gauge

Gauge ...

func (*Metrics) GaugeVec

func (m *Metrics) GaugeVec(subsystem, name string) *prometheus.GaugeVec

GaugeVec ...

func (*Metrics) Handler

func (m *Metrics) Handler() http.Handler

Handler ...

func (*Metrics) NewCounter

func (m *Metrics) NewCounter(subsystem, name, help string) prometheus.Counter

NewCounter ...

func (*Metrics) NewCounterFunc

func (m *Metrics) NewCounterFunc(subsystem, name, help string, f func() float64) prometheus.CounterFunc

NewCounterFunc ...

func (*Metrics) NewCounterVec

func (m *Metrics) NewCounterVec(subsystem, name, help string, labels []string) *prometheus.CounterVec

NewCounterVec ...

func (*Metrics) NewGauge

func (m *Metrics) NewGauge(subsystem, name, help string) prometheus.Gauge

NewGauge ...

func (*Metrics) NewGaugeFunc

func (m *Metrics) NewGaugeFunc(subsystem, name, help string, f func() float64) prometheus.GaugeFunc

NewGaugeFunc ...

func (*Metrics) NewGaugeVec

func (m *Metrics) NewGaugeVec(subsystem, name, help string, labels []string) *prometheus.GaugeVec

NewGaugeVec ...

func (*Metrics) NewSummary

func (m *Metrics) NewSummary(subsystem, name, help string) prometheus.Summary

NewSummary ...

func (*Metrics) NewSummaryVec

func (m *Metrics) NewSummaryVec(subsystem, name, help string, labels []string) *prometheus.SummaryVec

NewSummaryVec ...

func (*Metrics) Run

func (m *Metrics) Run(addr string)

Run ...

func (*Metrics) Summary

func (m *Metrics) Summary(subsystem, name string) prometheus.Summary

Summary ...

func (*Metrics) SummaryVec

func (m *Metrics) SummaryVec(subsystem, name string) *prometheus.SummaryVec

SummaryVec ...

type Options

type Options struct {
	Data    string
	Backlog int
	Threads int
}

Options ...

type Server

type Server struct {
	// contains filtered or unexported fields
}

Server ...

func NewServer

func NewServer(bind string, options *Options) *Server

NewServer ...

func (*Server) AddRoute

func (s *Server) AddRoute(method, path string, handler http.Handler)

func (*Server) CloseHandler

func (s *Server) CloseHandler() httprouter.Handle

CloseHandler ...

func (*Server) CreateHandler

func (s *Server) CreateHandler() httprouter.Handle

CreateHandler ...

func (*Server) IndexHandler

func (s *Server) IndexHandler() httprouter.Handle

IndexHandler ...

func (*Server) KillHandler

func (s *Server) KillHandler() httprouter.Handle

KillHandler ...

func (*Server) ListenAndServe

func (s *Server) ListenAndServe()

ListenAndServe ...

func (*Server) LogsHandler

func (s *Server) LogsHandler() httprouter.Handle

LogsHandler ...

func (*Server) OutputHandler

func (s *Server) OutputHandler() httprouter.Handle

OutputHandler ...

func (*Server) SearchHandler

func (s *Server) SearchHandler() httprouter.Handle

SearchHandler ...

func (*Server) Shutdown

func (s *Server) Shutdown()

func (*Server) WriteHandler

func (s *Server) WriteHandler() httprouter.Handle

WriteHandler ...

type State

type State int

State ...

const (
	STATE_CREATED State
	STATE_WAITING
	STATE_RUNNING
	STATE_STOPPED
	STATE_KILLED
	STATE_ERRORED
)

func ParseState

func ParseState(s string) State

func (State) String

func (s State) String() string

type Store

type Store interface {
	Close() error
	NextId() ID
	Save(job *Job) error
	Get(id ID) (*Job, error)
	Find(id ...ID) ([]*Job, error)
	All() ([]*Job, error)
	Search(q string) ([]*Job, error)
}

func InitDB

func InitDB(uri string) (Store, error)

func NewBoltStore

func NewBoltStore(dbpath string) (Store, error)

func NewMemoryStore

func NewMemoryStore() (Store, error)

type URI

type URI struct {
	Type string
	Path string
}

func ParseURI

func ParseURI(uri string) (*URI, error)

func (*URI) String

func (u *URI) String() string

Directories

Path Synopsis
cmd
je
job
Package codec contains sub-packages with different codecs that can be used to encode and decode entities in Storm.
Package codec contains sub-packages with different codecs that can be used to encode and decode entities in Storm.
gob
Package gob contains a codec to encode and decode entities in Gob format
Package gob contains a codec to encode and decode entities in Gob format
json
Package json contains a codec to encode and decode entities in JSON format
Package json contains a codec to encode and decode entities in JSON format
msgpack
Package msgpack contains a codec to encode and decode entities in msgpack format
Package msgpack contains a codec to encode and decode entities in msgpack format

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL