qmd

README

QMD

QMD (prounounced "command") is a service for executing arbitrary shell scripts over HTTP. We use it to compile javascript assets and upload them to s3.

Note: Exposing shell scripts over HTTP might be very dangerous, so be careful! QMD is intended to be used as an internal service, running in a non-privileged Docker container.

Usage

sudo docker run qmd -config /etc/qmd.conf

• qmd.conf see example file
• scripts directory where QMD looks for shell scripts to run, see examples

REST API

Create QMD job - Execute a script

POST /scripts/:filename

Request params (JSON):

• callback_url: (optional) execute the script in the background and send the output to the callback_url when the script finishes
• args: array of command line arguments to pass to the script upon execution
• files: JSON object containing filename : filedata pairs which are saved in $QMD_TMP for the script to use

Response (JSON):

• id: unique job ID
• script: the filename in the scripts directory
• args: the user given arguments if any
• files: the user given files if any
• callback_url: an endpoint to send the output
• output: the $QMD_OUT output
• exec_log: the piped STDOUT and STDERR script execution log
• status: the exit status of the script; either OK or ERR
• start_time: the time (in local system time) the script began to execute
• end_time: the time (in local system time) the script finished executing
• duration: the amount of time taken to run the script in seconds as a string

Example: Enqueue a script to execute in the background and send output to a callback URL

POST /scripts/bench.sh
{
    "callback_url": "http://...",
    "args": ["1", "20"],
}

{
    "id": 1,
    "script": "bench.sh",
    "args": [
        "1",
        "20"
    ],
    "callback_url": "http://..."
}

...the job then runs in the background.. and when finishes it will send the following response to callback_url:

{
    "id": 1,
    "script": "bench.sh",
    "args": [
        "1",
        "20"
    ],
    "callback_url": "http://...",
    "output": "",
    "exec_log": "Running under PID #16231\nMaking file in tmp dir at /home/vagrant/test/tmp/883\n/home/vagrant/test/scripts/bench.sh: line 10: Test #$PID: command not found\nMaking file in store dir at /home/vagrant/test/store\n/home/vagrant/test/scripts/bench.sh: line 13: Test #$PID: command not found\nRandomly selected 2\nzzzzzzzzzzzzzzzzz\nzzzzzz..I'M AWAKE\nRandomly selected 12\nzzzzzzzzzzzzzzzzz\nzzzzzz..I'M AWAKE\nRandomly selected 3\nzzzzzzzzzzzzzzzzz\nzzzzzz..I'M AWAKE\nRandomly selected 20\nAbandon ship! Abandon ship! Everyone for themselves!\n\nsignal: hangup",
    "status": "ERR",
    "start_time": "2014-06-24T17:26:39.643458173Z",
    "end_time": "2014-06-24T17:26:56.666502567Z",
    "duration": "17.023044394"
}

List QMD jobs and their state

GET /jobs/

Notes

• Scripts will have access to the following environment variables
  • QMD_TMP: the directory where your script is being run. All files here will be deleted unless keepTemp is set to true in your config. Located at workingDir/tmp/:id.
  • QMD_STORE: the directory set in your config as storeDir. All files written here will be left alone.
  • QMD_OUT: the output file for your script. Everything written here will be persisted to the response/log under the output key. Located at QMD_TMP/qmd.out.

Requirements

• Redis
• Disque

sudo docker run -d --name redis -v /data/redis:/data --restart=always -p 6379:6379 redis:latest redis-server --appendonly yes
sudo docker run -d --name disque -v /data/disque:/data --restart=always -p 7711:7711 richnorth/disque:latest disque-server --appendonly yes

LICENSE

Licensed under the MIT License.

Documentation

Index

• Variables
• type Cmd
  • func (cmd *Cmd) Cleanup() error
  • func (cmd *Cmd) Kill() error
  • func (cmd *Cmd) Run() error
  • func (cmd *Cmd) Start() error
  • func (cmd *Cmd) Wait() error
• type CmdState
  • func (s CmdState) String() string
• type DB
  • func NewDB(address string) (*DB, error)
  • func (db *DB) Close()
  • func (db *DB) GetResponse(ID string) ([]byte, error)
  • func (db *DB) Len() (int, error)
  • func (db *DB) Ping() error
  • func (db *DB) SaveResponse(resp *api.ScriptsResponse) error
  • func (db *DB) TotalLen() (int, error)
• type Priority
  • func (s Priority) String() string
• type Qmd
  • func New(conf *config.Config) (*Qmd, error)
  • func (qmd *Qmd) Close()
  • func (qmd *Qmd) ClosingResponder(h http.Handler) http.Handler
  • func (qmd *Qmd) Cmd(from *exec.Cmd) (*Cmd, error)
  • func (qmd *Qmd) Dequeue() (*disque.Job, error)
  • func (qmd *Qmd) Enqueue(data string, priority string) (*disque.Job, error)
  • func (qmd *Qmd) GetAsyncResponse(req *api.ScriptsRequest, ID string) ([]byte, error)
  • func (qmd *Qmd) GetResponse(ID string) ([]byte, error)
  • func (qmd *Qmd) GetScript(file string) (string, error)
  • func (qmd *Qmd) ListenQueue()
  • func (qmd *Qmd) PostResponseCallback(req *api.ScriptsRequest, ID string) error
  • func (qmd *Qmd) StartWorkers()
  • func (qmd *Qmd) WatchScripts()
• type Scripts
  • func (s *Scripts) Get(file string) (string, error)
  • func (s *Scripts) Update(dir string) error
• type SlackNotifier
  • func (s *SlackNotifier) Notify(msg string) error
• type Worker

Variables

var (
	ErrNotFound = errors.New("not found")
)

Types

type Cmd

type Cmd struct {
	*exec.Cmd `json:"cmd"`

	JobID       string
	State       CmdState
	StartTime   time.Time
	EndTime     time.Time
	Duration    time.Duration
	StatusCode  int
	CallbackURL string
	Err         error
	Priority    Priority

	CmdOut     bytes.Buffer
	QmdOut     bytes.Buffer
	QmdOutFile string

	StoreDir          string
	ExtraWorkDirFiles map[string]string

	// Started channel block until the cmd is started.
	Started chan struct{}
	// Finished channel block until the cmd is finished/killed/invalidated.
	Finished chan struct{}

	// WaitOnce guards the Wait() logic, so it can be called multiple times.
	WaitOnce sync.Once
	// StartOnce guards the Start() logic, so it can be called multiple times.
	StartOnce sync.Once
}

func (*Cmd) Cleanup

func (cmd *Cmd) Cleanup() error

func (*Cmd) Kill

func (cmd *Cmd) Kill() error

func (*Cmd) Run

func (cmd *Cmd) Run() error

func (*Cmd) Start

func (cmd *Cmd) Start() error

func (*Cmd) Wait

func (cmd *Cmd) Wait() error

Wait waits for cmd to finish. It closes the Stdout and Stderr pipes.

type CmdState

type CmdState int

const (
	Initialized CmdState = iota
	Running
	Finished
	Terminated
	Invalidated
	Failed
)

func (CmdState) String

func (s CmdState) String() string

type DB

type DB struct {
	// contains filtered or unexported fields
}

func NewDB

func NewDB(address string) (*DB, error)

func (*DB) Close

func (db *DB) Close()

func (*DB) GetResponse

func (db *DB) GetResponse(ID string) ([]byte, error)

func (*DB) Len

func (db *DB) Len() (int, error)

func (*DB) Ping

func (db *DB) Ping() error

func (*DB) SaveResponse

func (db *DB) SaveResponse(resp *api.ScriptsResponse) error

func (*DB) TotalLen

func (db *DB) TotalLen() (int, error)

type Priority

type Priority int

const (
	PriorityLow Priority = iota
	PriorityHigh
	PriorityUrgent
)

func (Priority) String

func (s Priority) String() string

type Qmd

type Qmd struct {
	Config  *config.Config
	DB      *DB
	Queue   *disque.Pool
	Scripts Scripts
	Workers chan Worker
	Slack   *SlackNotifier

	Closing            bool
	ClosingListenQueue chan struct{}
	WaitListenQueue    sync.WaitGroup
	ClosingWorkers     chan struct{}
	WaitWorkers        sync.WaitGroup
}

func New

func New(conf *config.Config) (*Qmd, error)

func (*Qmd) Close

func (qmd *Qmd) Close()

func (*Qmd) ClosingResponder

func (qmd *Qmd) ClosingResponder(h http.Handler) http.Handler

func (*Qmd) Cmd

func (qmd *Qmd) Cmd(from *exec.Cmd) (*Cmd, error)

func (*Qmd) Dequeue

func (qmd *Qmd) Dequeue() (*disque.Job, error)

func (*Qmd) Enqueue

func (qmd *Qmd) Enqueue(data string, priority string) (*disque.Job, error)

func (*Qmd) GetAsyncResponse

func (qmd *Qmd) GetAsyncResponse(req *api.ScriptsRequest, ID string) ([]byte, error)

func (*Qmd) GetResponse

func (qmd *Qmd) GetResponse(ID string) ([]byte, error)

func (*Qmd) GetScript

func (qmd *Qmd) GetScript(file string) (string, error)

func (*Qmd) ListenQueue

func (qmd *Qmd) ListenQueue()

func (*Qmd) PostResponseCallback

func (qmd *Qmd) PostResponseCallback(req *api.ScriptsRequest, ID string) error

func (*Qmd) StartWorkers

func (qmd *Qmd) StartWorkers()

func (*Qmd) WatchScripts

func (qmd *Qmd) WatchScripts()

TODO: Use fsnotify.

type Scripts

type Scripts struct {
	Running bool

	sync.Mutex // guards files
	// contains filtered or unexported fields
}

func (*Scripts) Get

func (s *Scripts) Get(file string) (string, error)

func (*Scripts) Update

func (s *Scripts) Update(dir string) error

Update walks ScriptDir directory for shell scripts and updates the files cache.

type SlackNotifier

type SlackNotifier struct {
	WebhookURL string
	Channel    string
	Prefix     string
}

func (*SlackNotifier) Notify

func (s *SlackNotifier) Notify(msg string) error

type Worker

type Worker chan *disque.Job