Help us shape the future of Go! Take the 2026 Go Developer Survey

gogridengine

package module
v0.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 27, 2020 License: MIT Imports: 16 Imported by: 1

README

Quality Gate Status

Coverage Status

SunGrid Engine Serialization

This library will allow you to take the SGE XML output from queue status and serialize it programmatically into Go objects. As a note, natively unserializing will provide a ResourceList object containing all of the dynamic resources serialized into strings.

That's not exactly sane for programming, so several helper / receiver methods have been added to the ReceiverList allowing you to get structured values back for the important ones.

Example

source := `<?xml version='1.0'?>
 <job_info  xmlns:xsd="http://arc.liv.ac.uk/repos/darcs/sge/source/dist/util/resources/schemas/qstat/qstat.xsd">
   <queue_info>
	 <Queue-List>
	   <name>all.q@magicalhostname</name>
	   <qtype>BIP</qtype>
	   <slots_used>32</slots_used>
	   <slots_resv>0</slots_resv>
	   <slots_total>36</slots_total>
	   <load_avg>31.63000</load_avg>
	   <arch>lx-amd64</arch>
	   <resource name="load_avg" type="hl">31.630000</resource>
	   <resource name="load_short" type="hl">31.700000</resource>
	   <resource name="load_medium" type="hl">31.630000</resource>
	   <resource name="load_long" type="hl">31.680000</resource>
	   <resource name="arch" type="hl">lx-amd64</resource>
	   <resource name="num_proc" type="hl">36</resource>
	   <resource name="mem_free" type="hl">57.353G</resource>
	   <resource name="swap_free" type="hl">0.000</resource>
	   <resource name="virtual_free" type="hl">57.353G</resource>
	   <resource name="mem_total" type="hl">58.973G</resource>
	   <resource name="swap_total" type="hl">0.000</resource>
	   <resource name="virtual_total" type="hl">58.973G</resource>
	   <resource name="mem_used" type="hl">1.619G</resource>
	   <resource name="swap_used" type="hl">0.000</resource>
	   <resource name="virtual_used" type="hl">1.619G</resource>
	   <resource name="cpu" type="hl">89.100000</resource>
	   <resource name="m_topology" type="hl">SCTTCTTCTTCTTCTTCTTCTTCTTCTTSCTTCTTCTTCTTCTTCTTCTTCTTCTT</resource>
	   <resource name="m_topology_inuse" type="hl">SCTTCTTCTTCTTCTTCTTCTTCTTCTTSCTTCTTCTTCTTCTTCTTCTTCTTCTT</resource>
	   <resource name="m_socket" type="hl">2</resource>
	   <resource name="m_core" type="hl">18</resource>
	   <resource name="m_thread" type="hl">36</resource>
	   <resource name="np_load_avg" type="hl">0.878611</resource>
	   <resource name="np_load_short" type="hl">0.880556</resource>
	   <resource name="np_load_medium" type="hl">0.878611</resource>
	   <resource name="np_load_long" type="hl">0.880000</resource>
	   <resource name="qname" type="qf">all.q</resource>
	   <resource name="hostname" type="qf">ip-172-16-2-102.us-west-2.compute.internal</resource>
	   <resource name="slots" type="qc">4</resource>
	   <resource name="tmpdir" type="qf">/tmp</resource>
	   <resource name="seq_no" type="qf">0</resource>
	   <resource name="rerun" type="qf">0.000000</resource>
	   <resource name="calendar" type="qf">NONE</resource>
	   <resource name="s_rt" type="qf">infinity</resource>
	   <resource name="h_rt" type="qf">infinity</resource>
	   <resource name="s_cpu" type="qf">infinity</resource>
	   <resource name="h_cpu" type="qf">infinity</resource>
	   <resource name="s_fsize" type="qf">infinity</resource>
	   <resource name="h_fsize" type="qf">infinity</resource>
	   <resource name="s_data" type="qf">infinity</resource>
	   <resource name="h_data" type="qf">infinity</resource>
	   <resource name="s_stack" type="qf">infinity</resource>
	   <resource name="h_stack" type="qf">infinity</resource>
	   <resource name="s_core" type="qf">infinity</resource>
	   <resource name="h_core" type="qf">infinity</resource>
	   <resource name="s_rss" type="qf">infinity</resource>
	   <resource name="h_rss" type="qf">infinity</resource>
	   <resource name="s_vmem" type="qf">infinity</resource>
	   <resource name="h_vmem" type="qf">infinity</resource>
	   <resource name="min_cpu_interval" type="qf">00:05:00</resource>
	   <job_list state="running">
		 <JB_job_number>4282</JB_job_number>
		 <JAT_prio>0.50500</JAT_prio>
		 <JB_name>Run478</JB_name>
		 <JB_owner>user</JB_owner>
		 <state>r</state>
		 <JAT_start_time>2019-09-15T15:26:36</JAT_start_time>
		 <slots>1</slots>
	   </job_list>
	 </Queue-List>
   </queue_info>
   <job_info>
   </job_info>
 </job_info>`

	var info JobInfo
	xml.Unmarshal([]byte(source), &info)

#Environment Variables GOGRIDENGINE_TEST : If set to "true", will trigger test mode where the library will look to generated content and not try to use qstat GOGRIDENGINE_TEST_SOURCE: If set, should be a URL to XML from qstat -xml output which will be used for testing

Documentation

Index

Constants

View Source 
const ErrInvalidTaskRangeIdentifier = Error("The provided job does not actually indicate a range or group of tasks")

ErrInvalidTaskRangeIdentifier is an error that identifies jobs with a non-range conformant task attribute. Basically means you're trying to extrapolate jobs from a task range that isn't really a task range.

View Source 
const (
	//TASKRANGEIDENTIFIERREGEX is a regex string used for identifying whether <tasks> objects indicate a range of tasks (normally only expressed on pending tasks)
	TASKRANGEIDENTIFIERREGEX string = `[0-9]{1,}-[0-9]{1,}:[0-9]`
)

Variables

View Source 
var TaskRangeRegex *regexp.Regexp = regexp.MustCompile(TASKRANGEIDENTIFIERREGEX)

TaskRangeRegex is the compiled regular expression used for identifying Tasks that define a pending / unscheduled range

Functions

func DeleteQueuedJobByID 

func DeleteQueuedJobByID(targets []string) (string, error)

DeleteQueuedJobByID is used to delete (1 or many) jobs by concatenating their IDs together and passing them to qdel

func DeleteQueuedJobByUsernames 

func DeleteQueuedJobByUsernames(targets []string) (string, error)

DeleteQueuedJobByUsernames is used to delete (1 or many) jobs by concatenating usernames together and feeding them to qdel

func DoesJobContainTaskGroup 

func DoesJobContainTaskGroup(j Job) bool

DoesJobContainTaskGroup evaluates whether the XML marshalled tasks value contains a group rather than a range of tasks

func DoesJobContainTaskRange 

func DoesJobContainTaskRange(j Job) bool

DoesJobContainTaskRange evaluates whether the XML marshalled tasks value contains the regex indicating a sequence of tasks.

func GetQstatOutput 

func GetQstatOutput(filters map[string]string) (string, error)

GetQstatOutput is used to pull in XML content from either the QSTAT command or generated data for testing purpoes

func IsJobInErrorState 

func IsJobInErrorState(job Job) int

func IsJobRunning 

func IsJobRunning(job Job) int

IsJobRunning returns a int (1 - running) (0 - not)

Types

type Error 

type Error string

Error allows us to define constant errors

func (Error) Error 

func (e Error) Error() string

type Host 

type Host struct {
	XMLName       xml.Name     `xml:"Queue-List" json:"-"`
	Name          string       `xml:"name" json:"name"`
	QType         string       `xml:"qtype" json:"qtype"`
	SlotsUsed     int32        `xml:"slots_used" json:"slots_used"`
	SlotsReserved int32        `xml:"slots_rsv" json:"slots_reserved"`
	SlotsTotal    int32        `xml:"slots_total" json:"slots_total"`
	LoadAverage   float64      `xml:"load_avg" json:"load_average"`
	Resources     ResourceList `xml:"resource" json:"resources"`
	JobList       []Job        `xml:"job_list" json:"job_list"`
}

Host is the top-level object (per host) that includes all subsequent data including jobs, resources etc

type Job 

type Job struct {
	//Because this is a node, we still need the XMLName identifier
	XMLName        xml.Name `xml:"job_list" json:"-"`
	StateAttribute string   `xml:"state,attr" json:"state_attribute_text"`
	State          string   `xml:"state" json:"state"`
	JBJobNumber    int64    `xml:"JB_job_number" json:"jb_job_number"`
	JATPriority    float64  `xml:"JAT_prio" json:"jat_prio"`
	JobName        string   `xml:"JB_name" json:"jb_name"`
	JobOwner       string   `xml:"JB_owner" json:"jb_owner"`
	StartTime      string   `xml:"JAT_start_time,omitempty" json:"start_time"`
	SubmittedTime  string   `xml:"JB_submission_time,omitempty" json:"submitted_time"`
	Slots          int32    `xml:"slots" json:"slots"`
	Tasks          Task     `xml:"tasks,omitempty" json:"tasks,omitempty"`
}

Job is the Sun Grid Engine XML Definition for a job running on a specific host, its details and current status

type JobInfo 

type JobInfo struct {
	XMLName     xml.Name   `xml:"job_info" json:"-"`
	QueueInfo   QueueInfo  `xml:"queue_info" json:"queue_info"`
	PendingJobs PendingJob `xml:"job_info,omitempty" json:"pending_jobs"`
}

JobInfo is the top level object for the SGE Qstat output

func NewJobInfo 

func NewJobInfo(input string) (JobInfo, error)

NewJobInfo returns the go struct of the qstat output

func (JobInfo) GetXML 

func (q JobInfo) GetXML() (string, error)

GetXML renders down the XML with UTF-8 opening tags to ensure feasability for testing of output

type JobList 

type JobList []Job

JobList is a slice of Jobs that is filterable and otherwise actionable via receiver.

func ExtrapolateTasksToJobs 

func ExtrapolateTasksToJobs(original Job) (JobList, error)

ExtrapolateTasksToJobs takes the role of finding the range identifier and returning a job list from it (Extrapolated from the task list)

func FilterJobs 

func FilterJobs(jobs JobList, filter func(j Job) bool) JobList

FilterJobs is a function allowing you to manually provide a JobList and a filter function to limit the content down.

func GetJobs 

func GetJobs() (JobList, error)

GetJobs returns a slice of only jobs from both scheduled and unscheduled queues

func GetJobsWithFilter 

func GetJobsWithFilter(filterfunc func(j Job) bool) (JobList, error)

GetJobsWithFilter allows you to specify a filter at the time of retrieving the JobList

func (JobList) Filter 

func (jl JobList) Filter(filter func(j Job) bool) JobList

Filter allows for the passage of any function taking a JobList and Filtering its contents down. Should be usable in fluent fashion as long as JobList is being returned

func (JobList) Sort 

func (jl JobList) Sort(sorter func(i, j int) bool) JobList

Sort allows you to provide your own Less function to handle sorting the list directly

type PendingJob 

type PendingJob struct {
	XMLName xml.Name `xml:"job_info" json:"-"`
	JobList []Job    `xml:"job_list" json:"job_list"`
}

PendingJob is a sub tag of job_info (also labeled job_info) which details jobs not yet executing.

type QueueInfo 

type QueueInfo struct {
	XMLName xml.Name `xml:"queue_info" json:"-"`
	Queues  []Host   `xml:"Queue-List" json:"queue_list"`
}

QueueInfo is the child object for qstat job output

type Resource 

type Resource struct {
	XMLName xml.Name `xml:"resource" json:"-"`
	Name    string   `xml:"name,attr" json:"name"`
	Type    string   `xml:"type,attr" json:"type"`
	Value   string   `xml:",innerxml"`
}

Resource is a general representation of a system resource attached to a host, identified purely by Key-value pairs. These can be best thought of as Unix Load, Memory resource allocations, etc

type ResourceList 

type ResourceList []Resource

ResourceList is a slice of resources primarily used for sourcing internally and setup of receiver based functions

func (ResourceList) CPU 

func (r ResourceList) CPU() (float64, error)

CPU returns utilization type cast as a float

func (ResourceList) FreeMemory 

func (r ResourceList) FreeMemory() (StorageValue, error)

FreeMemory returns the type safe values for Memory free (in bytes)

func (ResourceList) FreeSwap 

func (r ResourceList) FreeSwap() (StorageValue, error)

FreeSwap returns the type casted value for Free Swap Bytes

func (ResourceList) FreeVirtualMemory 

func (r ResourceList) FreeVirtualMemory() (StorageValue, error)

FreeVirtualMemory returns the type casted value for Free Virtual Memory Bytes

func (ResourceList) Load 

func (r ResourceList) Load(window string) (float64, error)

Load returns the formatted, type safe float value for the Short Load resource. Provide the window of length: load_short load_medium load_long

func (ResourceList) MCoreCount 

func (r ResourceList) MCoreCount() (int64, error)

MCoreCount returns the core count type cast as an int64

func (ResourceList) MSocketCount 

func (r ResourceList) MSocketCount() (int64, error)

MSocketCount returns the socket count type cast as an int64

func (ResourceList) MThreadCount 

func (r ResourceList) MThreadCount() (int64, error)

MThreadCount returns the thread count type cast as an int64

func (ResourceList) MemoryUsed 

func (r ResourceList) MemoryUsed() (StorageValue, error)

MemoryUsed returns the type cast value for Memory used in bytes

func (ResourceList) NPLoadAverage 

func (r ResourceList) NPLoadAverage() (float64, error)

NPLoadAverage NP Load type cast as float 64

func (ResourceList) NPLoadLong 

func (r ResourceList) NPLoadLong() (float64, error)

NPLoadLong NP Load type cast as float 64

func (ResourceList) NPLoadMedium 

func (r ResourceList) NPLoadMedium() (float64, error)

NPLoadMedium NP Load type cast as float 64

func (ResourceList) NPLoadShort 

func (r ResourceList) NPLoadShort() (float64, error)

NPLoadShort NP Load type cast as float 64

func (ResourceList) NumberofProcessors 

func (r ResourceList) NumberofProcessors() (int32, error)

NumberofProcessors is the formatted, type-safe value for the num_proc xml attribute

func (ResourceList) ProcessorCount 

func (r ResourceList) ProcessorCount() (int64, error)

ProcessorCount returns the number of processors type cast as integer64

func (ResourceList) SwapUsed 

func (r ResourceList) SwapUsed() (StorageValue, error)

SwapUsed returns the type cast value for swap used in bytes

func (ResourceList) TotalMemory 

func (r ResourceList) TotalMemory() (StorageValue, error)

TotalMemory returns the Type cast value for Total Memory in bytes

func (ResourceList) TotalSwap 

func (r ResourceList) TotalSwap() (StorageValue, error)

TotalSwap returns the Type cast value for Total Swap in bytes

func (ResourceList) TotalVirtual 

func (r ResourceList) TotalVirtual() (StorageValue, error)

TotalVirtual returns the Type cast value for the virtual memory total in bytes

func (ResourceList) VirtualUsed 

func (r ResourceList) VirtualUsed() (StorageValue, error)

VirtualUsed returns the type cast value for swap used in bytes

type StorageValue 

type StorageValue struct {
	Size  float64 `json:"size"`
	Scale string  `json:"scale"`
	Bytes int64   `json:"bytes"`
}

StorageValue breaks down string metrics from a computer storage standpoint (ie 10.2G) so that it can be calculated to bytes

type Task 

type Task struct {
	//Mixed type. Can be either a string representation of an int64 OR a string range identifier, eg: 40-55:1 (Jobs 40-55 incremented by 1)
	Source string
	//Typed representation of the Source if mapped to an integer
	TaskID int64
}

Task is an element used for handling task arrays from the grid engine. Here we'll store the raw value (Source) and the TaskID if an individual identifier.

func (*Task) MarshalXML 

func (t *Task) MarshalXML(e *xml.Encoder, start xml.StartElement) error

MarshalXML renders the value back down to the XML structure

func (*Task) UnmarshalXML 

func (t *Task) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error

UnmarshalXML is a custom marshaller for handling complex logic surrounding task data.

type XMLDataSource 

type XMLDataSource struct {
	// contains filtered or unexported fields
}

func (*XMLDataSource) Get 

func (d *XMLDataSource) Get() (string, error)

type XmlContentReader 

type XmlContentReader struct {
	// contains filtered or unexported fields
}

func (*XmlContentReader) Read 

func (x *XmlContentReader) Read() (string, error)

type XmlResourceGetter 

type XmlResourceGetter interface {
	Get() (string, error)
}

type XmlResourceReader 

type XmlResourceReader interface {
	Read() (string, error)
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.