web

Documentation

Index

• Variables
• func AcquisitionErasCiHandler(w http.ResponseWriter, r *http.Request)
• func AcquisitionErasHandler(w http.ResponseWriter, r *http.Request)
• func ApisHandler(w http.ResponseWriter, r *http.Request)
• func BlockChildrenHandler(w http.ResponseWriter, r *http.Request)
• func BlockDumpHandler(w http.ResponseWriter, r *http.Request)
• func BlockOriginHandler(w http.ResponseWriter, r *http.Request)
• func BlockParentsHandler(w http.ResponseWriter, r *http.Request)
• func BlockSummariesHandler(w http.ResponseWriter, r *http.Request)
• func BlockTrioHandler(w http.ResponseWriter, r *http.Request)
• func BlocksHandler(w http.ResponseWriter, r *http.Request)
• func BulkBlocksHandler(w http.ResponseWriter, r *http.Request)
• func DBSGetHandler(w http.ResponseWriter, r *http.Request, a string)
• func DBSPostHandler(w http.ResponseWriter, r *http.Request, a string)
• func DBSPutHandler(w http.ResponseWriter, r *http.Request, a string)
• func DBStatsHandler(w http.ResponseWriter, r *http.Request)
• func DataTypesHandler(w http.ResponseWriter, r *http.Request)
• func DatasetAccessTypesHandler(w http.ResponseWriter, r *http.Request)
• func DatasetChildrenHandler(w http.ResponseWriter, r *http.Request)
• func DatasetListHandler(w http.ResponseWriter, r *http.Request)
• func DatasetParentsHandler(w http.ResponseWriter, r *http.Request)
• func DatasetsHandler(w http.ResponseWriter, r *http.Request)
• func DatatiersHandler(w http.ResponseWriter, r *http.Request)
• func DummyHandler(w http.ResponseWriter, r *http.Request)
• func Etag(str string, weak bool) string
• func FileArrayHandler(w http.ResponseWriter, r *http.Request)
• func FileChildrenHandler(w http.ResponseWriter, r *http.Request)
• func FileLumisHandler(w http.ResponseWriter, r *http.Request)
• func FileParentsByLumiHandler(w http.ResponseWriter, r *http.Request)
• func FileParentsHandler(w http.ResponseWriter, r *http.Request)
• func FileSummariesHandler(w http.ResponseWriter, r *http.Request)
• func FilesHandler(w http.ResponseWriter, r *http.Request)
• func Handlers() *mux.Router
• func MainHandler(w http.ResponseWriter, r *http.Request)
• func MetricsHandler(w http.ResponseWriter, r *http.Request)
• func MigrationCancelHandler(w http.ResponseWriter, r *http.Request)
• func MigrationProcessHandler(w http.ResponseWriter, r *http.Request)
• func MigrationRemoveHandler(w http.ResponseWriter, r *http.Request)
• func MigrationStatusHandler(w http.ResponseWriter, r *http.Request)
• func MigrationSubmitHandler(w http.ResponseWriter, r *http.Request)
• func MigrationTotalHandler(w http.ResponseWriter, r *http.Request)
• func NotImplementedHandler(w http.ResponseWriter, r *http.Request, api string)
• func OutputConfigsHandler(w http.ResponseWriter, r *http.Request)
• func ParentDSTrioHandler(w http.ResponseWriter, r *http.Request)
• func ParseConfig(configFile string) error
• func PhysicsGroupsHandler(w http.ResponseWriter, r *http.Request)
• func PrimaryDSTypesHandler(w http.ResponseWriter, r *http.Request)
• func PrimaryDatasetsHandler(w http.ResponseWriter, r *http.Request)
• func ProcessingErasHandler(w http.ResponseWriter, r *http.Request)
• func QueryHandler(w http.ResponseWriter, r *http.Request)
• func ReleaseVersionsHandler(w http.ResponseWriter, r *http.Request)
• func RunSummariesHandler(w http.ResponseWriter, r *http.Request)
• func RunsHandler(w http.ResponseWriter, r *http.Request)
• func Server(configFile string)
• func ServerInfoHandler(w http.ResponseWriter, r *http.Request)
• func StatusHandler(w http.ResponseWriter, r *http.Request)
• type Configuration
  • func (c *Configuration) String() string
• type HTTPError
• type Mem
• type Memory
• type Metrics
• type RequestStats
  • func (r *RequestStats) Update()
• type ServerError
• type Templates
  • func (q Templates) Tmpl(tdir, tfile string, tmplData map[string]interface{}) string
• type TmplRecord

Variables

var AvgGetRequestTime float64

AvgGetRequestTime represents average GET request time

var AvgPostRequestTime float64

AvgPostRequestTime represents average POST request time

var AvgPutRequestTime float64

AvgPutRequestTime represents average PUT request time

var CMSAuth cmsauth.CMSAuth

CMSAuth structure to create CMS Auth headers

var GitVersion string

GitVersion defines git version of the server

var GraphQLSchema *graphql.Schema

GraphQLSchema holds graphql schema

var LimiterMiddleware *stdlib.Middleware

LimiterMiddleware provides limiter middleware pointer

var MetricsLastUpdateTime time.Time

MetricsLastUpdateTime keeps track of last update time of the metrics

var NumLogicalCores int

NumLogicalCores represents number of cores in our node

var NumPhysicalCores int

NumPhysicalCores represents number of cores in our node

var RPS float64

RPS represents requests per second for a given server

var RPSLogical float64

RPSLogical represents requests per second for a given server times number of logical CPU cores

var RPSPhysical float64

RPSPhysical represents requests per second for a given server times number of physical CPU cores

var ServerInfo string

ServerInfo defines dbs server info

var StartTime time.Time

StartTime represents initial time when we started the server

var TotalGetRequests uint64

TotalGetRequests counts total number of GET requests received by the server

var TotalPostRequests uint64

TotalPostRequests counts total number of POST requests received by the server

var TotalPutRequests uint64

TotalPutRequests counts total number of PUT requests received by the server

Functions

func AcquisitionErasCiHandler

func AcquisitionErasCiHandler(w http.ResponseWriter, r *http.Request)

AcquisitionErasCiHandler provides access to AcquisitionErasCi DBS API. Takes the following arguments: acquisition_era_name

func AcquisitionErasHandler

func AcquisitionErasHandler(w http.ResponseWriter, r *http.Request)

AcquisitionErasHandler provides access to AcquisitionEras DBS API. Takes the following arguments: acquisition_era_name

func ApisHandler

func ApisHandler(w http.ResponseWriter, r *http.Request)

ApisHandler provides list of supporter apis by the DBS server

func BlockChildrenHandler

func BlockChildrenHandler(w http.ResponseWriter, r *http.Request)

BlockChildrenHandler provides access to BlockChildren DBS API. Takes the following arguments: block_name

func BlockDumpHandler

func BlockDumpHandler(w http.ResponseWriter, r *http.Request)

BlockDumpHandler provides access to BlockDump DBS API

func BlockOriginHandler

func BlockOriginHandler(w http.ResponseWriter, r *http.Request)

BlockOriginHandler provides access to BlockOrigin DBS API. Takes the following arguments: origin_site_name, dataset, block_name

func BlockParentsHandler

func BlockParentsHandler(w http.ResponseWriter, r *http.Request)

BlockParentsHandler provides access to BlockParents DBS API. Takes the following arguments: block_name

func BlockSummariesHandler

func BlockSummariesHandler(w http.ResponseWriter, r *http.Request)

BlockSummariesHandler provides access to BlockSummaries DBS API. Takes the following arguments: block_name, dataset, detail

func BlockTrioHandler

func BlockTrioHandler(w http.ResponseWriter, r *http.Request)

BlockTrioHandler provides access to BlockTrio DBS API. Takes the following arguments: block_name, list of lfns

func BlocksHandler

func BlocksHandler(w http.ResponseWriter, r *http.Request)

BlocksHandler provides access to Blocks DBS API. Takes the following arguments: dataset, block_name, data_tier_name, origin_site_name, logical_file_name, run_num, min_cdate, max_cdate, min_ldate, max_ldate, cdate, ldate, open_for_writing, detail

func BulkBlocksHandler

func BulkBlocksHandler(w http.ResponseWriter, r *http.Request)

BulkBlocksHandler provides access to BulkBlocks DBS API POST API takes no argument, the payload should be supplied as JSON

func DBSGetHandler

func DBSGetHandler(w http.ResponseWriter, r *http.Request, a string)

DBSGetHandler is a generic Get handler to call DBS Get APIs.

func DBSPostHandler

func DBSPostHandler(w http.ResponseWriter, r *http.Request, a string)

DBSPostHandler is a generic Post Handler to call DBS Post APIs

func DBSPutHandler

func DBSPutHandler(w http.ResponseWriter, r *http.Request, a string)

DBSPutHandler is a generic Post Handler to call DBS Post APIs

func DBStatsHandler

func DBStatsHandler(w http.ResponseWriter, r *http.Request)

DBStatsHandler provides metrics

func DataTypesHandler

func DataTypesHandler(w http.ResponseWriter, r *http.Request)

DataTypesHandler provides access to DataTypes DBS API. Takes the following arguments: datatype, dataset

func DatasetAccessTypesHandler

func DatasetAccessTypesHandler(w http.ResponseWriter, r *http.Request)

DatasetAccessTypesHandler provides access to DatasetAccessTypes DBS API. Takes the following arguments: dataset_access_type

func DatasetChildrenHandler

func DatasetChildrenHandler(w http.ResponseWriter, r *http.Request)

DatasetChildrenHandler provides access to DatasetChildren DBS API. Takes the following arguments: dataset

func DatasetListHandler

func DatasetListHandler(w http.ResponseWriter, r *http.Request)

DatasetListHandler provides access to DatasetList DBS API POST API takes no argument, the payload should be supplied as JSON

func DatasetParentsHandler

func DatasetParentsHandler(w http.ResponseWriter, r *http.Request)

DatasetParentsHandler provides access to DatasetParents DBS API. Takes the following arguments: dataset

func DatasetsHandler

func DatasetsHandler(w http.ResponseWriter, r *http.Request)

DatasetsHandler provides access to Datasets DBS API. Takes the following arguments: dataset, parent_dataset, release_version, pset_hash, app_name, output_module_label, global_tag, processing_version, acquisition_era_name, run_num, physics_group_name, logical_file_name, primary_ds_name, primary_ds_type, processed_ds_name, data_tier_name, dataset_access_type, prep_id, create_by, last_modified_by, min_cdate, max_cdate, min_ldate, max_ldate, cdate, ldate, detail, dataset_id

func DatatiersHandler

func DatatiersHandler(w http.ResponseWriter, r *http.Request)

DatatiersHandler provides access to DataTiers DBS API. Takes the following arguments: data_tier_name

func DummyHandler

func DummyHandler(w http.ResponseWriter, r *http.Request)

DummyHandler provides example how to write GET/POST handler

func Etag

func Etag(str string, weak bool) string

Generates an Etag for given string, provided by https://github.com/amalfra/etag

func FileArrayHandler

func FileArrayHandler(w http.ResponseWriter, r *http.Request)

FileArrayHandler provides access to FileArray DBS API POST API takes no argument, the payload should be supplied as JSON

func FileChildrenHandler

func FileChildrenHandler(w http.ResponseWriter, r *http.Request)

FileChildrenHandler provides access to FileChildren DBS API. Takes the following arguments: logical_file_name, block_name, block_id

func FileLumisHandler

func FileLumisHandler(w http.ResponseWriter, r *http.Request)

FileLumisHandler provides access to FileLumis DBS API GET API takes the following arguments: logical_file_name, block_name, run_num, validFileOnly POST API takes no argument, the payload should be supplied as JSON

func FileParentsByLumiHandler

func FileParentsByLumiHandler(w http.ResponseWriter, r *http.Request)

FileParentsByLumiHandler provides access to FileParentsByLumi DBS API POST API takes no argument, the payload should be supplied as JSON

func FileParentsHandler

func FileParentsHandler(w http.ResponseWriter, r *http.Request)

FileParentsHandler provides access to FileParent DBS API. Takes the following arguments: logical_file_name, block_id, block_name

func FileSummariesHandler

func FileSummariesHandler(w http.ResponseWriter, r *http.Request)

FileSummariesHandler provides access to FileSummaries DBS API. Takes the following arguments: block_name, dataset, run_num, validFileOnly, sumOverLumi

func FilesHandler

func FilesHandler(w http.ResponseWriter, r *http.Request)

FilesHandler provides access to Files DBS API. Takes the following arguments: dataset, block_name, logical_file_name, release_version, pset_hash, app_name, output_module_label, run_num, origin_site_name, lumi_list, detail, validFileOnly, sumOverLumi

func Handlers

func Handlers() *mux.Router

Handlers provides helper function to setup all HTTP routes

func MainHandler

func MainHandler(w http.ResponseWriter, r *http.Request)

MainHandler provides access to main page of DBS server

func MetricsHandler

func MetricsHandler(w http.ResponseWriter, r *http.Request)

MetricsHandler provides metrics

func MigrationCancelHandler

func MigrationCancelHandler(w http.ResponseWriter, r *http.Request)

MigrationCancelHandler provides access to CancelMigration DBS API POST API takes no argument, the payload should be supplied as JSON

func MigrationProcessHandler

func MigrationProcessHandler(w http.ResponseWriter, r *http.Request)

MigrationProcessHandler provides access to ProcessMigration DBS API POST API takes no argument, the payload should be supplied as JSON

func MigrationRemoveHandler

func MigrationRemoveHandler(w http.ResponseWriter, r *http.Request)

MigrationRemoveHandler provides access to RemoveMigration DBS API POST API takes no argument, the payload should be supplied as JSON

func MigrationStatusHandler

func MigrationStatusHandler(w http.ResponseWriter, r *http.Request)

MigrationStatusHandler provides access to StatusMigration DBS API

func MigrationSubmitHandler

func MigrationSubmitHandler(w http.ResponseWriter, r *http.Request)

MigrationSubmitHandler provides access to SubmitMigration DBS API POST API takes no argument, the payload should be supplied as JSON

func MigrationTotalHandler

func MigrationTotalHandler(w http.ResponseWriter, r *http.Request)

MigrationTotalHandler provides access to TotalMigration DBS API

func NotImplementedHandler

func NotImplementedHandler(w http.ResponseWriter, r *http.Request, api string)

NotImplementedHandler returns server status error

func OutputConfigsHandler

func OutputConfigsHandler(w http.ResponseWriter, r *http.Request)

OutputConfigsHandler provides access to OutputConfigs DBS API. Takes the following arguments: dataset, logical_file_name, release_version, pset_hash, app_name, output_module_label, block_id, global_tag

func ParentDSTrioHandler

func ParentDSTrioHandler(w http.ResponseWriter, r *http.Request)

ParentDSTrioHandler provides access to ParentDSTrio DBS API. Takes the following arguments: dataset

func ParseConfig

func ParseConfig(configFile string) error

ParseConfig parses given configuration file and initialize Config object

func PhysicsGroupsHandler

func PhysicsGroupsHandler(w http.ResponseWriter, r *http.Request)

PhysicsGroupsHandler provides access to PhysicsGroups DBS API Takes the following arguments: physics_group_name

func PrimaryDSTypesHandler

func PrimaryDSTypesHandler(w http.ResponseWriter, r *http.Request)

PrimaryDSTypesHandler provides access to PrimaryDSTypes DBS API. Takes the following arguments: primary_ds_type, dataset

func PrimaryDatasetsHandler

func PrimaryDatasetsHandler(w http.ResponseWriter, r *http.Request)

PrimaryDatasetsHandler provides access to PrimaryDatasets DBS API. Takes the following arguments: primary_ds_name, primary_ds_type

func ProcessingErasHandler

func ProcessingErasHandler(w http.ResponseWriter, r *http.Request)

ProcessingErasHandler provices access to ProcessingEras DBS API. Takes the following arguments: processing_version

func QueryHandler

func QueryHandler(w http.ResponseWriter, r *http.Request)

QueryHandler provides access to graph ql query

func ReleaseVersionsHandler

func ReleaseVersionsHandler(w http.ResponseWriter, r *http.Request)

ReleaseVersionsHandler provides access to ReleaseVersions DBS API. Takes the following arguments: release_version, dataset, logical_file_name

func RunSummariesHandler

func RunSummariesHandler(w http.ResponseWriter, r *http.Request)

RunSummariesHandler provides access to RunSummaries DBS API. Takes the following arguments: dataset, run_num

func RunsHandler

func RunsHandler(w http.ResponseWriter, r *http.Request)

RunsHandler provides access to Runs DBS API. Takes the following arguments: run_num, logical_file_name, block_name, dataset

func Server

func Server(configFile string)

Server represents main web server for DBS service

func ServerInfoHandler

func ServerInfoHandler(w http.ResponseWriter, r *http.Request)

ServerInfoHandler provides basic functionality of status response

func StatusHandler

func StatusHandler(w http.ResponseWriter, r *http.Request)

StatusHandler provides basic functionality of status response

Types

type Configuration

type Configuration struct {
	Port          int    `json:"port"`           // dbs port number
	StaticDir     string `json:"staticdir"`      // location of static directory
	Base          string `json:"base"`           // dbs base path
	Verbose       int    `json:"verbose"`        // verbosity level
	LogFile       string `json:"log_file"`       // server log file (should ends with .log) or log area
	UTC           bool   `json:"utc"`            // report logger time in UTC
	MonitType     string `json:"monit_type"`     // monit record type
	MonitProducer string `json:"monit_producer"` // monit record producer
	Hmac          string `json:"hmac"`           // cmsweb hmac file location
	LimiterPeriod string `json:"limiter_rate"`   // limiter rate value
	LimiterHeader string `json:"limiter_header"` // limiter header to use
	MetricsPrefix string `json:"metrics_prefix"` // metrics prefix used for prometheus
	ServerType    string `json:"server_type"`    // DBS server type to start: DBSReader, DBSWriter, DBSMigrate, DBSMigration
	Etag          string `json:"etag"`           // etag value to use for ETag generation
	CacheControl  string `json:"cache_control"`  // Cache-Control value, e.g. max-age=300
	CMSRole       string `json:"cms_role"`       // cms role for write access
	CMSGroup      string `json:"cms_group"`      // cms group for write access

	// Migration server settings
	MigrationDBFile          string `json:"migration_dbfile"`           // dbfile with secrets
	MigrationServerInterval  int    `json:"migration_server_interval"`  // migration process interval
	MigrationProcessTimeout  int    `json:"migration_process_timeout"`  // migration process timeout
	MigrationCleanupInterval int    `json:"migration_cleanup_interval"` // migration cleanup interval
	MigrationCleanupOffset   int64  `json:"migration_cleanup_offset"`   // migration cleanup offset

	// db related configuration
	DBFile               string `json:"dbfile"`                  // dbs db file with secrets
	MaxDBConnections     int    `json:"max_db_connections"`      // maximum number of DB connections
	MaxIdleConnections   int    `json:"max_idle_connections"`    // maximum number of idle connections
	DBMonitoringInterval int    `json:"db_monitoring_interval"`  // db mon interval in seconds
	LexiconFile          string `json:"lexicon_file"`            // lexicon json file
	FileChunkSize        int    `json:"file_chunk_size"`         // chunk size for []File insertion
	FileLumiChunkSize    int    `json:"file_lumi_chunk_size"`    // chunk size for []FileLumi insertion
	FileLumiMaxSize      int    `json:"file_lumi_max_size"`      // max size for []FileLumi insertion
	FileLumiInsertMethod string `json:"file_lumi_insert_method"` // insert method for FileLumi list
	ConcurrentBulkBlocks bool   `json:"concurrent_bulkblocks"`   // use concurrent BulkBlocks API

	// server static parts
	Templates string `json:"templates"` // location of server templates
	Jscripts  string `json:"jscripts"`  // location of server JavaScript files
	Images    string `json:"images"`    // location of server images
	Styles    string `json:"styles"`    // location of server CSS styles

	// security parts
	ServerKey  string `json:"serverkey"`  // server key for https
	ServerCrt  string `json:"servercrt"`  // server certificate for https
	RootCA     string `json:"rootCA"`     // RootCA file
	CSRFKey    string `json:"csrfKey"`    // CSRF 32-byte-long-auth-key
	Production bool   `json:"production"` // production server or not

	// GraphQL parts
	GraphQLSchema string `json:"graphqlSchema"` // graph ql schema file name
}

Configuration stores dbs configuration parameters

var Config Configuration

Config represents global configuration object

func (*Configuration) String

func (c *Configuration) String() string

String returns string representation of dbs Config

type HTTPError

type HTTPError struct {
	Method         string `json:"method"`           // HTTP method
	HTTPCode       int    `json:"code"`             // HTTP status code from IANA
	Timestamp      string `json:"timestamp"`        // timestamp of the error
	Path           string `json:"path"`             // URL path
	UserAgent      string `json:"user_agent"`       // http user-agent field
	XForwardedHost string `json:"x_forwarded_host"` // http.Request X-Forwarded-Host
	XForwardedFor  string `json:"x_forwarded_for"`  // http.Request X-Forwarded-For
	RemoteAddr     string `json:"remote_addr"`      // http.Request remote address
}

HTTPError represents HTTP error structure

type Mem

type Mem struct {
	Virtual Memory `json:"virtual"` // virtual memory metrics from gopsutils
	Swap    Memory `json:"swap"`    // swap memory metrics from gopsutils
}

Mem structure keeps track of virtual/swap memory of the server

type Memory

type Memory struct {
	Total       uint64  `json:"total"`
	Free        uint64  `json:"free"`
	Used        uint64  `json:"used"`
	UsedPercent float64 `json:"usedPercent"`
}

Memory structure keeps track of server memory

type Metrics

type Metrics struct {
	CPU          []float64               `json:"cpu"`          // cpu metrics from gopsutils
	CpuPercent   float64                 `json:"cpu_pct"`      // cpu percent
	Connections  []net.ConnectionStat    `json:"connections"`  // connections metrics from gopsutils
	Load         load.AvgStat            `json:"load"`         // load metrics from gopsutils
	Memory       Mem                     `json:"memory"`       // memory metrics from gopsutils
	OpenFiles    []process.OpenFilesStat `json:"openFiles"`    // open files metrics from gopsutils
	GoRoutines   uint64                  `json:"goroutines"`   // total number of go routines at run-time
	Uptime       float64                 `json:"uptime"`       // uptime of the server
	GetRequests  uint64                  `json:"getRequests"`  // total number of get requests across all services
	PostRequests uint64                  `json:"postRequests"` // total number of post requests across all services
	PutRequests  uint64                  `json:"putRequests"`  // total number of post requests across all services
	AvgGetTime   float64                 `json:"avgGetTime"`   // avg GET request time
	AvgPostTime  float64                 `json:"avgPostTime"`  // avg POST request time
	AvgPutTime   float64                 `json:"avgPutTime"`   // avg PUT request time
	RPS          float64                 `json:"rps"`          // throughput req/sec
	RPSPhysical  float64                 `json:"rpsPhysical"`  // throughput req/sec using physical cpu
	RPSLogical   float64                 `json:"rpsLogical"`   // throughput req/sec using logical cpu
	ProcFS       utils.ProcFS            `json:"procfs"`       // metrics from prometheus procfs
}

Metrics provide various metrics about our server

type RequestStats

type RequestStats struct {
	TotalGetRequests  uint64
	TotalPostRequests uint64
	TotalPutRequests  uint64
	Time              time.Time
	NumPhysicalCores  int
	NumLogicalCores   int
}

RequestStats holds metrics related to number of requests on a server

func (*RequestStats) Update

func (r *RequestStats) Update()

Update RequestStatus metrics

type ServerError

type ServerError struct {
	DBSError  error     `json:"error"`     // DBS error
	HTTPError HTTPError `json:"http"`      // HTTP section of the error
	Exception int       `json:"exception"` // for compatibility with Python server
	Type      string    `json:"type"`      // for compatibility with Python server
	Message   string    `json:"message"`   // for compatibility with Python server
}

ServerError represents HTTP server error structure

type Templates

type Templates struct {
	// contains filtered or unexported fields
}

Templates structure

func (Templates) Tmpl

func (q Templates) Tmpl(tdir, tfile string, tmplData map[string]interface{}) string

Tmpl method for ServerTemplates structure

type TmplRecord

type TmplRecord map[string]interface{}

TmplRecord represent template record