s3

package module
v0.0.0-...-1e3e343 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 7, 2021 License: CC0-1.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	W3C string = "w3c"
	S3  string = "s3"
)
View Source 
const (
	NETWORK    string = "network"
	FILESYSTEM string = "filesystem"
)
View Source 
const (
	BUCKETOWNER = iota + 1
	BUCKET
	TIME
	REMOTEIP
	REQUESTER
	REQUESTID
	OPERATION
	KEY
	REQUESTURI
	HTTPSTATUSCODE
	ERRORCODE
	BYTESSENT
	OBJECTSIZE
	TOTALTIME
	TURNAROUNDTIME
	REFERRER
	USERAGENT
	VERSIONID
)

Constants for indexing into a regex

View Source 
const (
	RXBucketOwner    string = `^(\w*)\s`
	RXBucket         string = `([a-zA-Z0-9\-]*)\s`
	RXTime           string = `(\[.*\])\s`
	RXRemoteIP       string = `([0-9\.]*)\s`
	RXRequester      string = `(\w*)\s`
	RXRequestId      string = `(\w*)\s`
	RXOperation      string = `([a-zA-Z0-9\.]*)\s`
	RXKey            string = `([a-zA-Z0-9\S]*)\s`
	RXRequestURI     string = `"(.*?)"\s`
	RXHttpStatusCode string = `(\d*)\s`
	RXErrorCode      string = `([\w-]*)\s`
	RXBytesSent      string = `([0-9-]*)\s`
	RXObjectSize     string = `([0-9-]*)\s`
	RXTotalTime      string = `([0-9-]*)\s`
	RXTurnAroundTime string = `([0-9-]*)\s`
	RXReferrer       string = `"(.*?)"\s`
	RXUserAgent      string = `"(.*?)"\s`
	RXVersionId      string = `(.*)$`
)

Regex match patterns for each field

View Source 
const S3Regex string = RXBucketOwner + RXBucket + RXTime +
	RXRemoteIP + RXRequester + RXRequestId + RXOperation + RXKey +
	RXRequestURI + RXHttpStatusCode + RXErrorCode + RXBytesSent +
	RXObjectSize + RXTotalTime + RXTurnAroundTime + RXReferrer +
	RXUserAgent + RXVersionId

Regex for match an entire log line

Variables

This section is empty.

Functions

func Encode64 

func Encode64(inputStr string) string

Encode64 for encoding secrets in manifests

func GetArgoManifest 

func GetArgoManifest(ml []argoManifests, mtype string) (man argoManifests, err error)

GetManifest returns an argoManifest for a given type

func GetObject 

func GetObject(client *minio.Client, bucket string, object string, opts minio.GetObjectOptions) (file string, er error)

func ListBucketObjects 

func ListBucketObjects(client *minio.Client, bucket string, opts minio.ListObjectsOptions) (objects []minio.ObjectInfo, err error)

func ListBuckets 

func ListBuckets(c *minio.Client) ([]minio.BucketInfo, error)

func NewClient 

func NewClient(p Provider) (*minio.Client, error)

func RFC1123 

func RFC1123(inputStr string) string

RFC1123 & 1035 is sanity check for label names used in docker and k8s - contain at most 63 characters - contain only lowercase alphanumeric characters or '-' - start with an alphanumeric character - end with an alphanumeric character

func ValidK8sValue 

func ValidK8sValue(inputStr string) string

ValidK8sValue converts strings to rfc1123 and add quotes if it an integer

Types

type Customer 

type Customer struct {
	// ID unique ID for this customer
	ID uuid.UUID `yaml:"id" json:"customersuuid"`

	// Customer Name
	Name string `yaml:"name" json:"name"`

	// Logs to monitor
	Logs []LogBuckets `yaml:"logs" json:"logs"`

	// Providers associated with logs
	Providers Providers `yaml:"providers" json:"providers"`

	// Syncconfiguration
	Configuration SyncConfiguration `yaml:"config" json:"configuration"`
	// contains filtered or unexported fields
}

Customer configuration and information

func (*Customer) GetCustomer 

func (c *Customer) GetCustomer(url, id string) (Customer, error)

func (*Customer) LoadFromDisk 

func (c *Customer) LoadFromDisk(file string) ([]Customer, error)

func (*Customer) LoadFromNetwork 

func (c *Customer) LoadFromNetwork(url string) ([]Customer, error)

func (*Customer) ShortName 

func (c *Customer) ShortName() string

type Customers 

type Customers struct {
	Customers []Customer `yaml:"customers"`
}

type Dependencies 

type Dependencies struct {
	Name            string `yaml:"name"`
	EventSourceName string `yaml:"eventSourceName"`
	EventName       string `yaml:"eventName"`
}

type Environment 

type Environment struct {
	LoadFrom             string `yaml:"loadFrom"`
	EnvironmentName      string `yaml:"environmentName"`
	EventBridgeConfigURL string `yaml:"eventBridgeConfigURL"`
	EventBridgePlogsURL  string `yaml:"eventBridgePlogsURL"`
	EventBridgePostHost  string `yaml:"eventBridgePostHost"`
	ConfigFile           string `yaml:"configFile"`
	K8SService           string `yaml:"k8sService"`
}

Environment i,e. dev/test/staging/production

func (*Environment) Get 

func (e *Environment) Get() Environment

func (*Environment) LoadFromDisk 

func (e *Environment) LoadFromDisk(file string) (Environment, error)

func (*Environment) Patch 

func (e *Environment) Patch()

Patch overload defaults from environment variables

type Label 

type Label struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type LambdaTrigger 

type LambdaTrigger struct {
	// Name for this trigger
	Name string `yaml:"name"`

	// FunctionName name of lambda function to call
	FunctionName string `yaml:"functionName"`

	// Provider credentials
	Provider string `yaml:"provider"`

	// Region to execute function in
	//   TODO: Pull from provider if available
	Region string `yaml:"region"`
}

type LogBuckets 

type LogBuckets struct {

	// Name of the bucket
	Name string `yaml:"name" json:"name"`

	// LogFormat S3, w3c, etc
	LogFormat string `yaml:"logFormat" json:"logFormat"`

	// Provider credentials
	Provider string `yaml:"provider" json:"provider"`

	// PruneAfterProcessing aka delete when down
	PruneAfterProcessing bool `yaml:"pruneAfterProcessing" json:"pruneAfterProcessing"`

	// FilterEvents to apply to the log
	FilterEvents S3Filter `yaml:"filter" json:"filterevents"`
}

LogBuckets Information on a bucket to monitor

type LogConfig 

type LogConfig struct {
	LoadFrom string `json:"loadFrom"` // network or filesystem
	LoadURL  string `json:"loadURL"`

	CustID       string `json:"custID"`       // Is prefixed to the file name on disk
	PlogConfigID string `json:"plogConfigID"` // ID when loading logs from the network
}

LogConfig passed data allowing processed logs to be read from a disk file or a eest end point

type LogFormat 

type LogFormat string

func (*LogFormat) String 

func (lf *LogFormat) String() string

type LogQueue 

type LogQueue []LogQueueItem

type LogQueueItem 

type LogQueueItem struct {
	ID           string        `json:"id"`
	Bucket       string        `json:"bucket"`
	Webhook      WebHookConfig `json:"webhook"`
	Filter       S3Filter      `json:"filter"`
	Name         string        `json:"name"`
	Created      time.Time     `json:"created"`
	Location     string        `json:"location"`
	LogFormat    string        `json:"logFormat"`
	Processed    bool          `json:"processed"`
	PlogConfigID string        `json:"plogConfigID"`
	Prune        bool          `json:"prune"`
}

type ProcessedLogItem 

type ProcessedLogItem struct {
	// Date log was processed
	Date time.Time `json:"date"`

	// Name of bucket containing the log
	Bucket string `json:"bucket"`

	// Name of log in the bucket
	Name string `json:"name"`

	// FileName of downloaded log
	FileName string `json:"fileName"`

	// Pruned if log was delete, FileName, after processing
	Pruned bool `json:"pruned"`
}

ProcessedLogItem information on a processed log

type ProcessedLogs 

type ProcessedLogs struct {
	// ID customer ID
	ID uuid.UUID `json:"plogsuuid"`

	// Processed list of processed logs
	ProcessedItems []ProcessedLogItem `json:"processeditems"`

	// Processed list of processed logs
	Created time.Time `json:"created"`
	Updated time.Time `json:"updated"`
}

ProcessedLogs for a given customer

func (*ProcessedLogs) AddProcessLog 

func (pls *ProcessedLogs) AddProcessLog(ID string, log ProcessedLogItem, conf LogConfig) error

func (*ProcessedLogs) Load 

func (pls *ProcessedLogs) Load(conf LogConfig) error

func (*ProcessedLogs) LoadFromDisk 

func (pls *ProcessedLogs) LoadFromDisk(ID string) error

func (*ProcessedLogs) LoadFromNetwork 

func (pls *ProcessedLogs) LoadFromNetwork(conf LogConfig) error

func (*ProcessedLogs) Processed 

func (pls *ProcessedLogs) Processed(bucket, name string) bool

Processed returns true if we've processed this log before 

false if we have not

func (*ProcessedLogs) Save 

func (pls *ProcessedLogs) Save(conf LogConfig) error

func (*ProcessedLogs) SaveToDisk 

func (pls *ProcessedLogs) SaveToDisk(ID string) error

func (*ProcessedLogs) SaveToNetwork 

func (pls *ProcessedLogs) SaveToNetwork(conf LogConfig) error

type Provider 

type Provider struct {
	// Name AWS, GCP, etc
	Name string `yaml:"name" json:"name"`

	// Credentials key name
	Credentials string `yaml:"credentials" json:"credentials"`
	// Key value
	Key string `yaml:"key" json:"key"`

	// Region us-west-1, etc
	Region string `yaml:"region" json:"region"`

	// Endpoint s3.aws.com
	Endpoint string `yaml:"endpoint" json:"endpoint"`
}

Provider for creating new client

type Providers 

type Providers []Provider

func (*Providers) Lookup 

func (ps *Providers) Lookup(pName string) (Provider, error)

type S3Filter 

type S3Filter struct {
	MatchedAPI          []string `yaml:"matchedAPI" json:"matchedAPI"`
	MatchedHTTPMethods  []string `yaml:"matchedHTTPMethods" json:"matchedHTTPMethods"`
	MatchedResouceTypes []string `yaml:"matchedResouceTypes" json:"matchedResouceTypes"`
}

S3Filter use for filtering logic

type S3LogLine 

type S3LogLine struct {
	BucketOwner    string `json:"bucketOwner"`    //0
	Bucket         string `json:"bucket"`         //1
	Time           string `json:"time"`           //2
	RemoteIP       string `json:"remoteIP"`       //3
	Requester      string `json:"requester"`      //4
	RequestId      string `json:"requestId"`      //5
	Operation      string `json:"operation"`      //6
	Key            string `json:"key"`            //7
	RequestURI     string `json:"requestURI"`     //8
	HttpStatusCode int    `json:"httpStatusCode"` //9
	ErrorCode      string `json:"errorCode"`      //10
	BytesSent      int    `json:"bytesSent"`      //11
	ObjectSize     int    `json:"objectSize"`     //12
	TotalTime      int    `json:"totalTime"`      //13
	TurnAroundTime int    `json:"turnAroundTime"` //14
	Referrer       string `json:"referrer"`       //15
	UserAgent      string `json:"userAgent"`      //16
	VersionId      string `json:"versionId"`      //17
}

S3LogLine format of S3 buckets logs

func ParseS3 

func ParseS3(file string) ([]S3LogLine, error)

ParseS3 S3 log file

func (*S3LogLine) GetOperation 

func (li *S3LogLine) GetOperation() S3Operation

func (*S3LogLine) String 

func (li *S3LogLine) String() string

String print a log line as a string

type S3LogLines 

type S3LogLines []S3LogLine

type S3Operation 

type S3Operation struct {
	API          string `json:"api"`
	HTTPMethod   string `json:"httpMethod"`
	ResourceType string `json:"resourceType"`
}

S3Operation parsed operations method of a log line

func (*S3Operation) FilterLine 

func (s3opt *S3Operation) FilterLine(s3line S3LogLine, filter S3Filter) bool

type S3PatternFilter 

type S3PatternFilter struct {
	Match   string
	Type    string
	ApplyTo string
}

Future use for filtering logic

type S3RequestURI 

type S3RequestURI struct {
	HTTPMethod string `json:"httpMethod"`
	Path       string `json:"path"`
	Protocol   string `json:"protocol"`
}

S3RequestURI 

Future use for parsing sub-objects

type SyncConfiguration 

type SyncConfiguration struct {
	// Name of this sync configuration
	Name string `yaml:"name" json:"name"`

	// Environment of this sync configuration
	Environment string `yaml:"env" json:"environment"`

	// Version of this configuration
	Version string `yaml:"version" json:"version"`

	// Hook web hook to post events to
	Hook WebHookConfig `yaml:"hook" json:"hook"`

	// TODO: Move to environment
	Kubectx string `yaml:"kubectx" json:"kubectx"`

	// Dependencies for triggers to listen for
	//  Basically the name of the webhook
	Dependencies Dependencies `yaml:"dependencies" json:"dependencies"`

	// Triggers to fire for sync event
	Triggers Triggers `yaml:"triggers" json:"triggers"`

	// ManifestDirectory to save manifests in
	ManifestDirectory string `yaml:"manifests" json:"manifests"`

	// TemplateDirctory to load templates from
	TemplateDirctory string `yaml:"templates" json:"templates"`

	// PlogConfigID
	PlogConfigID string `yaml:"plogConfigID" json:"plogConfigID"`
}

func (*SyncConfiguration) DeleteDeployment 

func (sc *SyncConfiguration) DeleteDeployment(cf *Customer, caller SyncInitiator) (data []byte, err error)

DeleteDeployment to using the Kubectx provided

func (*SyncConfiguration) DeployManifests 

func (sc *SyncConfiguration) DeployManifests(cf *Customer, caller SyncInitiator) (data []byte, err error)

DeployManifests to using the Kubectx provided

func (*SyncConfiguration) GenerateManifests 

func (sc *SyncConfiguration) GenerateManifests(cf *Customer, caller SyncInitiator)

GenerateManifests

func (*SyncConfiguration) GetDeployments 

func (sc *SyncConfiguration) GetDeployments(cf *Customer, caller SyncInitiator) (data []byte, err error)

GetDeployments to using the Kubectx provided

func (*SyncConfiguration) KubeExec 

func (sc *SyncConfiguration) KubeExec(options ...string) (data []byte, err error)

type SyncInitiator 

type SyncInitiator struct {
	// Mandatory
	// CustomerID
	CustomerID string

	// UserID requesting sync
	UserID string

	// ConfID configuration being synced
	ConfID string

	// Optional
	// AuthorizationToken
	AuthorizationToken string

	// ReferenceID
	ReferenceID string
}

SyncInitiator 

Is used to label generated structures and authenticate
synchronization requests if required

func (*SyncInitiator) GenerateLables 

func (si *SyncInitiator) GenerateLables() []Label

type Triggers 

type Triggers struct {
	Lambda []LambdaTrigger `yaml:"lambda"`
}

type WebHookConfig 

type WebHookConfig struct {
	Name string `yaml:"name"`
	Host string `yaml:"host"`
	Port string `yaml:"port"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.