sftprelay

Documentation

Index

• Constants
• Variables
• func GetFileSize(filepath string) (int64, error)
• func NewRemoteFileInfo(path string, fileInfo os.FileInfo) agaveproto.RemoteFileInfo
• func NewSSHClientCallback(ctx context.Context, id string) (*ssh.Client, error)
• func ProtobufFromJSON(jsonSystemConfig string) (*agaveproto.RemoteSystemConfig, error)
• func ProtobufToJSON(systemConfig *agaveproto.RemoteSystemConfig) (string, error)
• type SSHConfig
  • func NewSSHConfig(systemConfig *agaveproto.RemoteSystemConfig) (*SSHConfig, error)
  • func (c *SSHConfig) BaseURI() string
  • func (c *SSHConfig) String() string
• type Server
  • func (s *Server) AuthCheck(ctx context.Context, req *agaveproto.AuthenticationCheckRequest) (*agaveproto.EmptyResponse, error)
  • func (s *Server) Disconnect(ctx context.Context, req *agaveproto.AuthenticationCheckRequest) (*agaveproto.EmptyResponse, error)
  • func (s *Server) Get(ctx context.Context, req *agaveproto.SrvGetRequest) (*agaveproto.TransferResponse, error)
  • func (s *Server) InitMetrics()
  • func (s *Server) List(ctx context.Context, req *agaveproto.SrvListRequest) (*agaveproto.FileInfoListResponse, error)
  • func (s *Server) Mkdir(ctx context.Context, req *agaveproto.SrvMkdirRequest) (*agaveproto.FileInfoResponse, error)
  • func (s *Server) Put(ctx context.Context, req *agaveproto.SrvPutRequest) (*agaveproto.TransferResponse, error)
  • func (s *Server) Remove(ctx context.Context, req *agaveproto.SrvRemoveRequest) (*agaveproto.EmptyResponse, error)
  • func (s *Server) Rename(ctx context.Context, req *agaveproto.SrvRenameRequest) (*agaveproto.FileInfoResponse, error)
  • func (s *Server) SetLogLevel(level logrus.Level)
  • func (s *Server) Stat(ctx context.Context, req *agaveproto.SrvStatRequest) (*agaveproto.FileInfoResponse, error)

Constants

const MAX_CONNECTION_RETRIES = 1

Variables

var (

	// dynamic metrics showing various pool dimensions.
	// these metrics are capturned every time the metrics endpoint is called by invoking the callback functions
	ConnectionPoolMaxSizeMetric = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
		Namespace: "sftprelay",
		Name:      "connection_pool_max_size",
		Help:      "Connection pool max size",
	}, func() float64 {
		return float64(viper.GetInt("poolSize"))
	})
	SessionPoolMaxSizeMetric = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
		Namespace: "sftprelay",
		Name:      "session_pool_max_size",
		Help:      "Maximum size of a session client pool",
	}, func() float64 {
		return float64(-1)
	})

	ConnectionPoolResetCounterMetric = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "sftprelay",
		Name:      "connection_pool_active_clients",
		Help:      "Total complete pool resets",
	}, nil)
	SessionPoolExhaustedCounterMetric = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "sftprelay",
		Name:      "session_pool_exhausted_total",
		Help:      "Total number of failed client creations due to session pool exhaustion",
	}, []string{"host", "port", "user"})
	SessionInvalidationCounterMetric = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "sftprelay",
		Name:      "session_invalidation_total",
		Help:      "Total number of failed client creations due to session pool exhaustion",
	}, []string{"host", "port", "user"})

	// queue dimensions capacity metrics populated through a goroutine running checking queue status every 5 seconds
	SessionPoolActiveClientsMetric = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "sftprelay",
		Name:      "session_pool_active_clients",
		Help:      "Total active clients for a session",
	}, []string{"host", "port", "user"})

	// Number of downloads by file size magnitude
	DownloadCounterMetric = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "sftprelay",
		Name:      "get_total_count",
		Help:      "File downloads to date",
	}, []string{"host", "port", "user", "magnitude"})

	// Summary stats of download file size
	DownloadBytesSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: "sftprelay",
		Name:      "get_total_bytes",
		Help:      "File download size summary distribution in bytes",
	}, []string{"host", "port", "user"})

	// Summary stats of file download duration in sec
	DownloadDurationSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: "sftprelay",
		Name:      "get_duration_seconds",
		Help:      "File download duration by file magnitude",
	}, []string{"host", "port", "user", "magnitude"})

	// Number of uploads by file size magnitude
	UploadCounterMetric = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "sftprelay",
		Name:      "put_total_count",
		Help:      "File uploads to date",
	}, []string{"host", "port", "user", "magnitude"})

	// Summary stats of upload file size
	UploadBytesSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: "sftprelay",
		Name:      "put_total_bytes",
		Help:      "File upload size summary distribution in bytes",
	}, []string{"host", "port", "user"})

	// Summary stats of file upload duration in sec
	UploadDurationSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace: "sftprelay",
		Name:      "put_duration_seconds",
		Help:      "File uploaded duration by file magnitude",
	}, []string{"host", "port", "user", "magnitude"})

	// Individual method invocation metrics
	MethodDurationSummary = prometheus.NewSummaryVec(prometheus.SummaryOpts{
		Namespace:   "sftprelay",
		Name:        "duration_seconds",
		Help:        "Operation duration summary in seconds",
		ConstLabels: map[string]string{"grpc_service": "sftpproto.SftpRelay", "grpc_type": "unary"},
	}, []string{"grpc_method", "host", "port", "user"})
)

The Pool and AgentSocket variables are created here and instantiated in the init function.

Functions

func GetFileSize

func GetFileSize(filepath string) (int64, error)

This will return the size of the file in bytes. It is not presently used.

func NewRemoteFileInfo

func NewRemoteFileInfo(path string, fileInfo os.FileInfo) agaveproto.RemoteFileInfo

func NewSSHClientCallback

func NewSSHClientCallback(ctx context.Context, id string) (*ssh.Client, error)

Implements the callback function to create a new ssh.Client from the SSHConfig

func ProtobufFromJSON

func ProtobufFromJSON(jsonSystemConfig string) (*agaveproto.RemoteSystemConfig, error)

deserializes remote system config proto from json for use creating connections in the ssh client connection pool

func ProtobufToJSON

func ProtobufToJSON(systemConfig *agaveproto.RemoteSystemConfig) (string, error)

serializes the remote system config to json for use as an id in the ssh client connection pool

Types

type SSHConfig

type SSHConfig struct {
	User     string
	Host     string
	Port     int
	Auth     []ssh.AuthMethod
	HashSalt string

	// Timeout is the maximum amount of time for the TCP connection to establish.
	Timeout time.Duration

	// TCPKeepAlive specifies whether to send TCP keepalive messages
	// to the other side.
	TCPKeepAlive bool
	// TCPKeepAlivePeriod specifies the TCP keepalive frequency.
	TCPKeepAlivePeriod time.Duration

	// AgentSocket is the path to the socket of authentication agent.
	AgentSocket string
	// ForwardAgent specifies whether the connection to the authentication agent
	// (if any) will be forwarded to the remote machine.
	ForwardAgent bool

	HostKeyCallback func(hostname string, remote net.Addr, key ssh.PublicKey) error
}

SSHConn defines the configuration options of the SSH connection.

func NewSSHConfig

func NewSSHConfig(systemConfig *agaveproto.RemoteSystemConfig) (*SSHConfig, error)

generates a new SSHConfig from the *agaveproto.RemoteSystemConfig included in every server request

func (*SSHConfig) BaseURI

func (c *SSHConfig) BaseURI() string

BaseURI returns the base portion of the remote host URI up to the path

func (*SSHConfig) String

func (c *SSHConfig) String() string

String returns a hash string generated from the SSH config parameters.

type Server

type Server struct {
	Registry    prometheus.Registry
	GrpcMetrics grpc_prometheus.ServerMetrics
	Pool        *clientpool.ClientPool
}

func (*Server) AuthCheck

func (s *Server) AuthCheck(ctx context.Context, req *agaveproto.AuthenticationCheckRequest) (*agaveproto.EmptyResponse, error)

Performs a basic authentication handshake to the remote system

func (*Server) Disconnect

func (s *Server) Disconnect(ctx context.Context, req *agaveproto.AuthenticationCheckRequest) (*agaveproto.EmptyResponse, error)

Invalidates the current session. This will kill all currently operating clients bound to the session pool

func (*Server) Get

func (s *Server) Get(ctx context.Context, req *agaveproto.SrvGetRequest) (*agaveproto.TransferResponse, error)

Fetches a file from the remote system and stores it locally

func (*Server) InitMetrics

func (s *Server) InitMetrics()

Initializes the metrics for this service

func (*Server) List

func (s *Server) List(ctx context.Context, req *agaveproto.SrvListRequest) (*agaveproto.FileInfoListResponse, error)

Transfers a file from the local system to the remote system

func (*Server) Mkdir

func (s *Server) Mkdir(ctx context.Context, req *agaveproto.SrvMkdirRequest) (*agaveproto.FileInfoResponse, error)

Performs a mkdirs operation on a remote system.

func (*Server) Put

func (s *Server) Put(ctx context.Context, req *agaveproto.SrvPutRequest) (*agaveproto.TransferResponse, error)

Transfers a file from the local system to the remote system

func (*Server) Remove

func (s *Server) Remove(ctx context.Context, req *agaveproto.SrvRemoveRequest) (*agaveproto.EmptyResponse, error)

Removes the remote path. If the remote path is a directory, the entire tree is deleted.

func (*Server) Rename

func (s *Server) Rename(ctx context.Context, req *agaveproto.SrvRenameRequest) (*agaveproto.FileInfoResponse, error)

Fetches file info for a remote path

func (*Server) SetLogLevel

func (s *Server) SetLogLevel(level logrus.Level)

Sets the log level on the current logger of this relay instance

func (*Server) Stat

func (s *Server) Stat(ctx context.Context, req *agaveproto.SrvStatRequest) (*agaveproto.FileInfoResponse, error)

Fetches file info for a remote path