file_system

Documentation

Index

• Constants
• func SyncRepo(fs *RepoFs, done chan int) error
• type CountWriter
  • func (c *CountWriter) Write(b []byte) (int, error)
• type FileSyncMode
• type LocalFs
• type RepoFs
  • func NewRepoFs(basePath string, concurrency int, repoName string, branchName string, ...) *RepoFs
  • func (fs *RepoFs) CheckCommit() error
  • func (fs *RepoFs) CloseCommit() error
  • func (fs *RepoFs) Delete(path string) error
  • func (fs *RepoFs) Exists(path string) bool
  • func (fs *RepoFs) GetCommitId() string
  • func (fs *RepoFs) GetLocalFilePath(fileName string) string
  • func (fs *RepoFs) GetRepoId() string
  • func (fs *RepoFs) GetWorkingDir() string
  • func (fs *RepoFs) ListFiles(rel_path string, r bool) ([]os.FileInfo, error)
  • func (fs *RepoFs) Mount() error
  • func (fs *RepoFs) PrintFileList(subpath string) error
  • func (fs *RepoFs) PushDir(subpath string) (size int64, fnError error)
  • func (fs *RepoFs) PushModelDir() (size int64, fnError error)
  • func (fs *RepoFs) PushOutputDir() (size int64, fnError error)
  • func (fs *RepoFs) Reader(path string, offset uint64, size uint64) (io.ReadCloser, error)
  • func (fs *RepoFs) RecordSyncTime(last_mod time.Time)
  • func (fs *RepoFs) Writer(path string) (io.WriteCloser, error)
• type SyncOpLimiter
  • func NewOpLimiter(ops int) SyncOpLimiter

Constants

const (
	RepoSyncTicker                 = 900 // secs
	DefaultFileOpLimit             = 5
	DefaultRepoDirPerm os.FileMode = 0775

	DefaultModelPath             = "/saved_models"
	DefaultModelPerm os.FileMode = 0775

	DefaultOutPath             = "/out"
	DefaultOutPerm os.FileMode = 0775

	UserPath = "/wh_data"
)

Functions

func SyncRepo

func SyncRepo(fs *RepoFs, done chan int) error

Types

type CountWriter

type CountWriter struct {
	// contains filtered or unexported fields
}

func (*CountWriter) Write

func (c *CountWriter) Write(b []byte) (int, error)

type FileSyncMode

type FileSyncMode string

const (
	SyncModeD  FileSyncMode = "DIRECT"
	SyncModeTS FileSyncMode = "THROUGH_SERVER"
)

type LocalFs

type LocalFs interface {
	GetWorkingDir() string
	MakeFile(fpath string, f func(io.Writer) error) (int64, error)
	Writer(path string) (io.WriteCloser, error)
	Reader(path string, offset uint64, size uint64) (io.ReadCloser, error)
	Delete(path string) error
	Exists(path string) bool
}

type RepoFs

type RepoFs struct {
	// contains filtered or unexported fields
}

func NewRepoFs

func NewRepoFs(basePath string, concurrency int, repoName string, branchName string, commitId string, wc *api_client.WorkerClient) *RepoFs

func (*RepoFs) CheckCommit

func (fs *RepoFs) CheckCommit() error

func (*RepoFs) CloseCommit

func (fs *RepoFs) CloseCommit() error

func (*RepoFs) Delete

func (fs *RepoFs) Delete(path string) error

func (*RepoFs) Exists

func (fs *RepoFs) Exists(path string) bool

func (*RepoFs) GetCommitId

func (fs *RepoFs) GetCommitId() string

func (*RepoFs) GetLocalFilePath

func (fs *RepoFs) GetLocalFilePath(fileName string) string

func (*RepoFs) GetRepoId

func (fs *RepoFs) GetRepoId() string

func (*RepoFs) GetWorkingDir

func (fs *RepoFs) GetWorkingDir() string

func (*RepoFs) ListFiles

func (fs *RepoFs) ListFiles(rel_path string, r bool) ([]os.FileInfo, error)

func (*RepoFs) Mount

func (fs *RepoFs) Mount() error

Mount files from remote to local file system

func (*RepoFs) PrintFileList

func (fs *RepoFs) PrintFileList(subpath string) error

func (*RepoFs) PushDir

func (fs *RepoFs) PushDir(subpath string) (size int64, fnError error)

func (*RepoFs) PushModelDir

func (fs *RepoFs) PushModelDir() (size int64, fnError error)

func (*RepoFs) PushOutputDir

func (fs *RepoFs) PushOutputDir() (size int64, fnError error)

Description: Upload files from out directory to commit map

func (*RepoFs) Reader

func (fs *RepoFs) Reader(path string, offset uint64, size uint64) (io.ReadCloser, error)

func (*RepoFs) RecordSyncTime

func (fs *RepoFs) RecordSyncTime(last_mod time.Time)

func (*RepoFs) Writer

func (fs *RepoFs) Writer(path string) (io.WriteCloser, error)

type SyncOpLimiter

type SyncOpLimiter interface {
	Ask()
	Release()
	Wait()
}

func NewOpLimiter

func NewOpLimiter(ops int) SyncOpLimiter