weed_server

package
Version: v0.0.0-...-94a01fc Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 16, 2021 License: Apache-2.0 Imports: 94 Imported by: 0

Documentation

Index

Constants

View Source
const (
	SequencerType        = "master.sequencer.type"
	SequencerEtcdUrls    = "master.sequencer.sequencer_etcd_urls"
	SequencerSnowflakeId = "master.sequencer.sequencer_snowflake_id"
)
View Source
const BufferSizeLimit = 1024 * 1024 * 2
View Source
const (
	LockDuration = 10 * time.Second
)
View Source
const (
	// MaxUnsyncedEvents send empty notification with timestamp when certain amount of events have been filtered
	MaxUnsyncedEvents = 1e3
)

Variables

View Source
var (
	OS_UID = uint32(os.Getuid())
	OS_GID = uint32(os.Getgid())

	ErrReadOnly = errors.New("read only")
)
View Source
var StaticFS fs.FS

Functions

func NewWebDavFileSystem

func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error)

func OptionsHandler

func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool)

func SaveAmzMetaData

func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool) (metadata map[string][]byte)

Types

type AdminLock

type AdminLock struct {
	// contains filtered or unexported fields
}

type AdminLocks

type AdminLocks struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewAdminLocks

func NewAdminLocks() *AdminLocks

type ClusterStatusResult

type ClusterStatusResult struct {
	IsLeader    bool             `json:"IsLeader,omitempty"`
	Leader      pb.ServerAddress `json:"Leader,omitempty"`
	Peers       []string         `json:"Peers,omitempty"`
	MaxVolumeId needle.VolumeId  `json:"MaxVolumeId,omitempty"`
}

type FileInfo

type FileInfo struct {
	// contains filtered or unexported fields
}

func (*FileInfo) IsDir

func (fi *FileInfo) IsDir() bool

func (*FileInfo) ModTime

func (fi *FileInfo) ModTime() time.Time

func (*FileInfo) Mode

func (fi *FileInfo) Mode() os.FileMode

func (*FileInfo) Name

func (fi *FileInfo) Name() string

func (*FileInfo) Size

func (fi *FileInfo) Size() int64

func (*FileInfo) Sys

func (fi *FileInfo) Sys() interface{}

type FilerOption

type FilerOption struct {
	Masters            []pb.ServerAddress
	Collection         string
	DefaultReplication string
	DisableDirListing  bool
	MaxMB              int
	DirListingLimit    int
	DataCenter         string
	Rack               string
	DefaultLevelDbDir  string
	DisableHttp        bool
	Host               pb.ServerAddress

	Cipher                bool
	SaveToFilerLimit      int64
	Filers                []pb.ServerAddress
	ConcurrentUploadLimit int64
	// contains filtered or unexported fields
}

type FilerPostResult

type FilerPostResult struct {
	Name  string `json:"name,omitempty"`
	Size  int64  `json:"size,omitempty"`
	Error string `json:"error,omitempty"`
}

type FilerServer

type FilerServer struct {
	// contains filtered or unexported fields
}

func NewFilerServer

func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error)

func (*FilerServer) AppendToEntry

func (*FilerServer) AssignVolume

func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error)

func (*FilerServer) CollectionList

func (*FilerServer) CreateEntry

func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error)

func (*FilerServer) DeleteCollection

func (*FilerServer) DeleteEntry

func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error)

func (*FilerServer) DeleteTaggingHandler

func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Request)

remove all Seaweed- prefixed attributes curl -X DELETE http://localhost:8888/path/to/a/file?tagging

func (*FilerServer) DownloadToLocal

func (*FilerServer) GetFilerConfiguration

func (*FilerServer) GetOrHeadHandler

func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)

func (*FilerServer) KeepConnected

func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error

func (*FilerServer) KvGet

func (*FilerServer) KvPut

KvPut sets the key~value. if empty value, delete the kv entry

func (*FilerServer) ListEntries

func (*FilerServer) LocateBroker

func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error)

func (*FilerServer) LookupVolume

func (*FilerServer) PostHandler

func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, contentLength int64)

func (*FilerServer) PutTaggingHandler

func (fs *FilerServer) PutTaggingHandler(w http.ResponseWriter, r *http.Request)

add or replace one file Seaweed- prefixed attributes curl -X PUT -H "Seaweed-Name1: value1" http://localhost:8888/path/to/a/file?tagging

func (*FilerServer) Statistics

func (fs *FilerServer) Statistics(ctx context.Context, req *filer_pb.StatisticsRequest) (resp *filer_pb.StatisticsResponse, err error)

func (*FilerServer) UpdateEntry

type MasterOption

type MasterOption struct {
	Master            pb.ServerAddress
	MetaFolder        string
	VolumeSizeLimitMB uint32
	VolumePreallocate bool
	// PulseSeconds            int
	DefaultReplicaPlacement string
	GarbageThreshold        float64
	WhiteList               []string
	DisableHttp             bool
	MetricsAddress          string
	MetricsIntervalSec      int
	IsFollower              bool
}

type MasterServer

type MasterServer struct {
	Topo *topology.Topology

	MasterClient *wdclient.MasterClient
	// contains filtered or unexported fields
}

func NewMasterServer

func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer

func (*MasterServer) Assign

func (*MasterServer) CollectionList

func (*MasterServer) KeepConnected

func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error

KeepConnected keep a stream gRPC call to the master. Used by clients to know the master is up. And clients gets the up-to-date list of volume locations

func (*MasterServer) LeaseAdminToken

func (*MasterServer) LookupEcVolume

func (*MasterServer) LookupVolume

func (*MasterServer) ProcessGrowRequest

func (ms *MasterServer) ProcessGrowRequest()

func (*MasterServer) SendHeartbeat

func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error

func (*MasterServer) SetRaftServer

func (ms *MasterServer) SetRaftServer(raftServer *RaftServer)

func (*MasterServer) Statistics

func (*MasterServer) VacuumVolume

func (*MasterServer) VolumeList

type RaftServer

type RaftServer struct {
	*raft.GrpcServer
	// contains filtered or unexported fields
}

func NewRaftServer

func NewRaftServer(grpcDialOption grpc.DialOption, peers []pb.ServerAddress, serverAddr pb.ServerAddress, dataDir string, topo *topology.Topology, raftResumeState bool) (*RaftServer, error)

func (*RaftServer) DoJoinCommand

func (s *RaftServer) DoJoinCommand()

func (*RaftServer) Peers

func (s *RaftServer) Peers() (members []string)

func (*RaftServer) StatusHandler

func (s *RaftServer) StatusHandler(w http.ResponseWriter, r *http.Request)

type StateMachine

type StateMachine struct {
	raft.StateMachine
	// contains filtered or unexported fields
}

func (StateMachine) Recovery

func (s StateMachine) Recovery(data []byte) error

func (StateMachine) Save

func (s StateMachine) Save() ([]byte, error)

type VolumeFileScanner4Tailing

type VolumeFileScanner4Tailing struct {
	// contains filtered or unexported fields
}

generate the volume idx

func (*VolumeFileScanner4Tailing) ReadNeedleBody

func (scanner *VolumeFileScanner4Tailing) ReadNeedleBody() bool

func (*VolumeFileScanner4Tailing) VisitNeedle

func (scanner *VolumeFileScanner4Tailing) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error

func (*VolumeFileScanner4Tailing) VisitSuperBlock

func (scanner *VolumeFileScanner4Tailing) VisitSuperBlock(superBlock super_block.SuperBlock) error

type VolumeServer

type VolumeServer struct {
	SeedMasterNodes []pb.ServerAddress

	FixJpgOrientation bool
	ReadMode          string
	// contains filtered or unexported fields
}

func NewVolumeServer

func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
	port int, grpcPort int, publicUrl string,
	folders []string, maxCounts []int, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType,
	idxFolder string,
	needleMapKind storage.NeedleMapKind,
	masterNodes []pb.ServerAddress, pulseSeconds int,
	dataCenter string, rack string,
	whiteList []string,
	fixJpgOrientation bool,
	readMode string,
	compactionMBPerSecond int,
	fileSizeLimitMB int,
	concurrentUploadLimit int64,
	concurrentDownloadLimit int64,
) *VolumeServer

func (*VolumeServer) CopyFile

CopyFile client pulls the volume related file from the source server. if req.CompactionRevision != math.MaxUint32, it ensures the compact revision is as expected The copying still stop at req.StopOffset, but you can set it to math.MaxUint64 in order to read all data.

func (*VolumeServer) DeleteHandler

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request)

func (*VolumeServer) GetMaster

func (vs *VolumeServer) GetMaster() pb.ServerAddress

func (*VolumeServer) GetOrHeadHandler

func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)

func (*VolumeServer) HandleTcpConnection

func (vs *VolumeServer) HandleTcpConnection(c net.Conn)

func (*VolumeServer) PostHandler

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request)

func (*VolumeServer) ReadNeedleBlob

func (*VolumeServer) SetStopping

func (vs *VolumeServer) SetStopping()

func (*VolumeServer) Shutdown

func (vs *VolumeServer) Shutdown()

func (*VolumeServer) StopHeartbeat

func (vs *VolumeServer) StopHeartbeat() (isAlreadyStopping bool)

func (*VolumeServer) VolumeCopy

VolumeCopy copy the .idx .dat .vif files, and mount the volume

func (*VolumeServer) VolumeEcShardsCopy

VolumeEcShardsCopy copy the .ecx and some ec data slices

func (*VolumeServer) VolumeEcShardsDelete

VolumeEcShardsDelete local delete the .ecx and some ec data slices if not needed the shard should not be mounted before calling this.

func (*VolumeServer) VolumeEcShardsGenerate

VolumeEcShardsGenerate generates the .ecx and .ec00 ~ .ec13 files

func (*VolumeServer) VolumeEcShardsRebuild

VolumeEcShardsRebuild generates the any of the missing .ec00 ~ .ec13 files

func (*VolumeServer) VolumeEcShardsToVolume

VolumeEcShardsToVolume generates the .idx, .dat files from .ecx, .ecj and .ec01 ~ .ec14 files

func (*VolumeServer) VolumeTierMoveDatFromRemote

VolumeTierMoveDatFromRemote copy dat file from a remote tier to local volume server

func (*VolumeServer) VolumeTierMoveDatToRemote

VolumeTierMoveDatToRemote copy dat file to a remote tier

type WebDavFile

type WebDavFile struct {
	// contains filtered or unexported fields
}

func (*WebDavFile) Close

func (f *WebDavFile) Close() error

func (*WebDavFile) Read

func (f *WebDavFile) Read(p []byte) (readSize int, err error)

func (*WebDavFile) Readdir

func (f *WebDavFile) Readdir(count int) (ret []os.FileInfo, err error)

func (*WebDavFile) Seek

func (f *WebDavFile) Seek(offset int64, whence int) (int64, error)

func (*WebDavFile) Stat

func (f *WebDavFile) Stat() (os.FileInfo, error)

func (*WebDavFile) Write

func (f *WebDavFile) Write(buf []byte) (int, error)

type WebDavFileSystem

type WebDavFileSystem struct {
	// contains filtered or unexported fields
}

func (*WebDavFileSystem) AdjustedUrl

func (fs *WebDavFileSystem) AdjustedUrl(location *filer_pb.Location) string

func (*WebDavFileSystem) Mkdir

func (fs *WebDavFileSystem) Mkdir(ctx context.Context, fullDirPath string, perm os.FileMode) error

func (*WebDavFileSystem) OpenFile

func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, flag int, perm os.FileMode) (webdav.File, error)

func (*WebDavFileSystem) RemoveAll

func (fs *WebDavFileSystem) RemoveAll(ctx context.Context, name string) error

func (*WebDavFileSystem) Rename

func (fs *WebDavFileSystem) Rename(ctx context.Context, oldName, newName string) error

func (*WebDavFileSystem) Stat

func (fs *WebDavFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error)

func (*WebDavFileSystem) WithFilerClient

func (fs *WebDavFileSystem) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error

type WebDavOption

type WebDavOption struct {
	Filer          pb.ServerAddress
	DomainName     string
	BucketsPath    string
	GrpcDialOption grpc.DialOption
	Collection     string
	Replication    string
	DiskType       string
	Uid            uint32
	Gid            uint32
	Cipher         bool
	CacheDir       string
	CacheSizeMB    int64
}

type WebDavServer

type WebDavServer struct {
	Handler *webdav.Handler
	// contains filtered or unexported fields
}

func NewWebDavServer

func NewWebDavServer(option *WebDavOption) (ws *WebDavServer, err error)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
t or T : Toggle theme light dark auto
y or Y : Canonical URL