Documentation ¶
Overview ¶
Package recoverylog specifies a finite state machine for recording and replaying observed filesystem operations into a Gazette journal.
Index ¶
- Variables
- type Author
- type FSM
- type FSMHints
- func (*FSMHints) Descriptor() ([]byte, []int)
- func (m FSMHints) LiveLogSegments() ([]Fnode, SegmentSet, error)
- func (m *FSMHints) Marshal() (dAtA []byte, err error)
- func (m *FSMHints) MarshalTo(dAtA []byte) (int, error)
- func (m *FSMHints) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*FSMHints) ProtoMessage()
- func (m *FSMHints) ProtoSize() (n int)
- func (m *FSMHints) Reset()
- func (m *FSMHints) String() string
- func (m *FSMHints) Unmarshal(dAtA []byte) error
- func (h *FSMHints) Validate() error
- func (m *FSMHints) XXX_DiscardUnknown()
- func (m *FSMHints) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *FSMHints) XXX_Merge(src proto.Message)
- func (m *FSMHints) XXX_Size() int
- func (m *FSMHints) XXX_Unmarshal(b []byte) error
- type FileRecorder
- type Fnode
- type FnodeSegments
- func (*FnodeSegments) Descriptor() ([]byte, []int)
- func (m *FnodeSegments) Marshal() (dAtA []byte, err error)
- func (m *FnodeSegments) MarshalTo(dAtA []byte) (int, error)
- func (m *FnodeSegments) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*FnodeSegments) ProtoMessage()
- func (m *FnodeSegments) ProtoSize() (n int)
- func (m *FnodeSegments) Reset()
- func (m *FnodeSegments) String() string
- func (m *FnodeSegments) Unmarshal(dAtA []byte) error
- func (m *FnodeSegments) XXX_DiscardUnknown()
- func (m *FnodeSegments) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *FnodeSegments) XXX_Merge(src proto.Message)
- func (m *FnodeSegments) XXX_Size() int
- func (m *FnodeSegments) XXX_Unmarshal(b []byte) error
- type Player
- type Property
- func (*Property) Descriptor() ([]byte, []int)
- func (m *Property) Marshal() (dAtA []byte, err error)
- func (m *Property) MarshalTo(dAtA []byte) (int, error)
- func (m *Property) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*Property) ProtoMessage()
- func (m *Property) ProtoSize() (n int)
- func (m *Property) Reset()
- func (m *Property) String() string
- func (m *Property) Unmarshal(dAtA []byte) error
- func (m *Property) XXX_DiscardUnknown()
- func (m *Property) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *Property) XXX_Merge(src proto.Message)
- func (m *Property) XXX_Size() int
- func (m *Property) XXX_Unmarshal(b []byte) error
- type RecordedAferoFS
- func (r RecordedAferoFS) Create(name string) (afero.File, error)
- func (r RecordedAferoFS) OpenFile(name string, flags int, perm os.FileMode) (afero.File, error)
- func (r RecordedAferoFS) Remove(name string) error
- func (r RecordedAferoFS) RemoveAll(path string) error
- func (r RecordedAferoFS) Rename(oldname, newname string) error
- type RecordedOp
- func (*RecordedOp) Descriptor() ([]byte, []int)
- func (m *RecordedOp) Marshal() (dAtA []byte, err error)
- func (m *RecordedOp) MarshalTo(dAtA []byte) (int, error)
- func (m *RecordedOp) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*RecordedOp) ProtoMessage()
- func (m *RecordedOp) ProtoSize() (n int)
- func (m *RecordedOp) Reset()
- func (m *RecordedOp) String() string
- func (m *RecordedOp) Unmarshal(dAtA []byte) error
- func (m *RecordedOp) XXX_DiscardUnknown()
- func (m *RecordedOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *RecordedOp) XXX_Merge(src proto.Message)
- func (m *RecordedOp) XXX_Size() int
- func (m *RecordedOp) XXX_Unmarshal(b []byte) error
- type RecordedOp_Create
- func (*RecordedOp_Create) Descriptor() ([]byte, []int)
- func (m *RecordedOp_Create) Marshal() (dAtA []byte, err error)
- func (m *RecordedOp_Create) MarshalTo(dAtA []byte) (int, error)
- func (m *RecordedOp_Create) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*RecordedOp_Create) ProtoMessage()
- func (m *RecordedOp_Create) ProtoSize() (n int)
- func (m *RecordedOp_Create) Reset()
- func (m *RecordedOp_Create) String() string
- func (m *RecordedOp_Create) Unmarshal(dAtA []byte) error
- func (m *RecordedOp_Create) XXX_DiscardUnknown()
- func (m *RecordedOp_Create) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *RecordedOp_Create) XXX_Merge(src proto.Message)
- func (m *RecordedOp_Create) XXX_Size() int
- func (m *RecordedOp_Create) XXX_Unmarshal(b []byte) error
- type RecordedOp_Link
- func (*RecordedOp_Link) Descriptor() ([]byte, []int)
- func (m *RecordedOp_Link) Marshal() (dAtA []byte, err error)
- func (m *RecordedOp_Link) MarshalTo(dAtA []byte) (int, error)
- func (m *RecordedOp_Link) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*RecordedOp_Link) ProtoMessage()
- func (m *RecordedOp_Link) ProtoSize() (n int)
- func (m *RecordedOp_Link) Reset()
- func (m *RecordedOp_Link) String() string
- func (m *RecordedOp_Link) Unmarshal(dAtA []byte) error
- func (m *RecordedOp_Link) XXX_DiscardUnknown()
- func (m *RecordedOp_Link) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *RecordedOp_Link) XXX_Merge(src proto.Message)
- func (m *RecordedOp_Link) XXX_Size() int
- func (m *RecordedOp_Link) XXX_Unmarshal(b []byte) error
- type RecordedOp_Write
- func (*RecordedOp_Write) Descriptor() ([]byte, []int)
- func (m *RecordedOp_Write) Marshal() (dAtA []byte, err error)
- func (m *RecordedOp_Write) MarshalTo(dAtA []byte) (int, error)
- func (m *RecordedOp_Write) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*RecordedOp_Write) ProtoMessage()
- func (m *RecordedOp_Write) ProtoSize() (n int)
- func (m *RecordedOp_Write) Reset()
- func (m *RecordedOp_Write) String() string
- func (m *RecordedOp_Write) Unmarshal(dAtA []byte) error
- func (m *RecordedOp_Write) XXX_DiscardUnknown()
- func (m *RecordedOp_Write) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *RecordedOp_Write) XXX_Merge(src proto.Message)
- func (m *RecordedOp_Write) XXX_Size() int
- func (m *RecordedOp_Write) XXX_Unmarshal(b []byte) error
- type Recorder
- func (r *Recorder) Barrier(waitFor client.OpFutures) *client.AsyncAppend
- func (r *Recorder) BuildHints() (FSMHints, error)
- func (r *Recorder) Dir() string
- func (r *Recorder) DisableRegisterChecks()
- func (r *Recorder) RecordCreate(path string) Fnode
- func (r *Recorder) RecordLink(src, target string)
- func (r *Recorder) RecordOpenFile(path string, flags int) Fnode
- func (r *Recorder) RecordRemove(path string)
- func (r *Recorder) RecordRename(src, target string)
- func (r *Recorder) RecordWriteAt(fnode Fnode, data []byte, offset int64)
- type Segment
- func (*Segment) Descriptor() ([]byte, []int)
- func (m *Segment) Marshal() (dAtA []byte, err error)
- func (m *Segment) MarshalTo(dAtA []byte) (int, error)
- func (m *Segment) MarshalToSizedBuffer(dAtA []byte) (int, error)
- func (*Segment) ProtoMessage()
- func (m *Segment) ProtoSize() (n int)
- func (m *Segment) Reset()
- func (m *Segment) String() string
- func (m *Segment) Unmarshal(dAtA []byte) error
- func (s Segment) Validate() error
- func (m *Segment) XXX_DiscardUnknown()
- func (m *Segment) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (m *Segment) XXX_Merge(src proto.Message)
- func (m *Segment) XXX_Size() int
- func (m *Segment) XXX_Unmarshal(b []byte) error
- type SegmentSet
Constants ¶
This section is empty.
Variables ¶
var ( ErrChecksumMismatch = fmt.Errorf("checksum mismatch") ErrExpectedHintedFnode = fmt.Errorf("op is not create, and an Fnode was hinted") ErrFnodeNotTracked = fmt.Errorf("fnode not tracked") ErrLinkExists = fmt.Errorf("link exists") ErrNoSuchLink = fmt.Errorf("fnode has no such link") ErrNotHintedAuthor = fmt.Errorf("op author does not match the next hinted author") ErrPropertyExists = fmt.Errorf("property exists") ErrWrongSeqNo = fmt.Errorf("wrong sequence number") )
Functions ¶
This section is empty.
Types ¶
type Author ¶
type Author uint32
Author is a random, unique ID which identifies a processes that creates RecordedOps. It's used by FSM to allow for reconciliation of divergent histories in the recovery log, through hints as to which Authors produced which Segments in the final history. Authors are also used for cooperative write-fencing between Player.InjectHandoff and Recorder.
func NewRandomAuthor ¶ added in v0.83.1
func NewRandomAuthor() Author
NewRandomAuthor creates and returns a new, randomized Author in the range [1, math.MaxUint32].
type FSM ¶
type FSM struct { // Expected sequence number and checksum of next operation. NextSeqNo int64 NextChecksum uint32 // Log of the last sequenced operation. LastLog pb.Journal // Target paths and contents of small files which are managed outside of // regular Fnode tracking. Property updates are triggered upon rename of // a tracked Fnode to a well-known property file path. // // Property paths must be "sinks" which: // * Are never directly written to. // * Are never renamed or linked from. // * Have exactly one hard-link (eg, only "rename" to the property path is // supported; "link" is not as it would introduce a second hard-link). Properties map[string]string // Maps from Fnode to current state of the node. LiveNodes map[Fnode]*fnodeState // Indexes current target paths of LiveNodes. Links map[string]Fnode // contains filtered or unexported fields }
FSM implements a finite state machine over RecordedOp. In particular FSM applies RecordedOp in order, verifying the SeqNo and Checksum of each operation. This ensures that only operations which are linear and consistent are applied.
func (*FSM) Apply ¶
func (m *FSM) Apply(op *RecordedOp, frame []byte) error
Apply attempts to transition the FSMs state by |op| & |frame|. It either performs a transition, or returns an error detailing how the operation is not consistent with prior applied operations or hints. A state transition occurs if and only if the returned err is nil, with one exception: ErrFnodeNotTracked may be returned to indicate that the operation is consistent and a transition occurred, but that FSMHints also indicate the Fnode no longer exists at the point-in-time at which the FSMHints were created, and the caller may therefor want to skip corresponding local playback actions.
type FSMHints ¶
type FSMHints struct { // Log is the implied recovery log of any contained |live_nodes| Segments // which omit a |log| value. This implied behavior is both for backward- // compatibility (Segments didn't always have a |log| field) and also for // compacting the representation in the common case of Segments mostly or // entirely addressing a single log. Log go_gazette_dev_core_broker_protocol.Journal `protobuf:"bytes,1,opt,name=log,proto3,casttype=go.gazette.dev/core/broker/protocol.Journal" json:"log,omitempty"` // Live Fnodes and their Segments as-of the generation of these FSMHints. LiveNodes []FnodeSegments `protobuf:"bytes,2,rep,name=live_nodes,json=liveNodes,proto3" json:"live_nodes"` // Property files and contents as-of the generation of these FSMHints. Properties []Property `protobuf:"bytes,3,rep,name=properties,proto3" json:"properties"` }
FSMHints represents a manifest of Fnodes which were still live (eg, having remaining links) at the time the FSMHints were produced, as well as any Properties. It allows a Player of the log to identify minimal Segments which must be read to recover all Fnodes, and also contains sufficient metadata for a Player to resolve all possible conflicts it could encounter while reading the log, to arrive at a consistent view of file state which exactly matches that of the Recorder producing the FSMHints. Next tag: 4.
func (*FSMHints) Descriptor ¶
func (FSMHints) LiveLogSegments ¶
func (m FSMHints) LiveLogSegments() ([]Fnode, SegmentSet, error)
LiveLogSegments flattens hinted LiveNodes into an ordered list of Fnodes, and the set of recovery log Segments which fully contain them.
func (*FSMHints) MarshalToSizedBuffer ¶ added in v0.86.1
func (*FSMHints) ProtoMessage ¶
func (*FSMHints) ProtoMessage()
func (*FSMHints) XXX_DiscardUnknown ¶
func (m *FSMHints) XXX_DiscardUnknown()
func (*FSMHints) XXX_Marshal ¶
func (*FSMHints) XXX_Unmarshal ¶
type FileRecorder ¶
FileRecorder is a convenience which associates an Fnode with its recorder and written offset.
func (*FileRecorder) RecordWrite ¶
func (r *FileRecorder) RecordWrite(data []byte)
RecordWrite records the write of |data| at the current FileRecorder Offset, which is then incremented to reflect |data|.
func (*FileRecorder) RecordWriteAt ¶ added in v0.84.1
func (r *FileRecorder) RecordWriteAt(data []byte, offset int64)
RecordWriteAt records the write of |data| at |offset|. The current FileRecorder offset is unchanged.
type Fnode ¶
type Fnode int64
An Fnode is an identifier which represents a file across its renames, links, and un-links within a file-system. When a file is created, it's assigned an Fnode value equal to the RecordedOp.SeqNo which created it.
type FnodeSegments ¶
type FnodeSegments struct { // Fnode being hinted. Fnode Fnode `protobuf:"varint,1,opt,name=fnode,proto3,casttype=Fnode" json:"fnode,omitempty"` // Segments of the Fnode in the log. Currently, FSM tracks only a single // Segment per Fnode per Author & Log. A specific implication of this is that Fnodes // modified over long periods of time will result in Segments spanning large // chunks of the log. For best performance, Fnodes should be opened & written // once, and then never be modified again (this is RocksDB's behavior). // If supporting this case is desired, FSM will have to be a bit smarter about // not extending Segments which gap over significant portions of the log // (eg, there's a trade-off to make over size of the hinted manifest, vs // savings incurred on playback by being able to skip portions of the log). Segments []Segment `protobuf:"bytes,2,rep,name=segments,proto3" json:"segments"` }
FnodeSegments captures log Segments containing all RecordedOps of the Fnode.
func (*FnodeSegments) Descriptor ¶
func (*FnodeSegments) Descriptor() ([]byte, []int)
func (*FnodeSegments) Marshal ¶
func (m *FnodeSegments) Marshal() (dAtA []byte, err error)
func (*FnodeSegments) MarshalToSizedBuffer ¶ added in v0.86.1
func (m *FnodeSegments) MarshalToSizedBuffer(dAtA []byte) (int, error)
func (*FnodeSegments) ProtoMessage ¶
func (*FnodeSegments) ProtoMessage()
func (*FnodeSegments) ProtoSize ¶
func (m *FnodeSegments) ProtoSize() (n int)
func (*FnodeSegments) Reset ¶
func (m *FnodeSegments) Reset()
func (*FnodeSegments) String ¶
func (m *FnodeSegments) String() string
func (*FnodeSegments) Unmarshal ¶
func (m *FnodeSegments) Unmarshal(dAtA []byte) error
func (*FnodeSegments) XXX_DiscardUnknown ¶
func (m *FnodeSegments) XXX_DiscardUnknown()
func (*FnodeSegments) XXX_Marshal ¶
func (m *FnodeSegments) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*FnodeSegments) XXX_Merge ¶
func (m *FnodeSegments) XXX_Merge(src proto.Message)
func (*FnodeSegments) XXX_Size ¶
func (m *FnodeSegments) XXX_Size() int
func (*FnodeSegments) XXX_Unmarshal ¶
func (m *FnodeSegments) XXX_Unmarshal(b []byte) error
type Player ¶
type Player struct { // Resolved outputs which may be read once the Play completes. Resolved struct { Log pb.Journal // Primary Log which was played back (i.e., FSMHints.Log). FSM *FSM // FSM recovered at Play completion. Dir string // Local directory into which the log is recovered. } // contains filtered or unexported fields }
Player reads from a log to rebuild encoded file operations onto the local filesystem. Player is a future: on completion of a successful Play (or equivalently, upon Done selecting) |Resolved| may be inspected for recovered playback state. |Resolved| is not valid before Play completes, or if Play returns an error.
func (*Player) Done ¶
func (p *Player) Done() <-chan struct{}
Done returns a channel which selects when Play has completed. If Play returned no error, then Player.Resolved will hold the recovered playback state. Otherwise, Player.Resolved will be zero-valued.
func (*Player) FinishAtWriteHead ¶
func (p *Player) FinishAtWriteHead()
FinishAtWriteHead requests that playback complete upon reaching the current write head. Only one invocation of FinishAtWriteHead or InjectHandoff may be made of a Player instance.
func (*Player) InjectHandoff ¶
InjectHandoff requests that playback complete upon injecting a no-op handoff of the given |author| at the log head. Only one invocation of InjectHandoff or FinishAtWriteHead may be made of a Player instance. |author| must be non- zero or InjectHandoff panics.
func (*Player) Play ¶
func (p *Player) Play(ctx context.Context, hints FSMHints, dir string, ajc client.AsyncJournalClient) error
Play uses the prepared Player to play back the FSMHints. It returns on the first encountered unrecoverable error, including context cancellation, or upon a successful FinishAtWriteHead or InjectHandoff.
type Property ¶
type Property struct { // Filesystem path of this property, relative to the common base directory. Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` // Complete file content of this property. Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` }
Property is a small file which rarely changes, and is thus managed outside of regular Fnode tracking. See FSM.Properties.
func (*Property) Descriptor ¶
func (*Property) MarshalToSizedBuffer ¶ added in v0.86.1
func (*Property) ProtoMessage ¶
func (*Property) ProtoMessage()
func (*Property) XXX_DiscardUnknown ¶
func (m *Property) XXX_DiscardUnknown()
func (*Property) XXX_Marshal ¶
func (*Property) XXX_Unmarshal ¶
type RecordedAferoFS ¶
RecordedAferoFS adapts a Recorder to wrap a afero.Fs instance.
func (RecordedAferoFS) Remove ¶
func (r RecordedAferoFS) Remove(name string) error
func (RecordedAferoFS) RemoveAll ¶
func (r RecordedAferoFS) RemoveAll(path string) error
func (RecordedAferoFS) Rename ¶
func (r RecordedAferoFS) Rename(oldname, newname string) error
type RecordedOp ¶
type RecordedOp struct { // Monotonically-increasing sequence number of this operation. SeqNo int64 `protobuf:"varint,1,opt,name=seq_no,json=seqNo,proto3" json:"seq_no,omitempty"` // Previous FSM checksum to which this operation should be applied (eg, the // expected checksum arrived at after applying the previous operation. Checksum uint32 `protobuf:"fixed32,2,opt,name=checksum,proto3" json:"checksum,omitempty"` // Author is the unique ID of the Recorder which wrote this RecordedOp. // Each Recorder randomly generates an Author ID at startup, and thereafter // applies it to all operations it records. Author Author `protobuf:"fixed32,3,opt,name=author,proto3,casttype=Author" json:"author,omitempty"` // First and last byte offset (exclusive) of this RecordedOp, and the journal // addressed by those offsets. // // These are meta-fields: they're not literally serialized into written messages. // The offsets of a particular message will also vary over its lifetime: // - When first recorded, the offsets at which the write will land within the journal // cannot be known ahead of time, and Recorders use an approximate lower bound // as |first_offset|. // - During playback, players have the benefit of inspecting the committed log and // attach exact byte offsets as they deserialized RecordedOps. FirstOffset int64 `protobuf:"varint,9,opt,name=first_offset,json=firstOffset,proto3" json:"first_offset,omitempty"` LastOffset int64 `protobuf:"varint,10,opt,name=last_offset,json=lastOffset,proto3" json:"last_offset,omitempty"` Log go_gazette_dev_core_broker_protocol.Journal `protobuf:"bytes,11,opt,name=log,proto3,casttype=go.gazette.dev/core/broker/protocol.Journal" json:"log,omitempty"` Create *RecordedOp_Create `protobuf:"bytes,4,opt,name=create,proto3" json:"create,omitempty"` Link *RecordedOp_Link `protobuf:"bytes,5,opt,name=link,proto3" json:"link,omitempty"` Unlink *RecordedOp_Link `protobuf:"bytes,6,opt,name=unlink,proto3" json:"unlink,omitempty"` Write *RecordedOp_Write `protobuf:"bytes,7,opt,name=write,proto3" json:"write,omitempty"` // Property indicates a property file has been created or updated. // DEPRECATED. Properties are no longer created, // but will still be applied from a written log. Property *Property `protobuf:"bytes,8,opt,name=property,proto3" json:"property,omitempty"` }
RecordedOp records states changes occuring within a local file-system. Next tag: 12.
func (*RecordedOp) Descriptor ¶
func (*RecordedOp) Descriptor() ([]byte, []int)
func (*RecordedOp) Marshal ¶
func (m *RecordedOp) Marshal() (dAtA []byte, err error)
func (*RecordedOp) MarshalToSizedBuffer ¶ added in v0.86.1
func (m *RecordedOp) MarshalToSizedBuffer(dAtA []byte) (int, error)
func (*RecordedOp) ProtoMessage ¶
func (*RecordedOp) ProtoMessage()
func (*RecordedOp) ProtoSize ¶
func (m *RecordedOp) ProtoSize() (n int)
func (*RecordedOp) Reset ¶
func (m *RecordedOp) Reset()
func (*RecordedOp) String ¶
func (m *RecordedOp) String() string
func (*RecordedOp) Unmarshal ¶
func (m *RecordedOp) Unmarshal(dAtA []byte) error
func (*RecordedOp) XXX_DiscardUnknown ¶
func (m *RecordedOp) XXX_DiscardUnknown()
func (*RecordedOp) XXX_Marshal ¶
func (m *RecordedOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*RecordedOp) XXX_Merge ¶
func (m *RecordedOp) XXX_Merge(src proto.Message)
func (*RecordedOp) XXX_Size ¶
func (m *RecordedOp) XXX_Size() int
func (*RecordedOp) XXX_Unmarshal ¶
func (m *RecordedOp) XXX_Unmarshal(b []byte) error
type RecordedOp_Create ¶
type RecordedOp_Create struct { // Filesystem path of this file, relative to the common base directory. Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` }
Create a new "File Node" (Fnode), initially linked to |path|. Fnodes play a similar role to Posix inodes: they identify a specific file object while being invariant to (and spanning across) its current or future path links. The assigned Fnode ID is the |seq_no| of this RecordedOp.
func (*RecordedOp_Create) Descriptor ¶
func (*RecordedOp_Create) Descriptor() ([]byte, []int)
func (*RecordedOp_Create) Marshal ¶
func (m *RecordedOp_Create) Marshal() (dAtA []byte, err error)
func (*RecordedOp_Create) MarshalTo ¶
func (m *RecordedOp_Create) MarshalTo(dAtA []byte) (int, error)
func (*RecordedOp_Create) MarshalToSizedBuffer ¶ added in v0.86.1
func (m *RecordedOp_Create) MarshalToSizedBuffer(dAtA []byte) (int, error)
func (*RecordedOp_Create) ProtoMessage ¶
func (*RecordedOp_Create) ProtoMessage()
func (*RecordedOp_Create) ProtoSize ¶
func (m *RecordedOp_Create) ProtoSize() (n int)
func (*RecordedOp_Create) Reset ¶
func (m *RecordedOp_Create) Reset()
func (*RecordedOp_Create) String ¶
func (m *RecordedOp_Create) String() string
func (*RecordedOp_Create) Unmarshal ¶
func (m *RecordedOp_Create) Unmarshal(dAtA []byte) error
func (*RecordedOp_Create) XXX_DiscardUnknown ¶
func (m *RecordedOp_Create) XXX_DiscardUnknown()
func (*RecordedOp_Create) XXX_Marshal ¶
func (m *RecordedOp_Create) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*RecordedOp_Create) XXX_Merge ¶
func (m *RecordedOp_Create) XXX_Merge(src proto.Message)
func (*RecordedOp_Create) XXX_Size ¶
func (m *RecordedOp_Create) XXX_Size() int
func (*RecordedOp_Create) XXX_Unmarshal ¶
func (m *RecordedOp_Create) XXX_Unmarshal(b []byte) error
type RecordedOp_Link ¶
type RecordedOp_Link struct { // Fnode being linked or unlinked. Fnode Fnode `protobuf:"varint,1,opt,name=fnode,proto3,casttype=Fnode" json:"fnode,omitempty"` // Filesystem path being un/linked, relative to the common base directory. Path string `protobuf:"bytes,2,opt,name=path,proto3" json:"path,omitempty"` }
Link or unlink an Fnode to a filesystem path.
func (*RecordedOp_Link) Descriptor ¶
func (*RecordedOp_Link) Descriptor() ([]byte, []int)
func (*RecordedOp_Link) Marshal ¶
func (m *RecordedOp_Link) Marshal() (dAtA []byte, err error)
func (*RecordedOp_Link) MarshalToSizedBuffer ¶ added in v0.86.1
func (m *RecordedOp_Link) MarshalToSizedBuffer(dAtA []byte) (int, error)
func (*RecordedOp_Link) ProtoMessage ¶
func (*RecordedOp_Link) ProtoMessage()
func (*RecordedOp_Link) ProtoSize ¶
func (m *RecordedOp_Link) ProtoSize() (n int)
func (*RecordedOp_Link) Reset ¶
func (m *RecordedOp_Link) Reset()
func (*RecordedOp_Link) String ¶
func (m *RecordedOp_Link) String() string
func (*RecordedOp_Link) Unmarshal ¶
func (m *RecordedOp_Link) Unmarshal(dAtA []byte) error
func (*RecordedOp_Link) XXX_DiscardUnknown ¶
func (m *RecordedOp_Link) XXX_DiscardUnknown()
func (*RecordedOp_Link) XXX_Marshal ¶
func (m *RecordedOp_Link) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*RecordedOp_Link) XXX_Merge ¶
func (m *RecordedOp_Link) XXX_Merge(src proto.Message)
func (*RecordedOp_Link) XXX_Size ¶
func (m *RecordedOp_Link) XXX_Size() int
func (*RecordedOp_Link) XXX_Unmarshal ¶
func (m *RecordedOp_Link) XXX_Unmarshal(b []byte) error
type RecordedOp_Write ¶
type RecordedOp_Write struct { // Fnode being written to. Fnode Fnode `protobuf:"varint,1,opt,name=fnode,proto3,casttype=Fnode" json:"fnode,omitempty"` // Byte-offset within the file to which this write is applied. Offset int64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` // Length of the write. Length int64 `protobuf:"varint,3,opt,name=length,proto3" json:"length,omitempty"` }
Write indicates |length| bytes should be written at |offset| to |fnode|. In a serialization stream, we expect |length| raw bytes of content to immediately follow this operation.
func (*RecordedOp_Write) Descriptor ¶
func (*RecordedOp_Write) Descriptor() ([]byte, []int)
func (*RecordedOp_Write) Marshal ¶
func (m *RecordedOp_Write) Marshal() (dAtA []byte, err error)
func (*RecordedOp_Write) MarshalToSizedBuffer ¶ added in v0.86.1
func (m *RecordedOp_Write) MarshalToSizedBuffer(dAtA []byte) (int, error)
func (*RecordedOp_Write) ProtoMessage ¶
func (*RecordedOp_Write) ProtoMessage()
func (*RecordedOp_Write) ProtoSize ¶
func (m *RecordedOp_Write) ProtoSize() (n int)
func (*RecordedOp_Write) Reset ¶
func (m *RecordedOp_Write) Reset()
func (*RecordedOp_Write) String ¶
func (m *RecordedOp_Write) String() string
func (*RecordedOp_Write) Unmarshal ¶
func (m *RecordedOp_Write) Unmarshal(dAtA []byte) error
func (*RecordedOp_Write) XXX_DiscardUnknown ¶
func (m *RecordedOp_Write) XXX_DiscardUnknown()
func (*RecordedOp_Write) XXX_Marshal ¶
func (m *RecordedOp_Write) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*RecordedOp_Write) XXX_Merge ¶
func (m *RecordedOp_Write) XXX_Merge(src proto.Message)
func (*RecordedOp_Write) XXX_Size ¶
func (m *RecordedOp_Write) XXX_Size() int
func (*RecordedOp_Write) XXX_Unmarshal ¶
func (m *RecordedOp_Write) XXX_Unmarshal(b []byte) error
type Recorder ¶
type Recorder struct {
// contains filtered or unexported fields
}
Recorder observes a sequence of changes to a file-system and preserves those changes via a written journal of file-system operations.
Recorder, and the recovery log protocol, allow for the possibility of /multiple/ active recorders each sequencing to the same recovery log. The design allows for disambiguating which Recorder's "view" of the log should be reconstructed on playback by using FSMHints produced by the appropriate Recorder instance. Taken to an extreme, it's possible (though silly) to have _many_ distinct file-systems all recorded into a single combined journal, and to rely exclusively on FSMHints to advertise the specific, consistent file-system to be recovered and further recorded. Put differently, N concurrent Recorders will encode a "tree" to the log with N self-consistent branches, where FSMHints then inform readers of a specific branch to be recovered.
Note that journal registers & checks can be used to prevent the multiple- writers scenario from occurring in the first place. However recovery log behavior (which pre-dates registers) will remain as-is for compatibility and for the additional resilience and explicitness it provides.
Recorder expects to record only operations that have already happened (and were consistent with the local file-system). For this reason, Recorder is crash-only in its handling of inconsistent file operations (eg, an attempt to remove a path that isn't already known to the Recorder).
func NewRecorder ¶
func NewRecorder(journal pb.Journal, fsm *FSM, author Author, dir string, ajc client.AsyncJournalClient) *Recorder
NewRecorder builds and returns a new *Recorder. |dir| must be a clean, absolute path or NewRecorder panics.
func (*Recorder) Barrier ¶ added in v0.83.1
func (r *Recorder) Barrier(waitFor client.OpFutures) *client.AsyncAppend
Barrier issues a zero-byte append which will not commence until |waitFor| operations have successfully resolved.
func (*Recorder) BuildHints ¶
BuildHints returns FSMHints which may be played back to fully reconstruct the local filesystem state observed by this Recorder. It may block while pending operations sync to the log.
func (*Recorder) Dir ¶ added in v0.83.1
Dir returns the directory in which this Recorder is recording.
func (*Recorder) DisableRegisterChecks ¶ added in v0.89.0
func (r *Recorder) DisableRegisterChecks()
DisableRegisterChecks disables the register checks which Recorder normally does with each and every append operation. The only practical reason for wanting to do this is for testing recovery log operation recording lineage and conflict handling -- handling which predates register checks, but is still useful as a supplemental integrity check of recovery log correctness.
func (*Recorder) RecordCreate ¶
RecordCreate records the creation or truncation of file |path|, and returns its FNode.
func (*Recorder) RecordLink ¶
RecordLink records the creation of a hard link from |src| to |target|.
func (*Recorder) RecordOpenFile ¶ added in v0.84.1
RecordOpenFile records the open of file |path| with flags |flags|, and returns its FNode. Flags values must match os.OpenFile() semantics or RecordOpenFile panics.
func (*Recorder) RecordRemove ¶
RecordRemove records the removal of the file at |path|.
func (*Recorder) RecordRename ¶
RecordRename records the rename of |src| to |target|.
type Segment ¶
type Segment struct { // Author which wrote RecordedOps of this Segment. Author Author `protobuf:"fixed32,1,opt,name=author,proto3,casttype=Author" json:"author,omitempty"` // First (lowest) sequence number of RecordedOps within this Segment. FirstSeqNo int64 `protobuf:"varint,2,opt,name=first_seq_no,json=firstSeqNo,proto3" json:"first_seq_no,omitempty"` // First byte offset of the Segment, where |first_seq_no| is recorded. // If this Segment was produced by a Recorder, this is guaranteed only to be a // lower-bound (eg, a Player reading at this offset may encounter irrelevant // operations prior to the RecordedOp indicated by the tuple // (|author|, |first_seq_no|, |first_checksum|). If a Player produced the Segment, // first_offset is exact. FirstOffset int64 `protobuf:"varint,3,opt,name=first_offset,json=firstOffset,proto3" json:"first_offset,omitempty"` // Checksum of the RecordedOp having |first_seq_no|. FirstChecksum uint32 `protobuf:"fixed32,4,opt,name=first_checksum,json=firstChecksum,proto3" json:"first_checksum,omitempty"` // Last (highest, inclusive) sequence number of RecordedOps within this Segment. LastSeqNo int64 `protobuf:"varint,5,opt,name=last_seq_no,json=lastSeqNo,proto3" json:"last_seq_no,omitempty"` // Last offset (exclusive) of the Segment. Zero means the offset is not known // (eg, because the Segment was produced by a Recorder). LastOffset int64 `protobuf:"varint,6,opt,name=last_offset,json=lastOffset,proto3" json:"last_offset,omitempty"` // Log is the Journal holding this Segment's data, and to which offsets are relative. Log go_gazette_dev_core_broker_protocol.Journal `protobuf:"bytes,7,opt,name=log,proto3,casttype=go.gazette.dev/core/broker/protocol.Journal" json:"log,omitempty"` }
Segment is a contiguous chunk of recovery log written by a single Author. Recorders track Segments they have written, for use in providing hints to future readers of the log. A key point to understand is that Gazette append semantics mean that Recorders *cannot know* exactly what offsets their writes are applied to in the log, nor guarantee that their operations are not being interleaved with those of other writers. Log Players are aware of these limitations, and use Segments to resolve conflicts of possible interpretation of the log. Segments produced by a Player are exact, since Players observe all recorded operations at their exact offsets. Next tag: 8.
func (*Segment) Descriptor ¶
func (*Segment) MarshalToSizedBuffer ¶ added in v0.86.1
func (*Segment) ProtoMessage ¶
func (*Segment) ProtoMessage()
func (*Segment) XXX_DiscardUnknown ¶
func (m *Segment) XXX_DiscardUnknown()
func (*Segment) XXX_Marshal ¶
func (*Segment) XXX_Unmarshal ¶
type SegmentSet ¶
type SegmentSet []Segment
SegmentSet is a collection of Segment with the following invariants:
- Entries have strictly increasing SeqNo, and are non-overlapping (s[i].LastSeqNo < s[i+1].SeqNo; note this implies a single author for any covered SeqNo).
- Entries of the same Log have monotonically increasing FirstOffset.
- Entries of the same Log have monotonically increasing LastOffset, however a strict suffix of entries are permitted to have a LastOffset of zero (implied infinite LastOffset).
func (*SegmentSet) Add ¶
func (s *SegmentSet) Add(segment Segment) error
Add a Segment to this SegmentSet. An error is returned if |segment| would result in an inconsistent SegmentSet.
func (SegmentSet) Intersect ¶
func (s SegmentSet) Intersect(log pb.Journal, firstOffset, lastOffset int64) SegmentSet
Intersect returns a slice of this SegmentSet which overlaps with the provided [firstOffset, lastOffset) range. All Segments of this set MUST have the provided |log|, or Intersect panics. Intersect also panics if the first/lastOffset range is mis-ordered.