backups

README

Backups

Note that this document describes the technical details of the backup system

Format

A backup is an https://github.com/infinitybotlist/iblfile with the standard AutoEncryptedFile format and has the following fields:

• backup_opts - JSON containing a types.BackupCreateOpts object
• core/guild - The core guild data (discordgo.Guild)
• assets/guildIcon - The guild icon data ([]byte)
• messages/{channel_id} - The messages in a channel along with basic attachment metadata ([]types.BackupMessage).
• dbg/* - Debug information. This may vary across backups and MUST NOT be used in restoring a backup.
• attachments/{attachment_id} - The attachments data itself

Documentation

Index

• Variables
• type AttachmentMetadata
• type AttachmentStorageFormat
• type BackupConstraints
• type BackupCreateConstraints
• type BackupCreateOpts
• type BackupMessage
• type BackupRestoreConstraints
• type BackupRestoreOpts
• type ChannelRestoreMode
• type RestoreMessage
• type RoleRestoreMode
• type ServerBackupCreateTask
  • func (t *ServerBackupCreateTask) Exec(l *zap.Logger, tcr *types.TaskCreateResponse, state taskstate.TaskState) (*types.TaskOutput, error)
  • func (t *ServerBackupCreateTask) Info() *types.TaskInfo
  • func (t *ServerBackupCreateTask) Validate(state taskstate.TaskState) error
• type ServerBackupRestoreTask
  • func (t *ServerBackupRestoreTask) Exec(l *zap.Logger, tcr *types.TaskCreateResponse, state taskstate.TaskState) (*types.TaskOutput, error)
  • func (t *ServerBackupRestoreTask) Info() *types.TaskInfo
  • func (t *ServerBackupRestoreTask) Validate(state taskstate.TaskState) error

Variables

var FreePlanBackupConstraints = &BackupConstraints{
	Create: &BackupCreateConstraints{
		TotalMaxMessages:          500,
		MaxAttachmentFileSize:     8_000_000,
		FileSizeWarningThreshold:  50_000_000,
		MinPerChannel:             50,
		DefaultPerChannel:         100,
		JpegReencodeQuality:       75,
		GuildAssetReencodeQuality: 85,
	},
	Restore: &BackupRestoreConstraints{
		RoleDeleteSleep:    3 * timex.Second,
		RoleCreateSleep:    3 * timex.Second,
		ChannelDeleteSleep: 3 * timex.Second,
		ChannelCreateSleep: 3 * timex.Second,
		ChannelEditSleep:   1 * timex.Second,
		SendMessageSleep:   3 * timex.Second,
		HttpClientTimeout:  10 * timex.Second,
		MaxBodySize:        100_000_000,
	},
	MaxServerBackupTasks: 1,
	FileType:             "backup.server",
}

Types

type AttachmentMetadata

type AttachmentMetadata struct {
	ID            string                  `json:"id"`             // ID of the attachment within the ticket
	URL           string                  `json:"url"`            // URL of the attachment
	ProxyURL      string                  `json:"proxy_url"`      // URL (cached) of the attachment
	Name          string                  `json:"name"`           // Name of the attachment
	ContentType   string                  `json:"content_type"`   // Content type of the attachment
	StorageFormat AttachmentStorageFormat `json:"storage_format"` // Storage format of the attachment
	Size          int                     `json:"size"`           // Size of the attachment in bytes
	Errors        []string                `json:"errors"`         // Non-fatal errors that occurred while uploading the attachment
}

Attachment contains metadata about an attachment

type AttachmentStorageFormat

type AttachmentStorageFormat string

const (
	AttachmentStorageFormatUnknownOrUnsaved AttachmentStorageFormat = ""
	AttachmentStorageFormatUncompressed     AttachmentStorageFormat = "uncompressed"
	AttachmentStorageFormatGzip             AttachmentStorageFormat = "gzip"
	AttachmentStorageFormatJpegEncoded      AttachmentStorageFormat = "jpeg_encoded"
	AttachmentStorageFormatRemote           AttachmentStorageFormat = "remote"
)

type BackupConstraints

type BackupConstraints struct {
	Create               *BackupCreateConstraints
	Restore              *BackupRestoreConstraints
	MaxServerBackupTasks int    // How many backup tasks can run concurrently per server
	FileType             string // The file type to use for backups
}

type BackupCreateConstraints

type BackupCreateConstraints struct {
	TotalMaxMessages          int // The maximum number of messages to backup
	MaxAttachmentFileSize     int // The maximum size of an attachment
	FileSizeWarningThreshold  int // The warning threshold for the total file size
	MinPerChannel             int // The minimum number of messages per channel
	DefaultPerChannel         int // The default number of messages per channel
	JpegReencodeQuality       int // The quality to use when reencoding to JPEGs
	GuildAssetReencodeQuality int // The quality to use when reencoding guild assets
}

type BackupCreateOpts

type BackupCreateOpts struct {
	PerChannel                int            `description:"The number of messages per channel"`
	MaxMessages               int            `description:"The maximum number of messages to backup"`
	BackupMessages            bool           `description:"Whether to backup messages or not"`
	BackupAttachments         bool           `description:"Whether to backup attachments or not"`
	BackupGuildAssets         []string       `description:"What assets to back up"`
	IgnoreMessageBackupErrors bool           `description:"Whether to ignore errors while backing up messages or not and skip these channels"`
	RolloverLeftovers         bool           `description:"Whether to attempt rollover of leftover message quota to another channels or not"`
	SpecialAllocations        map[string]int `description:"Specific channel allocation overrides"`
	Encrypt                   string         `description:"The key to encrypt backups with, if any"`
}

Options that can be set when creatng a backup

type BackupMessage

type BackupMessage struct {
	Message            *discordgo.Message   `json:"message"`
	AttachmentMetadata []AttachmentMetadata `json:"attachment_metadata"`
	// contains filtered or unexported fields
}

Represents a backed up message

type BackupRestoreConstraints

type BackupRestoreConstraints struct {
	RoleDeleteSleep    timex.Duration // How long to sleep between role deletes
	RoleCreateSleep    timex.Duration // How long to sleep between role creates
	ChannelDeleteSleep timex.Duration // How long to sleep between channel deletes
	ChannelCreateSleep timex.Duration // How long to sleep between channel creates
	ChannelEditSleep   timex.Duration // How long to sleep between channel edits
	SendMessageSleep   timex.Duration // How long to sleep between message sends
	HttpClientTimeout  timex.Duration // How long to wait for HTTP requests to complete
	MaxBodySize        int64          // The maximum size of the backup file to download/use
}

type BackupRestoreOpts

type BackupRestoreOpts struct {
	IgnoreRestoreErrors bool               `description:"Whether to ignore errors while restoring or not and skip these channels/roles"`
	ProtectedChannels   []string           `description:"Channels to protect from being deleted"`
	ProtectedRoles      []string           `description:"Roles to protect from being deleted"`
	BackupSource        string             `description:"The source of the backup"`
	Decrypt             string             `description:"The key to decrypt backups with, if any"`
	ChannelRestoreMode  ChannelRestoreMode `description:"Channel backup restore method. Use 'full' if unsure"`
	RoleRestoreMode     RoleRestoreMode    `description:"Role backup restore method. Use 'full' if unsure"`
}

Options that can be set when restoring a backup

type ChannelRestoreMode

type ChannelRestoreMode string

const (
	ChannelRestoreModeFull           ChannelRestoreMode = "full"
	ChannelRestoreModeDiff           ChannelRestoreMode = "diff" // TODO
	ChannelRestoreModeIgnoreExisting ChannelRestoreMode = "ignore_existing"
)

type RestoreMessage

type RestoreMessage struct {
	MessageSend *discordgo.MessageSend
	Author      *discordgo.User
	SmallFiles  []*discordgo.File
}

INTERNAL: Represents a message to be restored

type RoleRestoreMode

type RoleRestoreMode string

const (
	RoleRestoreModeFull RoleRestoreMode = "full"
)

type ServerBackupCreateTask

type ServerBackupCreateTask struct {
	// The ID of the server
	ServerID string

	// Constraints, this is auto-set by the task in jobserver and hence not configurable in this mode.
	Constraints *BackupConstraints

	// Backup options
	Options BackupCreateOpts
	// contains filtered or unexported fields
}

A task to create backup a server

func (*ServerBackupCreateTask) Exec

func (t *ServerBackupCreateTask) Exec(l *zap.Logger, tcr *types.TaskCreateResponse, state taskstate.TaskState) (*types.TaskOutput, error)

func (*ServerBackupCreateTask) Info

func (t *ServerBackupCreateTask) Info() *types.TaskInfo

func (*ServerBackupCreateTask) Validate

func (t *ServerBackupCreateTask) Validate(state taskstate.TaskState) error

type ServerBackupRestoreTask

type ServerBackupRestoreTask struct {
	// The ID of the server
	ServerID string

	// Constraints, this is auto-set by the task in jobserver and hence not configurable in this mode.
	Constraints *BackupConstraints

	// Backup options
	Options BackupRestoreOpts
	// contains filtered or unexported fields
}

A task to restore a backup of a server

func (*ServerBackupRestoreTask) Exec

func (t *ServerBackupRestoreTask) Exec(l *zap.Logger, tcr *types.TaskCreateResponse, state taskstate.TaskState) (*types.TaskOutput, error)

func (*ServerBackupRestoreTask) Info

func (t *ServerBackupRestoreTask) Info() *types.TaskInfo

func (*ServerBackupRestoreTask) Validate

func (t *ServerBackupRestoreTask) Validate(state taskstate.TaskState) error

Validate validates the task and sets up state if needed