Documentation
¶
Overview ¶
Package mirror periodically mirrors (bare clones) remote repositories locally. The mirror is created with `--mirror=fetch` hence everything in `refs/*` on the remote will be directly mirrored into `refs/*` in the local repository. it can also maintain multiple mirrored checked out worktrees on different references.
The implementation borrows heavily from kubernetes/git-sync. If you want to sync single repository on one reference then you are probably better off with kubernetes/git-sync, as it provides a lot more customisation. `git-mirror` should be used if multiple mirrored repositories with multiple checked out branches (worktrees) is required.
Usages ¶
please see examples below
Logging: ¶
package takes slog reference for logging and prints logs up to 'trace' level
Example:
loggerLevel = new(slog.LevelVar) levelStrings = map[string]slog.Level{ "trace": slog.Level(-8), "debug": slog.LevelDebug, "info": slog.LevelInfo, "warn": slog.LevelWarn, "error": slog.LevelError, } logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ Level: loggerLevel, })) loggerLevel.Set(levelStrings["trace"]) repos, err := NewRepoPool(conf, logger.With("logger", "git-mirror"), nil) if err != nil { panic(err) }
Example (WithOutWorktree) ¶
tmpRoot, err := os.MkdirTemp("", "git-mirror-without-worktree-example-*") if err != nil { panic(err) } defer os.RemoveAll(tmpRoot) config := ` defaults: root: interval: 30s mirror_timeout: 2m git_gc: always repositories: - remote: https://github.com/utilitywarehouse/git-mirror.git ` ctx := context.Background() conf := mirror.RepoPoolConfig{} err = yaml.Unmarshal([]byte(config), &conf) if err != nil { panic(err) } conf.Defaults.Root = tmpRoot repos, err := mirror.NewRepoPool(ctx, conf, slog.Default(), nil) if err != nil { panic(err) } // perform 1st mirror to ensure all repositories // initial mirror might take longer if err := repos.MirrorAll(ctx, 5*time.Minute); err != nil { panic(err) } // start mirror Loop repos.StartLoop() hash, err := repos.Hash(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "") if err != nil { panic(err) } fmt.Println("last commit hash at main", "hash", hash) msg, err := repos.Subject(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main") if err != nil { panic(err) } fmt.Println("last commit msg at main", "msg", msg)
Example (Worktree) ¶
tmpRoot, err := os.MkdirTemp("", "git-mirror-worktree-example-*") if err != nil { panic(err) } defer os.RemoveAll(tmpRoot) config := ` defaults: root: /tmp/git-mirror interval: 30s mirror_timeout: 2m git_gc: always repositories: - remote: https://github.com/utilitywarehouse/git-mirror.git worktrees: - link: main ` ctx := context.Background() conf := mirror.RepoPoolConfig{} err = yaml.Unmarshal([]byte(config), &conf) if err != nil { panic(err) } conf.Defaults.Root = tmpRoot repos, err := mirror.NewRepoPool(ctx, conf, slog.Default(), nil) if err != nil { panic(err) } // perform 1st mirror to ensure all repositories // initial mirror might take longer if err := repos.MirrorAll(ctx, 5*time.Minute); err != nil { panic(err) } // start mirror Loop repos.StartLoop() hash, err := repos.Hash(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "") if err != nil { panic(err) } fmt.Println("last commit hash at main", "hash", hash) msg, err := repos.Subject(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main") if err != nil { panic(err) } fmt.Println("last commit msg at main", "msg", msg) // make sure file exists in the tree _, err = os.Stat(tmpRoot + "/main/pkg/mirror/repository.go") if err != nil { panic(err) }
Index ¶
- Variables
- func DefaultRepoDir(root string) string
- func EnableMetrics(metricsNamespace string, registerer prometheus.Registerer)
- func IsCommitHash(hash string) bool
- func IsFullCommitHash(hash string) bool
- type Auth
- type CommitInfo
- type DefaultConfig
- type RepoPool
- func (rp *RepoPool) AddRepository(repoConf RepositoryConfig) error
- func (rp *RepoPool) AddWorktreeLink(remote string, wt WorktreeConfig) error
- func (rp *RepoPool) BranchCommits(ctx context.Context, remote, branch string) ([]CommitInfo, error)
- func (rp *RepoPool) ChangedFiles(ctx context.Context, remote, hash string) ([]string, error)
- func (rp *RepoPool) Clone(ctx context.Context, remote, dst, branch string, pathspecs []string, ...) (string, error)
- func (rp *RepoPool) Hash(ctx context.Context, remote, ref, path string) (string, error)
- func (rp *RepoPool) ListCommitsWithChangedFiles(ctx context.Context, remote, ref1, ref2 string) ([]CommitInfo, error)
- func (rp *RepoPool) MergeCommits(ctx context.Context, remote, mergeCommitHash string) ([]CommitInfo, error)
- func (rp *RepoPool) Mirror(ctx context.Context, remote string) error
- func (rp *RepoPool) MirrorAll(ctx context.Context, timeout time.Duration) error
- func (rp *RepoPool) ObjectExists(ctx context.Context, remote, obj string) error
- func (rp *RepoPool) RemoveRepository(remote string) error
- func (rp *RepoPool) RemoveWorktreeLink(remote string, wtLink string) error
- func (rp *RepoPool) RepositoriesDirPath() []string
- func (rp *RepoPool) RepositoriesRemote() []string
- func (rp *RepoPool) Repository(remote string) (*Repository, error)
- func (rp *RepoPool) StartLoop()
- func (rp *RepoPool) Subject(ctx context.Context, remote, hash string) (string, error)
- type RepoPoolConfig
- type Repository
- func (r *Repository) AddWorktreeLink(wtc WorktreeConfig) error
- func (r *Repository) BranchCommits(ctx context.Context, branch string) ([]CommitInfo, error)
- func (r *Repository) ChangedFiles(ctx context.Context, hash string) ([]string, error)
- func (r *Repository) Clone(ctx context.Context, dst, ref string, pathspecs []string, rmGitDir bool) (string, error)
- func (r *Repository) Hash(ctx context.Context, ref, path string) (string, error)
- func (r *Repository) IsRunning() bool
- func (r *Repository) ListCommitsWithChangedFiles(ctx context.Context, ref1, ref2 string) ([]CommitInfo, error)
- func (r *Repository) MergeCommits(ctx context.Context, mergeCommitHash string) ([]CommitInfo, error)
- func (r *Repository) Mirror(ctx context.Context) error
- func (r *Repository) ObjectExists(ctx context.Context, obj string) error
- func (r *Repository) RemoveWorktreeLink(link string) error
- func (r *Repository) StartLoop(ctx context.Context)
- func (r *Repository) StopLoop()
- func (r *Repository) Subject(ctx context.Context, hash string) (string, error)
- func (r *Repository) WorktreeLinks() map[string]*WorkTreeLink
- type RepositoryConfig
- type WorkTreeLink
- type WorktreeConfig
Examples ¶
Constants ¶
This section is empty.
Variables ¶
var ( ErrExist = errors.New("repo already exist") ErrNotExist = errors.New("repo does not exist") )
var ( ErrRepoMirrorFailed = errors.New("repository mirror failed") ErrRepoWTUpdateFailed = errors.New("repository worktree update failed") )
Functions ¶
func DefaultRepoDir ¶ added in v0.3.0
DefaultRepoDir returns path of dir where all repositories mirrors are cloned
func EnableMetrics ¶
func EnableMetrics(metricsNamespace string, registerer prometheus.Registerer)
EnableMetrics will enable metrics collection for git mirrors. Available metrics are...
- git_last_mirror_timestamp - (tags: repo) A Gauge that captures the Timestamp of the last successful git sync per repo.
- git_mirror_count - (tags: repo,success) A Counter for each repo sync, incremented with each sync attempt and tagged with the result (success=true|false)
- git_mirror_latency_seconds - (tags: repo) A Summary that keeps track of the git sync latency per repo.
func IsCommitHash ¶ added in v0.1.1
IsCommitHash returns whether or not a string is a abbreviated Hash or 40 char SHA-1 or 64 char SHA-256 hash
func IsFullCommitHash ¶ added in v0.1.1
IsCommitHash returns whether or not a string is a 40 char SHA-1 or 64 char SHA-256 hash
Types ¶
type Auth ¶
type Auth struct { // path to the ssh key used to fetch remote SSHKeyPath string `yaml:"ssh_key_path"` // path to the known hosts of the remote host SSHKnownHostsPath string `yaml:"ssh_known_hosts_path"` }
Auth represents authentication config of the repository
type CommitInfo ¶ added in v0.2.4
func ParseCommitWithChangedFilesList ¶ added in v0.2.4
func ParseCommitWithChangedFilesList(output string) []CommitInfo
ParseCommitWithChangedFilesList will parse following output of 'show/log' command with `--name-only`, `--pretty=format:%H` flags
72ea9c9de6963e97ac472d9ea996e384c6923cca
80e11d114dd3aa135c18573402a8e688599c69e0 one/readme.yaml one/hello.tf two/readme.yaml
type DefaultConfig ¶
type DefaultConfig struct { // Root is the absolute path to the root dir where all repositories directories // will be created all repos worktree links will be created here if not // specified in repo config Root string `yaml:"root"` // LinkRoot is the absolute path to the dir which is the root for the worktree links // if link is a relative path it will be relative to this dir // if link is not specified it will be constructed from repo name and worktree ref // and it will be placed in this dir // if not specified it will be same as root LinkRoot string `yaml:"link_root"` // Interval is time duration for how long to wait between mirrors Interval time.Duration `yaml:"interval"` // MirrorTimeout represents the total time allowed for the complete mirror loop MirrorTimeout time.Duration `yaml:"mirror_timeout"` // GitGC garbage collection string. valid values are // 'auto', 'always', 'aggressive' or 'off' GitGC string `yaml:"git_gc"` // Auth config to fetch remote repos Auth Auth `yaml:"auth"` }
DefaultConfig is the default config for repositories if not set at repo level
type RepoPool ¶
type RepoPool struct { Stopped chan bool // contains filtered or unexported fields }
RepoPool represents the collection of mirrored repositories it provides simple wrapper around Repository methods. A RepoPool is safe for concurrent use by multiple goroutines.
func NewRepoPool ¶
func NewRepoPool(ctx context.Context, conf RepoPoolConfig, log *slog.Logger, commonENVs []string) (*RepoPool, error)
NewRepoPool will create mirror repositories based on given config. Remote repo will not be mirrored until either Mirror() or StartLoop() is called
func (*RepoPool) AddRepository ¶
func (rp *RepoPool) AddRepository(repoConf RepositoryConfig) error
AddRepository will add given repository to repoPool. Remote repo will not be mirrored until either Mirror() or StartLoop() is called
func (*RepoPool) AddWorktreeLink ¶
func (rp *RepoPool) AddWorktreeLink(remote string, wt WorktreeConfig) error
AddWorktreeLink is wrapper around repositories AddWorktreeLink method
func (*RepoPool) BranchCommits ¶ added in v0.2.4
BranchCommits is wrapper around repositories BranchCommits method
func (*RepoPool) ChangedFiles ¶ added in v0.1.1
ChangedFiles is wrapper around repositories ChangedFiles method
func (*RepoPool) Clone ¶
func (rp *RepoPool) Clone(ctx context.Context, remote, dst, branch string, pathspecs []string, rmGitDir bool) (string, error)
Clone is wrapper around repositories Clone method
func (*RepoPool) ListCommitsWithChangedFiles ¶ added in v0.2.4
func (rp *RepoPool) ListCommitsWithChangedFiles(ctx context.Context, remote, ref1, ref2 string) ([]CommitInfo, error)
ListCommitsWithChangedFiles is wrapper around repositories ListCommitsWithChangedFiles method
func (*RepoPool) MergeCommits ¶ added in v0.2.4
func (rp *RepoPool) MergeCommits(ctx context.Context, remote, mergeCommitHash string) ([]CommitInfo, error)
MergeCommits is wrapper around repositories MergeCommits method
func (*RepoPool) MirrorAll ¶ added in v0.2.1
MirrorAll will trigger mirror on every repo in foreground with given timeout. It will error out if any of the repository mirror errors. Ideally MirrorAll should be used for the first mirror cycle to ensure repositories are successfully mirrored
func (*RepoPool) ObjectExists ¶ added in v0.1.1
ObjectExists is wrapper around repositories ObjectExists method
func (*RepoPool) RemoveRepository ¶ added in v0.3.0
RemoveRepository will remove given repository from the repoPool.
func (*RepoPool) RemoveWorktreeLink ¶ added in v0.3.0
RemoveWorktreeLink is wrapper around repositories RemoveWorktreeLink method
func (*RepoPool) RepositoriesDirPath ¶ added in v0.3.0
RepositoriesDirPath returns local paths of all the mirrored repositories
func (*RepoPool) RepositoriesRemote ¶ added in v0.3.0
RepositoriesRemote returns remote URLs of all the repositories
func (*RepoPool) Repository ¶
func (rp *RepoPool) Repository(remote string) (*Repository, error)
Repository will return Repository object based on given remote URL
type RepoPoolConfig ¶
type RepoPoolConfig struct { // default config for all the repositories if not set Defaults DefaultConfig `yaml:"defaults"` // List of mirrored repositories. Repositories []RepositoryConfig `yaml:"repositories"` }
RepoPoolConfig is the configuration to create repoPool
func (*RepoPoolConfig) ValidateAndApplyDefaults ¶ added in v0.3.0
func (conf *RepoPoolConfig) ValidateAndApplyDefaults() error
ValidateAndApplyDefaults will validate link paths and default and apply defaults
type Repository ¶
type Repository struct {
// contains filtered or unexported fields
}
Repository represents the mirrored repository of the given remote. The implementation borrows heavily from https://github.com/kubernetes/git-sync. A Repository is safe for concurrent use by multiple goroutines.
func NewRepository ¶
func NewRepository(repoConf RepositoryConfig, envs []string, log *slog.Logger) (*Repository, error)
NewRepository creates new repository from the given config. Remote repo will not be mirrored until either Mirror() or StartLoop() is called.
func (*Repository) AddWorktreeLink ¶
func (r *Repository) AddWorktreeLink(wtc WorktreeConfig) error
AddWorktreeLink adds workTree link to the mirror repository.
func (*Repository) BranchCommits ¶ added in v0.2.4
func (r *Repository) BranchCommits(ctx context.Context, branch string) ([]CommitInfo, error)
BranchCommits lists commits from the tip of the branch but not from the HEAD of the repository in chronological order. (latest to oldest)
func (*Repository) ChangedFiles ¶ added in v0.1.1
ChangedFiles returns path of the changed files for given commit hash
func (*Repository) Clone ¶
func (r *Repository) Clone(ctx context.Context, dst, ref string, pathspecs []string, rmGitDir bool) (string, error)
Clone creates a single-branch local clone of the mirrored repository to a new location on disk. On success, it returns the hash of the new repository clone's HEAD. if pathspec is provided only those paths will be checked out. if ref is commit hash then pathspec will be ignored. if rmGitDir is true `.git` folder will be deleted after the clone. if dst not empty all its contents will be removed.
func (*Repository) Hash ¶
Hash returns the hash of the given revision and for the path if specified.
func (*Repository) IsRunning ¶ added in v0.3.0
func (r *Repository) IsRunning() bool
IsRunning returns if repositories mirror loop is running or not
func (*Repository) ListCommitsWithChangedFiles ¶ added in v0.2.4
func (r *Repository) ListCommitsWithChangedFiles(ctx context.Context, ref1, ref2 string) ([]CommitInfo, error)
ListCommitsWithChangedFiles returns path of the changed files for given commit hash list all the commits and files which are reachable from 'ref2', but not from 'ref1' The output is given in reverse chronological order.
func (*Repository) MergeCommits ¶ added in v0.2.4
func (r *Repository) MergeCommits(ctx context.Context, mergeCommitHash string) ([]CommitInfo, error)
MergeCommits lists commits from the mergeCommitHash but not from the first parent of mergeCommitHash (mergeCommitHash^) in chronological order. (latest to oldest)
func (*Repository) Mirror ¶
func (r *Repository) Mirror(ctx context.Context) error
Mirror will run mirror loop of the repository
- init and validate if existing repo dir
- fetch remote
- ensure worktrees
- cleanup if needed
func (*Repository) ObjectExists ¶ added in v0.1.1
func (r *Repository) ObjectExists(ctx context.Context, obj string) error
ObjectExists returns error is given object is not valid or if it doesn't exists
func (*Repository) RemoveWorktreeLink ¶ added in v0.3.0
func (r *Repository) RemoveWorktreeLink(link string) error
RemoveWorktreeLink removes workTree link from the mirror repository. it will remove published link as well even (if it failed to remove worktree)
func (*Repository) StartLoop ¶
func (r *Repository) StartLoop(ctx context.Context)
StartLoop mirrors repository periodically based on repo's mirror interval
func (*Repository) StopLoop ¶ added in v0.3.0
func (r *Repository) StopLoop()
StopLoop stops sync loop gracefully
func (*Repository) WorktreeLinks ¶ added in v0.3.0
func (r *Repository) WorktreeLinks() map[string]*WorkTreeLink
WorktreeLinks returns current clone of worktree maps
type RepositoryConfig ¶
type RepositoryConfig struct { // git URL of the remote repo to mirror Remote string `yaml:"remote"` // Root is the absolute path to the root dir where repo dir // will be created. Worktree links will be created here if // absolute path is not provided Root string `yaml:"root"` // LinkRoot is the absolute path to the dir which is the root for the worktree links // if link is a relative path it will be relative to this dir // if link is not specified it will be constructed from repo name and worktree ref // and it will be placed in this dir // if not specified it will be same as root LinkRoot string `yaml:"link_root"` // Interval is time duration for how long to wait between mirrors Interval time.Duration `yaml:"interval"` // MirrorTimeout represents the total time allowed for the complete mirror loop MirrorTimeout time.Duration `yaml:"mirror_timeout"` // GitGC garbage collection string. valid values are // 'auto', 'always', 'aggressive' or 'off' GitGC string `yaml:"git_gc"` // Auth config to fetch remote repos Auth Auth `yaml:"auth"` // Worktrees contains list of worktrees links which will be maintained. // worktrees are optional repo can be mirrored without worktree Worktrees []WorktreeConfig `yaml:"worktrees"` }
RepositoryConfig represents the config for the mirrored repository of the given remote.
func (*RepositoryConfig) PopulateEmptyLinkPaths ¶ added in v0.3.0
func (repo *RepositoryConfig) PopulateEmptyLinkPaths() error
PopulateEmptyLinkPaths will try and generate missing link paths
type WorkTreeLink ¶
type WorkTreeLink struct {
// contains filtered or unexported fields
}
func (*WorkTreeLink) Equals ¶ added in v0.3.0
func (wt *WorkTreeLink) Equals(wtc WorktreeConfig) bool
Equals returns if given worktree and its config is equal they are considered equal only if link, ref and pathspecs are matching. order of pothspecs is ignored
type WorktreeConfig ¶
type WorktreeConfig struct { // Link is the path at which to create a symlink to the worktree dir // if path is not absolute it will be created under repository link_root // if link is not specified it will be constructed from repo name and worktree ref // and it will be placed in link_root dir Link string `yaml:"link"` // Ref represents the git reference of the worktree branch, tags or hash // are supported. default is HEAD Ref string `yaml:"ref"` // Pathspecs of the dirs to checkout if required Pathspecs []string `yaml:"pathspecs"` }
Worktree represents maintained worktree on given link.