mirror

package
v0.2.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 3, 2024 License: MIT Imports: 18 Imported by: 0

Documentation

Overview

Package mirror periodically mirrors (bare clones) remote repositories locally. The mirror is created with `--mirror=fetch` hence everything in `refs/*` on the remote will be directly mirrored into `refs/*` in the local repository. it can also maintain multiple mirrored checked out worktrees on different references.

The implementation borrows heavily from kubernetes/git-sync. If you want to sync single repository on one reference then you are probably better off with kubernetes/git-sync, as it provides a lot more customisation. `git-mirror` should be used if multiple mirrored repositories with multiple checked out branches (worktrees) is required.

Usages

please see examples below

Logging:

package takes slog reference for logging and prints logs up to 'trace' level

Example: 

loggerLevel  = new(slog.LevelVar)
levelStrings = map[string]slog.Level{
	"trace": slog.Level(-8),
	"debug": slog.LevelDebug,
	"info":  slog.LevelInfo,
	"warn":  slog.LevelWarn,
	"error": slog.LevelError,
}

logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
	Level: loggerLevel,
}))
loggerLevel.Set(levelStrings["trace"])

repos, err := NewRepoPool(conf, logger.With("logger", "git-mirror"), nil)
if err != nil {
	panic(err)
}
Example (WithOutWorktree) 
tmpRoot, err := os.MkdirTemp("", "git-mirror-without-worktree-example-*")
if err != nil {
	panic(err)
}
defer os.RemoveAll(tmpRoot)

config := `
defaults:
  root:
  interval: 30s
  mirror_timeout: 2m
  git_gc: always
repositories:
  - remote: https://github.com/utilitywarehouse/git-mirror.git
`
ctx := context.Background()

conf := mirror.RepoPoolConfig{}
err = yaml.Unmarshal([]byte(config), &conf)
if err != nil {
	panic(err)
}
conf.Defaults.Root = tmpRoot

repos, err := mirror.NewRepoPool(conf, slog.Default(), nil)
if err != nil {
	panic(err)
}

// perform 1st mirror to ensure all repositories
// initial mirror might take longer
if err := repos.MirrorAll(ctx, 5*time.Minute); err != nil {
	panic(err)
}

// start mirror Loop
repos.StartLoop()

hash, err := repos.Hash(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "")
if err != nil {
	panic(err)
}
fmt.Println("last commit hash at main", "hash", hash)

msg, err := repos.LogMsg(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "")
if err != nil {
	panic(err)
}
fmt.Println("last commit msg at main", "msg", msg)
Example (Worktree) 
tmpRoot, err := os.MkdirTemp("", "git-mirror-worktree-example-*")
if err != nil {
	panic(err)
}
defer os.RemoveAll(tmpRoot)

config := `
defaults:
  root: /tmp/git-mirror
  interval: 30s
  mirror_timeout: 2m
  git_gc: always
repositories:
  - remote: https://github.com/utilitywarehouse/git-mirror.git
    worktrees:
    - link: main
`
ctx := context.Background()

conf := mirror.RepoPoolConfig{}
err = yaml.Unmarshal([]byte(config), &conf)
if err != nil {
	panic(err)
}

conf.Defaults.Root = tmpRoot

repos, err := mirror.NewRepoPool(conf, slog.Default(), nil)
if err != nil {
	panic(err)
}

// perform 1st mirror to ensure all repositories
// initial mirror might take longer
if err := repos.MirrorAll(ctx, 5*time.Minute); err != nil {
	panic(err)
}

// start mirror Loop
repos.StartLoop()

hash, err := repos.Hash(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "")
if err != nil {
	panic(err)
}
fmt.Println("last commit hash at main", "hash", hash)

msg, err := repos.LogMsg(ctx, "https://github.com/utilitywarehouse/git-mirror.git", "main", "")
if err != nil {
	panic(err)
}
fmt.Println("last commit msg at main", "msg", msg)

// make sure file exists in the tree
_, err = os.Stat(tmpRoot + "/main/pkg/mirror/repository.go")
if err != nil {
	panic(err)
}

Index

Examples

Constants

This section is empty.

Variables

View Source 
var (
	ErrExist    = fmt.Errorf("repo already exist")
	ErrNotExist = fmt.Errorf("repo does not exist")
)

Functions

func EnableMetrics 

func EnableMetrics(metricsNamespace string, registerer prometheus.Registerer)

EnableMetrics will enable metrics collection for git mirrors. Available metrics are...

  • git_last_mirror_timestamp - (tags: repo) A Gauge that captures the Timestamp of the last successful git sync per repo.
  • git_mirror_count - (tags: repo,success) A Counter for each repo sync, incremented with each sync attempt and tagged with the result (success=true|false)
  • git_mirror_latency_seconds - (tags: repo) A Summary that keeps track of the git sync latency per repo.

func IsCommitHash added in v0.1.1 

func IsCommitHash(hash string) bool

IsCommitHash returns whether or not a string is a abbreviated Hash or 40 char SHA-1 or 64 char SHA-256 hash

func IsFullCommitHash added in v0.1.1 

func IsFullCommitHash(hash string) bool

IsCommitHash returns whether or not a string is a 40 char SHA-1 or 64 char SHA-256 hash

Types

type Auth 

type Auth struct {
	// path to the ssh key used to fetch remote
	SSHKeyPath string `yaml:"ssh_key_path"`

	// path to the known hosts of the remote host
	SSHKnownHostsPath string `yaml:"ssh_known_hosts_path"`
}

Auth represents authentication config of the repository

type DefaultConfig 

type DefaultConfig struct {
	// Root is the absolute path to the root dir where all repositories directories
	// will be created all repos worktree links will be created here if not
	// specified in repo config
	Root string `yaml:"root"`

	// Interval is time duration for how long to wait between mirrors
	Interval time.Duration `yaml:"interval"`

	// MirrorTimeout represents the total time allowed for the complete mirror loop
	MirrorTimeout time.Duration `yaml:"mirror_timeout"`

	// GitGC garbage collection string. valid values are
	// 'auto', 'always', 'aggressive' or 'off'
	GitGC string `yaml:"git_gc"`

	// Auth config to fetch remote repos
	Auth Auth `yaml:"auth"`
}

DefaultConfig is the default config for repositories if not set at repo level

type RepoPool 

type RepoPool struct {
	// contains filtered or unexported fields
}

RepoPool represents the collection of mirrored repositories it provides simple wrapper around Repository methods. A RepoPool is safe for concurrent use by multiple goroutines.

func NewRepoPool 

func NewRepoPool(conf RepoPoolConfig, log *slog.Logger, commonENVs []string) (*RepoPool, error)

NewRepoPool will create mirror repositories based on given config. Remote repo will not be mirrored until either Mirror() or StartLoop() is called

func (*RepoPool) AddRepository 

func (rp *RepoPool) AddRepository(repo *Repository) error

AddRepository will add given repository to repoPool. Remote repo will not be mirrored until either Mirror() or StartLoop() is called 

func (rp *RepoPool) AddWorktreeLink(remote string, link, ref, pathspec string) error

AddWorktreeLink is wrapper around repositories AddWorktreeLink method

func (*RepoPool) ChangedFiles added in v0.1.1 

func (rp *RepoPool) ChangedFiles(ctx context.Context, remote, hash string) ([]string, error)

ChangedFiles is wrapper around repositories ChangedFiles method

func (*RepoPool) Clone 

func (rp *RepoPool) Clone(ctx context.Context, remote, dst, branch, pathspec string, rmGitDir bool) (string, error)

Clone is wrapper around repositories Clone method

func (*RepoPool) Hash 

func (rp *RepoPool) Hash(ctx context.Context, remote, ref, path string) (string, error)

Hash is wrapper around repositories hash method

func (*RepoPool) LogMsg 

func (rp *RepoPool) LogMsg(ctx context.Context, remote, ref, path string) (string, error)

LogMsg is wrapper around repositories LogMsg method

func (*RepoPool) Mirror 

func (rp *RepoPool) Mirror(ctx context.Context, remote string) error

Mirror is wrapper around repositories Mirror method

func (*RepoPool) MirrorAll added in v0.2.1 

func (rp *RepoPool) MirrorAll(ctx context.Context, timeout time.Duration) error

MirrorAll will trigger mirror on every repo in foreground with given timeout. It will error out if any of the repository mirror errors. Ideally MirrorAll should be used for the first mirror cycle to ensure repositories are successfully mirrored

func (*RepoPool) ObjectExists added in v0.1.1 

func (rp *RepoPool) ObjectExists(ctx context.Context, remote, obj string) error

ObjectExists is wrapper around repositories ObjectExists method

func (*RepoPool) Repository 

func (rp *RepoPool) Repository(remote string) (*Repository, error)

Repository will return Repository object based on given remote URL

func (*RepoPool) StartLoop 

func (rp *RepoPool) StartLoop()

StartLoop will start mirror loop on all repositories if its not already started

func (*RepoPool) Subject added in v0.1.1 

func (rp *RepoPool) Subject(ctx context.Context, remote, hash string) (string, error)

Subject is wrapper around repositories Subject method

type RepoPoolConfig 

type RepoPoolConfig struct {
	// default config for all the repositories if not set
	Defaults DefaultConfig `yaml:"defaults"`
	// List of mirrored repositories.
	Repositories []RepositoryConfig `yaml:"repositories"`
}

RepoPoolConfig is the configuration to create repoPool

func (*RepoPoolConfig) ApplyDefaults 

func (rpc *RepoPoolConfig) ApplyDefaults()

ApplyDefaults will add given default config to repository config if where needed

func (*RepoPoolConfig) ValidateDefaults 

func (rpc *RepoPoolConfig) ValidateDefaults() error

ValidateDefaults will verify default config

func (*RepoPoolConfig) ValidateLinkPaths 

func (rpc *RepoPoolConfig) ValidateLinkPaths() error

It is possible that same root is used for multiple repositories since Links are placed at the root, we need to make sure that all link's name (path) are diff. ValidateLinkPaths makes sures all link's absolute paths are different.

type Repository 

type Repository struct {
	// contains filtered or unexported fields
}

Repository represents the mirrored repository of the given remote. The implementation borrows heavily from https://github.com/kubernetes/git-sync. A Repository is safe for concurrent use by multiple goroutines.

func NewRepository 

func NewRepository(repoConf RepositoryConfig, envs []string, log *slog.Logger) (*Repository, error)

NewRepository creates new repository from the given config. Remote repo will not be mirrored until either Mirror() or StartLoop() is called. 

func (r *Repository) AddWorktreeLink(link, ref, pathspec string) error

AddWorktreeLink adds add workTree link to the mirror repository.

func (*Repository) ChangedFiles added in v0.1.1 

func (r *Repository) ChangedFiles(ctx context.Context, hash string) ([]string, error)

ChangedFiles returns path of the changed files for given commit hash

func (*Repository) Clone 

func (r *Repository) Clone(ctx context.Context, dst, ref, pathspec string, rmGitDir bool) (string, error)

Clone creates a single-branch local clone of the mirrored repository to a new location on disk. On success, it returns the hash of the new repository clone's HEAD. if pathspec is provided only those paths will be checked out. if ref is commit hash then pathspec will be ignored. if rmGitDir is true `.git` folder will be deleted after the clone. if dst not empty all its contents will be removed.

func (*Repository) Hash 

func (r *Repository) Hash(ctx context.Context, ref, path string) (string, error)

Hash returns the hash of the given revision and for the path if specified.

func (*Repository) LogMsg 

func (r *Repository) LogMsg(ctx context.Context, ref, path string) (string, error)

LogMsg returns the formatted log subject with author info of the given revision and for the path if specified.

func (*Repository) Mirror 

func (r *Repository) Mirror(ctx context.Context) error

Mirror will run mirror loop of the repository

  1. init and validate if existing repo dir
  2. fetch remote
  3. ensure worktrees
  4. cleanup if needed

func (*Repository) ObjectExists added in v0.1.1 

func (r *Repository) ObjectExists(ctx context.Context, obj string) error

ObjectExists returns error is given object is not valid or if it doesn't exists

func (*Repository) StartLoop 

func (r *Repository) StartLoop(ctx context.Context)

StartLoop mirrors repository periodically based on repo's mirror interval

func (*Repository) Subject added in v0.1.1 

func (r *Repository) Subject(ctx context.Context, hash string) (string, error)

Subject returns commit subject of given commit hash

type RepositoryConfig 

type RepositoryConfig struct {
	// git URL of the remote repo to mirror
	Remote string `yaml:"remote"`

	// Root is the absolute path to the root dir where repo dir
	// will be created. Worktree links will be created here if
	// absolute path is not provided
	Root string `yaml:"root"`

	// Interval is time duration for how long to wait between mirrors
	Interval time.Duration `yaml:"interval"`

	// MirrorTimeout represents the total time allowed for the complete mirror loop
	MirrorTimeout time.Duration `yaml:"mirror_timeout"`

	// GitGC garbage collection string. valid values are
	// 'auto', 'always', 'aggressive' or 'off'
	GitGC string `yaml:"git_gc"`

	// Auth config to fetch remote repos
	Auth Auth `yaml:"auth"`

	// Worktrees contains list of worktrees links which will be maintained.
	// worktrees are optional repo can be mirrored without worktree
	Worktrees []WorktreeConfig `yaml:"worktrees"`
}

RepositoryConfig represents the config for the mirrored repository of the given remote. 

type WorkTreeLink struct {
	// contains filtered or unexported fields
}

type WorktreeConfig 

type WorktreeConfig struct {
	// Link is the path at which to create a symlink to the worktree dir
	// if path is not absolute it will be created under repository root
	Link string `yaml:"link"`

	// Ref represents the git reference of the worktree branch, tags or hash
	// are supported. default is HEAD
	Ref string `yaml:"ref"`

	// Pathspec of the dirs to checkout if required
	Pathspec string `yaml:"pathspec"`
}

Worktree represents maintained worktree on given link.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.