chaparral

README

chaparral

Chaparral is an API service for storing and accessing versioned datasets. It uses the Oxford Common File Layout (OCFL) specification for storing, versioning, and validating content. File system and S3 backends are supported.

[!Warning] This project is in early development. Expect bugs and breaking changes.

Server

The server is distributed as a container image: srerickson/chaparral). See config.yaml for an example configuration

API

Chaparral's server API is defined using protocol buffers/gRPC and implemented using connect-go. It supports both gRPC and http/1.1 requests. Documentation is available through the Buf schema registry.

Building Images

Images are built with ko

 VERSION=0.x.y KO_DOCKER_REPO=srerickson ko -B build ./cmd/chaparral

Authorization

Chaparral can use signed web tokens for authentication by setting the CHAPARRAL_PUBKEY_FILE environment variable or the pubkey_file config value to the path of a PEM-encoded RSA public key.

Generate a new RSA key pair:

# generate a new key
$ openssl genrsa -out auth.pem 2048

# export public key
$openssl pkey -in auth.pem -pubout > auth-pub.pem

About the name

Chaparral is a shrubland plant community found primarily in California, in southern Oregon and in the northern portion of the Baja California Peninsula in Mexico. It is shaped by a Mediterranean climate (mild wet winters and hot dry summers) and infrequent, high-intensity crown fires. (Wikipedia)

Documentation

Index

• Constants
• Variables
• func IsNotFound(err error) bool
• type Client
  • func NewClient(c *http.Client, baseurl string) *Client
  • func (cli Client) Commit(ctx context.Context, commit *Commit) error
  • func (cli Client) DeleteObject(ctx context.Context, storeID string, objectID string) error
  • func (cli Client) DeleteUploader(ctx context.Context, id string) error
  • func (cli Client) GetContent(ctx context.Context, storeID, objectID, digest string) (*Content, error)
  • func (cli Client) GetContentPath(ctx context.Context, storeID, objectID, contentPath string) (*Content, error)
  • func (cli Client) GetObjectManifest(ctx context.Context, storeID string, objectID string) (*ObjectManifest, error)
  • func (cli Client) GetObjectVersion(ctx context.Context, storeID string, objectID string, ver int) (*ObjectVersion, error)
  • func (cli Client) GetUploader(ctx context.Context, id string) (up *Uploader, err error)
  • func (cli Client) ListUploaders(ctx context.Context) ([]UploaderListItem, error)
  • func (cli Client) NewUploader(ctx context.Context, algs []string, desc string) (up *Uploader, err error)
  • func (cli Client) Upload(ctx context.Context, uploadPath string, r io.Reader) (result Upload, err error)
• type Commit
• type Content
• type FileInfo
• type Manifest
  • func (m Manifest) DigestMap() ocfl.DigestMap
  • func (m Manifest) PathMap() ocfl.PathMap
• type ObjectManifest
• type ObjectRef
• type ObjectVersion
• type Upload
• type Uploader
• type UploaderListItem
• type UploaderRef

Constants

const (
	RouteDownload = "/" + chapv1connect.AccessServiceName + "/" + "download"
	RouteUpload   = `/` + chapv1connect.CommitServiceName + "/" + "upload"

	QueryDigest      = "digest"
	QueryContentPath = "content_path"
	QueryObjectID    = "object_id"
	QueryUploaderID  = "uploader"
	QueryStorageRoot = "storage_root"
)

http routes for upload/download

Variables

var CODE_VERSION = func() string {
	if info, ok := debug.ReadBuildInfo(); ok {
		revision := ""

		localmods := false
		for _, setting := range info.Settings {
			switch setting.Key {
			case "vcs.revision":
				revision = setting.Value

			case "vcs.modified":
				localmods = setting.Value == "true"
			}
		}
		if !localmods {
			return revision
		}
	}
	return "none"
}()

var VERSION string

VERSION is set with ldflags -X

Functions

func IsNotFound

func IsNotFound(err error) bool

Types

type Client

type Client struct {
	*http.Client
	// contains filtered or unexported fields
}

func NewClient

func NewClient(c *http.Client, baseurl string) *Client

func (Client) Commit

func (cli Client) Commit(ctx context.Context, commit *Commit) error

CommitFork creates or updates an object using an existing object as its content source. If the commit's state is empty, then the source object's current version state is used. If srcStore is empty strings, commit.GroupID and commit.StorageRootID are used.

func (Client) DeleteObject

func (cli Client) DeleteObject(ctx context.Context, storeID string, objectID string) error

func (Client) DeleteUploader

func (cli Client) DeleteUploader(ctx context.Context, id string) error

func (Client) GetContent

func (cli Client) GetContent(ctx context.Context, storeID, objectID, digest string) (*Content, error)

Download

func (Client) GetContentPath

func (cli Client) GetContentPath(ctx context.Context, storeID, objectID, contentPath string) (*Content, error)

func (Client) GetObjectManifest

func (cli Client) GetObjectManifest(ctx context.Context, storeID string, objectID string) (*ObjectManifest, error)

func (Client) GetObjectVersion

func (cli Client) GetObjectVersion(ctx context.Context, storeID string, objectID string, ver int) (*ObjectVersion, error)

func (Client) GetUploader

func (cli Client) GetUploader(ctx context.Context, id string) (up *Uploader, err error)

func (Client) ListUploaders

func (cli Client) ListUploaders(ctx context.Context) ([]UploaderListItem, error)

func (Client) NewUploader

func (cli Client) NewUploader(ctx context.Context, algs []string, desc string) (up *Uploader, err error)

func (Client) Upload

func (cli Client) Upload(ctx context.Context, uploadPath string, r io.Reader) (result Upload, err error)

type Commit

type Commit struct {
	To             ObjectRef
	Version        int
	User           ocfl.User
	Message        string
	State          map[string]string
	Alg            string
	ContentSources []any
}

type Content

type Content struct {
	io.ReadCloser
	Size int64
}

type FileInfo

type FileInfo struct {
	Size   int64          // number of bytes
	Paths  []string       // sorted slice of path names
	Fixity ocfl.DigestSet // other digests associated witht the content
}

type Manifest

type Manifest map[string]FileInfo

Manifest maps digest to FileInfo

func (Manifest) DigestMap

func (m Manifest) DigestMap() ocfl.DigestMap

func (Manifest) PathMap

func (m Manifest) PathMap() ocfl.PathMap

type ObjectManifest

type ObjectManifest struct {
	ObjectRef
	Path            string
	Spec            string
	DigestAlgorithm string
	Manifest        Manifest
}

ObjectManifest corresponds to GetObjectManifestResponse proto

type ObjectRef

type ObjectRef struct {
	StorageRootID string `json:"storage_root_id"`
	ID            string `json:"object_id"`
}

type ObjectVersion

type ObjectVersion struct {
	ObjectRef
	Spec            string
	Version         int
	Head            int
	DigestAlgorithm string
	State           Manifest
	Message         string
	User            *ocfl.User
	Created         time.Time
}

ObjectVersion corresponds to GetObjectVersionResponse proto

type Upload

type Upload struct {
	Size    int64          `json:"size"`
	Digests ocfl.DigestSet `json:"digests"`
}

type Uploader

type Uploader struct {
	UploaderRef
	UploadPath       string    `json:"upload_path"`
	DigestAlgorithms []string  `json:"digest_algorithms"`
	Description      string    `json:"description"`
	Created          time.Time `json:"created"`
	UserID           string    `json:"user_id"`
	Uploads          []Upload  `json:"uploads,omitempty"`
}

type UploaderListItem

type UploaderListItem struct {
	UploaderRef
	Created     time.Time
	UserID      string
	Description string
}

type UploaderRef

type UploaderRef struct {
	ID string `json:"uploader_id"`
}