cdr

package module
v0.0.0-...-721b47d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 11, 2022 License: Apache-2.0 Imports: 21 Imported by: 0

README

Common Data Refs

NOTE: This is not an officially supported Pachyderm product.

Common Data Refs are a standard intermediary format for passing references to data between systems.

CLI

There is a command line tool cdr which can be built with make targets/cdr. It allows users to inpsect and dereference CDRs. Run it with no arguments to see the help message.

Contributing

To run the tests use make run_minio to run a minio container. Then run the tests with make test

To create a test reference with arbitrary data using minio run ./targets/create-test-ref. The result is base64 encoded

make targets/create-test-ref
echo "test data" | ./targets/create-test-ref 127.0.0.1:9000

Documentation

Overview

package cdr provides Common Data Refs

Index

Constants

This section is empty.

Variables

View Source 
var (
	HashAlgo_name = map[int32]string{
		0:     "UNKNOWN_HASH",
		45600: "BLAKE2b_256",
	}
	HashAlgo_value = map[string]int32{
		"UNKNOWN_HASH": 0,
		"BLAKE2b_256":  45600,
	}
)

Enum value maps for HashAlgo.

View Source 
var (
	CipherAlgo_name = map[int32]string{
		0: "UNKNOWN_CIPHER",
		1: "CHACHA20",
	}
	CipherAlgo_value = map[string]int32{
		"UNKNOWN_CIPHER": 0,
		"CHACHA20":       1,
	}
)

Enum value maps for CipherAlgo.

View Source 
var (
	CompressAlgo_name = map[int32]string{
		0: "UNKNOWN_COMPRESS",
		1: "GZIP",
	}
	CompressAlgo_value = map[string]int32{
		"UNKNOWN_COMPRESS": 0,
		"GZIP":             1,
	}
)

Enum value maps for CompressAlgo.

View Source 
var File_cdr_proto protoreflect.FileDescriptor

Functions

func IsImmutable 

func IsImmutable(ref *Ref) bool

func MaxSize 

func MaxSize(ref *Ref) int64

func MinSize 

func MinSize(ref *Ref) int64

Types

type Cipher 

type Cipher struct {
	Inner *Ref       `protobuf:"bytes,1,opt,name=inner,proto3" json:"inner,omitempty"`
	Algo  CipherAlgo `protobuf:"varint,2,opt,name=algo,proto3,enum=pachyderm.cdr.CipherAlgo" json:"algo,omitempty"`
	Key   []byte     `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"`
	Nonce []byte     `protobuf:"bytes,4,opt,name=nonce,proto3" json:"nonce,omitempty"`
	// contains filtered or unexported fields
}

1:1 Transforms

func (*Cipher) Descriptor deprecated 

func (*Cipher) Descriptor() ([]byte, []int)

Deprecated: Use Cipher.ProtoReflect.Descriptor instead.

func (*Cipher) GetAlgo 

func (x *Cipher) GetAlgo() CipherAlgo

func (*Cipher) GetInner 

func (x *Cipher) GetInner() *Ref

func (*Cipher) GetKey 

func (x *Cipher) GetKey() []byte

func (*Cipher) GetNonce 

func (x *Cipher) GetNonce() []byte

func (*Cipher) ProtoMessage 

func (*Cipher) ProtoMessage()

func (*Cipher) ProtoReflect 

func (x *Cipher) ProtoReflect() protoreflect.Message

func (*Cipher) Reset 

func (x *Cipher) Reset()

func (*Cipher) String 

func (x *Cipher) String() string

type CipherAlgo 

type CipherAlgo int32
const (
	CipherAlgo_UNKNOWN_CIPHER CipherAlgo = 0
	CipherAlgo_CHACHA20       CipherAlgo = 1
)

func (CipherAlgo) Descriptor 

func (CipherAlgo) Descriptor() protoreflect.EnumDescriptor

func (CipherAlgo) Enum 

func (x CipherAlgo) Enum() *CipherAlgo

func (CipherAlgo) EnumDescriptor deprecated 

func (CipherAlgo) EnumDescriptor() ([]byte, []int)

Deprecated: Use CipherAlgo.Descriptor instead.

func (CipherAlgo) Number 

func (x CipherAlgo) Number() protoreflect.EnumNumber

func (CipherAlgo) String 

func (x CipherAlgo) String() string

func (CipherAlgo) Type 

func (CipherAlgo) Type() protoreflect.EnumType

type Compress 

type Compress struct {
	Inner *Ref         `protobuf:"bytes,1,opt,name=inner,proto3" json:"inner,omitempty"`
	Algo  CompressAlgo `protobuf:"varint,2,opt,name=algo,proto3,enum=pachyderm.cdr.CompressAlgo" json:"algo,omitempty"`
	// contains filtered or unexported fields
}

func (*Compress) Descriptor deprecated 

func (*Compress) Descriptor() ([]byte, []int)

Deprecated: Use Compress.ProtoReflect.Descriptor instead.

func (*Compress) GetAlgo 

func (x *Compress) GetAlgo() CompressAlgo

func (*Compress) GetInner 

func (x *Compress) GetInner() *Ref

func (*Compress) ProtoMessage 

func (*Compress) ProtoMessage()

func (*Compress) ProtoReflect 

func (x *Compress) ProtoReflect() protoreflect.Message

func (*Compress) Reset 

func (x *Compress) Reset()

func (*Compress) String 

func (x *Compress) String() string

type CompressAlgo 

type CompressAlgo int32
const (
	CompressAlgo_UNKNOWN_COMPRESS CompressAlgo = 0
	CompressAlgo_GZIP             CompressAlgo = 1
)

func (CompressAlgo) Descriptor 

func (CompressAlgo) Descriptor() protoreflect.EnumDescriptor

func (CompressAlgo) Enum 

func (x CompressAlgo) Enum() *CompressAlgo

func (CompressAlgo) EnumDescriptor deprecated 

func (CompressAlgo) EnumDescriptor() ([]byte, []int)

Deprecated: Use CompressAlgo.Descriptor instead.

func (CompressAlgo) Number 

func (x CompressAlgo) Number() protoreflect.EnumNumber

func (CompressAlgo) String 

func (x CompressAlgo) String() string

func (CompressAlgo) Type 

func (CompressAlgo) Type() protoreflect.EnumType

type Concat 

type Concat struct {
	Refs []*Ref `protobuf:"bytes,1,rep,name=refs,proto3" json:"refs,omitempty"`
	// contains filtered or unexported fields
}

Many:1 Transforms

func (*Concat) Descriptor deprecated 

func (*Concat) Descriptor() ([]byte, []int)

Deprecated: Use Concat.ProtoReflect.Descriptor instead.

func (*Concat) GetRefs 

func (x *Concat) GetRefs() []*Ref

func (*Concat) ProtoMessage 

func (*Concat) ProtoMessage()

func (*Concat) ProtoReflect 

func (x *Concat) ProtoReflect() protoreflect.Message

func (*Concat) Reset 

func (x *Concat) Reset()

func (*Concat) String 

func (x *Concat) String() string

type ContentHash 

type ContentHash struct {
	Inner *Ref     `protobuf:"bytes,1,opt,name=inner,proto3" json:"inner,omitempty"`
	Algo  HashAlgo `protobuf:"varint,2,opt,name=algo,proto3,enum=pachyderm.cdr.HashAlgo" json:"algo,omitempty"`
	Hash  []byte   `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
	// contains filtered or unexported fields
}

Contraints

func (*ContentHash) Descriptor deprecated 

func (*ContentHash) Descriptor() ([]byte, []int)

Deprecated: Use ContentHash.ProtoReflect.Descriptor instead.

func (*ContentHash) GetAlgo 

func (x *ContentHash) GetAlgo() HashAlgo

func (*ContentHash) GetHash 

func (x *ContentHash) GetHash() []byte

func (*ContentHash) GetInner 

func (x *ContentHash) GetInner() *Ref

func (*ContentHash) ProtoMessage 

func (*ContentHash) ProtoMessage()

func (*ContentHash) ProtoReflect 

func (x *ContentHash) ProtoReflect() protoreflect.Message

func (*ContentHash) Reset 

func (x *ContentHash) Reset()

func (*ContentHash) String 

func (x *ContentHash) String() string

type Creator 

type Creator struct {
	// contains filtered or unexported fields
}

func NewCreator 

func NewCreator(layers []Middleware, postFunc PostFunc) *Creator

func (*Creator) MakeRef 

func (c *Creator) MakeRef(ctx context.Context, data []byte) (*Ref, error)

type HTTP 

type HTTP struct {
	Url     string            `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
	Headers map[string]string `` /* 155-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*HTTP) Descriptor deprecated 

func (*HTTP) Descriptor() ([]byte, []int)

Deprecated: Use HTTP.ProtoReflect.Descriptor instead.

func (*HTTP) GetHeaders 

func (x *HTTP) GetHeaders() map[string]string

func (*HTTP) GetUrl 

func (x *HTTP) GetUrl() string

func (*HTTP) ProtoMessage 

func (*HTTP) ProtoMessage()

func (*HTTP) ProtoReflect 

func (x *HTTP) ProtoReflect() protoreflect.Message

func (*HTTP) Reset 

func (x *HTTP) Reset()

func (*HTTP) String 

func (x *HTTP) String() string

type HashAlgo 

type HashAlgo int32

Uses numeric values from this table https://github.com/multiformats/multicodec/blob/master/table.csv 

const (
	HashAlgo_UNKNOWN_HASH HashAlgo = 0
	HashAlgo_BLAKE2b_256  HashAlgo = 45600
)

func (HashAlgo) Descriptor 

func (HashAlgo) Descriptor() protoreflect.EnumDescriptor

func (HashAlgo) Enum 

func (x HashAlgo) Enum() *HashAlgo

func (HashAlgo) EnumDescriptor deprecated 

func (HashAlgo) EnumDescriptor() ([]byte, []int)

Deprecated: Use HashAlgo.Descriptor instead.

func (HashAlgo) Number 

func (x HashAlgo) Number() protoreflect.EnumNumber

func (HashAlgo) String 

func (x HashAlgo) String() string

func (HashAlgo) Type 

func (HashAlgo) Type() protoreflect.EnumType

type Middleware 

type Middleware = func(ctx context.Context, in []byte, next PostFunc) (*Ref, error)

type Option 

type Option func(r *Resolver)

func WithHTTPClient 

func WithHTTPClient(c *http.Client) Option

type PostFunc 

type PostFunc = func(ctx context.Context, data []byte) (*Ref, error)

type Ref 

type Ref struct {

	// Types that are assignable to Body:
	//	*Ref_Http
	//	*Ref_ContentHash
	//	*Ref_SizeLimits
	//	*Ref_Cipher
	//	*Ref_Compress
	//	*Ref_Slice
	//	*Ref_Concat
	Body isRef_Body `protobuf_oneof:"body"`
	// contains filtered or unexported fields
}

func CompressGzip 

func CompressGzip(ctx context.Context, in []byte, next PostFunc) (*Ref, error)

func CreateAWSRef 

func CreateAWSRef(ctx context.Context, client interface{}, bucket, key string, ttl time.Duration) (*Ref, error)

func CreateAzureRef 

func CreateAzureRef(ctx context.Context, client interface{}, bucket, key string, ttl time.Duration) (*Ref, error)

func CreateConcatRef 

func CreateConcatRef(refs []*Ref) *Ref

func CreateGCSRef 

func CreateGCSRef(ctx context.Context, client interface{}, bucket, key string, ttl time.Duration) (*Ref, error)

func CreateHTTPRef 

func CreateHTTPRef(url string, headers map[string]string) *Ref

func CreateMinioRef 

func CreateMinioRef(ctx context.Context, client *minio.Client, bucket, key string, ttl time.Duration) (*Ref, error)

func CreateSliceRef 

func CreateSliceRef(x *Ref, start, end uint64) *Ref

func EncryptChaCha20 

func EncryptChaCha20(ctx context.Context, data []byte, next PostFunc) (*Ref, error)

EncryptChaCha20 encrypts the data with a random key (stored in the ref) Does not protect the data from tampering.

func HashBlake2b256 

func HashBlake2b256(ctx context.Context, in []byte, next PostFunc) (*Ref, error)

func (*Ref) Descriptor deprecated 

func (*Ref) Descriptor() ([]byte, []int)

Deprecated: Use Ref.ProtoReflect.Descriptor instead.

func (*Ref) GetBody 

func (m *Ref) GetBody() isRef_Body

func (*Ref) GetCipher 

func (x *Ref) GetCipher() *Cipher

func (*Ref) GetCompress 

func (x *Ref) GetCompress() *Compress

func (*Ref) GetConcat 

func (x *Ref) GetConcat() *Concat

func (*Ref) GetContentHash 

func (x *Ref) GetContentHash() *ContentHash

func (*Ref) GetHttp 

func (x *Ref) GetHttp() *HTTP

func (*Ref) GetSizeLimits 

func (x *Ref) GetSizeLimits() *SizeLimits

func (*Ref) GetSlice 

func (x *Ref) GetSlice() *Slice

func (*Ref) MarshalBase64 

func (r *Ref) MarshalBase64() []byte

func (*Ref) ProtoMessage 

func (*Ref) ProtoMessage()

func (*Ref) ProtoReflect 

func (x *Ref) ProtoReflect() protoreflect.Message

func (*Ref) Reset 

func (x *Ref) Reset()

func (*Ref) String 

func (x *Ref) String() string

func (*Ref) UnmarshalBase64 

func (r *Ref) UnmarshalBase64(x []byte) error

type Ref_Cipher 

type Ref_Cipher struct {
	// 1:1 Transforms
	Cipher *Cipher `protobuf:"bytes,64,opt,name=cipher,proto3,oneof"`
}

type Ref_Compress 

type Ref_Compress struct {
	Compress *Compress `protobuf:"bytes,65,opt,name=compress,proto3,oneof"`
}

type Ref_Concat 

type Ref_Concat struct {
	// Many:1 Transforms
	Concat *Concat `protobuf:"bytes,96,opt,name=concat,proto3,oneof"`
}

type Ref_ContentHash 

type Ref_ContentHash struct {
	// Constraints
	ContentHash *ContentHash `protobuf:"bytes,32,opt,name=content_hash,json=contentHash,proto3,oneof"`
}

type Ref_Http 

type Ref_Http struct {
	// Sources
	Http *HTTP `protobuf:"bytes,1,opt,name=http,proto3,oneof"`
}

type Ref_SizeLimits 

type Ref_SizeLimits struct {
	SizeLimits *SizeLimits `protobuf:"bytes,33,opt,name=size_limits,json=sizeLimits,proto3,oneof"`
}

type Ref_Slice 

type Ref_Slice struct {
	Slice *Slice `protobuf:"bytes,66,opt,name=slice,proto3,oneof"`
}

type Resolver 

type Resolver struct {
	// contains filtered or unexported fields
}

func NewResolver 

func NewResolver(opts ...Option) *Resolver

func (*Resolver) Deref 

func (r *Resolver) Deref(ctx context.Context, ref *Ref) (io.ReadCloser, error)

type SizeLimits 

type SizeLimits struct {
	Inner *Ref  `protobuf:"bytes,1,opt,name=inner,proto3" json:"inner,omitempty"`
	Min   int64 `protobuf:"varint,2,opt,name=min,proto3" json:"min,omitempty"`
	Max   int64 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"`
	// contains filtered or unexported fields
}

func (*SizeLimits) Descriptor deprecated 

func (*SizeLimits) Descriptor() ([]byte, []int)

Deprecated: Use SizeLimits.ProtoReflect.Descriptor instead.

func (*SizeLimits) GetInner 

func (x *SizeLimits) GetInner() *Ref

func (*SizeLimits) GetMax 

func (x *SizeLimits) GetMax() int64

func (*SizeLimits) GetMin 

func (x *SizeLimits) GetMin() int64

func (*SizeLimits) ProtoMessage 

func (*SizeLimits) ProtoMessage()

func (*SizeLimits) ProtoReflect 

func (x *SizeLimits) ProtoReflect() protoreflect.Message

func (*SizeLimits) Reset 

func (x *SizeLimits) Reset()

func (*SizeLimits) String 

func (x *SizeLimits) String() string

type Slice 

type Slice struct {
	Inner *Ref   `protobuf:"bytes,1,opt,name=inner,proto3" json:"inner,omitempty"`
	Start uint64 `protobuf:"varint,2,opt,name=start,proto3" json:"start,omitempty"`
	End   uint64 `protobuf:"varint,3,opt,name=end,proto3" json:"end,omitempty"`
	// contains filtered or unexported fields
}

1:1 Transforms

func (*Slice) Descriptor deprecated 

func (*Slice) Descriptor() ([]byte, []int)

Deprecated: Use Slice.ProtoReflect.Descriptor instead.

func (*Slice) GetEnd 

func (x *Slice) GetEnd() uint64

func (*Slice) GetInner 

func (x *Slice) GetInner() *Ref

func (*Slice) GetStart 

func (x *Slice) GetStart() uint64

func (*Slice) ProtoMessage 

func (*Slice) ProtoMessage()

func (*Slice) ProtoReflect 

func (x *Slice) ProtoReflect() protoreflect.Message

func (*Slice) Reset 

func (x *Slice) Reset()

func (*Slice) String 

func (x *Slice) String() string

Source Files

View all Source files

Directories

Path Synopsis
cmd
cdr
etc
create-test-ref

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.