catalog

package
v0.0.0-...-9742f5a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 29, 2020 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ChunkCount 

func ChunkCount(size uint64, chunkSize int) int

func NewCatalog 

func NewCatalog(l *logrus.Logger, upstream Upstream) (*catalog, error)

Types

type ChunkSource 

type ChunkSource int
const (
	ChunkSourceInline ChunkSource = 0
	ChunkSourceHTTP   ChunkSource = 1
)

type ContentCatalog 

type ContentCatalog interface {
	// XXX: Returning a pointer to the underlying ObjectMetadata object seems like it will produce concurrency issues.
	GetData(ctx context.Context, req *ccmsg.ContentRequest) (*ObjectMetadata, error)

	GetMetadata(ctx context.Context, path string) (*ObjectMetadata, error)

	ChunkSource(ctx context.Context, req *ccmsg.CacheMissRequest, path string, metadata *ObjectMetadata) (*ccmsg.Chunk, error)

	Upstream(path string) (Upstream, error)
}

type ContentCatalogMock 

type ContentCatalogMock struct {
	mock.Mock
}

ContentCatalogMock mocks the ContentCatalog interface for testing.

func NewContentCatalogMock 

func NewContentCatalogMock() *ContentCatalogMock

NewContentCatalogMock creates a new mock

func (*ContentCatalogMock) ChunkSource 

func (ccm *ContentCatalogMock) ChunkSource(ctx context.Context, req *ccmsg.CacheMissRequest, path string, metadata *ObjectMetadata) (*ccmsg.Chunk, error)

ChunkSource is defined on the ContentCatalog interface

func (*ContentCatalogMock) GetData 

func (ccm *ContentCatalogMock) GetData(ctx context.Context, req *ccmsg.ContentRequest) (*ObjectMetadata, error)

GetData is defined on the ContentCatalog interface

func (*ContentCatalogMock) GetMetadata 

func (ccm *ContentCatalogMock) GetMetadata(ctx context.Context, path string) (*ObjectMetadata, error)

GetMetadata is defined on the ContentCatalog interface

func (*ContentCatalogMock) Upstream 

func (ccm *ContentCatalogMock) Upstream(path string) (Upstream, error)

Upstream is defined on the ContentCatalog interface

type ContentLocator 

type ContentLocator interface {
	GetContentSource(ctx context.Context, req *ccmsg.CacheMissRequest) (*ccmsg.CacheMissResponse, error)
}

XXX: This is not currently used. Either use it or remove it.

type FetchResult 

type FetchResult struct {
	// contains filtered or unexported fields
}

FetchResult describes the metadata, data, and/or errors returned by an Upstream in response to a single request. They are consumed by the catalog, which uses them to update its cache.

func (*FetchResult) ObjectSize 

func (r *FetchResult) ObjectSize() (int, error)

ObjectSize returns the size of the entire object; the response might actually contain data for only some part of it. TODO: Support '*' in the Content-Range header.

Valid formats for the 'Content-Range' header: 

'bytes 0-49/500'    // The first 50 bytes of a 500-byte object.
'bytes 0-49/*'      // The first 50 bytes of an object whose length is unknown.
'bytes */1234'      // Used in 416 (Range Not Satisfiable) responses.

type MockUpstream 

type MockUpstream struct {
	Objects map[string][]byte
	// contains filtered or unexported fields
}

func NewMockUpstream 

func NewMockUpstream(l *logrus.Logger) (*MockUpstream, error)

func (*MockUpstream) AddRandomObject 

func (up *MockUpstream) AddRandomObject(path string, size uint)

func (*MockUpstream) ChunkSource 

func (up *MockUpstream) ChunkSource(req *ccmsg.CacheMissRequest, path string, meta *ccmsg.ObjectMetadata,
	policy *ObjectPolicy) (*ccmsg.CacheMissResponse, error)

func (*MockUpstream) FetchData 

func (up *MockUpstream) FetchData(ctx context.Context, path string, metadata *ObjectMetadata, rangeBegin, rangeEnd uint) (*FetchResult, error)

type ObjectMetadata 

type ObjectMetadata struct {
	Status      ObjectStatus
	LastUpdate  time.Time
	LastAttempt time.Time
	ValidUntil  *time.Time
	Immutable   bool

	HTTPLastModified *string
	HTTPEtag         *string
	// contains filtered or unexported fields
}

func (*ObjectMetadata) ChunkCount 

func (m *ObjectMetadata) ChunkCount() int

ChunkCount returns the number of chunks in this object. XXX: This is a problem; m.metadata may be nil if we don't know anything about the object.

func (*ObjectMetadata) ChunkRange 

func (m *ObjectMetadata) ChunkRange(rangeBegin uint64, rangeEnd uint64) ([][]byte, error)

func (*ObjectMetadata) ChunkSize 

func (m *ObjectMetadata) ChunkSize(chunkIdx uint32) (int, error)

ChunkSize returns the size of a particular chunk in bytes. N.B.: It's important that this return the actual size of the indicated chunk; otherwise, if we are generating a 

puzzle that includes the last chunk in an object (which may be shorter than PolicyChunkSize() would suggest)
the colocation puzzle code may generate unsolvable puzzles (e.g. when the initial offset is chosen to be past
the end of the actual chunk).

func (*ObjectMetadata) Fresh 

func (m *ObjectMetadata) Fresh() bool

func (*ObjectMetadata) GetChunk 

func (m *ObjectMetadata) GetChunk(chunkIdx uint32) ([]byte, error)

func (*ObjectMetadata) Metadata 

func (m *ObjectMetadata) Metadata() *ccmsg.ObjectMetadata

XXX: Is this a concurrency issue?

func (*ObjectMetadata) ObjectSize 

func (m *ObjectMetadata) ObjectSize() uint64

func (*ObjectMetadata) PolicyChunkSize 

func (m *ObjectMetadata) PolicyChunkSize() uint64

type ObjectPolicy 

type ObjectPolicy struct {
	ChunkSize            int
	DefaultCacheDuration time.Duration
}

ObjectPolicy contains publisher-determined metadata such as chunk size. This is distinct from ccmsg.ObjectMetadata, which contains metadata cached from the upstream.

func (*ObjectPolicy) SplitIntoChunks 

func (policy *ObjectPolicy) SplitIntoChunks(buf []byte) [][]byte

type ObjectStatus 

type ObjectStatus int
const (
	StatusUnknown ObjectStatus = iota
	StatusOK
	StatusNotFound
	StatusNotModified
	StatusInternalError
	StatusUpstreamUnreachable
	StatusUpstreamError
)

func (ObjectStatus) String 

func (i ObjectStatus) String() string

type Upstream 

type Upstream interface {
	// FetchData ensures that metadata is fresh, and also that the indicated chunks are available in the cache.  An
	// empty list of chunk indices is allowed; this ensures metadata freshness but does not pull any chunks.
	//
	// forceMetadata requires that object metadata be fetched even if it would not otherwise be fetched.
	//
	// rangeEnd must be >= rangeBegin.  rangeEnd == 0 means "continue to he end of the object".
	//
	// Cases:
	// - We want to fetch metadata only.
	// - We want to fetch metadata *and* a series of chunks.
	// - We have metadata that we already believe to be be valid, so we don't necessarily need to fetch it, if that's
	//   any extra effort.  We want to fetch a series of chunks.
	//
	// Questions:
	// - Some upstreams will require CacheCash (not upstream) metadata for the object.  For example, the HTTP upstream
	//   will need to know chunk sizes in order to translate chunk indices into byte ranges.  How should this be done?
	//
	FetchData(ctx context.Context, path string, metadata *ObjectMetadata, rangeBegin, rangeEnd uint) (*FetchResult, error)
}

func NewHTTPUpstream 

func NewHTTPUpstream(l *logrus.Logger, baseURL string, defaultCacheDuration time.Duration) (Upstream, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.