storage

Documentation

Index

• type Crdb
• type DBClient
  • func NewDB(conn string) (*DBClient, error)
  • func (db *DBClient) CreateJob(ctx context.Context, cidStr string, fname string, timestamp *int64, ...) error
  • func (db *DBClient) UnfinishedJobs(ctx context.Context) ([]UnfinishedJob, error)
  • func (db *DBClient) UpdateJobStatus(ctx context.Context, cid []byte, activation time.Time) error
• type FileUploader
  • func NewFileUploader(ctx context.Context, eventData []byte, cfg *UploaderConfig) (*FileUploader, error)
  • func (u *FileUploader) Upload(ctx context.Context) error
• type GCS
• type GCSClient
  • func NewGCSClient(ctx context.Context, data []byte) (*GCSClient, error)
  • func (r *GCSClient) GetObjectMetadata(ctx context.Context, bucketName, objectName string) (map[string]string, error)
  • func (r *GCSClient) GetObjectReader(ctx context.Context, bucketName, objectName string) (io.ReadCloser, error)
  • func (r *GCSClient) ParseEvent() (string, string, error)
• type IntermediateFile
  • func NewIntermediateFile(data []byte, name string) *IntermediateFile
  • func (f *IntermediateFile) Close() error
  • func (f *IntermediateFile) Read(p []byte) (n int, err error)
  • func (f *IntermediateFile) Stat() (fs.FileInfo, error)
• type MockBasinStorage
  • func (c *MockBasinStorage) AddCID(_ context.Context, _ string, cids string, _ int64, _ *bind.TransactOpts) error
  • func (c *MockBasinStorage) EstimateGas(_ context.Context, _ string, _ string, _ int64) (*bind.TransactOpts, error)
  • func (c *MockBasinStorage) GetPendingNonce(_ context.Context) (uint64, error)
• type MockReadCloser
  • func (mrc *MockReadCloser) Close() error
• type Pub
• type StatusChecker
  • func NewStatusChecker(ctx context.Context, cfg *StatusCheckerConfig) (*StatusChecker, error)
  • func (sc *StatusChecker) ProcessJobs(ctx context.Context) error
• type StatusCheckerConfig
• type UnfinishedJob
• type UploaderConfig

Types

type Crdb

type Crdb interface {
	CreateJob(
		ctx context.Context,
		cidStr string,
		fileName string,
		timestamp *int64,
		cacheDuration int64,
		sign string,
		hash string,
	) error
	UnfinishedJobs(ctx context.Context) ([]UnfinishedJob, error)
	UpdateJobStatus(ctx context.Context, cid []byte, activation time.Time) error
}

Crdb is an interface that defines the methods to interact with CockroachDB.

type DBClient

type DBClient struct {
	// Initialize cockroachdb client to store metadata
	DB *sql.DB
}

DBClient is a Crdb implementation.

func NewDB

func NewDB(conn string) (*DBClient, error)

NewDB creates a new DBClient.

func (*DBClient) CreateJob

func (db *DBClient) CreateJob(
	ctx context.Context,
	cidStr string,
	fname string,
	timestamp *int64,
	cacheDuration int64,
	sign string,
	hash string,
) error

CreateJob creates a new job in the DB.

func (*DBClient) UnfinishedJobs

func (db *DBClient) UnfinishedJobs(ctx context.Context) ([]UnfinishedJob, error)

UnfinishedJobs returns all currently unfinished jobs in the db.

func (*DBClient) UpdateJobStatus

func (db *DBClient) UpdateJobStatus(ctx context.Context, cid []byte, activation time.Time) error

UpdateJobStatus updates the job status in the DB.

type FileUploader

type FileUploader struct {
	StorageClient GCS        // StorageClient is a GCS instance used to interact with GCS.
	DealClient    w3s.Client // DealClient is a w3s.Client instance used to interact with W3S.
	DBClient      Crdb       // DBClient is a Crdb instance used to interact with CockroachDB.
}

FileUploader download a file from GCS and uploads to web3.storage.

func NewFileUploader

func NewFileUploader(ctx context.Context, eventData []byte, cfg *UploaderConfig) (*FileUploader, error)

NewFileUploader creates a new FileUploader.

func (*FileUploader) Upload

func (u *FileUploader) Upload(ctx context.Context) error

Upload downloads a file from GCS and uploads it to web3.storage.

type GCS

type GCS interface {
	GetObjectReader(ctx context.Context, bName, oName string) (io.ReadCloser, error)
	GetObjectMetadata(ctx context.Context, bName, oName string) (map[string]string, error)
	ParseEvent() (string, string, error)
}

GCS defines the interface for interacting with Google Cloud Storage (GCS).

type GCSClient

type GCSClient struct {
	Client    *storage.Client
	EventData []byte
}

GCSClient implements the GCSOps interface.

func NewGCSClient

func NewGCSClient(ctx context.Context, data []byte) (*GCSClient, error)

NewGCSClient creates a new GCSClient.

func (*GCSClient) GetObjectMetadata

func (r *GCSClient) GetObjectMetadata(ctx context.Context, bucketName, objectName string) (map[string]string, error)

GetObjectMetadata returns the metadata for the specified object in the specified bucket.

func (*GCSClient) GetObjectReader

func (r *GCSClient) GetObjectReader(ctx context.Context, bucketName, objectName string) (io.ReadCloser, error)

GetObjectReader returns a reader for the specified object in the specified bucket.

func (*GCSClient) ParseEvent

func (r *GCSClient) ParseEvent() (string, string, error)

ParseEvent parses the CloudEvent data to get the bucket name and object path.

type IntermediateFile

type IntermediateFile struct {
	// contains filtered or unexported fields
}

IntermediateFile implements fs.File and is used to wrap the data read from GCS. This is needed because the Put method of the w3s.Client interface expects a fs.File. The Put method is used to upload the data to web3.storage.

func NewIntermediateFile

func NewIntermediateFile(data []byte, name string) *IntermediateFile

NewIntermediateFile creates a new IntermediateFile instance.

func (*IntermediateFile) Close

func (f *IntermediateFile) Close() error

Close noop for an in memory virtual file.

func (*IntermediateFile) Read

func (f *IntermediateFile) Read(p []byte) (n int, err error)

Read reads up to len(p) bytes into p.

func (*IntermediateFile) Stat

func (f *IntermediateFile) Stat() (fs.FileInfo, error)

Stat returns a fs.FileInfo describing the file.

type MockBasinStorage

type MockBasinStorage struct {
	// contains filtered or unexported fields
}

MockBasinStorage is the mock type for BasinStorage Contract.

func (*MockBasinStorage) AddCID

func (c *MockBasinStorage) AddCID(
	_ context.Context,
	_ string,
	cids string,
	_ int64,
	_ *bind.TransactOpts,
) error

AddCID is a mock implementation of BasinStorage.AddCID.

func (*MockBasinStorage) EstimateGas

func (c *MockBasinStorage) EstimateGas(
	_ context.Context,
	_ string,
	_ string,
	_ int64,
) (*bind.TransactOpts, error)

EstimateGas is a mock implementation of BasinStorage.EstimateGas.

func (*MockBasinStorage) GetPendingNonce

func (c *MockBasinStorage) GetPendingNonce(
	_ context.Context,
) (uint64, error)

GetPendingNonce is a mock implementation of BasinStorage.GetPendingNonce.

type MockReadCloser

type MockReadCloser struct {
	*bytes.Reader
}

MockReadCloser is a mock type for crdb.DBClient.

func (*MockReadCloser) Close

func (mrc *MockReadCloser) Close() error

Close is a mock implementation of io.Closer.

type Pub

type Pub struct {
	Namespace string
	Relation  string
}

Pub represents a ns and table/relation name.

type StatusChecker

type StatusChecker struct {
	// StatusClient is a w3s.Client instance used to interact with W3S.
	StatusClient w3s.Client
	// DBClient is a Crdb instance used to interact with CockroachDB.
	DBClient Crdb
	// contains filtered or unexported fields
}

StatusChecker checks the status of a job and updates the status in the DB.

func NewStatusChecker

func NewStatusChecker(ctx context.Context, cfg *StatusCheckerConfig) (*StatusChecker, error)

NewStatusChecker creates a new StatusChecker.

func (*StatusChecker) ProcessJobs

func (sc *StatusChecker) ProcessJobs(ctx context.Context) error

ProcessJobs checks the status of all unfinished jobs. If a job has active deals, it adds the "CID" to the BasinStorage contract. If a job has no active deals, it does nothing. If a job has already been activated, it does nothing. Finally, it updates the job status in the DB.

type StatusCheckerConfig

type StatusCheckerConfig struct {
	W3SToken         string
	CrdbConn         string
	PrivateKey       string
	BackendURL       string
	BasinStorageAddr string
	ChainID          string
}

StatusCheckerConfig defines the configuration for a StatusChecker.

type UnfinishedJob

type UnfinishedJob struct {
	Pub       Pub
	Cid       []byte
	Activated time.Time
	Timestamp *int64
	CachePath string
	ExpiresAt time.Time
}

UnfinishedJob represents a job in db that tracks deals on Filecoin.

type UploaderConfig

type UploaderConfig struct {
	W3SToken string
	CrdbConn string
}

UploaderConfig defines the configuration for a FileUploader.