bacalhau

package
v0.3.22 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 22, 2023 License: Apache-2.0 Imports: 62 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	JSONFormat                         string = "json"
	YAMLFormat                         string = "yaml"
	DefaultDockerRunWaitSeconds               = 600
	PrintoutCanceledButRunningNormally string = "printout canceled but running normally"
	// AutoDownloadFolderPerm is what permissions we give to a folder we create when downloading results
	AutoDownloadFolderPerm       = 0755
	HowFrequentlyToUpdateTicker  = 50 * time.Millisecond
	DefaultSpinnerFormatDuration = 30 * time.Millisecond
	DefaultTimeout               = 30 * time.Minute
)

Variables

View Source 
var DefaultShortenStringLength = 20
View Source 
var DefaultSwarmPort = 1235
View Source 
var Fatal = FatalErrorHandler
View Source 
var ShutdownSignals = []os.Signal{
	os.Interrupt,
	syscall.SIGTERM,
}

Functions

func CreateJob 

func CreateJob(cmdArgs []string, odr *DockerRunOptions) (*model.Job, error)

CreateJob creates a job object from the given command line arguments and options.

func DockerImageContainsTag 

func DockerImageContainsTag(image string) bool

DockerImageContainsTag checks if the image contains a tag or a digest

func Execute 

func Execute()

func ExecuteJob 

func ExecuteJob(ctx context.Context,
	cm *system.CleanupManager,
	cmd *cobra.Command,
	j *model.Job,
	runtimeSettings RunTimeSettings,
	downloadSettings model.DownloaderSettings,
) error

func ExecuteTestCobraCommand 

func ExecuteTestCobraCommand(t *testing.T, args ...string) (c *cobra.Command, output string, err error)

func ExecuteTestCobraCommandWithStdin 

func ExecuteTestCobraCommandWithStdin(_ *testing.T, stdin io.Reader, args ...string) (
	c *cobra.Command, output string, err error,
)

func FakeFatalErrorHandler 

func FakeFatalErrorHandler(cmd *cobra.Command, msg string, code int)

FakeFatalErrorHandler captures the error for testing, responsibility of the test to handle the exit (if any) NOTE: If your test is not idempotent, you can cause side effects (the underlying function will continue to run) Returned as text JSON to wherever RootCmd is printing.

func FatalErrorHandler 

func FatalErrorHandler(cmd *cobra.Command, msg string, code int)

func GenerateJobJSONSchema 

func GenerateJobJSONSchema() ([]byte, error)

func GetAPIClient 

func GetAPIClient() *publicapi.RequesterAPIClient

func NewIPFSDownloadFlags 

func NewIPFSDownloadFlags(settings *model.DownloaderSettings) *pflag.FlagSet

func NewRootCmd 

func NewRootCmd() *cobra.Command

func NewRunTimeSettingsFlags 

func NewRunTimeSettingsFlags(settings *RunTimeSettings) *pflag.FlagSet

func ReadFromStdinIfAvailable 

func ReadFromStdinIfAvailable(cmd *cobra.Command, args []string) ([]byte, error)

func WaitForJobAndPrintResultsToUser added in v0.3.22 

func WaitForJobAndPrintResultsToUser(ctx context.Context, cmd *cobra.Command, j *model.Job, quiet bool) error

WaitForJobAndPrintResultsToUser uses events to decide what to output to the terminal using a spinner to show long-running tasks. When the job is complete (or the user triggers SIGINT) then the function will complete and stop outputting to the terminal.

Types

type ArrayValueFlag 

type ArrayValueFlag[T any] struct {
	// contains filtered or unexported fields
}

An ArrayValueFlag is like a ValueFlag except it will add the command line value into a slice of values, and hence can be used for flags that are meant to appear multiple times.

func ExcludedTagFlag 

func ExcludedTagFlag(value *[]model.ExcludedTag) *ArrayValueFlag[model.ExcludedTag]

func IncludedTagFlag 

func IncludedTagFlag(value *[]model.IncludedTag) *ArrayValueFlag[model.IncludedTag]

func NewIPFSStorageSpecArrayFlag 

func NewIPFSStorageSpecArrayFlag(value *[]model.StorageSpec) *ArrayValueFlag[model.StorageSpec]

func NewURLStorageSpecArrayFlag 

func NewURLStorageSpecArrayFlag(value *[]model.StorageSpec) *ArrayValueFlag[model.StorageSpec]

func (*ArrayValueFlag[T]) Set 

func (s *ArrayValueFlag[T]) Set(input string) error

Set implements pflag.Value

func (*ArrayValueFlag[T]) String 

func (s *ArrayValueFlag[T]) String() string

String implements pflag.Value

func (*ArrayValueFlag[T]) Type 

func (s *ArrayValueFlag[T]) Type() string

Type implements pflag.Value

type CancelOptions added in v0.3.21 

type CancelOptions struct{}

func NewCancelOptions added in v0.3.21 

func NewCancelOptions() *CancelOptions

type ColumnEnum 

type ColumnEnum string

From: https://stackoverflow.com/questions/50824554/permitted-flag-values-for-cobra 

const (
	ColumnID        ColumnEnum = "id"
	ColumnCreatedAt ColumnEnum = "created_at"
)

func (*ColumnEnum) Set 

func (c *ColumnEnum) Set(v string) error

Set must have pointer receiver so it doesn't change the value of a copy

func (*ColumnEnum) String 

func (c *ColumnEnum) String() string

func (*ColumnEnum) Type 

func (c *ColumnEnum) Type() string

Type is only used in help text

type CreateOptions 

type CreateOptions struct {
	Filename        string                   // Filename for job (can be .json or .yaml)
	Concurrency     int                      // Number of concurrent jobs to run
	Confidence      int                      // Minimum number of nodes that must agree on a verification result
	RunTimeSettings RunTimeSettings          // Run time settings for execution (e.g. wait, get, etc after submission)
	DownloadFlags   model.DownloaderSettings // Settings for running Download
	DryRun          bool
}

func NewCreateOptions 

func NewCreateOptions() *CreateOptions

type DescribeOptions 

type DescribeOptions struct {
	Filename      string // Filename for job (can be .json or .yaml)
	IncludeEvents bool   // Include events in the description
	OutputSpec    bool   // Print Just the jobspec to stdout
	JSON          bool   // Print description as JSON
}

func NewDescribeOptions 

func NewDescribeOptions() *DescribeOptions

type DockerRunOptions 

type DockerRunOptions struct {
	Engine           string   // Executor - executor.Executor
	Verifier         string   // Verifier - verifier.Verifier
	Publisher        string   // Publisher - publisher.Publisher
	Inputs           []string // Array of input CIDs
	InputUrls        []string // Array of input URLs (will be copied to IPFS)
	InputVolumes     []string // Array of input volumes in 'CID:mount point' form
	OutputVolumes    []string // Array of output volumes in 'name:mount point' form
	Env              []string // Array of environment variables
	IDOnly           bool     // Only print the job ID
	Concurrency      int      // Number of concurrent jobs to run
	Confidence       int      // Minimum number of nodes that must agree on a verification result
	MinBids          int      // Minimum number of bids before they will be accepted (at random)
	Timeout          float64  // Job execution timeout in seconds
	CPU              string
	Memory           string
	GPU              string
	Networking       model.Network
	NetworkDomains   []string
	WorkingDirectory string   // Working directory for docker
	Labels           []string // Labels for the job on the Bacalhau network (for searching)
	NodeSelector     string   // Selector (label query) to filter nodes on which this job can be executed

	Image      string   // Image to execute
	Entrypoint []string // Entrypoint to the docker image

	SkipSyntaxChecking bool // Verify the syntax using shellcheck

	DryRun bool // Don't submit the jobspec, print it to STDOUT

	RunTimeSettings RunTimeSettings // Settings for running the job

	DownloadFlags model.DownloaderSettings // Settings for running Download

	ShardingGlobPattern string
	ShardingBasePath    string
	ShardingBatchSize   int

	FilPlus bool // add a "filplus" label to the job to grab the attention of fil+ moderators
}

DockerRunOptions declares the arguments accepted by the `docker run` command

func NewDockerRunOptions 

func NewDockerRunOptions() *DockerRunOptions

type FullLineMessage 

type FullLineMessage struct {
	Message     string
	TimerString string
	StopString  string
	Width       int
}

FullLineMessage has to be global so that multiple routines can access

func (*FullLineMessage) PrintDone 

func (f *FullLineMessage) PrintDone() string

func (*FullLineMessage) PrintError 

func (f *FullLineMessage) PrintError() string

func (*FullLineMessage) String 

func (f *FullLineMessage) String() string

type GetOptions 

type GetOptions struct {
	IPFSDownloadSettings *model.DownloaderSettings
}

func NewGetOptions 

func NewGetOptions() *GetOptions

type IDInfo 

type IDInfo struct {
	ID string `json:"ID"`
}

type KeyValueParser 

type KeyValueParser[K comparable, V any] func(string) (K, V, error)

A KeyValueParser is like a Parser except that it returns two values representing a key and a value.

type KeyValueStringer 

type KeyValueStringer[K comparable, V any] func(*K, *V) string

A KeyValueStringer is like a Stringer except that it converts native objects representing a key and a value into a string.

type LanguageRunOptions 

type LanguageRunOptions struct {
	Deterministic bool     // Execute this job deterministically
	Inputs        []string // Array of input CIDs
	InputUrls     []string // Array of input URLs (will be copied to IPFS)
	InputVolumes  []string // Array of input volumes in 'CID:mount point' form
	OutputVolumes []string // Array of output volumes in 'name:mount point' form
	Env           []string // Array of environment variables
	Concurrency   int      // Number of concurrent jobs to run
	Confidence    int      // Minimum number of nodes that must agree on a verification result
	MinBids       int      // Minimum number of bids that must be received before any are accepted (at random)
	Timeout       float64  // Job execution timeout in seconds
	Labels        []string // Labels for the job on the Bacalhau network (for searching)

	Command          string // Command to execute
	RequirementsPath string // Path for requirements.txt for executing with Python
	ContextPath      string // ContextPath (code) for executing with Python

	RuntimeSettings  RunTimeSettings
	DownloadSettings model.DownloaderSettings
}

LanguageRunOptions declares the arguments accepted by the `'language' run` command

func NewLanguageRunOptions 

func NewLanguageRunOptions() *LanguageRunOptions

type ListOptions 

type ListOptions struct {
	HideHeader   bool                // Hide the column headers
	IDFilter     string              // Filter by Job List to IDs matching substring.
	IncludeTags  []model.IncludedTag // Only return jobs with these annotations
	ExcludeTags  []model.ExcludedTag // Only return jobs without these annotations
	NoStyle      bool                // Remove all styling from table output.
	MaxJobs      int                 // Print the first NUM jobs instead of the first 10.
	OutputFormat string              // The output format for the list of jobs (json or text)
	SortReverse  bool                // Reverse order of table - for time sorting, this will be newest first.
	SortBy       ColumnEnum          // Sort by field, defaults to creation time, with newest first [Allowed "id", "created_at"].
	OutputWide   bool                // Print full values in the table results
	ReturnAll    bool                // Return all jobs, not just those that belong to the user
}

func NewListOptions 

func NewListOptions() *ListOptions

type MapValueFlag 

type MapValueFlag[K comparable, V any] struct {
	// contains filtered or unexported fields
}

A MapValueFlag is like a ValueFlag except it will add the command line value into a map of values, and hence can be used for flags that are meant to appear multiple times and represent a key-value structure.

func EnvVarMapFlag 

func EnvVarMapFlag(value *map[string]string) *MapValueFlag[string, string]

func (*MapValueFlag[K, V]) Set 

func (s *MapValueFlag[K, V]) Set(input string) error

Set implements pflag.Value

func (*MapValueFlag[K, V]) String 

func (s *MapValueFlag[K, V]) String() string

String implements pflag.Value

func (*MapValueFlag[K, V]) Type 

func (s *MapValueFlag[K, V]) Type() string

Type implements pflag.Value

type Parser 

type Parser[T any] func(string) (T, error)

A Parser is a function that can convert a string into a native object.

type RunTimeSettings 

type RunTimeSettings struct {
	AutoDownloadResults   bool // Automatically download the results after finishing
	IPFSGetTimeOut        int  // Timeout for IPFS in seconds
	IsLocal               bool // Job should be executed locally
	WaitForJobToFinish    bool // Wait for the job to finish before returning
	WaitForJobTimeoutSecs int  // Timeout for waiting for the job to finish
	PrintJobIDOnly        bool // Only print the Job ID as output
	PrintNodeDetails      bool // Print the node details as output
}

func NewRunTimeSettings 

func NewRunTimeSettings() *RunTimeSettings

type ServeOptions 

type ServeOptions struct {
	NodeType                              []string          // "compute", "requester" node or both
	PeerConnect                           string            // The libp2p multiaddress to connect to.
	IPFSConnect                           string            // The multiaddress to connect to for IPFS.
	FilecoinUnsealedPath                  string            // Go template to turn a Filecoin CID into a local filepath with the unsealed data.
	EstuaryAPIKey                         string            // The API key used when using the estuary API.
	HostAddress                           string            // The host address to listen on.
	SwarmPort                             int               // The host port for libp2p network.
	JobSelectionDataLocality              string            // The data locality to use for job selection.
	JobSelectionDataRejectStateless       bool              // Whether to reject jobs that don't specify any data.
	JobSelectionDataAcceptNetworked       bool              // Whether to accept jobs that require network access.
	JobSelectionProbeHTTP                 string            // The HTTP URL to use for job selection.
	JobSelectionProbeExec                 string            // The executable to use for job selection.
	LimitTotalCPU                         string            // The total amount of CPU the system can be using at one time.
	LimitTotalMemory                      string            // The total amount of memory the system can be using at one time.
	LimitTotalGPU                         string            // The total amount of GPU the system can be using at one time.
	LimitJobCPU                           string            // The amount of CPU the system can be using at one time for a single job.
	LimitJobMemory                        string            // The amount of memory the system can be using at one time for a single job.
	LimitJobGPU                           string            // The amount of GPU the system can be using at one time for a single job.
	LotusFilecoinStorageDuration          time.Duration     // How long deals should be for the Lotus Filecoin publisher
	LotusFilecoinPathDirectory            string            // The location of the Lotus configuration directory which contains config.toml, etc
	LotusFilecoinUploadDirectory          string            // Directory to put files when uploading to Lotus (optional)
	LotusFilecoinMaximumPing              time.Duration     // The maximum ping allowed when selecting a Filecoin miner
	JobExecutionTimeoutClientIDBypassList []string          // IDs of clients that can submit jobs more than the configured job execution timeout
	Labels                                map[string]string // Labels to apply to the node that can be used for node selection and filtering
	IPFSSwarmAddresses                    []string          // IPFS multiaddresses that the in-process IPFS should connect to
	PrivateInternalIPFS                   bool              // Whether the in-process IPFS should automatically discover other IPFS nodes
}

func NewServeOptions 

func NewServeOptions() *ServeOptions

type Stringer 

type Stringer[T any] func(*T) string

A Stringer is a function that can convert a native object into a string.

type ValidateOptions 

type ValidateOptions struct {
	Filename        string // Filename for job (can be .json or .yaml)
	OutputFormat    string // Output format (json or yaml)
	OutputSchema    bool   // Output the schema to stdout
	OutputDirectory string // Output directory for the job
}

func NewValidateOptions 

func NewValidateOptions() *ValidateOptions

type ValueFlag 

type ValueFlag[T any] struct {
	// contains filtered or unexported fields
}

A ValueFlag is a pflag.Value that knows how to take a command line value represented as a string and set it as a native object into a struct.

func LoggingFlag added in v0.3.20 

func LoggingFlag(value *logger.LogMode) *ValueFlag[logger.LogMode]

func NetworkFlag 

func NetworkFlag(value *model.Network) *ValueFlag[model.Network]

func PublisherFlag 

func PublisherFlag(value *model.Publisher) *ValueFlag[model.Publisher]

func VerifierFlag 

func VerifierFlag(value *model.Verifier) *ValueFlag[model.Verifier]

func (*ValueFlag[T]) Set 

func (s *ValueFlag[T]) Set(input string) error

Set implements pflag.Value

func (*ValueFlag[T]) String 

func (s *ValueFlag[T]) String() string

String implements pflag.Value

func (*ValueFlag[T]) Type 

func (s *ValueFlag[T]) Type() string

Type implements pflag.Value

type VersionOptions 

type VersionOptions struct {
	ClientOnly bool
	Output     string
	// contains filtered or unexported fields
}

VersionOptions is a struct to support version command

func NewVersionOptions 

func NewVersionOptions() *VersionOptions

NewVersionOptions returns initialized Options

func (*VersionOptions) Run 

func (oV *VersionOptions) Run(ctx context.Context, cmd *cobra.Command) error

Run executes version command

func (*VersionOptions) Validate 

func (oV *VersionOptions) Validate(*cobra.Command) error

Validate validates the provided options

type Versions 

type Versions struct {
	ClientVersion *model.BuildVersionInfo `json:"clientVersion,omitempty"`
	ServerVersion *model.BuildVersionInfo `json:"serverVersion,omitempty"`
}

Versions is a struct for version information

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.