Documentation ¶
Index ¶
- Constants
- Variables
- func CompileInstillAcceptFormats(sch *structpb.Struct) error
- func CompileInstillFormat(sch *structpb.Struct) error
- func ConvertFromStructpb(from *structpb.Struct, to any) error
- func ConvertToStructpb(from any) (*structpb.Struct, error)
- func FormatErrors(inputPath string, e jsonschema.Detailed, errors *[]string)
- func GetBase64FileExtension(b64 string) string
- func NewInputReader(input *structpb.Struct, schema string) *inputReader
- func NewOutputWriter(ow OutputWriter, schema string) *outputWriter
- func NewUnresolvedCredential(key string) error
- func ReadFromGlobalConfig(key string, secrets map[string]any) string
- func RenderJSON(tasksJSONBytes []byte, additionalJSONBytes map[string][]byte) ([]byte, error)
- func SequentialExecutor(ctx context.Context, jobs []*Job, ...) error
- func TaskIDToTitle(id string) string
- func TrimBase64Mime(b64 string) string
- func Validate(data *structpb.Struct, jsonSchema string, target string) error
- type Component
- func (c *Component) GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error)
- func (c *Component) GetDefinitionID() string
- func (c *Component) GetDefinitionUID() uuid.UUID
- func (c *Component) GetLogger() *zap.Logger
- func (c *Component) GetTaskInputSchemas() map[string]string
- func (c *Component) GetTaskOutputSchemas() map[string]string
- func (c *Component) HandleVerificationEvent(header map[string][]string, req *structpb.Struct, setup map[string]any) (isVerification bool, resp *structpb.Struct, err error)
- func (c *Component) IsSecretField(target string) bool
- func (c *Component) ListInputAcceptFormatsFields() (map[string]map[string][]string, error)
- func (c *Component) ListOutputFormatsFields() (map[string]map[string]string, error)
- func (c *Component) ListSecretFields() ([]string, error)
- func (c *Component) LoadDefinition(definitionJSONBytes, setupJSONBytes, tasksJSONBytes []byte, ...) error
- func (c *Component) ParseEvent(ctx context.Context, req *structpb.Struct, setup map[string]any) (parsed *structpb.Struct, err error)
- func (c *Component) Test(sysVars map[string]any, setup *structpb.Struct) error
- func (c *Component) UsageHandlerCreator() UsageHandlerCreator
- type ComponentConfig
- type ComponentExecution
- func (e *ComponentExecution) FillInDefaultValues(input *structpb.Struct) (*structpb.Struct, error)
- func (e *ComponentExecution) GetComponent() IComponent
- func (e *ComponentExecution) GetComponentID() string
- func (e *ComponentExecution) GetLogger() *zap.Logger
- func (e *ComponentExecution) GetSetup() *structpb.Struct
- func (e *ComponentExecution) GetSystemVariables() map[string]any
- func (e *ComponentExecution) GetTask() string
- func (e *ComponentExecution) GetTaskInputSchema() string
- func (e *ComponentExecution) GetTaskOutputSchema() string
- func (e *ComponentExecution) UsesInstillCredentials() bool
- type ErrorHandler
- type ExecutionWrapper
- type IComponent
- type IExecution
- type InputReader
- type InstillAcceptFormatsCompiler
- type InstillAcceptFormatsSchema
- type InstillExtension
- type InstillFormatCompiler
- type InstillFormatSchema
- type Job
- type OutputWriter
- type UsageHandler
- type UsageHandlerCreator
Constants ¶
const OpenAPITemplate = `` /* 1538-byte string literal not displayed */
const SecretKeyword = "__INSTILL_SECRET"
SecretKeyword is a keyword to reference a secret in a component configuration. When a component detects this value in a configuration parameter, it will used the pre-configured value, injected at initialization.
Variables ¶
var InstillAcceptFormatsMeta = jsonschema.MustCompileString("instillAcceptFormats.json", `{
"properties" : {
"instillAcceptFormats": {
"type": "array",
"items": {
"type": "string"
}
}
}
}`)
var InstillFormatMeta = jsonschema.MustCompileString("instillFormat.json", `{
"properties" : {
"instillFormat": {
"type": "string"
}
}
}`)
Functions ¶
func CompileInstillFormat ¶
func ConvertFromStructpb ¶
ConvertFromStructpb converts from structpb.Struct to a struct
func ConvertToStructpb ¶
ConvertToStructpb converts from a struct to structpb.Struct
func FormatErrors ¶
func GetBase64FileExtension ¶
return the extension of the file from the base64 string, in the "jpeg" , "png" format, check with provided header
func NewInputReader ¶
func NewOutputWriter ¶
func NewOutputWriter(ow OutputWriter, schema string) *outputWriter
func NewUnresolvedCredential ¶
NewUnresolvedCredential returns an end-user error signaling that the component setup contains credentials that reference a global secret that wasn't injected into the component.
func ReadFromGlobalConfig ¶
ReadFromGlobalConfig looks up a component credential field from a secret map that comes from the environment variable configuration.
Config parameters are defined with snake_case, but the environment variable configuration loader replaces underscores by dots, so we can't use the parameter key directly. TODO using camelCase in configuration fields would fix this issue.
func RenderJSON ¶
func SequentialExecutor ¶
func TaskIDToTitle ¶
TaskIDToTitle builds a Task title from its ID. This is used when the `title` key in the task definition isn't present.
func TrimBase64Mime ¶
Types ¶
type Component ¶
type Component struct { Logger *zap.Logger NewUsageHandler UsageHandlerCreator // contains filtered or unexported fields }
Component implements the common component methods.
func (*Component) GetDefinition ¶
func (c *Component) GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error)
func (*Component) GetDefinitionID ¶
GetDefinitionID returns the component definition ID.
func (*Component) GetDefinitionUID ¶
GetDefinitionUID returns the component definition UID.
func (*Component) GetLogger ¶
GetLogger returns the component's logger. If it hasn't been initialized, a no-op logger is returned.
func (*Component) GetTaskInputSchemas ¶
func (*Component) GetTaskOutputSchemas ¶
func (*Component) HandleVerificationEvent ¶
func (*Component) IsSecretField ¶
IsSecretField checks if the target field is secret field
func (*Component) ListInputAcceptFormatsFields ¶
func (*Component) ListOutputFormatsFields ¶
func (*Component) ListSecretFields ¶
ListSecretFields lists the secret fields by definition id
func (*Component) LoadDefinition ¶
func (c *Component) LoadDefinition(definitionJSONBytes, setupJSONBytes, tasksJSONBytes []byte, additionalJSONBytes map[string][]byte) error
LoadDefinition loads the component definitions from json files
func (*Component) ParseEvent ¶
func (*Component) UsageHandlerCreator ¶
func (c *Component) UsageHandlerCreator() UsageHandlerCreator
UsageHandlerCreator returns a function to initialize a UsageHandler. If the component doesn't have such function initialized, a no-op usage handler creator is returned.
type ComponentConfig ¶
type ComponentExecution ¶
type ComponentExecution struct { Component IComponent // Component ID is the ID of the component *as defined in the recipe*. This // identifies an instance of a component, which holds a given configuration // (task, setup, input parameters, etc.). // // NOTE: this is a property of the component not of the execution. However, // right now components are being created on startup and only executions // are created every time a pipeline is triggered. Therefore, at the moment // there's no intermediate entity reflecting "a component within a // pipeline". Since we need to access the component ID for e.g. logging / // metric collection purposes, for now this information will live in the // execution, but note that several executions might have the same // component ID. ComponentID string SystemVariables map[string]any Setup *structpb.Struct Task string }
ComponentExecution implements the common methods for component execution.
func (*ComponentExecution) FillInDefaultValues ¶
func (*ComponentExecution) GetComponent ¶
func (e *ComponentExecution) GetComponent() IComponent
GetComponent returns the component interface that is triggering the execution.
func (*ComponentExecution) GetComponentID ¶
func (e *ComponentExecution) GetComponentID() string
GetComponentID returns the ID of the component that's being executed.
func (*ComponentExecution) GetLogger ¶
func (e *ComponentExecution) GetLogger() *zap.Logger
func (*ComponentExecution) GetSetup ¶
func (e *ComponentExecution) GetSetup() *structpb.Struct
func (*ComponentExecution) GetSystemVariables ¶
func (e *ComponentExecution) GetSystemVariables() map[string]any
func (*ComponentExecution) GetTask ¶
func (e *ComponentExecution) GetTask() string
func (*ComponentExecution) GetTaskInputSchema ¶
func (e *ComponentExecution) GetTaskInputSchema() string
func (*ComponentExecution) GetTaskOutputSchema ¶
func (e *ComponentExecution) GetTaskOutputSchema() string
func (*ComponentExecution) UsesInstillCredentials ¶
func (e *ComponentExecution) UsesInstillCredentials() bool
UsesInstillCredentials indicates wether the component setup includes the use of global secrets (as opposed to a bring-your-own-key configuration) to connect to external services. Components should override this method when they have the ability to read global secrets and be executed without explicit credentials.
type ErrorHandler ¶
type ExecutionWrapper ¶
type ExecutionWrapper struct {
IExecution
}
ExecutionWrapper performs validation and usage collection around the execution of a component.
type IComponent ¶
type IComponent interface { GetDefinitionID() string GetDefinitionUID() uuid.UUID GetLogger() *zap.Logger GetTaskInputSchemas() map[string]string GetTaskOutputSchemas() map[string]string LoadDefinition(definitionJSON, setupJSON, tasksJSON []byte, additionalJSONBytes map[string][]byte) error // Note: Some content in the definition JSON schema needs to be generated // by sysVars or component setting. GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error) // CreateExecution takes a ComponentExecution that can be used to compose // the core component behaviour with the particular business logic in the // implmentation. CreateExecution(base ComponentExecution) (IExecution, error) Test(sysVars map[string]any, config *structpb.Struct) error IsSecretField(target string) bool // Note: These two functions are for the pipeline run-on-event feature, // which is still experimental and may change at any time. HandleVerificationEvent(header map[string][]string, req *structpb.Struct, setup map[string]any) (isVerification bool, resp *structpb.Struct, err error) ParseEvent(ctx context.Context, req *structpb.Struct, setup map[string]any) (parsed *structpb.Struct, err error) UsageHandlerCreator() UsageHandlerCreator }
IComponent is the interface that wraps the basic component methods. All component need to implement this interface.
type IExecution ¶
type IExecution interface { GetTask() string GetLogger() *zap.Logger GetTaskInputSchema() string GetTaskOutputSchema() string GetSystemVariables() map[string]any GetComponent() IComponent GetComponentID() string UsesInstillCredentials() bool Execute(context.Context, []*Job) error }
IExecution allows components to be executed.
type InputReader ¶
type InstillAcceptFormatsCompiler ¶
type InstillAcceptFormatsCompiler struct{}
func (InstillAcceptFormatsCompiler) Compile ¶
func (InstillAcceptFormatsCompiler) Compile(ctx jsonschema.CompilerContext, m map[string]interface{}) (jsonschema.ExtSchema, error)
type InstillAcceptFormatsSchema ¶
type InstillAcceptFormatsSchema []string
func (InstillAcceptFormatsSchema) Validate ¶
func (s InstillAcceptFormatsSchema) Validate(ctx jsonschema.ValidationContext, v interface{}) error
type InstillExtension ¶
type InstillExtension struct {
jsoniter.DummyExtension
}
func (*InstillExtension) UpdateStructDescriptor ¶
func (e *InstillExtension) UpdateStructDescriptor(structDescriptor *jsoniter.StructDescriptor)
type InstillFormatCompiler ¶
type InstillFormatCompiler struct{}
func (InstillFormatCompiler) Compile ¶
func (InstillFormatCompiler) Compile(ctx jsonschema.CompilerContext, m map[string]interface{}) (jsonschema.ExtSchema, error)
type InstillFormatSchema ¶
type InstillFormatSchema string
func (InstillFormatSchema) Validate ¶
func (s InstillFormatSchema) Validate(ctx jsonschema.ValidationContext, v interface{}) error
type Job ¶
type Job struct { Input InputReader Output OutputWriter Error ErrorHandler }
func GenerateMockJob ¶
func GenerateMockJob(c minimock.Tester) (ir *mock.InputReaderMock, ow *mock.OutputWriterMock, eh *mock.ErrorHandlerMock, job *Job)
type OutputWriter ¶
type UsageHandler ¶
type UsageHandler interface { Check(ctx context.Context, inputs []*structpb.Struct) error Collect(ctx context.Context, inputs, outputs []*structpb.Struct) error }
UsageHandler allows the component execution wrapper to add checks and collect usage metrics around a component execution.
func NewNoopUsageHandler ¶
func NewNoopUsageHandler(IExecution) (UsageHandler, error)
NewNoopUsageHandler is a no-op usage handler initializer.
type UsageHandlerCreator ¶
type UsageHandlerCreator func(IExecution) (UsageHandler, error)
UsageHandlerCreator returns a function to initialize a UsageHandler.