aitpipelines

package module
v0.0.0-...-1fa2ff5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 9, 2023 License: MIT Imports: 30 Imported by: 0

README 

= aitrailblazer-pipelines

golangci-lint run ./...

Documentation

Index

Constants

View Source 
const (
	OpenAIEndpointChatCompletions = "https://api.openai.com/v1/chat/completions"
	OpenAIEndpointEmbeddings      = "https://api.openai.com/v1/embeddings"
	AnthropicAIEndpointComplete   = "https://api.anthropic.com/v1/complete"

	// gpt-3.5-turbo	$0.002 / 1K tokens
	RateGPT3_5 = 0.002
	// Ada v2	$0.0004 / 1K tokens
	RateEmbedding = 0.0004
)

Variables

View Source 
var Cl100k_baseVocab vocab = vocab{}/* 100256 elements not displayed */

Functions

func Ask 

func Ask(apiKey string, queryEmbedding string, query string, df []EmbeddingAndText, model string, tokenBudget int, config ConfigAI, numTokensInTemplate int, systemMessage string, introduction string, topN int, articleFormat string, threshold float64) (MessagePromptTemplate, Usage, error)

func CopyContactExtractorToOutput 

func CopyContactExtractorToOutput(src *ContactExtractor, dst *ContactExtractorOutput)

func CosineSimilarity 

func CosineSimilarity(a, b *mat.VecDense) float64

func ExtractField 

func ExtractField(input string, fieldName string) (string, error)

func ExtractWebsiteText 

func ExtractWebsiteText(doc *goquery.Document) (string, error)

func IsValidURL 

func IsValidURL(toTest string) bool

IsValidURL tests a string to determine if it is a well-structured url or not.

func MarshalReplace 

func MarshalReplace(replacement interface{}) ([]byte, error)

func ReplaceField 

func ReplaceField(input string, fieldName string, newValue string) (string, error)

Types

type Choice 

type Choice struct {
	Message      MessagePromptTemplate `json:"message"`
	FinishReason string                `json:"finish_reason"`
	Index        int                   `json:"index"`
}

type CitationMetadataPaLM2 

type CitationMetadataPaLM2 struct {
	Citations []interface{} `json:"citations"`
}

type ClaudeResponse 

type ClaudeResponse struct {
	Completion string `json:"completion"`
	StopReason string `json:"stop_reason"`
	Truncated  bool   `json:"truncated"`
	Stop       string `json:"stop"`
	Model      string `json:"model"`
	LogID      string `json:"log_id"`
	Exception  string `json:"exception"`
}

type Codec 

type Codec struct {
	// contains filtered or unexported fields
}

Codec is a struct that represents a tokenizer codec. It contains the necessary information to perform encoding and decoding of text using a specific vocabulary. The vocabulary is represented as a map of tokens to their corresponding IDs. The Codec struct also includes a regular expression (splitRegexp) used to split the input text into tokens, as well as a map of special tokens and their IDs.

func NewCl100kBase 

func NewCl100kBase() *Codec

NewCl100kBase returns a new instance of the Codec struct with the "cl100k_base" encoding format. The function initializes the vocabulary with the Cl100k_baseVocab map, which contains the token-to-ID mappings for this encoding format. The splitRegexp field is set to a regular expression that defines how to tokenize the input text. The specialTokens field contains a map of special tokens and their corresponding IDs.

func (*Codec) Decode 

func (c *Codec) Decode(tokens []uint) (string, error)

This method takes a slice of token IDs and decodes them back into a string. It returns two values: the decoded string (string) and an error (error). If an error occurs during decoding, the error value will be non-nil.

func (*Codec) Encode 

func (c *Codec) Encode(input string) ([]uint, []string, error)

This method takes an input string and encodes it into a sequence of token IDs. It returns three values: a slice of token IDs ([]uint), a slice of tokens ([]string), and an error (error). If an error occurs during encoding, the error value will be non-nil.

func (*Codec) GetName 

func (c *Codec) GetName() string

This method returns the name of the tokenizer. The name is a string that identifies the tokenizer, such as "cl100k_base".

type ConfigAI 

type ConfigAI struct {
	Schema            int                         `json:"schema"`
	Type              string                      `json:"type"`
	Description       string                      `json:"description"`
	Completion        OpenAIChatCompletionsPrompt `json:"completion"`
	Prompt            string                      `json:"prompt,omitempty"`
	Model             string                      `json:"model,omitempty"`
	MaxTokensToSample int                         `json:"max_tokens_to_sample,omitempty"`
	StopSequences     []string                    `json:"stop_sequences,omitempty"`
	Temperature       float64                     `json:"temperature,omitempty"`
}

type ContactExtractor 

type ContactExtractor struct {
	Nm_GivenName  string
	Nm_FamilyName string
	Nm_Salutation string

	Jb_OrganizationName     string
	Ph_PhoneNumber          string
	Jb_JobTitle             string
	Ad_AddressLines         string
	Ad_City                 string
	Ad_StateOrRegion        string
	Ad_PostalCode           string
	Ad_CountryOrRegionCode  string
	Ad_TimeZone             string
	Ad_TimeZoneAbbreviation string
	Ph_FaxNumber            string
	Em_EmailAddress         string

	Ws_WebsiteAddress         string
	Ws_WebsiteTitle           string
	Ws_WebsiteDescription     string
	Ws_WebsiteImage           string
	Ws_WebsiteSiteName        string
	Ws_WebsiteType            string
	Ws_WebsiteCountry         string
	Ws_WebsiteContentLanguage string
	Ws_WebsiteAbout           string
	Ws_WebsitePrivacy         string

	EmailResponse  string
	EmailResponse2 string
	Memory         string
	Keypoints      string
	Followup       string
	// contains filtered or unexported fields
}

func (*ContactExtractor) ExecutePipelines 

func (c *ContactExtractor) ExecutePipelines(ctx context.Context, apiKey, input string, pipelineFileNames []string) (string, int, float64, error)

func (*ContactExtractor) ExecutePipelinesFromFile 

func (c *ContactExtractor) ExecutePipelinesFromFile(ctx context.Context, apiKey, inputFileName string, pipelineFileNames []string) (string, int, float64, error)

START: Pipeline execution functions executePipelineWithFile executes a pipeline using the pipeline definition in the specified file. Used for testing only.

func (*ContactExtractor) ExtractAndSetFields 

func (c *ContactExtractor) ExtractAndSetFields(output string) error

func (*ContactExtractor) GetAd_AddressLines 

func (c *ContactExtractor) GetAd_AddressLines() string

func (*ContactExtractor) GetAd_City 

func (c *ContactExtractor) GetAd_City() string

func (*ContactExtractor) GetAd_CountryOrRegionCode 

func (c *ContactExtractor) GetAd_CountryOrRegionCode() string

func (*ContactExtractor) GetAd_PostalCode 

func (c *ContactExtractor) GetAd_PostalCode() string

func (*ContactExtractor) GetAd_StateOrRegion 

func (c *ContactExtractor) GetAd_StateOrRegion() string

func (*ContactExtractor) GetAd_TimeZone 

func (c *ContactExtractor) GetAd_TimeZone() string

func (*ContactExtractor) GetAd_TimeZoneAbbreviation 

func (c *ContactExtractor) GetAd_TimeZoneAbbreviation() string

func (*ContactExtractor) GetEm_EmailAddress 

func (c *ContactExtractor) GetEm_EmailAddress() string

func (*ContactExtractor) GetEmail2Response 

func (c *ContactExtractor) GetEmail2Response() string

/

func (*ContactExtractor) GetEmailResponse 

func (c *ContactExtractor) GetEmailResponse() string

func (*ContactExtractor) GetFollowup 

func (c *ContactExtractor) GetFollowup() string

func (*ContactExtractor) GetJb_JobTitle 

func (c *ContactExtractor) GetJb_JobTitle() string

func (*ContactExtractor) GetJb_OrganizationName 

func (c *ContactExtractor) GetJb_OrganizationName() string

func (*ContactExtractor) GetKeypoints 

func (c *ContactExtractor) GetKeypoints() string

func (*ContactExtractor) GetMemory 

func (c *ContactExtractor) GetMemory() string

func (*ContactExtractor) GetNm_FamilyName 

func (c *ContactExtractor) GetNm_FamilyName() string

func (*ContactExtractor) GetNm_GivenName 

func (c *ContactExtractor) GetNm_GivenName() string

func (*ContactExtractor) GetNm_Salutation 

func (c *ContactExtractor) GetNm_Salutation() string

func (*ContactExtractor) GetPh_FaxNumber 

func (c *ContactExtractor) GetPh_FaxNumber() string

func (*ContactExtractor) GetPh_PhoneNumber 

func (c *ContactExtractor) GetPh_PhoneNumber() string

func (*ContactExtractor) GetWs_WebsiteAbout 

func (c *ContactExtractor) GetWs_WebsiteAbout() string

func (*ContactExtractor) GetWs_WebsiteAddress 

func (c *ContactExtractor) GetWs_WebsiteAddress() string

func (*ContactExtractor) GetWs_WebsiteContentLanguage 

func (c *ContactExtractor) GetWs_WebsiteContentLanguage() string

func (*ContactExtractor) GetWs_WebsiteCountry 

func (c *ContactExtractor) GetWs_WebsiteCountry() string

func (*ContactExtractor) GetWs_WebsiteDescription 

func (c *ContactExtractor) GetWs_WebsiteDescription() string

func (*ContactExtractor) GetWs_WebsiteImage 

func (c *ContactExtractor) GetWs_WebsiteImage() string

func (*ContactExtractor) GetWs_WebsitePrivacy 

func (c *ContactExtractor) GetWs_WebsitePrivacy() string

func (*ContactExtractor) GetWs_WebsiteSiteName 

func (c *ContactExtractor) GetWs_WebsiteSiteName() string

func (*ContactExtractor) GetWs_WebsiteTitle 

func (c *ContactExtractor) GetWs_WebsiteTitle() string

func (*ContactExtractor) GetWs_WebsiteType 

func (c *ContactExtractor) GetWs_WebsiteType() string

func (*ContactExtractor) SetAd_AddressLines 

func (c *ContactExtractor) SetAd_AddressLines(val string)

func (*ContactExtractor) SetAd_City 

func (c *ContactExtractor) SetAd_City(val string)

func (*ContactExtractor) SetAd_CountryOrRegionCode 

func (c *ContactExtractor) SetAd_CountryOrRegionCode(val string)

func (*ContactExtractor) SetAd_PostalCode 

func (c *ContactExtractor) SetAd_PostalCode(val string)

func (*ContactExtractor) SetAd_StateOrRegion 

func (c *ContactExtractor) SetAd_StateOrRegion(val string)

func (*ContactExtractor) SetAd_TimeZone 

func (c *ContactExtractor) SetAd_TimeZone(val string)

func (*ContactExtractor) SetAd_TimeZoneAbbreviation 

func (c *ContactExtractor) SetAd_TimeZoneAbbreviation(val string)

func (*ContactExtractor) SetEm_EmailAddress 

func (c *ContactExtractor) SetEm_EmailAddress(val string)

func (*ContactExtractor) SetEmailResponse 

func (c *ContactExtractor) SetEmailResponse(val string)

/

func (*ContactExtractor) SetEmailResponse2 

func (c *ContactExtractor) SetEmailResponse2(val string)

func (*ContactExtractor) SetFollowup 

func (c *ContactExtractor) SetFollowup(val string)

func (*ContactExtractor) SetJb_JobTitle 

func (c *ContactExtractor) SetJb_JobTitle(val string)

func (*ContactExtractor) SetJb_OrganizationName 

func (c *ContactExtractor) SetJb_OrganizationName(val string)

func (*ContactExtractor) SetKeypoints 

func (c *ContactExtractor) SetKeypoints(val string)

func (*ContactExtractor) SetMemory 

func (c *ContactExtractor) SetMemory(val string)

func (*ContactExtractor) SetNm_FamilyName 

func (c *ContactExtractor) SetNm_FamilyName(val string)

func (*ContactExtractor) SetNm_GivenName 

func (c *ContactExtractor) SetNm_GivenName(val string)

func (*ContactExtractor) SetNm_Salutation 

func (c *ContactExtractor) SetNm_Salutation(val string)

func (*ContactExtractor) SetPh_FaxNumber 

func (c *ContactExtractor) SetPh_FaxNumber(val string)

func (*ContactExtractor) SetPh_PhoneNumber 

func (c *ContactExtractor) SetPh_PhoneNumber(val string)

func (*ContactExtractor) SetWs_WebsiteAbout 

func (c *ContactExtractor) SetWs_WebsiteAbout(val string)

func (*ContactExtractor) SetWs_WebsiteAddress 

func (c *ContactExtractor) SetWs_WebsiteAddress(val string)

func (*ContactExtractor) SetWs_WebsiteContentLanguage 

func (c *ContactExtractor) SetWs_WebsiteContentLanguage(val string)

func (*ContactExtractor) SetWs_WebsiteCountry 

func (c *ContactExtractor) SetWs_WebsiteCountry(val string)

func (*ContactExtractor) SetWs_WebsiteDescription 

func (c *ContactExtractor) SetWs_WebsiteDescription(val string)

func (*ContactExtractor) SetWs_WebsiteImage 

func (c *ContactExtractor) SetWs_WebsiteImage(val string)

func (*ContactExtractor) SetWs_WebsitePrivacy 

func (c *ContactExtractor) SetWs_WebsitePrivacy(val string)

func (*ContactExtractor) SetWs_WebsiteSiteName 

func (c *ContactExtractor) SetWs_WebsiteSiteName(val string)

func (*ContactExtractor) SetWs_WebsiteTitle 

func (c *ContactExtractor) SetWs_WebsiteTitle(val string)

func (*ContactExtractor) SetWs_WebsiteType 

func (c *ContactExtractor) SetWs_WebsiteType(val string)

type ContactExtractorOutput 

type ContactExtractorOutput struct {
	Nm_GivenName  string
	Nm_FamilyName string
	Nm_Salutation string

	Jb_OrganizationName     string
	Ph_PhoneNumber          string
	Jb_JobTitle             string
	Ad_AddressLines         string
	Ad_City                 string
	Ad_StateOrRegion        string
	Ad_PostalCode           string
	Ad_CountryOrRegionCode  string
	Ad_TimeZone             string
	Ad_TimeZoneAbbreviation string
	Ph_FaxNumber            string
	Em_EmailAddress         string

	Ws_WebsiteAddress         string
	Ws_WebsiteTitle           string
	Ws_WebsiteDescription     string
	Ws_WebsiteImage           string
	Ws_WebsiteSiteName        string
	Ws_WebsiteType            string
	Ws_WebsiteCountry         string
	Ws_WebsiteContentLanguage string
	Ws_WebsiteAbout           string
	Ws_WebsitePrivacy         string

	EmailResponse  string
	EmailResponse2 string
	Memory         string
	Keypoints      string
	Followup       string
}

func (*ContactExtractorOutput) CreateOutput 

func (c *ContactExtractorOutput) CreateOutput() string

type Data 

type Data struct {
	Service            string
	Revision           string
	Project            string
	Region             string
	AuthenticatedEmail string
	Color              string
}

https://github.com/GoogleCloudPlatform/cloud-run-hello/blob/master/hello.go

type DataFrame 

type DataFrame struct {
	Text       []string
	Embeddings [][]float64
}

type Embedding 

type Embedding struct {
	Index     int       `json:"index"`
	Embedding []float64 `json:"embedding"`
	Object    string    `json:"object"`
}

type EmbeddingAndText 

type EmbeddingAndText struct {
	Text      string
	Embedding []float64
}

type EmbeddingResponse 

type EmbeddingResponse struct {
	Data   []Embedding `json:"data"`
	Model  string      `json:"model"`
	Object string      `json:"object"`
	Usage  Usage       `json:"usage"`
}

type ExamplePaLM2 

type ExamplePaLM2 struct {
	Input  MessagePaLM2 `json:"input"`
	Output MessagePaLM2 `json:"output"`
}

type InstancePaLM2 

type InstancePaLM2 struct {
	Context  *string         `json:"context,omitempty"`
	Examples *[]ExamplePaLM2 `json:"examples,omitempty"`
	Messages *[]MessagePaLM2 `json:"messages,omitempty"`
	Content  *string         `json:"content,omitempty"`
}

type MessagePaLM2 

type MessagePaLM2 struct {
	Author  string `json:"author"`
	Content string `json:"content"`
}

type MessagePromptTemplate 

type MessagePromptTemplate struct {
	Content string `json:"content"`
	Role    string `json:"role"`
	Name    string `json:"name,omitempty"`
}

type OpenAIChatCompletionsPrompt 

type OpenAIChatCompletionsPrompt struct {
	Model            string                  `json:"model"`
	Temperature      float64                 `json:"temperature"`
	N                int                     `json:"n"`
	TopP             float64                 `json:"top_p"`
	FrequencyPenalty float64                 `json:"frequency_penalty"`
	PresencePenalty  float64                 `json:"presence_penalty"`
	MaxTokens        int                     `json:"max_tokens"`
	User             string                  `json:"user"`
	StopSequences    []string                `json:"stop,omitempty"`
	Messages         []MessagePromptTemplate `json:"messages,omitempty"`
}

type OpenAIChatCompletionsResponse 

type OpenAIChatCompletionsResponse struct {
	ID      string   `json:"id"`
	Object  string   `json:"object"`
	Created int64    `json:"created"`
	Model   string   `json:"model"`
	Usage   Usage    `json:"usage"`
	Choices []Choice `json:"choices"`
}

Response from OpenAI API

type ParametersPaLM2 

type ParametersPaLM2 struct {
	Temperature     float32 `json:"temperature"`
	MaxOutputTokens int     `json:"maxOutputTokens"`
	TopP            float32 `json:"topP"`
	TopK            int     `json:"topK"`
}

type PayloadPaLM2 

type PayloadPaLM2 struct {
	Instances  []InstancePaLM2 `json:"instances"`
	Parameters ParametersPaLM2 `json:"parameters"`
}

func ReadAndReplaceJSONPaLM2 

func ReadAndReplaceJSONPaLM2(filepath string, replacementText string) (PayloadPaLM2, string, error)

type PipelineDefinition 

type PipelineDefinition struct {
	Name            string     `json:"name"`
	Steps           []string   `json:"steps,omitempty"`
	ConcurrentSteps [][]string `json:"concurrent_steps,omitempty"`
	PipelineOutput  []string   `json:"pipeline_output,omitempty"`
}

type PredictionPaLM2 

type PredictionPaLM2 struct {
	Content          string                `json:"content"`
	SafetyAttributes SafetyAttributesPaLM2 `json:"safetyAttributes"`
	CitationMetadata CitationMetadataPaLM2 `json:"citationMetadata"`
}

type ResponsePaLM2 

type ResponsePaLM2 struct {
	Predictions []PredictionPaLM2 `json:"predictions"`
}

type Result 

type Result struct {
	Text        string
	Relatedness float64
}

type RetryConfig 

type RetryConfig struct {
	MaxAttempts   int
	MinDelay      time.Duration
	MaxDelay      time.Duration
	BackoffFactor float64
	JitterFactor  float64
}

type SafetyAttributesPaLM2 

type SafetyAttributesPaLM2 struct {
	Scores     []float32 `json:"scores"`
	Blocked    bool      `json:"blocked"`
	Categories []string  `json:"categories"`
}

type StepResult 

type StepResult struct {
	Step   string
	Result string
}

type TemplateData 

type TemplateData struct {
	Name      string                  `json:"name"`
	StepLines []MessagePromptTemplate `json:"steplines"`
}

type Usage 

type Usage struct {
	PromptTokens     int `json:"prompt_tokens"`
	CompletionTokens int `json:"completion_tokens,omitempty"`
	TotalTokens      int `json:"total_tokens"`
}

func GetOpenAIEmbedding 

func GetOpenAIEmbedding(apiKey string, input string) ([]float64, Usage, error)

GetOpenAIEmbedding returns the embedding for the given input text. The input text should be a single sentence. The returned embedding is a n-dimensional vector.

func StringsRankedByRelatedness 

func StringsRankedByRelatedness(apiKey string, queryEmbedding string, df []EmbeddingAndText, topN int, threshold float64) ([]string, []float64, Usage, error)

type WebData 

type WebData struct {
	Title           string
	Description     string
	OGImage         string
	OgSiteName      string
	OgType          string
	Country         string
	ContentLanguage string
	CanonicalLink   string
	Text            string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.