provider

package

v0.0.0-...-390f9a4 Latest Latest Go to latest Published: Feb 27, 2024 License: MIT Imports: 7 Imported by: 0

Details

Valid go.mod file

The Go module system was introduced in Go 1.11 and is the official dependency management solution for Go.
Redistributable license

Redistributable licenses place minimal restrictions on how software can be used, modified, and redistributed.
Tagged version

Modules with tagged versions give importers more predictable builds.
Stable version

When a project reaches major version v1 it is considered stable.
Learn more about best practices

Repository

github.com/beyondblog/llm-api-gateway

Links

Open Source Insights

Documentation ¶

Index ¶

type CreateInstanceParam
type ExecuteCommand
type ExecuteCommandResponse
type Instance
type InstancePortInfo
type InstanceResponse
type LLMModelInfoResponse
type LLMProvider
type QueryBundleResponse
type ServerEndpoint
type VastAIProvider
- func NewVastAIProvider(apiKey, model, branch, label string) *VastAIProvider

Constants ¶

This section is empty.

Variables ¶

This section is empty.

Functions ¶

This section is empty.

Types ¶

type CreateInstanceParam ¶

type CreateInstanceParam struct {
	ClientId      string            `json:"client_id"`
	Image         string            `json:"image"`
	Env           map[string]string `json:"env"`
	Price         float64           `json:"price"`
	Disk          float64           `json:"disk"`
	Label         string            `json:"label"`
	Onstart       string            `json:"onstart"`
	RunType       string            `json:"runtype"`
	ImageLogin    string            `json:"image_login;omitempty"`
	PythonUtf8    bool              `json:"python_utf8"`
	LangUtf8      bool              `json:"lang_utf8"`
	UseJupyterLab bool              `json:"use_jupyter_lab"`
	JupyterDir    string            `json:"jupyter_dir"`
	CreateFrom    string            `json:"create_from"`
	Force         bool              `json:"force"`
}

type ExecuteCommand ¶

type ExecuteCommand struct {
	Command string `json:"command"`
}

type ExecuteCommandResponse ¶

type ExecuteCommandResponse struct {
	Success       bool   `json:"success"`
	WriteablePath string `json:"writeable_path"`
	ResultUrl     string `json:"result_url"`
	Msg           string `json:"msg"`
}

type Instance ¶

type Instance struct {
	IsBid             bool                          `json:"is_bid"`
	InetUpBilled      float64                       `json:"inet_up_billed"`
	InetDownBilled    float64                       `json:"inet_down_billed"`
	External          bool                          `json:"external"`
	Webpage           interface{}                   `json:"webpage"`
	Rentable          bool                          `json:"rentable"`
	ComputeCap        int                           `json:"compute_cap"`
	CreditBalance     interface{}                   `json:"credit_balance"`
	CreditDiscount    interface{}                   `json:"credit_discount"`
	CreditDiscountMax float64                       `json:"credit_discount_max"`
	DriverVersion     string                        `json:"driver_version"`
	CudaMaxGood       float64                       `json:"cuda_max_good"`
	MachineId         int                           `json:"machine_id"`
	HostingType       int                           `json:"hosting_type"`
	PublicIpaddr      string                        `json:"public_ipaddr"`
	Geolocation       string                        `json:"geolocation"`
	FlopsPerDphtotal  float64                       `json:"flops_per_dphtotal"`
	DlperfPerDphtotal float64                       `json:"dlperf_per_dphtotal"`
	Reliability2      float64                       `json:"reliability2"`
	HostRunTime       float64                       `json:"host_run_time"`
	ClientRunTime     float64                       `json:"client_run_time"`
	HostId            int                           `json:"host_id"`
	Id                int                           `json:"id"`
	BundleId          int                           `json:"bundle_id"`
	NumGpus           int                           `json:"num_gpus"`
	TotalFlops        float64                       `json:"total_flops"`
	MinBid            float64                       `json:"min_bid"`
	DphBase           float64                       `json:"dph_base"`
	DphTotal          float64                       `json:"dph_total"`
	GpuName           string                        `json:"gpu_name"`
	GpuRam            int                           `json:"gpu_ram"`
	GpuTotalram       int                           `json:"gpu_totalram"`
	VramCostperhour   float64                       `json:"vram_costperhour"`
	GpuDisplayActive  bool                          `json:"gpu_display_active"`
	GpuMemBw          float64                       `json:"gpu_mem_bw"`
	BwNvlink          float64                       `json:"bw_nvlink"`
	DirectPortCount   int                           `json:"direct_port_count"`
	GpuLanes          int                           `json:"gpu_lanes"`
	PcieBw            float64                       `json:"pcie_bw"`
	PciGen            float64                       `json:"pci_gen"`
	Dlperf            float64                       `json:"dlperf"`
	CpuName           string                        `json:"cpu_name"`
	MoboName          string                        `json:"mobo_name"`
	CpuRam            int                           `json:"cpu_ram"`
	CpuCores          int                           `json:"cpu_cores"`
	CpuCoresEffective float64                       `json:"cpu_cores_effective"`
	GpuFrac           float64                       `json:"gpu_frac"`
	HasAvx            int                           `json:"has_avx"`
	DiskSpace         float64                       `json:"disk_space"`
	DiskName          string                        `json:"disk_name"`
	DiskBw            float64                       `json:"disk_bw"`
	InetUp            float64                       `json:"inet_up"`
	InetDown          float64                       `json:"inet_down"`
	StartDate         float64                       `json:"start_date"`
	EndDate           *float64                      `json:"end_date"`
	Duration          *float64                      `json:"duration"`
	StorageCost       float64                       `json:"storage_cost"`
	InetUpCost        float64                       `json:"inet_up_cost"`
	InetDownCost      float64                       `json:"inet_down_cost"`
	StorageTotalCost  float64                       `json:"storage_total_cost"`
	OsVersion         string                        `json:"os_version"`
	Verification      string                        `json:"verification"`
	StaticIp          bool                          `json:"static_ip"`
	Score             float64                       `json:"score"`
	SshIdx            string                        `json:"ssh_idx"`
	SshHost           string                        `json:"ssh_host"`
	SshPort           int                           `json:"ssh_port"`
	ActualStatus      string                        `json:"actual_status"`
	IntendedStatus    string                        `json:"intended_status"`
	CurState          string                        `json:"cur_state"`
	NextState         string                        `json:"next_state"`
	ImageUuid         string                        `json:"image_uuid"`
	ImageArgs         []interface{}                 `json:"image_args"`
	ImageRuntype      string                        `json:"image_runtype"`
	ExtraEnv          [][]string                    `json:"extra_env"`
	Onstart           string                        `json:"onstart"`
	Label             string                        `json:"label"`
	JupyterToken      string                        `json:"jupyter_token"`
	StatusMsg         string                        `json:"status_msg"`
	GpuUtil           float64                       `json:"gpu_util"`
	DiskUtil          float64                       `json:"disk_util"`
	DiskUsage         float64                       `json:"disk_usage"`
	GpuTemp           float64                       `json:"gpu_temp"`
	LocalIpaddrs      string                        `json:"local_ipaddrs"`
	DirectPortEnd     int                           `json:"direct_port_end"`
	DirectPortStart   int                           `json:"direct_port_start"`
	CpuUtil           float64                       `json:"cpu_util"`
	MemUsage          float64                       `json:"mem_usage"`
	MemLimit          float64                       `json:"mem_limit"`
	VmemUsage         float64                       `json:"vmem_usage"`
	MachineDirSshPort int                           `json:"machine_dir_ssh_port"`
	Ports             map[string][]InstancePortInfo `json:"ports"`
}

type InstancePortInfo ¶

type InstancePortInfo struct {
	HostIP   string `json:"HostIp"`
	HostPort string `json:"HostPort"`
}

type InstanceResponse ¶

type InstanceResponse struct {
	Instances []Instance `json:"instances"`
}

type LLMModelInfoResponse ¶

type LLMModelInfoResponse struct {
	ModelName string        `json:"model_name"`
	LoraNames []interface{} `json:"lora_names"`
}

type LLMProvider ¶

type LLMProvider interface {
	GetEndpoints() ([]ServerEndpoint, error)
	AutoScaling(replica int) error
	GetModel() string
}

type QueryBundleResponse ¶

type QueryBundleResponse struct {
	Offers []struct {
		IsBid              bool        `json:"is_bid"`
		InetUpBilled       interface{} `json:"inet_up_billed"`
		InetDownBilled     interface{} `json:"inet_down_billed"`
		External           bool        `json:"external"`
		Webpage            interface{} `json:"webpage"`
		Logo               string      `json:"logo"`
		Rentable           bool        `json:"rentable"`
		ComputeCap         int         `json:"compute_cap"`
		CreditBalance      interface{} `json:"credit_balance"`
		CreditDiscount     interface{} `json:"credit_discount"`
		CreditDiscountMax  *float64    `json:"credit_discount_max"`
		DriverVersion      string      `json:"driver_version"`
		CudaMaxGood        float64     `json:"cuda_max_good"`
		MachineId          int         `json:"machine_id"`
		HostingType        *int        `json:"hosting_type"`
		PublicIpaddr       string      `json:"public_ipaddr"`
		Geolocation        string      `json:"geolocation"`
		FlopsPerDphtotal   float64     `json:"flops_per_dphtotal"`
		DlperfPerDphtotal  float64     `json:"dlperf_per_dphtotal"`
		Reliability2       float64     `json:"reliability2"`
		HostRunTime        float64     `json:"host_run_time"`
		ClientRunTime      float64     `json:"client_run_time"`
		HostId             int         `json:"host_id"`
		Id                 int         `json:"id"`
		BundleId           int         `json:"bundle_id"`
		NumGpus            int         `json:"num_gpus"`
		TotalFlops         float64     `json:"total_flops"`
		MinBid             float64     `json:"min_bid"`
		DphBase            float64     `json:"dph_base"`
		DphTotal           float64     `json:"dph_total"`
		GpuName            string      `json:"gpu_name"`
		GpuRam             int         `json:"gpu_ram"`
		GpuTotalram        int         `json:"gpu_totalram"`
		VramCostperhour    float64     `json:"vram_costperhour"`
		GpuDisplayActive   bool        `json:"gpu_display_active"`
		GpuMemBw           float64     `json:"gpu_mem_bw"`
		BwNvlink           float64     `json:"bw_nvlink"`
		DirectPortCount    int         `json:"direct_port_count"`
		GpuLanes           int         `json:"gpu_lanes"`
		PcieBw             float64     `json:"pcie_bw"`
		PciGen             float64     `json:"pci_gen"`
		Dlperf             float64     `json:"dlperf"`
		CpuName            string      `json:"cpu_name"`
		MoboName           string      `json:"mobo_name"`
		CpuRam             int         `json:"cpu_ram"`
		CpuCores           int         `json:"cpu_cores"`
		CpuCoresEffective  float64     `json:"cpu_cores_effective"`
		GpuFrac            float64     `json:"gpu_frac"`
		HasAvx             int         `json:"has_avx"`
		DiskSpace          float64     `json:"disk_space"`
		DiskName           string      `json:"disk_name"`
		DiskBw             float64     `json:"disk_bw"`
		InetUp             float64     `json:"inet_up"`
		InetDown           float64     `json:"inet_down"`
		StartDate          float64     `json:"start_date"`
		EndDate            *float64    `json:"end_date"`
		Duration           *float64    `json:"duration"`
		StorageCost        float64     `json:"storage_cost"`
		InetUpCost         float64     `json:"inet_up_cost"`
		InetDownCost       float64     `json:"inet_down_cost"`
		StorageTotalCost   float64     `json:"storage_total_cost"`
		OsVersion          string      `json:"os_version"`
		Verification       string      `json:"verification"`
		StaticIp           bool        `json:"static_ip"`
		Score              float64     `json:"score"`
		DiscountRate       *float64    `json:"discount_rate"`
		DiscountedHourly   float64     `json:"discounted_hourly"`
		DiscountedDphTotal float64     `json:"discounted_dph_total"`
		Rented             bool        `json:"rented"`
		BundledResults     int         `json:"bundled_results"`
		PendingCount       int         `json:"pending_count"`
	} `json:"offers"`
}

type ServerEndpoint ¶

type ServerEndpoint struct {
	ID      string
	Host    string
	Port    int
	CPUName string
	GPUName string
}

type VastAIProvider ¶

type VastAIProvider struct {
	// contains filtered or unexported fields
}

func NewVastAIProvider ¶

func NewVastAIProvider(apiKey, model, branch, label string) *VastAIProvider

func (*VastAIProvider) AutoScaling ¶

func (v *VastAIProvider) AutoScaling(replica int) error

func (*VastAIProvider) GetEndpoints ¶

func (v *VastAIProvider) GetEndpoints() ([]ServerEndpoint, error)

func (*VastAIProvider) GetModel ¶

func (v *VastAIProvider) GetModel() string

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL