inference

Documentation

Overview

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Copyright (c) Microsoft Corporation. Licensed under the MIT license.

Index

• Constants
• Variables
• func CreatePresetInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, ...) (client.Object, error)
• func CreateTemplateInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, ...) (client.Object, error)
• func GetImageInfo(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, ...) (string, []corev1.LocalObjectReference)

Constants

const (
	DefaultNnodes       = "1"
	DefaultNprocPerNode = "1"
	DefaultNodeRank     = "0"
	DefaultMasterAddr   = "localhost"
	DefaultMasterPort   = "29500"
)

const (
	DefaultMaxRestarts  = "3"
	DefaultRdzvId       = "rdzv_id"
	DefaultRdzvBackend  = "c10d"            // Pytorch Native Distributed data store
	DefaultRdzvEndpoint = "localhost:29500" // e.g. llama-2-13b-chat-0.llama-headless.default.svc.cluster.local:29500
)

Torch Rendezvous Params

const (
	DefaultConfigFile   = "config.yaml"
	DefaultNumProcesses = "1"
	DefaultNumMachines  = "1"
	DefaultMachineRank  = "0"
	DefaultGPUIds       = "all"
)

const (
	ProbePath              = "/healthz"
	Port5000               = int32(5000)
	InferenceFile          = "inference_api.py"
	DefaultVolumeMountPath = "/dev/shm"
)

Variables

var (
	DefaultTorchRunParams = map[string]string{
		"nnodes":         DefaultNnodes,
		"nproc_per_node": DefaultNprocPerNode,
		"node_rank":      DefaultNodeRank,
		"master_addr":    DefaultMasterAddr,
		"master_port":    DefaultMasterPort,
	}

	DefaultTorchRunRdzvParams = map[string]string{
		"max_restarts":  DefaultMaxRestarts,
		"rdzv_id":       DefaultRdzvId,
		"rdzv_backend":  DefaultRdzvBackend,
		"rdzv_endpoint": DefaultRdzvEndpoint,
	}

	DefaultAccelerateParams = map[string]string{
		"num_processes": DefaultNumProcesses,
		"num_machines":  DefaultNumMachines,
		"machine_rank":  DefaultMachineRank,
		"gpu_ids":       DefaultGPUIds,
	}

	DefaultImagePullSecrets = []corev1.LocalObjectReference{}
)

TODO: remove the above local variables starting with lower cases.

Functions

func CreatePresetInference

func CreatePresetInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace,
	inferenceObj *model.PresetInferenceParam, supportDistributedInference bool, kubeClient client.Client) (client.Object, error)

func CreateTemplateInference

func CreateTemplateInference(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, kubeClient client.Client) (client.Object, error)

func GetImageInfo

func GetImageInfo(ctx context.Context, workspaceObj *kaitov1alpha1.Workspace, inferenceObj *model.PresetInferenceParam) (string, []corev1.LocalObjectReference)