common

Documentation

Index

• Constants
• Variables
• func CompareFloat(a float64, b float64, tolerance float64) bool
• func CompareFloatArray(a interface{}, b interface{}, tolerance float64) bool
• func CompareIntArray(a interface{}, b interface{}) bool
• func CreateClientSet() *kubernetes.Clientset
• func DimProduct(dim []int64) int64
• func GetDimFromTensorProto(pb *tensor_go_proto.TensorProto) []int64
• func GetMasterPodName(jobName string) string
• func PodFinished(clientSet *kubernetes.Clientset, namespace string, podName string) bool
• func Slice(t *Tensor) interface{}
• type DataType
• type EmbeddingTable
  • func NewEmbeddingTable(dim int64, initializer string, dtype types_go_proto.DataType) *EmbeddingTable
  • func (e *EmbeddingTable) GetEmbeddingVector(index int64) *Tensor
  • func (e *EmbeddingTable) GetEmbeddingVectors(indices []int64) *Tensor
  • func (e *EmbeddingTable) SetEmbeddingVectors(idxslice *IndexedSlices) error
  • func (e *EmbeddingTable) ToIndexedSlices() *IndexedSlices
• type IndexedSlices
  • func DeserializeFromIndexedSliceProto(pb *proto.IndexedSlicesProto) *IndexedSlices
  • func MergeIndexedSlices(first *IndexedSlices, second *IndexedSlices) (*IndexedSlices, error)
  • func NewIndexedSlices(t *Tensor, ids []int64) *IndexedSlices
  • func (t *IndexedSlices) SerializeToIndexedSlicesProto() *proto.IndexedSlicesProto
• type Initializer
  • func Constant(n interface{}) Initializer
  • func RandomNorm(mean float64, std float64, seed int64) Initializer
  • func RandomUniform(min float64, max float64, seed int64) Initializer
  • func TruncatedNormal(mean float64, std float64, seed int64) Initializer
  • func Zero() Initializer
• type Tensor
  • func DeserializeFromTensorProto(pb *tensor_go_proto.TensorProto) *Tensor
  • func NewEmptyTensor(dim []int64, dtype types_go_proto.DataType) *Tensor
  • func NewEmptyVector(dim int64, dtype types_go_proto.DataType) *Tensor
  • func NewTensor(slice interface{}, dim []int64) *Tensor
  • func NewVector(slice interface{}) *Tensor
  • func (t *Tensor) GetRow(idx int64) *Tensor
  • func (t *Tensor) GetSubTensor(begin int64, length int64) *Tensor
  • func (t *Tensor) IsValid() bool
  • func (t *Tensor) SerializeToTensorProto() *tensor_go_proto.TensorProto
  • func (t *Tensor) SetRow(idx int64, vec *Tensor)
  • func (t *Tensor) SetSubTensor(begin int64, length int64, val *Tensor)

Constants

const (
	Invalid = types_go_proto.DataType_DT_INVALID
	Int8    = types_go_proto.DataType_DT_INT8
	Int16   = types_go_proto.DataType_DT_INT16
	Int32   = types_go_proto.DataType_DT_INT32
	Int64   = types_go_proto.DataType_DT_INT64
	Float16 = types_go_proto.DataType_DT_BFLOAT16
	Float32 = types_go_proto.DataType_DT_FLOAT
	Float64 = types_go_proto.DataType_DT_DOUBLE
	Bool    = types_go_proto.DataType_DT_BOOL
)

const alias for dtype

Variables

var DtypeSize = make(map[types_go_proto.DataType]int32)

DtypeSize Dtype -> size

var DtypeToSliceType = make(map[types_go_proto.DataType]reflect.Type)

DtypeToSliceType Dtype -> reflect.Type

var DtypeToValueType = make(map[types_go_proto.DataType]reflect.Type)

DtypeToValueType Dtype -> reflect.Type

var SliceTypeToDtype = make(map[reflect.Type]types_go_proto.DataType)

SliceTypeToDtype reflect.Type -> Dtype

Functions

func CompareFloat

func CompareFloat(a float64, b float64, tolerance float64) bool

CompareFloat compares two float32 number

func CompareFloatArray

func CompareFloatArray(a interface{}, b interface{}, tolerance float64) bool

CompareFloatArray compares two float32/64 array

func CompareIntArray

func CompareIntArray(a interface{}, b interface{}) bool

CompareIntArray compares two int8/16/32/64 array

func CreateClientSet

func CreateClientSet() *kubernetes.Clientset

CreateClientSet uses in-cluster config to create a clientset.

func DimProduct

func DimProduct(dim []int64) int64

DimProduct get the number of the elements of a tensor of this dim

func GetDimFromTensorProto

func GetDimFromTensorProto(pb *tensor_go_proto.TensorProto) []int64

GetDimFromTensorProto get dim from proto

func GetMasterPodName

func GetMasterPodName(jobName string) string

GetMasterPodName returns master pod name

func PodFinished

func PodFinished(clientSet *kubernetes.Clientset, namespace string, podName string) bool

PodFinished returns true if the pod is Succeeded/Failed, or still running but with metadata.labels["status"]==""Finished"

func Slice

func Slice(t *Tensor) interface{}

Slice gives a Slice interface to the Tensor data

Types

type DataType

type DataType = types_go_proto.DataType

DataType alias

type EmbeddingTable

type EmbeddingTable struct {
	Dim              int64
	Initializer      string
	EmbeddingVectors map[int64]*Tensor
	Dtype            types_go_proto.DataType
	// contains filtered or unexported fields
}

EmbeddingTable struct

func NewEmbeddingTable

func NewEmbeddingTable(dim int64, initializer string, dtype types_go_proto.DataType) *EmbeddingTable

NewEmbeddingTable creates an embedding table instance

func (*EmbeddingTable) GetEmbeddingVector

func (e *EmbeddingTable) GetEmbeddingVector(index int64) *Tensor

GetEmbeddingVector returns an REFERENCE of embedding vector giving an index

func (*EmbeddingTable) GetEmbeddingVectors

func (e *EmbeddingTable) GetEmbeddingVectors(indices []int64) *Tensor

GetEmbeddingVectors returns COPYS of embedding vectors giving an array of indices

func (*EmbeddingTable) SetEmbeddingVectors

func (e *EmbeddingTable) SetEmbeddingVectors(idxslice *IndexedSlices) error

SetEmbeddingVectors sets (indices, value) pair to embedding vector

func (*EmbeddingTable) ToIndexedSlices

func (e *EmbeddingTable) ToIndexedSlices() *IndexedSlices

ToIndexedSlices transforms embedding table format to indexed slices format

type IndexedSlices

type IndexedSlices struct {
	ConcatTensors *Tensor
	Ids           []int64
}

IndexedSlices : IndexedSlices in-memory representation

func DeserializeFromIndexedSliceProto

func DeserializeFromIndexedSliceProto(pb *proto.IndexedSlicesProto) *IndexedSlices

DeserializeFromIndexedSliceProto return common.IndexedTensor

func MergeIndexedSlices

func MergeIndexedSlices(first *IndexedSlices, second *IndexedSlices) (*IndexedSlices, error)

MergeIndexedSlices merges two indexed slices into one indexed slices

func NewIndexedSlices

func NewIndexedSlices(t *Tensor, ids []int64) *IndexedSlices

NewIndexedSlices return a IndexedTensor instance

func (*IndexedSlices) SerializeToIndexedSlicesProto

func (t *IndexedSlices) SerializeToIndexedSlicesProto() *proto.IndexedSlicesProto

SerializeToIndexedSlicesProto return proto.IndexedSlices

type Initializer

type Initializer = func(*Tensor) error

Initializer definition

func Constant

func Constant(n interface{}) Initializer

Constant return a constant Initializer

func RandomNorm

func RandomNorm(mean float64, std float64, seed int64) Initializer

RandomNorm return a normal Initializer

func RandomUniform

func RandomUniform(min float64, max float64, seed int64) Initializer

RandomUniform return a uniform Initializer

func TruncatedNormal

func TruncatedNormal(mean float64, std float64, seed int64) Initializer

TruncatedNormal return a truncated-normal Initializer

func Zero

func Zero() Initializer

Zero return a zero Initializer

type Tensor

type Tensor struct {
	Buffer []byte
	Dims   []int64
	Dtype  types_go_proto.DataType
}

Tensor definition

func DeserializeFromTensorProto

func DeserializeFromTensorProto(pb *tensor_go_proto.TensorProto) *Tensor

DeserializeFromTensorProto transforms pb to tensor

func NewEmptyTensor

func NewEmptyTensor(dim []int64, dtype types_go_proto.DataType) *Tensor

NewEmptyTensor create an empty n-dim tensor

func NewEmptyVector

func NewEmptyVector(dim int64, dtype types_go_proto.DataType) *Tensor

NewEmptyVector create an empty 1-dim tensor

func NewTensor

func NewTensor(slice interface{}, dim []int64) *Tensor

NewTensor create a n-dim tensor using exsiting slice

func NewVector

func NewVector(slice interface{}) *Tensor

NewVector create an empty 1-dim tensor

func (*Tensor) GetRow

func (t *Tensor) GetRow(idx int64) *Tensor

GetRow get the row reference of a 2-dim tensor

func (*Tensor) GetSubTensor

func (t *Tensor) GetSubTensor(begin int64, length int64) *Tensor

GetSubTensor get the part reference of the tensor

func (*Tensor) IsValid

func (t *Tensor) IsValid() bool

IsValid check tensor validity

func (*Tensor) SerializeToTensorProto

func (t *Tensor) SerializeToTensorProto() *tensor_go_proto.TensorProto

SerializeToTensorProto transforms tensor to pb

func (*Tensor) SetRow

func (t *Tensor) SetRow(idx int64, vec *Tensor)

SetRow set a vector to an index of tensor

func (*Tensor) SetSubTensor

func (t *Tensor) SetSubTensor(begin int64, length int64, val *Tensor)

SetSubTensor set a vector to an index of tensor