nnet

package
Version: v0.0.0-...-c5f678a Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 17, 2019 License: GPL-3.0 Imports: 21 Imported by: 0

Documentation

Overview

Package nnet contains routines for constructing, training and testing neural networks.

Index

Constants

This section is empty.

Variables

View Source
var (
	DataDir   = os.Getenv("GOPATH") + "/src/github.com/jnb666/deepthought2/data"
	LogDir    = os.Getenv("GOPATH") + "/src/github.com/jnb666/deepthought2/log"
	DataTypes = []string{"train", "test", "valid"}
)

Functions

func Bprop

func Bprop(q num.Queue, layers []Layer, grad *num.Array, work [3]num.Buffer) *num.Array

Back propagate gradient through the layers

func CheckErr

func CheckErr(err error)

Exit in case of error

func CopyParams

func CopyParams(q num.Queue, src, dst []Layer, sync bool)

Copy weights and bias arrays from src to dst

func Download

func Download(rawurl, dir string) error

Download data from url and save it to directory dir. if file suffix is .gz then gunzip the data. If file type is .tar then untar the files.

func FileExists

func FileExists(name string) bool

Check if file exists under DataDir

func FormatBytes

func FormatBytes(n int) string

Convert no. of bytes to string

func FormatDuration

func FormatDuration(d time.Duration) string

func Fprop

func Fprop(q num.Queue, layers []Layer, input *num.Array, work num.Buffer, trainMode bool) *num.Array

Feed forward the input to get the predicted output

func InitLogger

func InitLogger(model string, flags int) error

Init logger to write to file

func LoadData

func LoadData(model string) (d map[string]Data, err error)

Load data from disk given the model name.

func MemoryProfile

func MemoryProfile(verbose bool, train, test *Network)

Print memory profile

func NewData

func NewData(classes []string, shape []int, labels []int32, inputs []float32) *data

NewData function creates a new data set which implements the Data interface

func ParamLayers

func ParamLayers(desc string, layers []Layer, callback func(desc string, l ParamLayer))

Call function on each of the ParamLayers in the network

func SaveDataFile

func SaveDataFile(d Data, name string) error

Encode in gob format and save to file under DataDir

func SetSeed

func SetSeed(seed int64) *rand.Rand

Set random number seed, or random seed if seed <= 0

func StatsHeaders

func StatsHeaders(d map[string]Data) []string

func Train

func Train(net *Network, dset *Dataset, test Tester)

Train the network on the given training set by updating the weights

func TrainEpoch

func TrainEpoch(net *Network, dset *Dataset, epoch int, pred []int32) (batchLoss []float64, trainError float64)

Perform one training epoch on dataset, returns the current loss prior to updating the weights.

Types

type AMSGrad

type AMSGrad struct {
	LearningRate float32
	Beta1        float32
	Beta2        float32
	Epsilon      float32
	Work         num.Buffer
}

AMSGrad optimiser with adaptive learning rate

func (*AMSGrad) Release

func (o *AMSGrad) Release()

func (*AMSGrad) Update

func (o *AMSGrad) Update(q num.Queue, x, dx *num.Array, v []*num.Array)

Update weights using AMSGrad optimiser:

v1 = beta1*v1 + (1-beta1) * dx
v2 = beta2*v2 + (1-beta2) * dx**2
v3 = max(v3, v2)
x += alpha * v1 / (sqrt(v3) + epsilon)

type Activation

type Activation struct{ Atype string }

Sigmoid, tanh or relu activation layer, implements OutputLayer interface.

func (Activation) Marshal

func (c Activation) Marshal() LayerConfig

func (Activation) String

func (c Activation) String() string

func (Activation) Type

func (c Activation) Type() string

type Adam

type Adam struct {
	LearningRate float32
	Beta1        float32
	Beta2        float32
	Epsilon      float32
	Iter         int
	Work1        num.Buffer
	Work2        num.Buffer
}

Adam optimiser with adaptive learning rate

func (*Adam) Release

func (o *Adam) Release()

func (*Adam) Update

func (o *Adam) Update(q num.Queue, x, dx *num.Array, v []*num.Array)

Update weights using Adam optimiser:

v1 = beta1*v1 + (1-beta1) * dx
v2 = beta2*v2 + (1-beta2) * dx**2
v1_hat = v1 / (1 - beta1**t)
v2_hat = v2 / (1 - beta2**t)
x += alpha * v1_hat / (sqrt(v2_hat) + epsilon)

type Add

type Add struct {
	X, Y []LayerConfig
}

Add two sets of layers, used in residual network block, output is X(input) + Y(input) if Y is nil then outputput is X(input) + 1

func AddLayer

func AddLayer(X, Y []ConfigLayer) Add

func (Add) Marshal

func (c Add) Marshal() LayerConfig

func (Add) String

func (c Add) String() string

func (Add) Type

func (c Add) Type() string

type BatchNorm

type BatchNorm struct{ AvgFactor, Epsilon float64 }

Batch normalisation layer.

func (BatchNorm) Marshal

func (c BatchNorm) Marshal() LayerConfig

func (BatchNorm) String

func (c BatchNorm) String() string

func (BatchNorm) Type

func (c BatchNorm) Type() string

type BatchNormLayer

type BatchNormLayer interface {
	ParamLayer
	Stats() (w, b, runMean, runVar *num.Array)
}

BatchNormLayer stores the scale, shift, running mean and variance

type Config

type Config struct {
	DataSet      string
	Optimiser    OptimiserType
	Eta          float64
	EtaDecay     float64
	EtaDecayStep int
	Lambda       float64
	Momentum     float64
	WeightInit   InitType
	Bias         float64
	Shuffle      bool
	Normalise    bool
	Distort      bool
	TrainRuns    int
	TrainBatch   int
	TestBatch    int
	MaxEpoch     int
	MaxSeconds   int
	MaxSamples   int
	LogEvery     int
	StopAfter    int
	ExtraEpochs  int
	ValidEMA     float64
	MinLoss      float64
	RandSeed     int64
	DebugLevel   int
	UseGPU       bool
	Profile      bool
	MemProfile   bool
	Layers       []LayerConfig
}

Training configuration settings

func LoadConfig

func LoadConfig(name string) (c Config, err error)

Load network from json file under DataDir

func (Config) AddLayers

func (c Config) AddLayers(layers ...ConfigLayer) Config

Append layers to the config struct

func (Config) Copy

func (c Config) Copy() Config

func (Config) DatasetConfig

func (c Config) DatasetConfig(test bool) DatasetOptions

func (Config) Fields

func (c Config) Fields() []string

func (Config) Get

func (c Config) Get(key string) interface{}

func (Config) OptimiserParams

func (c Config) OptimiserParams(epoch, samples int) (learningRate, weightDecay float32)

Get learning rate and weight decay

func (Config) Save

func (c Config) Save(name string) error

Save config to JSON file under DataDir

func (Config) SetString

func (c Config) SetString(key, val string) (Config, error)

func (Config) String

func (c Config) String() string

type ConfigLayer

type ConfigLayer interface {
	Marshal() LayerConfig
	Type() string
	String() string
}

type Conv

type Conv struct {
	Nfeats int
	Size   int
	Stride int
	Pad    bool
	NoBias bool
}

Convolutional layer, implements ParamLayer interface.

func (Conv) Marshal

func (c Conv) Marshal() LayerConfig

func (Conv) String

func (c Conv) String() string

func (Conv) Type

func (c Conv) Type() string

type Data

type Data interface {
	Len() int
	Classes() []string
	ClassSize() int
	Shape() []int
	Label(index []int, label []int32)
	Input(index []int, buf []float32, t *img.Transformer)
	Image(ix int, channel string) *img.Image
	Encode(w io.Writer) error
	Decode(r io.Reader) error
}

Data interface type represents the raw data for a training or test set

func LoadDataFile

func LoadDataFile(name string) (Data, error)

Decode data from file in gob format under DataDir

type Dataset

type Dataset struct {
	Data
	Samples   int
	BatchSize int
	Batches   int

	sync.WaitGroup
	// contains filtered or unexported fields
}

Dataset type encapsulates a set of training, test or validation data.

func NewDataset

func NewDataset(dev num.Device, data Data, opts DatasetOptions, rng *rand.Rand) *Dataset

Create a new Dataset struct, allocate array buffers and set the batch size and maxSamples

func (*Dataset) NextBatch

func (d *Dataset) NextBatch() (x, y, yOneHot *num.Array)

Get next batch of data

func (*Dataset) NextEpoch

func (d *Dataset) NextEpoch()

Called at start of each epoch

func (*Dataset) Profiling

func (d *Dataset) Profiling(on bool, title string)

Enable profiling for the associated queue

func (*Dataset) Release

func (d *Dataset) Release()

release allocated buffers

func (*Dataset) SetTrans

func (d *Dataset) SetTrans(normalise, distort bool)

Set image transform

func (*Dataset) Shuffle

func (d *Dataset) Shuffle()

Shuffle the data set

func (*Dataset) Trans

func (d *Dataset) Trans() img.TransType

Transforms applied to input data

type DatasetOptions

type DatasetOptions struct {
	BatchSize  int
	MaxSamples int
	Normalise  bool
	Distort    bool
}

Config options for dataset

type Dropout

type Dropout struct{ Ratio float64 }

Dropout layer, randomly drops given ratio of nodes.

func (Dropout) Marshal

func (c Dropout) Marshal() LayerConfig

func (Dropout) String

func (c Dropout) String() string

func (Dropout) Type

func (c Dropout) Type() string

type Flatten

type Flatten struct{}

Flatten layer reshapes from 4 to 2 dimensions.

func (Flatten) Marshal

func (c Flatten) Marshal() LayerConfig

func (Flatten) String

func (c Flatten) String() string

func (Flatten) Type

func (c Flatten) Type() string

type InitType

type InitType int

Weight initialisation type

const (
	Zeros InitType = iota
	Ones
	RandomNormal
	RandomUniform
	LecunNormal
	GlorotUniform
	HeNormal
)

func (InitType) Options

func (t InitType) Options() []string

func (InitType) String

func (t InitType) String() string

func (InitType) WeightFunc

func (t InitType) WeightFunc(dims []int, rng *rand.Rand) func() float64

type Layer

type Layer interface {
	ConfigLayer
	Init(q num.Queue, inShape []int, opts num.LayerOpts, seed int64, cfg *Config) (workSize, inSize int)
	InShape() []int
	OutShape() []int
	Fprop(q num.Queue, in *num.Array, work num.Buffer, trainMode bool) *num.Array
	Bprop(q num.Queue, grad, dsrc *num.Array, work [3]num.Buffer) *num.Array
	Output() *num.Array
	Memory() (weights, outputs, temp int)
	BpropData() bool
	Release()
}

Layer interface type represents one layer of the neural net.

type LayerConfig

type LayerConfig struct {
	Type string
	Data json.RawMessage
}

Layer configuration details

func (LayerConfig) Unmarshal

func (l LayerConfig) Unmarshal() Layer

Unmarshal JSON data and construct new layer

type LayerGroup

type LayerGroup interface {
	Layer
	Layers() []Layer
	LayerDesc() []string
}

LayerGroup is a compound layer made up of multiple layers

type Linear

type Linear struct{ Nout int }

Linear fully connected layer, implements ParamLayer interface.

func (Linear) Marshal

func (c Linear) Marshal() LayerConfig

func (Linear) String

func (c Linear) String() string

func (Linear) Type

func (c Linear) Type() string

type Nesterov

type Nesterov struct {
	LearningRate float32
	Momentum     float32
	Work         num.Buffer
}

SGD with Nesterov momentum

func (*Nesterov) Release

func (o *Nesterov) Release()

func (*Nesterov) Update

func (o *Nesterov) Update(q num.Queue, x, dx *num.Array, v []*num.Array)

Update weights using Nesterov optimiser:

v = momentum*v + learningRate*dx
x += -momentum*vPrev + (1 + momentum)*v

type Network

type Network struct {
	Config
	Layers    []Layer
	InShape   []int
	WorkSpace [3]num.Buffer
	// contains filtered or unexported fields
}

Network type represents a multilayer neural network model.

func New

func New(queue num.Queue, conf Config, batchSize int, inShape []int, bprop bool, rng *rand.Rand) *Network

New function creates a new network with the given layers. If bprop is true then allocate memory for back propagation.

func (*Network) BatchError

func (n *Network) BatchError(batch int, dset *Dataset, y, yPred *num.Array, pred []int32) float64

get the total error for this batch, if pred is non-null then save predicted values

func (*Network) BatchLoss

func (n *Network) BatchLoss(yOneHot, yPred *num.Array) float64

get the average loss for this batch

func (*Network) Error

func (n *Network) Error(dset *Dataset, pred []int32) float64

Calculate the error from the predicted versus actual values if pred slice is not nil then also return the predicted output classes.

func (*Network) InitWeights

func (n *Network) InitWeights(rng *rand.Rand)

Initialise network weights using a linear or normal distribution. Weights for each layer are scaled by 1/sqrt(nin)

func (*Network) Memory

func (n *Network) Memory() int

Get total allocated memory in bytes

func (*Network) MemoryProfile

func (n *Network) MemoryProfile(name string) string

Print profile of allocated memory

func (*Network) OutLayer

func (n *Network) OutLayer() OutputLayer

Accessor for output layer

func (*Network) PrintWeights

func (n *Network) PrintWeights()

Print network weights

func (*Network) Release

func (n *Network) Release()

release allocated buffers

func (*Network) String

func (n *Network) String() string

Print network description

type Optimiser

type Optimiser interface {
	Update(q num.Queue, x, dx *num.Array, v []*num.Array)
	Release()
}

Optimiser updates the weights

func NewOptimiser

func NewOptimiser(net *Network, iter int, eta float32) Optimiser

Create new optimiser with given config settings.

type OptimiserType

type OptimiserType int

Optimiser type

const (
	SGDOpt OptimiserType = iota
	NesterovOpt
	RMSpropOpt
	AdamOpt
	AMSGradOpt
)

func NewOptType

func NewOptType(name string) (OptimiserType, error)

func (OptimiserType) ExtraArrays

func (t OptimiserType) ExtraArrays() int

func (OptimiserType) Options

func (t OptimiserType) Options() []string

func (OptimiserType) String

func (t OptimiserType) String() string

type OptionList

type OptionList interface {
	Options() []string
	String() string
}

type OutputLayer

type OutputLayer interface {
	Layer
	Loss(q num.Queue, yOneHot, yPred *num.Array) *num.Array
}

OutputLayer is the final layer in the stack

type ParamLayer

type ParamLayer interface {
	Layer
	Params() (W, B *num.Array)
	InitParams(q num.Queue, init InitType, bias float64, rng *rand.Rand)
	WeightDecay(q num.Queue, decay float32)
	UpdateParams(q num.Queue, opt Optimiser)
	Copy(q num.Queue, layer Layer)
	Export(q num.Queue) []uint32
	Import(q num.Queue, vec []uint32)
	NumWeights() int
}

ParamLayer is a layer which may have weight and bias parameters

type Pool

type Pool struct {
	Size    int
	Stride  int
	Pad     bool
	Average bool
}

Max pooling layer, should follow conv layer.

func (Pool) Marshal

func (c Pool) Marshal() LayerConfig

func (Pool) String

func (c Pool) String() string

func (Pool) Type

func (c Pool) Type() string

type RMSprop

type RMSprop struct {
	LearningRate float32
	Gamma        float32
	Epsilon      float32
	Work         num.Buffer
}

RMSprop optimiser with adaptive learning rate

func (*RMSprop) Release

func (o *RMSprop) Release()

func (*RMSprop) Update

func (o *RMSprop) Update(q num.Queue, x, dx *num.Array, v []*num.Array)

Update weights using RMSProp optimiser:

v = gamma*v + (1-gamma)*(dx**2)
x += learningRate * dx / (sqrt(v) + epsilon)

type SGD

type SGD struct {
	LearningRate float32
	Momentum     float32
}

Stochastic gradient descent with optional momentum.

func (*SGD) Release

func (o *SGD) Release()

func (*SGD) Update

func (o *SGD) Update(q num.Queue, x, dx *num.Array, v []*num.Array)

Update weights using SGD optimiser:

x += learningRate*dx , or
v = momentum*v + learningRate*dx; x += v

type Stats

type Stats struct {
	Epoch     int
	AvgLoss   float64
	Loss      []float64
	Error     []float64
	BestSince int
	TrainTime time.Duration
	Elapsed   time.Duration
}

Training statistics

func (Stats) Copy

func (s Stats) Copy() Stats

func (Stats) Format

func (s Stats) Format() []string

func (Stats) FormatElapsed

func (s Stats) FormatElapsed() string

func (Stats) FormatError

func (s Stats) FormatError(i int) string

func (Stats) FormatLoss

func (s Stats) FormatLoss() string

func (Stats) String

func (s Stats) String(headers []string) string

type TestBase

type TestBase struct {
	Net     *Network
	Data    map[string]*Dataset
	Pred    map[string][]int32
	Stats   []Stats
	Headers []string
	// contains filtered or unexported fields
}

Tester which evaluates the loss and error for each of the data sets and updates the stats.

func NewTestBase

func NewTestBase() *TestBase

Create a new base class which implements the Tester interface.

func (*TestBase) Epilogue

func (t *TestBase) Epilogue() bool

func (*TestBase) Init

func (t *TestBase) Init(dev num.Device, conf Config, data map[string]Data, rng *rand.Rand) *TestBase

Initialise the test dataset, network and other configuration.

func (*TestBase) Network

func (t *TestBase) Network() *Network

func (*TestBase) Predict

func (t *TestBase) Predict(train *Dataset) *TestBase

Generate the predicted results when test is next run.

func (*TestBase) Release

func (t *TestBase) Release()

Release allocated buffers

func (*TestBase) Reset

func (t *TestBase) Reset()

Reset stats prior to new run

func (*TestBase) Test

func (t *TestBase) Test(net *Network, epoch int, batchLoss []float64, trainError float64, start time.Time) bool

Test performance of the network, called from the Train function on completion of each epoch.

type Tester

type Tester interface {
	Test(net *Network, epoch int, batchLoss []float64, trainError float64, start time.Time) bool
	Epilogue() bool
	Release()
	Network() *Network
}

Tester interface to evaluate the performance after each epoch, Test method returns true if training should stop.

func NewTestLogger

func NewTestLogger(dev num.Device, conf Config, data map[string]Data, rng *rand.Rand) Tester

Create a new tester which logs stats to stdout.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL