cmd

README

Evaluation Commands

go build main.go

Run ./main -h for help.

To get help on a subcommand, accuracy for example, run go run main.go accuracy -h.

Model

• Model information across different batch sizes

  ./main model info --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --format=csv

Layer

• Layer information

  ./main layer info --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• Layer duration

  ./main layer duration --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Layer duration variance

  ./main layer duration --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --box_plot

• Layer memory

  ./main layer memory --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Layer occurrence

  ./main layer occurrence --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --pie_plot

• Layer aggregated duration based on operator type

  ./main layer aggre_duration --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --pie_plot

• Layer theoretical flops calculation using the layer operator type and shape

  TODO

GPU

• GPU kernel information

  ./main gpu_kernel info --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• GPU kernel information aggregated within each layer

  ./main gpu_kernel layer_aggre --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• GPU kernel information aggregated within the model

  ./main gpu_kernel model_aggre --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --format=csv

• Total flops of GPU kernels per layer

  ./main gpu_kernel layer_flops --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Total dram read of GPU kernels per layer

  ./main gpu_kernel layer_dram_read --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Total dram write of GPU kernels per layer

  ./main gpu_kernel layer_dram_write --database_name=$DATABASE_NAME --database_address=$DATABASE_ADDRESS --model_name=$MODEL_NAME --output=$OUTPUTFILE --batch_size=$BATCH_SIZE --bar_plot

• Layer GPU vs CPU time

  Use the information from gpu_kernel layer_aggre

• GPU kernel roofline analysis

  Use the information from gpu_kernel info

• Layer roofline analysis

  Use the information from gpu_kernel layer_aggre

• Model roofline analysis

  Use the information from gpu_kernel model_aggre

Documentation

Index

• Variables
• func TempFile(dir, pattern string) string
• type Rower
• type Rowers
• type Writer
  • func NewWriter(rower Rower, opts ...writer.Option) *Writer
  • func (w *Writer) Close()
  • func (w *Writer) Flush()
  • func (w *Writer) Header(rower Rower) error
  • func (w *Writer) Row(rower Rower) error
  • func (w *Writer) Rows(rower Rowers) error

Variables

var (
	AllCmds = []*cobra.Command{
		modelCmd,
		layerCmd,
		gpuKernelCmd,
		eventflowCmd,
		accuracyCmd,
	}
)

var EvaluationCmd = &cobra.Command{
	Use:   "evaluation",
	Short: "Get various information about the evaluation",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		fmt.Println("Running " + cmd.Name())
		return nil
	},
	PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
		safeClose := func(cls ...io.Closer) {
			for _, c := range cls {
				if c == nil {
					return
				}
				c.Close()
			}
		}
		safeClose(
			evaluationCollection,
			performanceCollection,
			inputPredictionCollection,
			modelAccuracyCollection,
			divergenceCollection,
			db,
		)

		return nil
	},
}

Functions

func TempFile

func TempFile(dir, pattern string) string

TempFile creates a new temporary file in the directory dir, opens the file for reading and writing, and returns the resulting *os.File. The filename is generated by taking pattern and adding a random string to the end. If pattern includes a "*", the random string replaces the last "*". If dir is the empty string, TempFile uses the default directory for temporary files (see os.TempDir). Multiple programs calling TempFile simultaneously will not choose the same file. The caller can use f.Name() to find the pathname of the file. It is the caller's responsibility to remove the file when no longer needed.

Types

type Rower

type Rower interface {
	Header(...writer.Option) []string
	Row(...writer.Option) []string
}

type Rowers

type Rowers interface {
	Rower
	Rows(...writer.Option) [][]string
}

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

func NewWriter

func NewWriter(rower Rower, opts ...writer.Option) *Writer

func (*Writer) Close

func (w *Writer) Close()

func (*Writer) Flush

func (w *Writer) Flush()

func (*Writer) Header

func (w *Writer) Header(rower Rower) error

func (*Writer) Row

func (w *Writer) Row(rower Rower) error

func (*Writer) Rows

func (w *Writer) Rows(rower Rowers) error