edasim

package
v1.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 14, 2021 License: CC-BY-4.0 Imports: 14 Imported by: 0

Documentation

Overview

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information. Package edasim implements the structures, methods, and functions used by the EDA simulator

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Copyright (C) Microsoft Corporation. All rights reserved. Licensed under the MIT License. See LICENSE-CODE in the project root for license information.

Index

Constants

View Source 
const (
	JobDir   = "/job"
	WorkDir  = "/work"
	StatsDir = "/stats"

	QueueJobReady    = "jobready"
	QueueJobComplete = "jobcomplete"
	QueueJobProcess  = "jobprocess"
	QueueUploader    = "uploader"

	DefaultFileSizeKB              = 384
	DefaultJobCount                = 10
	DefaultJobSubmitterThreadCount = 1

	DefaultOrchestratorThreads = 16
	DefaultWorkStartFiles      = 3
	DefaultJobEndFiles         = 12

	KB = 1024
	MB = KB * KB

	JobReaderLabel              = "JobReader"
	JobWriterLabel              = "JobWriter"
	JobCompleteReaderLabel      = "JobCompleteReader"
	JobCompleteWriterLabel      = "JobCompleteWriter"
	WorkStartFileReaderLabel    = "WorkStartFileReader"
	WorkStartFileWriterLabel    = "WorkStartFileWriter"
	WorkCompleteFileReaderLabel = "WorkCompleteFileReader"
	WorkCompleteFileWriterLabel = "WorkCompleteFileWriter"
)
View Source 
const (
	QueueMessageCount = 1
)

Variables

View Source 
var (
	// JobWriter is the writer used for job files
	JobWriter *file.ReaderWriter
	// JobReader is the reader used for readung job files
	JobReader *file.ReaderWriter

	// WorkStartFileWriter is the writer used for work start files
	WorkStartFileWriter *file.ReaderWriter
	// WorkStartFileReader is the reader used for work start files
	WorkStartFileReader *file.ReaderWriter

	// WorkCompleteFileWriter is the writer used for work complete files
	WorkCompleteFileWriter *file.ReaderWriter
	// WorkCompleteFileReader is the reader used for work complete files
	WorkCompleteFileReader *file.ReaderWriter

	// JobCompleteWriter is the writer used for job complete files
	JobCompleteWriter *file.ReaderWriter
	// JobCompleteReader is the reader used for job complete files
	JobCompleteReader *file.ReaderWriter
)

Functions

func GenerateBatchName 

func GenerateBatchName(jobCount int) string

GenerateBatchName generates a batchname based on time

func GenerateBatchName2 

func GenerateBatchName2(jobCount int) string

GenerateBatchName2 generates a batchname based on time

func GenerateBatchNameFromJobRun 

func GenerateBatchNameFromJobRun(uniqueName string, jobRunName string, batchid int) string

GenerateBatchNameFromJobRun generates a batch name from unique name and job run name and batch id

func GetBatchName 

func GetBatchName(fullFilePath string) string

GetBatchName returns the batch name, which is just the parent directory

func GetBatchNamePartsFromJobRun 

func GetBatchNamePartsFromJobRun(fullFilePath string) (string, string)

GetBatchNamePartsFromJobRun generates the parts of the batch name

func GetEventHubName 

func GetEventHubName(uniqueName string) string

GetEventHubName returns the event hub name

func GetJobCompleteQueueName 

func GetJobCompleteQueueName(uniqueName string) string

GetJobCompleteQueueName returns the job complete queue name

func GetJobRunQueueName 

func GetJobRunQueueName(uniqueName string) string

GetJobRunQueueName returns the job run queue name

func GetJobStartQueueName 

func GetJobStartQueueName(uniqueName string) string

GetJobStartQueueName returns the job run queue name

func GetWorkCompleteQueueName 

func GetWorkCompleteQueueName(uniqueName string) string

GetWorkCompleteQueueName returns the work complete queue name

func GetWorkStartQueueName 

func GetWorkStartQueueName(uniqueName string) string

GetWorkStartQueueName returns the work start queue name

func InitializeReaderWriters 

func InitializeReaderWriters(
	ctx context.Context,
	eventHubSenderName string,
	eventHubSenderKey string,
	eventHubNamespaceName string,
	eventHubHubName string) *azure.EventHubSender

InitializeReaderWriters initializes the reader writers with event hub profiling

func SetStatsChannel 

func SetStatsChannel(ctx context.Context) context.Context

SetStatsChannel adds the stats channel to the context

func StatsCollector 

func StatsCollector(ctx context.Context, syncWaitGroup *sync.WaitGroup)

StatsCollector is a go routine that prints the queue statistics on a schedule

Types

type EdasimFile 

type EdasimFile struct {
	// the job run details
	MountPath   string
	FullPath    string
	MountParity bool
}

EdasimFile breaks an edasimfile into three parts

func InitializeEdasimFileFromString 

func InitializeEdasimFileFromString(edasimFileString string) (*EdasimFile, error)

InitializeEdasimFileFromString reads a edasimFileString from json string

func (*EdasimFile) GetEdasimFileString 

func (e *EdasimFile) GetEdasimFileString() (string, error)

GetEdasimFileString returns the JSON representation of the edasimFileString

type JobConfigFile 

type JobConfigFile struct {
	Name           string
	IsCompleteFile bool
	JobRun         JobRun
	PaddedString   string
}

JobConfigFile represents a job configuration file

func InitializeJobCompleteFile 

func InitializeJobCompleteFile(name string, jobRun *JobRun) *JobConfigFile

InitializeJobCompleteFile sets the unique name of the job configuration and the batch name and is used to signify job completion

func InitializeJobConfigFile 

func InitializeJobConfigFile(name string, jobRun *JobRun) *JobConfigFile

InitializeJobConfigFile sets the unique name of the job configuration and the batch name

func ReadJobConfigFile 

func ReadJobConfigFile(reader *file.ReaderWriter, filename string) (*JobConfigFile, error)

ReadJobConfigFile reads a job config file from disk

func (*JobConfigFile) WriteJobConfigFile 

func (j *JobConfigFile) WriteJobConfigFile(writer *file.ReaderWriter, filepath string, fileSize int) (string, error)

WriteJobConfigFile writes the job configuration file to disk, padding it so it makes the necessary size

type JobRun 

type JobRun struct {
	// the unique name identifies the queue and eventhub, this avoids multiple people having colisions
	UniqueName string

	// the job run details
	JobRunName string
	JobCount   int
	BatchCount int
	BatchID    int

	// job start and end file information
	JobFileConfigSizeKB int

	// mount information
	MountParity bool

	// job queue information
	JobRunStartQueueName         string
	WorkStartFileSizeKB          int
	WorkStartFileCount           int
	WorkCompleteFileSizeKB       int
	WorkCompleteFileCount        int
	WorkCompleteFailedFileSizeKB int
	WorkFailedProbability        float64
	DeleteFiles                  bool
}

JobRun describes the details of a full job run including how many batches to break it into

func InitializeJobRunFromString 

func InitializeJobRunFromString(jobRunString string) (*JobRun, error)

InitializeJobRunFromString reads a jobrun from json string

func (*JobRun) SubmitBatches 

func (j *JobRun) SubmitBatches(ctx context.Context, storageAccount string, storageKey string)

SubmitBatches submits the necessary batches into the queue

type JobSubmitter 

type JobSubmitter struct {
	Context       context.Context
	UniqueName    string
	JobRunQueue   *azure.Queue
	JobStartQueue *azure.Queue
	ThreadCount   int
	PathManager   *file.RoundRobinPathManager
	DirManager    *file.DirectoryManager
}

JobSubmitter defines the structure used for the job submitter process

func InitializeJobSubmitter 

func InitializeJobSubmitter(
	ctx context.Context,
	storageAccount string,
	storageKey string,
	uniqueName string,
	mountPaths []string,
	threadCount int) *JobSubmitter

InitializeJobSubmitter initializes the job submitter structure

func (*JobSubmitter) JobSubmitterWorkerRun 

func (j *JobSubmitter) JobSubmitterWorkerRun(syncWaitGroup *sync.WaitGroup, id int, batchName string, jobRun *JobRun, jobCount int)

JobSubmitterWorkerRun is the entry point for the JobSubmitter go routine

func (*JobSubmitter) Run 

func (j *JobSubmitter) Run(syncWaitGroup *sync.WaitGroup)

type Orchestrator 

type Orchestrator struct {
	Context             context.Context
	UniqueName          string
	JobStartQueue       *azure.Queue
	WorkStartQueue      *azure.Queue
	WorkComplete        *azure.Queue
	JobComplete         *azure.Queue
	PathManager         *file.RoundRobinPathManager
	DirManager          *file.DirectoryManager
	OrchestratorThreads int
	ReadyCh             chan struct{}
	MsgCh               chan *azqueue.DequeuedMessage
}

Orchestrator defines the orchestrator structure

func InitializeOrchestrator 

func InitializeOrchestrator(
	ctx context.Context,
	storageAccount string,
	storageKey string,
	uniqueName string,
	mountPaths []string,
	orchestratorThreads int) *Orchestrator

InitializeOrchestrator initializes the Orchestrator

func (*Orchestrator) JobDispatcher 

func (o *Orchestrator) JobDispatcher(syncWaitGroup *sync.WaitGroup)

JobDispatcher dispatches jobs to workers based on input from the ready queue

func (*Orchestrator) Run 

func (o *Orchestrator) Run(syncWaitGroup *sync.WaitGroup)

Run implements the go routine entry point for the orchestrator. This starts the various go routines for managment of the queues

func (*Orchestrator) StartJobWorker 

func (o *Orchestrator) StartJobWorker(syncWaitGroup *sync.WaitGroup)

StartJobWorker implements the go routine of the worker that gets jobs from the queue

type StatsChannels 

type StatsChannels struct {
	ChJobProcessed          chan struct{}
	ChProcessedFilesWritten chan struct{}
	ChJobCompleted          chan struct{}
	ChUpload                chan struct{}
	ChError                 chan struct{}
}

StatsChannels defines the stats channels that counts queue stats

func GetStatsChannel 

func GetStatsChannel(ctx context.Context) *StatsChannels

GetStatsChannel gets the stats channel from the context

func InitializeStatsChannels 

func InitializeStatsChannels() *StatsChannels

InitializeStatsChannels initializes the stats channels

func (*StatsChannels) Error 

func (s *StatsChannels) Error()

Error signals that an error was encountered

func (*StatsChannels) JobCompleted 

func (s *StatsChannels) JobCompleted()

JobCompleted signals the job was completed and the file was written

func (*StatsChannels) JobProcessed 

func (s *StatsChannels) JobProcessed()

JobProcessed signals a job was processed

func (*StatsChannels) ProcessedFilesWritten 

func (s *StatsChannels) ProcessedFilesWritten()

ProcessedFilesWritten signals the worker start files were written

func (*StatsChannels) Upload 

func (s *StatsChannels) Upload()

Upload signals that an upload was queued

type WorkFileWriter 

type WorkFileWriter struct {
	JobConfigName string
	JobRun        JobRun
	PaddedString  string
}

WorkFileWriter handles the work start file and complete file creation for a single job

func InitializeWorkerFileWriter 

func InitializeWorkerFileWriter(
	jobConfigName string,
	jobRun *JobRun) *WorkFileWriter

InitializeWorkerFileWriter creates a work file writer for a single job

func ReadWorkFile 

func ReadWorkFile(reader *file.ReaderWriter, filename string) (*WorkFileWriter, error)

ReadWorkFile reads a work file from disk

func (*WorkFileWriter) FirstStartFile 

func (w *WorkFileWriter) FirstStartFile(filepath string) string

FirstStartFile returns the path of the first start file

func (*WorkFileWriter) WriteStartFiles 

func (w *WorkFileWriter) WriteStartFiles(writer *file.ReaderWriter, filepath string, fileSize int, fileCount int) error

WriteStartFiles writes the required number of start files

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.