mapreduce

Documentation

Index

• Constants
• func DPrintf(format string, a ...interface{}) (n int, err error)
• func DoMap(JobNumber int, fileName string, nreduce int, Map func(string) *list.List)
• func DoReduce(job int, fileName string, nmap int, Reduce func(string, *list.List) string)
• func MapName(fileName string, MapJob int) string
• func MergeName(fileName string, ReduceJob int) string
• func ReduceName(fileName string, MapJob int, ReduceJob int) string
• func Register(master string, me string)
• func RemoveFile(n string)
• func RunSingle(nMap int, nReduce int, file string, Map func(string) *list.List, ...)
• func RunWorker(MasterAddress string, me string, MapFunc func(string) *list.List, ...)
• type DoJobArgs
• type DoJobReply
• type JobType
• type KeyValue
• type MapReduce
  • func InitMapReduce(nmap int, nreduce int, file string, master string) *MapReduce
  • func MakeMapReduce(nmap int, nreduce int, file string, master string) *MapReduce
  • func (mr *MapReduce) CleanupFiles()
  • func (mr *MapReduce) CleanupRegistration()
  • func (mr *MapReduce) KillWorkers() *list.List
  • func (mr *MapReduce) Merge()
  • func (mr *MapReduce) Register(args *RegisterArgs, res *RegisterReply) error
  • func (mr *MapReduce) Run()
  • func (mr *MapReduce) RunMaster() *list.List
  • func (mr *MapReduce) RunWorker(w string)
  • func (mr *MapReduce) Shutdown(args *ShutdownArgs, res *ShutdownReply) error
  • func (mr *MapReduce) Split(fileName string)
  • func (mr *MapReduce) StartRegistrationServer()
• type RegisterArgs
• type RegisterReply
• type ShutdownArgs
• type ShutdownReply
• type Worker
  • func (wk *Worker) DoJob(arg *DoJobArgs, res *DoJobReply) error
  • func (wk *Worker) Shutdown(args *ShutdownArgs, res *ShutdownReply) error
• type WorkerInfo

Constants

const Debug = 0

Debug is here for Debugging

Functions

func DPrintf

func DPrintf(format string, a ...interface{}) (n int, err error)

DPrintf prints if debugging is enabled

func DoMap

func DoMap(JobNumber int, fileName string,
	nreduce int, Map func(string) *list.List)

Read split for job, call Map for that split, and create nreduce partitions.

func DoReduce

func DoReduce(job int, fileName string, nmap int,
	Reduce func(string, *list.List) string)

Read map outputs for partition job, sort them by key, call reduce for each key

func MapName

func MapName(fileName string, MapJob int) string

Name of the file that is the input for map job <MapJob>

func MergeName

func MergeName(fileName string, ReduceJob int) string

func ReduceName

func ReduceName(fileName string, MapJob int, ReduceJob int) string

func Register

func Register(master string, me string)

Tell the master we exist and ready to work

func RemoveFile

func RemoveFile(n string)

func RunSingle

func RunSingle(nMap int, nReduce int, file string,
	Map func(string) *list.List,
	Reduce func(string, *list.List) string)

Run jobs sequentially.

func RunWorker

func RunWorker(MasterAddress string, me string,
	MapFunc func(string) *list.List,
	ReduceFunc func(string, *list.List) string, nRPC int)

Set up a connection with the master, register with the master, and wait for jobs from the master

Types

type DoJobArgs

type DoJobArgs struct {
	File          string
	Operation     JobType
	JobNumber     int // this job's number
	NumOtherPhase int // total number of jobs in other phase (map or reduce)
}

type DoJobReply

type DoJobReply struct {
	OK bool
}

type JobType

type JobType string

JobType represents type of job :P

const (
	Map    JobType = "Map"
	Reduce JobType = "Reduce"
)

Map & Reduce JobType

type KeyValue

type KeyValue struct {
	Key   string
	Value string
}

KeyValue pairs are what Map and Reduce deal with

type MapReduce

type MapReduce struct {
	MasterAddress string

	DoneChannel chan bool

	// Map of registered workers that you need to keep up to date
	Workers map[string]*WorkerInfo
	// contains filtered or unexported fields
}

MapReduce contains information about the Map Reduce job

func InitMapReduce

func InitMapReduce(nmap int, nreduce int, file string, master string) *MapReduce

InitMapReduce initiates map reduce structure

func MakeMapReduce

func MakeMapReduce(nmap int, nreduce int, file string, master string) *MapReduce

MakeMapReduce creates map reduce structure and runs registration server

func (*MapReduce) CleanupFiles

func (mr *MapReduce) CleanupFiles()

func (*MapReduce) CleanupRegistration

func (mr *MapReduce) CleanupRegistration()

func (*MapReduce) KillWorkers

func (mr *MapReduce) KillWorkers() *list.List

KillWorkers clean up all workers by sending a Shutdown RPC to each one of them Collect the number of jobs each work has performed.

func (*MapReduce) Merge

func (mr *MapReduce) Merge()

Merge the results of the reduce jobs XXX use merge sort

func (*MapReduce) Register

func (mr *MapReduce) Register(args *RegisterArgs, res *RegisterReply) error

func (*MapReduce) Run

func (mr *MapReduce) Run()

Run jobs in parallel, assuming a shared file system

func (*MapReduce) RunMaster

func (mr *MapReduce) RunMaster() *list.List

RunMaster listen on register events and assign tasks

func (*MapReduce) RunWorker

func (mr *MapReduce) RunWorker(w string)

RunWorker handle single worker jobs and state

func (*MapReduce) Shutdown

func (mr *MapReduce) Shutdown(args *ShutdownArgs, res *ShutdownReply) error

func (*MapReduce) Split

func (mr *MapReduce) Split(fileName string)

Split bytes of input file into nMap splits, but split only on white space

func (*MapReduce) StartRegistrationServer

func (mr *MapReduce) StartRegistrationServer()

type RegisterArgs

type RegisterArgs struct {
	Worker string
}

type RegisterReply

type RegisterReply struct {
	OK bool
}

type ShutdownArgs

type ShutdownArgs struct {
}

type ShutdownReply

type ShutdownReply struct {
	Njobs int
	OK    bool
}

type Worker

type Worker struct {
	Reduce func(string, *list.List) string
	Map    func(string) *list.List
	// contains filtered or unexported fields
}

func (*Worker) DoJob

func (wk *Worker) DoJob(arg *DoJobArgs, res *DoJobReply) error

The master sent us a job

func (*Worker) Shutdown

func (wk *Worker) Shutdown(args *ShutdownArgs, res *ShutdownReply) error

The master is telling us to shutdown. Report the number of Jobs we have processed.

type WorkerInfo

type WorkerInfo struct {
	// contains filtered or unexported fields
}

WorkerInfo contains infomation about workers