mare

package module
v0.0.0-...-20ec7c3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 6, 2021 License: MIT Imports: 26 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	ResourceBackend_name = map[int32]string{
		0: "NULL",
		1: "FILE",
		2: "S3",
		3: "XDT",
	}
	ResourceBackend_value = map[string]int32{
		"NULL": 0,
		"FILE": 1,
		"S3":   2,
		"XDT":  3,
	}
)

Enum value maps for ResourceBackend.

View Source 
var File_mare_proto protoreflect.FileDescriptor

Functions

func Drive 

func Drive(
	ctx context.Context,
	workerURL,
	inputBack,
	interBack,
	interHint,
	outputBack,
	outputHint string,
	nReducers int,
	inputLocators []string) (backend string, locator string)

func EndSpan 

func EndSpan(s tracing.Span)

func MakeSpan 

func MakeSpan(spanName string) tracing.Span

func MapKeys 

func MapKeys(m map[string]interface{}) (keys []string)

MapKeys return the keys of map `m`.

func MarshalPairs 

func MarshalPairs(pairs []Pair) string

func RandString 

func RandString(n int) string

RandString generates a random string of length `n`.

func RegisterMareServer 

func RegisterMareServer(s grpc.ServiceRegistrar, srv MareServer)

func StartSpan 

func StartSpan(s tracing.Span, ctx context.Context) context.Context

func Work 

func Work(mapper Mapper, reducer Reducer) error

Types

type MapBatchRequest 

type MapBatchRequest struct {
	Input      *Resource     `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"`
	OutputHint *ResourceHint `protobuf:"bytes,2,opt,name=outputHint,proto3" json:"outputHint,omitempty"`
	// contains filtered or unexported fields
}

func (*MapBatchRequest) Descriptor deprecated 

func (*MapBatchRequest) Descriptor() ([]byte, []int)

Deprecated: Use MapBatchRequest.ProtoReflect.Descriptor instead.

func (*MapBatchRequest) GetInput 

func (x *MapBatchRequest) GetInput() *Resource

func (*MapBatchRequest) GetOutputHint 

func (x *MapBatchRequest) GetOutputHint() *ResourceHint

func (*MapBatchRequest) ProtoMessage 

func (*MapBatchRequest) ProtoMessage()

func (*MapBatchRequest) ProtoReflect 

func (x *MapBatchRequest) ProtoReflect() protoreflect.Message

func (*MapBatchRequest) Reset 

func (x *MapBatchRequest) Reset()

func (*MapBatchRequest) String 

func (x *MapBatchRequest) String() string

type MapBatchResponse 

type MapBatchResponse struct {
	Output *Resource `protobuf:"bytes,1,opt,name=output,proto3" json:"output,omitempty"`
	Keys   []string  `protobuf:"bytes,2,rep,name=keys,proto3" json:"keys,omitempty"`
	// contains filtered or unexported fields
}

func (*MapBatchResponse) Descriptor deprecated 

func (*MapBatchResponse) Descriptor() ([]byte, []int)

Deprecated: Use MapBatchResponse.ProtoReflect.Descriptor instead.

func (*MapBatchResponse) GetKeys 

func (x *MapBatchResponse) GetKeys() []string

func (*MapBatchResponse) GetOutput 

func (x *MapBatchResponse) GetOutput() *Resource

func (*MapBatchResponse) ProtoMessage 

func (*MapBatchResponse) ProtoMessage()

func (*MapBatchResponse) ProtoReflect 

func (x *MapBatchResponse) ProtoReflect() protoreflect.Message

func (*MapBatchResponse) Reset 

func (x *MapBatchResponse) Reset()

func (*MapBatchResponse) String 

func (x *MapBatchResponse) String() string

type Mapper 

type Mapper interface {
	Map(ctx context.Context, pair Pair) ([]Pair, error)
}

type MareClient 

type MareClient interface {
	MapBatch(ctx context.Context, in *MapBatchRequest, opts ...grpc.CallOption) (*MapBatchResponse, error)
	ReduceBatch(ctx context.Context, in *ReduceBatchRequest, opts ...grpc.CallOption) (*ReduceBatchResponse, error)
}

MareClient is the client API for Mare service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.

func NewMareClient 

func NewMareClient(cc grpc.ClientConnInterface) MareClient

type MareServer 

type MareServer interface {
	MapBatch(context.Context, *MapBatchRequest) (*MapBatchResponse, error)
	ReduceBatch(context.Context, *ReduceBatchRequest) (*ReduceBatchResponse, error)
	// contains filtered or unexported methods
}

MareServer is the server API for Mare service. All implementations must embed UnimplementedMareServer for forward compatibility

type Pair 

type Pair struct {
	Key   string `json:"K"`
	Value string `json:"V"`
}

func UnmarshalPairs 

func UnmarshalPairs(data string) (pairs []Pair)

type ReduceBatchRequest 

type ReduceBatchRequest struct {
	Keys       []string      `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"`
	Inputs     []*Resource   `protobuf:"bytes,2,rep,name=inputs,proto3" json:"inputs,omitempty"`
	OutputHint *ResourceHint `protobuf:"bytes,3,opt,name=outputHint,proto3" json:"outputHint,omitempty"`
	// contains filtered or unexported fields
}

func (*ReduceBatchRequest) Descriptor deprecated 

func (*ReduceBatchRequest) Descriptor() ([]byte, []int)

Deprecated: Use ReduceBatchRequest.ProtoReflect.Descriptor instead.

func (*ReduceBatchRequest) GetInputs 

func (x *ReduceBatchRequest) GetInputs() []*Resource

func (*ReduceBatchRequest) GetKeys 

func (x *ReduceBatchRequest) GetKeys() []string

func (*ReduceBatchRequest) GetOutputHint 

func (x *ReduceBatchRequest) GetOutputHint() *ResourceHint

func (*ReduceBatchRequest) ProtoMessage 

func (*ReduceBatchRequest) ProtoMessage()

func (*ReduceBatchRequest) ProtoReflect 

func (x *ReduceBatchRequest) ProtoReflect() protoreflect.Message

func (*ReduceBatchRequest) Reset 

func (x *ReduceBatchRequest) Reset()

func (*ReduceBatchRequest) String 

func (x *ReduceBatchRequest) String() string

type ReduceBatchResponse 

type ReduceBatchResponse struct {
	Output *Resource `protobuf:"bytes,1,opt,name=output,proto3" json:"output,omitempty"`
	// contains filtered or unexported fields
}

func (*ReduceBatchResponse) Descriptor deprecated 

func (*ReduceBatchResponse) Descriptor() ([]byte, []int)

Deprecated: Use ReduceBatchResponse.ProtoReflect.Descriptor instead.

func (*ReduceBatchResponse) GetOutput 

func (x *ReduceBatchResponse) GetOutput() *Resource

func (*ReduceBatchResponse) ProtoMessage 

func (*ReduceBatchResponse) ProtoMessage()

func (*ReduceBatchResponse) ProtoReflect 

func (x *ReduceBatchResponse) ProtoReflect() protoreflect.Message

func (*ReduceBatchResponse) Reset 

func (x *ReduceBatchResponse) Reset()

func (*ReduceBatchResponse) String 

func (x *ReduceBatchResponse) String() string

type Reducer 

type Reducer interface {
	Reduce(ctx context.Context, key string, values []string) ([]Pair, error)
}

type Resource 

type Resource struct {
	Backend ResourceBackend `protobuf:"varint,1,opt,name=backend,proto3,enum=mare.ResourceBackend" json:"backend,omitempty"`
	Locator string          `protobuf:"bytes,2,opt,name=locator,proto3" json:"locator,omitempty"`
	// contains filtered or unexported fields
}

func (*Resource) Descriptor deprecated 

func (*Resource) Descriptor() ([]byte, []int)

Deprecated: Use Resource.ProtoReflect.Descriptor instead.

func (*Resource) Get 

func (x *Resource) Get(ctx context.Context) (string, error)

func (*Resource) GetBackend 

func (x *Resource) GetBackend() ResourceBackend

func (*Resource) GetLocator 

func (x *Resource) GetLocator() string

func (*Resource) ProtoMessage 

func (*Resource) ProtoMessage()

func (*Resource) ProtoReflect 

func (x *Resource) ProtoReflect() protoreflect.Message

func (*Resource) Reset 

func (x *Resource) Reset()

func (*Resource) String 

func (x *Resource) String() string

type ResourceBackend 

type ResourceBackend int32
const (
	ResourceBackend_NULL ResourceBackend = 0
	ResourceBackend_FILE ResourceBackend = 1
	ResourceBackend_S3   ResourceBackend = 2
	ResourceBackend_XDT  ResourceBackend = 3
)

func (ResourceBackend) Descriptor 

func (ResourceBackend) Descriptor() protoreflect.EnumDescriptor

func (ResourceBackend) Enum 

func (x ResourceBackend) Enum() *ResourceBackend

func (ResourceBackend) EnumDescriptor deprecated 

func (ResourceBackend) EnumDescriptor() ([]byte, []int)

Deprecated: Use ResourceBackend.Descriptor instead.

func (ResourceBackend) Number 

func (x ResourceBackend) Number() protoreflect.EnumNumber

func (ResourceBackend) String 

func (x ResourceBackend) String() string

func (ResourceBackend) Type 

func (ResourceBackend) Type() protoreflect.EnumType

type ResourceHint 

type ResourceHint struct {
	Backend ResourceBackend `protobuf:"varint,1,opt,name=backend,proto3,enum=mare.ResourceBackend" json:"backend,omitempty"`
	Hint    string          `protobuf:"bytes,2,opt,name=hint,proto3" json:"hint,omitempty"`
	// contains filtered or unexported fields
}

func (*ResourceHint) Descriptor deprecated 

func (*ResourceHint) Descriptor() ([]byte, []int)

Deprecated: Use ResourceHint.ProtoReflect.Descriptor instead.

func (*ResourceHint) GetBackend 

func (x *ResourceHint) GetBackend() ResourceBackend

func (*ResourceHint) GetHint 

func (x *ResourceHint) GetHint() string

func (*ResourceHint) ProtoMessage 

func (*ResourceHint) ProtoMessage()

func (*ResourceHint) ProtoReflect 

func (x *ResourceHint) ProtoReflect() protoreflect.Message

func (*ResourceHint) Put 

func (x *ResourceHint) Put(ctx context.Context, data string) (*Resource, error)

func (*ResourceHint) Reset 

func (x *ResourceHint) Reset()

func (*ResourceHint) String 

func (x *ResourceHint) String() string

type UnimplementedMareServer 

type UnimplementedMareServer struct {
}

UnimplementedMareServer must be embedded to have forward compatible implementations.

func (UnimplementedMareServer) MapBatch 

func (UnimplementedMareServer) MapBatch(context.Context, *MapBatchRequest) (*MapBatchResponse, error)

func (UnimplementedMareServer) ReduceBatch 

func (UnimplementedMareServer) ReduceBatch(context.Context, *ReduceBatchRequest) (*ReduceBatchResponse, error)

type UnsafeMareServer 

type UnsafeMareServer interface {
	// contains filtered or unexported methods
}

UnsafeMareServer may be embedded to opt out of forward compatibility for this service. Use of this interface is not recommended, as added methods to MareServer will result in compilation errors.

Source Files

View all Source files

Directories

Path Synopsis
examples
amplab1
amplab2
word-count

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.