gohadoop

README

gohadoop

gohadoop is native go clients for Apache Hadoop YARN.

It includes an early version of Hadoop IPC client and requisite YARN client libraries to implement YARN applications completely in go (both YARN application-client and application-master).

###Notes:

• Set HADOOP_CONF_DIR environment variable, and ensure the conf directory contains both *-default.xml and *-site.xml files.
• hadoop_yarn/examples/dist_shell is an example go YARN application: client.go is the submission client and applicationmaster.go is the application-master.

To run:

Run regular dist-shell with AM in-cluster

$ HADOOP_CONF_DIR=conf go run hadoop_yarn/examples/dist_shell/client.go

Run dist-shell with unmanaged AM

$ HADOOP_CONF_DIR=conf go run hadoop_yarn/examples/dist_shell/unmanaged.go

Documentation

Index

• Variables
• func ConvertBytesToFixed(rawBytes []byte, data interface{}) error
• func ConvertFixedToBytes(data interface{}) ([]byte, error)
• func CreateSimpleUGIProto() (*hadoop_common.UserInformationProto, error)
• func GetCalleeRPCRequestHeaderProto(protocolName *string) *hadoop_common.RequestHeaderProto
• type AuthMethod
  • func (authmethod AuthMethod) String() string
• type AuthProtocol
  • func (authprotocol AuthProtocol) String() string

Variables

var (
	RPC_HEADER        []byte = []byte("hrpc")
	VERSION           []byte = []byte{0x09}
	RPC_SERVICE_CLASS byte   = 0x00

	RPC_PROTOCOL_BUFFFER    hadoop_common.RpcKindProto                         = hadoop_common.RpcKindProto_RPC_PROTOCOL_BUFFER
	RPC_FINAL_PACKET        hadoop_common.RpcRequestHeaderProto_OperationProto = hadoop_common.RpcRequestHeaderProto_RPC_FINAL_PACKET
	RPC_DEFAULT_RETRY_COUNT int32                                              = hadoop_common.Default_RpcRequestHeaderProto_RetryCount
	CLIENT_PROTOCOL_VERSION uint64                                             = 1
)

Functions

func ConvertBytesToFixed

func ConvertBytesToFixed(rawBytes []byte, data interface{}) error

func ConvertFixedToBytes

func ConvertFixedToBytes(data interface{}) ([]byte, error)

func CreateSimpleUGIProto

func CreateSimpleUGIProto() (*hadoop_common.UserInformationProto, error)

func GetCalleeRPCRequestHeaderProto

func GetCalleeRPCRequestHeaderProto(protocolName *string) *hadoop_common.RequestHeaderProto

Types

type AuthMethod

type AuthMethod byte

const (
	AUTH_SIMPLE   AuthMethod = 0x50
	AUTH_KERBEROS AuthMethod = 0x51
	AUTH_TOKEN    AuthMethod = 0x52
	AUTH_PLAIN    AuthMethod = 0x53
)

func (AuthMethod) String

func (authmethod AuthMethod) String() string

type AuthProtocol

type AuthProtocol byte

const (
	AUTH_PROTOCOL_NONE AuthProtocol = 0x00
	AUTH_PROTOCOL_SASL AuthProtocol = 0xDF
)

func (AuthProtocol) String

func (authprotocol AuthProtocol) String() string