gohadoop

README

gohadoop

gohadoop is native go clients for Apache Hadoop YARN.

It includes an early version of Hadoop IPC client and requisite YARN client libraries to implement YARN applications completely in go (both YARN application-client and application-master).

###Notes:

• Set HADOOP_CONF_DIR environment variable, and ensure the conf directory contains both *-default.xml and *-site.xml files.
• Currently the go hadoop IPC client doesn't support secure Hadoop IPC (SASL). As a result, you'll need to apply patches/gohadoop.patch to switch off the default token-auth in effect for YARN applications. Also, you'll need to set yarn.rpc.auth to false in yarn-site.xml.
• hadoop_yarn/examples/dist_shell is an example go YARN application: client.go is the submission client and applicationmaster.go is the application-master.

To run:

HADOOP_CONF_DIR=conf go run hadoop_yarn/examples/dist_shell/client.go

Documentation

Index

• Variables
• func ConvertBytesToFixed(rawBytes []byte, data interface{}) error
• func ConvertFixedToBytes(data interface{}) ([]byte, error)
• func CreateSimpleUGIProto() (*hadoop_common.UserInformationProto, error)
• func GetCalleeRPCRequestHeaderProto(protocolName *string) *hadoop_common.RequestHeaderProto
• type AuthMethod
  • func (authmethod AuthMethod) String() string
• type AuthProtocol
  • func (authprotocol AuthProtocol) String() string

Variables

var (
	RPC_HEADER        []byte = []byte("hrpc")
	VERSION           []byte = []byte{0x09}
	RPC_SERVICE_CLASS byte   = 0x00

	RPC_PROTOCOL_BUFFFER    hadoop_common.RpcKindProto                         = hadoop_common.RpcKindProto_RPC_PROTOCOL_BUFFER
	RPC_FINAL_PACKET        hadoop_common.RpcRequestHeaderProto_OperationProto = hadoop_common.RpcRequestHeaderProto_RPC_FINAL_PACKET
	RPC_DEFAULT_RETRY_COUNT int32                                              = hadoop_common.Default_RpcRequestHeaderProto_RetryCount
	CLIENT_PROTOCOL_VERSION uint64                                             = 1
)

Functions

func ConvertBytesToFixed

func ConvertBytesToFixed(rawBytes []byte, data interface{}) error

func ConvertFixedToBytes

func ConvertFixedToBytes(data interface{}) ([]byte, error)

func CreateSimpleUGIProto

func CreateSimpleUGIProto() (*hadoop_common.UserInformationProto, error)

func GetCalleeRPCRequestHeaderProto

func GetCalleeRPCRequestHeaderProto(protocolName *string) *hadoop_common.RequestHeaderProto

Types

type AuthMethod

type AuthMethod byte

const (
	AUTH_SIMPLE   AuthMethod = 0x50
	AUTH_KERBEROS AuthMethod = 0x51
	AUTH_TOKEN    AuthMethod = 0x52
	AUTH_PLAIN    AuthMethod = 0x53
)

func (AuthMethod) String

func (authmethod AuthMethod) String() string

type AuthProtocol

type AuthProtocol byte

const (
	AUTH_PROTOCOL_NONE AuthProtocol = 0x00
	AUTH_PROTOCOL_SASL AuthProtocol = 0xDF
)

func (AuthProtocol) String

func (authprotocol AuthProtocol) String() string