extractor

README

Berserker Extractor

Berserker Extractor is in charge of obtain all the files from specific repositories, reading rooted repositories in siva files. It sends the data to the Spark processes using gRPC.

How to install

• To generate the vendor folder use glide install or glide up if you want to update the dependencies.
• To regenerate .proto files, execute proteus -p github.com/src-d/berserker/extractor -f proto
• To build a gRPC server go build -o extractor-server ./server/*.go
• To build a CLI (useful for debugging&profiling) go build -o extractor ./cli/*.go

Documentation

Overview

Package extractor is a generated protocol buffer package.

It is generated from these files:

github.com/src-d/berserker/extractor/generated.proto

It has these top-level messages:

File
RepositoryData
Request
Service_GetRepositoriesDataResponse

Index

• Variables
• func NewExtractorServiceServer(limit uint64, maxGrpcMsgSize int) *extractorServiceServer
• func RegisterExtractorServiceServer(s *grpc.Server, srv ExtractorServiceServer)
• type ExtractorServiceClient
  • func NewExtractorServiceClient(cc *grpc.ClientConn) ExtractorServiceClient
• type ExtractorServiceServer
• type File
  • func (*File) Descriptor() ([]byte, []int)
  • func (m *File) Marshal() (dAtA []byte, err error)
  • func (m *File) MarshalTo(dAtA []byte) (int, error)
  • func (*File) ProtoMessage()
  • func (m *File) ProtoSize() (n int)
  • func (m *File) Reset()
  • func (m *File) String() string
  • func (m *File) Unmarshal(dAtA []byte) error
• type RepositoryData
  • func (*RepositoryData) Descriptor() ([]byte, []int)
  • func (m *RepositoryData) Marshal() (dAtA []byte, err error)
  • func (m *RepositoryData) MarshalTo(dAtA []byte) (int, error)
  • func (*RepositoryData) ProtoMessage()
  • func (m *RepositoryData) ProtoSize() (n int)
  • func (m *RepositoryData) Reset()
  • func (m *RepositoryData) String() string
  • func (m *RepositoryData) Unmarshal(dAtA []byte) error
• type RepositoryIDs
• type Request
  • func (*Request) Descriptor() ([]byte, []int)
  • func (m *Request) Marshal() (dAtA []byte, err error)
  • func (m *Request) MarshalTo(dAtA []byte) (int, error)
  • func (*Request) ProtoMessage()
  • func (m *Request) ProtoSize() (n int)
  • func (m *Request) Reset()
  • func (m *Request) String() string
  • func (m *Request) Unmarshal(dAtA []byte) error
• type Service
  • func NewService(n uint64, maxGrpcMsgSize int) *Service
  • func (s *Service) GetRepositoriesData(r *Request) ([]*RepositoryData, error)
  • func (s *Service) GetRepositoryData(r *Request) (*RepositoryData, error)
• type Service_GetRepositoriesDataResponse
  • func (*Service_GetRepositoriesDataResponse) Descriptor() ([]byte, []int)
  • func (m *Service_GetRepositoriesDataResponse) GetResult1() []*RepositoryData
  • func (m *Service_GetRepositoriesDataResponse) Marshal() (dAtA []byte, err error)
  • func (m *Service_GetRepositoriesDataResponse) MarshalTo(dAtA []byte) (int, error)
  • func (*Service_GetRepositoriesDataResponse) ProtoMessage()
  • func (m *Service_GetRepositoriesDataResponse) ProtoSize() (n int)
  • func (m *Service_GetRepositoriesDataResponse) Reset()
  • func (m *Service_GetRepositoriesDataResponse) String() string
  • func (m *Service_GetRepositoriesDataResponse) Unmarshal(dAtA []byte) error

Variables

var (
	ErrInvalidLengthGenerated = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowGenerated   = fmt.Errorf("proto: integer overflow")
)

Functions

func NewExtractorServiceServer

func NewExtractorServiceServer(limit uint64, maxGrpcMsgSize int) *extractorServiceServer

func RegisterExtractorServiceServer

func RegisterExtractorServiceServer(s *grpc.Server, srv ExtractorServiceServer)

Types

type ExtractorServiceClient

type ExtractorServiceClient interface {
	Service_GetRepositoriesData(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Service_GetRepositoriesDataResponse, error)
	Service_GetRepositoryData(ctx context.Context, in *Request, opts ...grpc.CallOption) (*RepositoryData, error)
}

func NewExtractorServiceClient

func NewExtractorServiceClient(cc *grpc.ClientConn) ExtractorServiceClient

type ExtractorServiceServer

type ExtractorServiceServer interface {
	Service_GetRepositoriesData(context.Context, *Request) (*Service_GetRepositoriesDataResponse, error)
	Service_GetRepositoryData(context.Context, *Request) (*RepositoryData, error)
}

type File

type File struct {
	Language string
	Path     string
	UAST     []byte
	Hash     string
}

func (*File) Descriptor

func (*File) Descriptor() ([]byte, []int)

func (*File) Marshal

func (m *File) Marshal() (dAtA []byte, err error)

func (*File) MarshalTo

func (m *File) MarshalTo(dAtA []byte) (int, error)

func (*File) ProtoMessage

func (*File) ProtoMessage()

func (*File) ProtoSize

func (m *File) ProtoSize() (n int)

func (*File) Reset

func (m *File) Reset()

func (*File) String

func (m *File) String() string

func (*File) Unmarshal

func (m *File) Unmarshal(dAtA []byte) error

type RepositoryData

type RepositoryData struct {
	RepositoryID string
	URL          string
	Files        []File
}

proteus:generate

func (*RepositoryData) Descriptor

func (*RepositoryData) Descriptor() ([]byte, []int)

func (*RepositoryData) Marshal

func (m *RepositoryData) Marshal() (dAtA []byte, err error)

func (*RepositoryData) MarshalTo

func (m *RepositoryData) MarshalTo(dAtA []byte) (int, error)

func (*RepositoryData) ProtoMessage

func (*RepositoryData) ProtoMessage()

func (*RepositoryData) ProtoSize

func (m *RepositoryData) ProtoSize() (n int)

func (*RepositoryData) Reset

func (m *RepositoryData) Reset()

func (*RepositoryData) String

func (m *RepositoryData) String() string

func (*RepositoryData) Unmarshal

func (m *RepositoryData) Unmarshal(dAtA []byte) error

type RepositoryIDs

type RepositoryIDs []string

type Request

type Request struct {
	RepositoryIDs  RepositoryIDs
	RootCommitHash []byte
	Reference      string
}

proteus:generate

func (*Request) Descriptor

func (*Request) Descriptor() ([]byte, []int)

func (*Request) Marshal

func (m *Request) Marshal() (dAtA []byte, err error)

func (*Request) MarshalTo

func (m *Request) MarshalTo(dAtA []byte) (int, error)

func (*Request) ProtoMessage

func (*Request) ProtoMessage()

func (*Request) ProtoSize

func (m *Request) ProtoSize() (n int)

func (*Request) Reset

func (m *Request) Reset()

func (*Request) String

func (m *Request) String() string

func (*Request) Unmarshal

func (m *Request) Unmarshal(dAtA []byte) error

type Service

type Service struct {
	// contains filtered or unexported fields
}

func NewService

func NewService(n uint64, maxGrpcMsgSize int) *Service

func (*Service) GetRepositoriesData

func (s *Service) GetRepositoriesData(r *Request) ([]*RepositoryData, error)

func (*Service) GetRepositoryData

func (s *Service) GetRepositoryData(r *Request) (*RepositoryData, error)

type Service_GetRepositoriesDataResponse

type Service_GetRepositoriesDataResponse struct {
	Result1 []*RepositoryData `protobuf:"bytes,1,rep,name=result1" json:"result1,omitempty"`
}

func (*Service_GetRepositoriesDataResponse) Descriptor

func (*Service_GetRepositoriesDataResponse) Descriptor() ([]byte, []int)

func (*Service_GetRepositoriesDataResponse) GetResult1

func (m *Service_GetRepositoriesDataResponse) GetResult1() []*RepositoryData

func (*Service_GetRepositoriesDataResponse) Marshal

func (m *Service_GetRepositoriesDataResponse) Marshal() (dAtA []byte, err error)

func (*Service_GetRepositoriesDataResponse) MarshalTo

func (m *Service_GetRepositoriesDataResponse) MarshalTo(dAtA []byte) (int, error)

func (*Service_GetRepositoriesDataResponse) ProtoMessage

func (*Service_GetRepositoriesDataResponse) ProtoMessage()

func (*Service_GetRepositoriesDataResponse) ProtoSize

func (m *Service_GetRepositoriesDataResponse) ProtoSize() (n int)

func (*Service_GetRepositoriesDataResponse) Reset

func (m *Service_GetRepositoriesDataResponse) Reset()

func (*Service_GetRepositoriesDataResponse) String

func (m *Service_GetRepositoriesDataResponse) String() string

func (*Service_GetRepositoriesDataResponse) Unmarshal

func (m *Service_GetRepositoriesDataResponse) Unmarshal(dAtA []byte) error