poseidon

Documentation

Overview

package main

Index

• Constants
• Variables
• func GetArchiveDay() string
• func MetaIdHdfsFilePart(gzMetaId string) string
• func NewPool(server string, password string, IdleTimeout time.Duration, ...) *redis.Pool
• func ParseDocRow(data []byte, rowIndex uint32) []byte
• func SliceItemToInterface(src DocItemList) []interface{}
• func SymcGet(symcurl string, keys map[string]string) (map[string]string, error)
• type AddrMemcachePool
  • func (self *AddrMemcachePool) Get(addr string) *MemcachePool
  • func (self *AddrMemcachePool) Init()
• type BusinessTrait
  • func BusinessTraitInstance(business string) BusinessTrait
• type DocDataResult
  • func NewDocDataResult() *DocDataResult
  • func (self *DocDataResult) FilterColumns(seperator string, columns []int)
  • func (self *DocDataResult) LogId() uint64
  • func (self *DocDataResult) ToJson(business string) string
• type DocDataResultList
  • func (a DocDataResultList) Len() int
  • func (a DocDataResultList) Less(i, j int) bool
  • func (a DocDataResultList) Swap(i, j int)
• type DocGzClient
  • func (self *DocGzClient) DocIdIntersect(tokenDocItems *[]DocItemList) DocItemList
  • func (self *DocGzClient) FetchDocItems(day string, business string, docItems *DocItemList) (dataList DocDataResultList)
• type DocIdList
  • func (a DocIdList) Len() int
  • func (a DocIdList) Less(i, j int) bool
  • func (a DocIdList) Swap(i, j int)
• type DocItemList
  • func SliceInterfaceToItem(src []interface{}) DocItemList
  • func (a DocItemList) Len() int
  • func (a DocItemList) Less(i, j int) bool
  • func (a DocItemList) Swap(i, j int)
• type HDFSReader
  • func (self *HDFSReader) Read(filePath string, offset int64, length int64) (data []byte, err error)
  • func (self *HDFSReader) ReadZip(filePath string, offset int64, length int64) (data []byte, err error)
• type IndexDataResult
• type InvertedIndexGzClient
  • func (self *InvertedIndexGzClient) FetchIndex(day string, business string, keywords map[string]string) (indexDataList []*IndexDataResult)
• type MemcachePool
  • func NewPoolInstance(addr string, limit int) *MemcachePool
  • func (self *MemcachePool) Alloc() (client *memcache.Client, err error)
  • func (self *MemcachePool) Release(client *memcache.Client, success bool)
• type Pool
  • func (p *Pool) Init(server string, password string, IdleTimeout time.Duration, ...)
• type TestTrait
  • func (self *TestTrait) BusinessName() string
  • func (self *TestTrait) DocFilePath(day string, name string) string
  • func (self *TestTrait) DocIdStoredKey(day string, docId uint64) string
  • func (self *TestTrait) DocStoredAddr() string
  • func (self *TestTrait) FieldShortName(field string) string
  • func (self *TestTrait) IndexFilePath(day string, field string, token string) string
  • func (self *TestTrait) IndexStoredAddr() string
  • func (self *TestTrait) IndexStoredKey(day string, field string, token string) string

Constants

const InvertedIndexGzCount = 200

const (
	PROTOCOL = "tcp" //connection protocol
)

Variables

var (
	MaxIdle     int           = 100
	MaxActive   int           = 1000
	IdleTimeout time.Duration = time.Duration(28 * time.Second)
)

var RedisPool *redis.Pool

var Rp = &Pool{}

var SimgoFramework *simgo.Framework

var ToEscapeFields = map[string]int{
	"fname": 1,
}

Functions

func GetArchiveDay

func GetArchiveDay() string

*

• 计算归档日期 默认一个月

func MetaIdHdfsFilePart

func MetaIdHdfsFilePart(gzMetaId string) string

func NewPool

func NewPool(server string, password string, IdleTimeout time.Duration, MaxIdle, MaxActive int) *redis.Pool

*

• Redis Pool *
• server serverAddress 127.0.0.1:6379
• password password 127.0.0.1:6379:password
• IdleTimeout 超时
• MaxIdle 连接池最大容量
• MaxActive 最大活跃数量
• dbno 选择db127.0.0.1:6379:password:1 *

func ParseDocRow

func ParseDocRow(data []byte, rowIndex uint32) []byte

func SliceItemToInterface

func SliceItemToInterface(src DocItemList) []interface{}

func SymcGet

func SymcGet(symcurl string, keys map[string]string) (map[string]string, error)

Types

type AddrMemcachePool

type AddrMemcachePool struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func (*AddrMemcachePool) Get

func (self *AddrMemcachePool) Get(addr string) *MemcachePool

func (*AddrMemcachePool) Init

func (self *AddrMemcachePool) Init()

type BusinessTrait

type BusinessTrait interface {
	BusinessName() string
	FieldShortName(field string) string

	IndexStoredAddr() string
	IndexStoredKey(day string, field string, token string) string
	IndexFilePath(day string, field string, token string) string

	DocStoredAddr() string
	DocIdStoredKey(day string, docId uint64) string
	DocFilePath(day string, name string) string
}

func BusinessTraitInstance

func BusinessTraitInstance(business string) BusinessTrait

type DocDataResult

type DocDataResult struct {
	DocId    uint64
	RowIndex uint32
	Data     []byte
	Err      error
}

func NewDocDataResult

func NewDocDataResult() *DocDataResult

func (*DocDataResult) FilterColumns

func (self *DocDataResult) FilterColumns(seperator string, columns []int)

func (*DocDataResult) LogId

func (self *DocDataResult) LogId() uint64

func (*DocDataResult) ToJson

func (self *DocDataResult) ToJson(business string) string

type DocDataResultList

type DocDataResultList []*DocDataResult

func (DocDataResultList) Len

func (a DocDataResultList) Len() int

func (DocDataResultList) Less

func (a DocDataResultList) Less(i, j int) bool

func (DocDataResultList) Swap

func (a DocDataResultList) Swap(i, j int)

type DocGzClient

type DocGzClient struct {
}

func (*DocGzClient) DocIdIntersect

func (self *DocGzClient) DocIdIntersect(tokenDocItems *[]DocItemList) DocItemList

func (*DocGzClient) FetchDocItems

func (self *DocGzClient) FetchDocItems(day string, business string, docItems *DocItemList) (dataList DocDataResultList)

type DocIdList

type DocIdList []int32

func (DocIdList) Len

func (a DocIdList) Len() int

func (DocIdList) Less

func (a DocIdList) Less(i, j int) bool

func (DocIdList) Swap

func (a DocIdList) Swap(i, j int)

type DocItemList

type DocItemList []poseidon_if.DocId

func SliceInterfaceToItem

func SliceInterfaceToItem(src []interface{}) DocItemList

func (DocItemList) Len

func (a DocItemList) Len() int

func (DocItemList) Less

func (a DocItemList) Less(i, j int) bool

func (DocItemList) Swap

func (a DocItemList) Swap(i, j int)

type HDFSReader

type HDFSReader struct{}

func (*HDFSReader) Read

func (self *HDFSReader) Read(filePath string, offset int64, length int64) (data []byte, err error)

func (*HDFSReader) ReadZip

func (self *HDFSReader) ReadZip(filePath string, offset int64, length int64) (data []byte, err error)

type IndexDataResult

type IndexDataResult struct {
	Token    string
	Pv, Uv   int
	DocItems DocItemList
	Err      error
}

type InvertedIndexGzClient

type InvertedIndexGzClient struct{}

func (*InvertedIndexGzClient) FetchIndex

func (self *InvertedIndexGzClient) FetchIndex(day string, business string, keywords map[string]string) (indexDataList []*IndexDataResult)

type MemcachePool

type MemcachePool struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewPoolInstance

func NewPoolInstance(addr string, limit int) *MemcachePool

func (*MemcachePool) Alloc

func (self *MemcachePool) Alloc() (client *memcache.Client, err error)

func (*MemcachePool) Release

func (self *MemcachePool) Release(client *memcache.Client, success bool)

type Pool

type Pool struct{}

func (*Pool) Init

func (p *Pool) Init(server string, password string, IdleTimeout time.Duration, MaxIdle, MaxActive int)

type TestTrait

type TestTrait struct {
	// contains filtered or unexported fields
}

func (*TestTrait) BusinessName

func (self *TestTrait) BusinessName() string

func (*TestTrait) DocFilePath

func (self *TestTrait) DocFilePath(day string, name string) string

func (*TestTrait) DocIdStoredKey

func (self *TestTrait) DocIdStoredKey(day string, docId uint64) string

func (*TestTrait) DocStoredAddr

func (self *TestTrait) DocStoredAddr() string

func (*TestTrait) FieldShortName

func (self *TestTrait) FieldShortName(field string) string

*

• 字段映射

func (*TestTrait) IndexFilePath

func (self *TestTrait) IndexFilePath(day string, field string, token string) string

func (*TestTrait) IndexStoredAddr

func (self *TestTrait) IndexStoredAddr() string

func (*TestTrait) IndexStoredKey

func (self *TestTrait) IndexStoredKey(day string, field string, token string) string