hlc18

package
v0.0.0-...-8a34706 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 13, 2022 License: MIT, MIT Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BIRTH_YEARS  = 56
	JOINED_YEARS = 8
	STATUSES     = 3
)
View Source 
const (
	HUGE_BLOCK_SIZE   = 32 * 1024 // для очень неселективных индексов (пол)
	LARGE_BLOCK_SIZE  = 4 * 1024  // для средне-селективных индексов (десятки-сотни бакетов)
	MEDIUM_BLOCK_SIZE = 1024      // для высоко-средне-селективных
	SMALL_BLOCK_SIZE  = 32        // для высоко-селективных

	LIKES_INDEX_VECTOR_CAPACITY = 24
)
View Source 
const BIRTH_YEARS_LEN = 56
View Source 
const DEFAULT_ITER_CAPACITY = 16
View Source 
const LIKE_MOD_FLAG = 2000000
View Source 
const TEST_BITS_32 = 4 * 8
View Source 
const TEST_BITS_64 = 8 * 8

Variables

View Source 
var (
	LikesIndexCompact [][]uint32

	InterestsIndexCompact    [][]uint32
	InterestsSexIndexCompact [][]uint32
	Interests2xIndexCompact  [][]uint32

	BirthYearIndexCompact [][]uint32
	CityIndexCompact      [][]uint32
	CountryIndexCompact   [][]uint32
	FnameIndexCompact     [][]uint32
	SexIndexCompact       [][]uint32
	StatusIndexCompact    [][]uint32

	//sex, premium, status, interest [,country|city]
	RecommendIndexCompact        map[int][]uint32
	RecommendIndexCountryCompact map[int][]uint32
	RecommendIndexCityCompact    map[int][]uint32

	// имена которые мы видели для полов [пол][имя] - 1:0
	SexNames [2][]byte

	BirthYearCityIndexCompact [][]uint32
	EmailPrefixIndexCompact   [][]uint32

	CountryPhoneCodeSexIndex map[int][]uint32
	CityPhoneCodeSexIndex    map[int][]uint32
)
View Source 
var (
	Store *Storage

	FnameDict    = makeDict8()
	SnameDict    = makeDict16()
	DomainDict   = makeDict8()
	InterestDict = makeDict8()
	CityDict     = makeDict16()
	CountryDict  = makeDict8()

	EmailMap = make(map[string]uint32, 1340000)
	PhoneMap = make(map[string]uint32, 600000)

	MinJoined           = timeStampOf("2011-01-01T00:00:00Z")
	MaxJoinedMinPremium = timeStampOf("2018-01-01T00:00:00Z")
	BirthMin            = timeStampOf("1950-01-01T00:00:00Z")
	BirthMax            = timeStampOf("2005-01-01T00:00:00Z")
)
View Source 
var (
	STATUS_OFFSET = uint(1)
	STATUS_MASK   = mask(STATUS_OFFSET, 2)

	DOMAIN_OFFSET = uint(3)
	DOMAIN_MASK   = mask(DOMAIN_OFFSET, 4)

	COUNTRY_OFFSET = uint(7)
	COUNTRY_MASK   = mask(COUNTRY_OFFSET, 7)

	CITY_OFFSET = uint(14)
	CITY_MASK   = mask(CITY_OFFSET, 10)

	FNAME_OFFSET = uint(24)
	FNAME_MASK   = mask(FNAME_OFFSET, 7)

	SNAME_OFFSET = uint(31)
	SNAME_MASK   = mask(SNAME_OFFSET, 11)

	BIRTH_Y_OFFSET = uint(42)
	BIRTH_Y_MASK   = mask(BIRTH_Y_OFFSET, 6)

	PHONECODE_OFFSET = uint(48)
	PHONECODE_MASK   = mask(PHONECODE_OFFSET, 10)

	JOINED_Y_OFFSET = uint(58)
	JOINED_Y_MASK   = mask(JOINED_Y_OFFSET, 3)

	IS_PREMIUM_OFFSET = uint(61)
	IS_PREMIUM_MASK   = mask(IS_PREMIUM_OFFSET, 1)
)
View Source 
var CityCount int
View Source 
var CountryCount int
View Source 
var InterestsCount int
View Source 
var TOKEN_MASK = uint64(mask(0, 24))

Functions

func AppendAccountLikesIndex 

func AppendAccountLikesIndex(id uint32, likes []Like)

func AppendHttpResponse 

func AppendHttpResponse(b []byte, status, headers string, body []byte) []byte

func B2s 

func B2s(b []byte) string

func Bitset128Any 

func Bitset128Any(t0 *uint64, t1 *uint64, p2 *Bitset128) bool

func Bitset128Contains 

func Bitset128Contains(t0 *uint64, t1 *uint64, p2 *Bitset128) bool

func Bitset128Get 

func Bitset128Get(bit byte, t0 *uint64, t1 *uint64) bool

func Bitset128Set 

func Bitset128Set(bit byte, t0 *uint64, t1 *uint64)

func CalculateGroups 

func CalculateGroups()

func CalculateGroupsParallel 

func CalculateGroupsParallel(wg *sync.WaitGroup)

func EvioServer 

func EvioServer(port int, handler func(c evio.Conn, in []byte) (out []byte, action evio.Action))

func GetFilterCache 

func GetFilterCache(token string) []byte

func GetGroupCache 

func GetGroupCache(token string) []byte

func GetRecommendCache 

func GetRecommendCache(token string) []byte

func GetSuggestCache 

func GetSuggestCache(token string) []byte

func LoadOptions 

func LoadOptions(dir string) (int32, int32)

func MakeIndexes 

func MakeIndexes(capacity int)

func Parsereq 

func Parsereq(data []byte, req *Request) (leftover []byte, err error)

func PrintMemUsage 

func PrintMemUsage(info string)

func ioutil.ReadFile(filename string) ([]byte, error)

---- 

file, err := os.Open("binary.dat")

     if err != nil {
         fmt.Println(err)
         return
     }

     defer file.Close()

     info, err := file.Stat()
      if err != nil {
        return nil, err
       }

     // calculate the bytes size
     var size int64 = info.Size()
     bytes := make([]byte, size)

     // read into buffer
     buffer := bufio.NewReader(file)
     _,err = buffer.Read(bytes)

func LoadDataArray2(dir string, arr *[]Account) (maxId uint32, runType int32, err error) { 

	if Store.Now, runType = LoadOptions(dir); Store.Now == 0 {
		return 0, 0, fmt.Errorf("error load %soptions.txt", dir)
	}

	r, err := zip.OpenReader(dir + "data.zip")
	if err != nil {
		return 0, runType, err
	}
	defer r.Close()

	//bytes := make([]byte, 20*1024*1024)

	// Iterate through the files in the archive,
	fmt.Printf("%v\tloading\n", Timenow())
	for _, f := range r.File {
		fmt.Print(".")

		rc, err := f.Open()
		if err != nil {
			return 0, runType, err
		}
		bytes, err := ioutil.ReadAll(rc)
		var p fastjson.Parser
		acc, err := p.ParseBytes(bytes)
		if err != nil {
			log.Fatal(err)
		}
		for _, v := range acc.GetArray("accounts") {
			id := uint32(v.GetInt("id"))
			if maxId < id {
				maxId = id
			}
			if err = CompressJsonToAccount(id, &(*arr)[id-1], v); err != nil {
				return 0, runType, err
			}
		}
		_ = rc.Close()
	}
	Rebuild(arr, maxId)
	return maxId, runType, nil
}

PrintMemUsage outputs the current, total and OS memory being used. As well as the number of garage collection cycles completed.

func PutFilterCache 

func PutFilterCache(token string, value []byte)

func PutGroupCache 

func PutGroupCache(token string, value []byte)

func PutRecommendCache 

func PutRecommendCache(token string, value []byte)

func PutSuggestCache 

func PutSuggestCache(token string, value []byte)

func RebuildAggregates 

func RebuildAggregates(accounts *[]Account, maxId uint32)

func RebuildIndexes 

func RebuildIndexes(accounts *[]Account, maxId uint32)

func RebuildRecommendIndexes 

func RebuildRecommendIndexes(accounts *[]Account, maxId uint32)

func Recommend 

func Recommend(myId uint32, limit int, params map[string]string) []uint32

func ResetCaches 

func ResetCaches()

func ResetIndexes 

func ResetIndexes()

func S2b 

func S2b(s string) []byte

func SearchBytes 

func SearchBytes(a []byte, x byte) int

func SearchLikes 

func SearchLikes(a []Like, id uint32) int

func SearchUint32 

func SearchUint32(a []uint32, x uint32) int

func Suggest 

func Suggest(myId uint32, limit int, params map[string]string) []uint32

todo: !!! "утоптать" лайки после фазы-2

func Timenow 

func Timenow() string

func UnmarshalNew 

func UnmarshalNew(account *AccountJson, bytes []byte) error

func UnmarshalUpdate 

func UnmarshalUpdate(account *AccountJson, bytes []byte) error

func ValidateLikes 

func ValidateLikes(likes []LikeUpdate) error

func ValidateNew 

func ValidateNew(accountJson *AccountJson) error

func ValidateUpdate 

func ValidateUpdate(accountJson *AccountJson, id uint32) error

func WaitRebuildComplete 

func WaitRebuildComplete()

func WriteAccountOut 

func WriteAccountOut(out []byte, separate bool, fields map[string]bool, account *Account, accountId uint32) []byte

* отдавать нужно id, email и поля из запроса (кроме interests и likes)

func WriteAccountOutRecommend 

func WriteAccountOutRecommend(out []byte, separate bool, account *Account, accountId uint32) []byte

var recommendFields = map[string]bool{"status": true, "fname": true, "sname": true, "birth": true, "premium": true}

func WriteAccountOutSuggest 

func WriteAccountOutSuggest(out []byte, separate bool, account *Account, accountId uint32) []byte

var suggestFields = map[string]bool{"status": true, "fname": true, "sname": true }

Types

type Account 

type Account struct {
	// contains filtered or unexported fields
}

func (*Account) IsEmpty 

func (a *Account) IsEmpty() bool

func (*Account) IsPremium 

func (a *Account) IsPremium() byte

type Account2 

type Account2 struct {
	// contains filtered or unexported fields
}

func (*Account2) IsPremium 

func (a *Account2) IsPremium(nowTimestamp int32) byte

type AccountJson 

type AccountJson struct {
	Id        uint32   `json:"id"`
	Fname     string   `json:"fname"`
	Sname     string   `json:"sname"`
	Email     string   `json:"email"`
	Interests []string `json:"interests"`
	Status    string   `json:"status"`
	Premium   Premium  `json:"premium"`
	Sex       string   `json:"sex"`
	Phone     string   `json:"phone"`
	Likes     []Like   `json:"likes"`
	Birth     int32    `json:"birth"`
	City      string   `json:"city"`
	Country   string   `json:"country"`
	Joined    int32    `json:"joined"`
}

func (AccountJson) MarshalEasyJSON 

func (v AccountJson) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (AccountJson) MarshalJSON 

func (v AccountJson) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*AccountJson) UnmarshalEasyJSON 

func (v *AccountJson) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*AccountJson) UnmarshalJSON 

func (v *AccountJson) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type Aggregate 

type Aggregate interface {
	Index(id uint32, account *Account)
	Aggregate(group *Group) (map[uint64]int32, bool)
	GroupKey(x int, y int, groups []bool) uint64
}

type Bitset128 

type Bitset128 [2]uint64

func (*Bitset128) Any 

func (p *Bitset128) Any(p2 *Bitset128) bool

func (*Bitset128) Contains 

func (p *Bitset128) Contains(p2 *Bitset128) bool

func (*Bitset128) Count 

func (p *Bitset128) Count() (count int)

func (*Bitset128) Get 

func (p *Bitset128) Get(bit byte) bool

func (*Bitset128) IsEmpty 

func (p *Bitset128) IsEmpty() bool

func (*Bitset128) IsNotEmpty 

func (p *Bitset128) IsNotEmpty() bool

func (*Bitset128) Reset 

func (p *Bitset128) Reset()

func (*Bitset128) Set 

func (p *Bitset128) Set(bit byte)

type Bitset256 

type Bitset256 [4]uint64

func (*Bitset256) Any 

func (p *Bitset256) Any(p2 *Bitset256) bool

func (*Bitset256) Contains 

func (p *Bitset256) Contains(p2 *Bitset256) bool

func (*Bitset256) Get 

func (p *Bitset256) Get(bit byte) bool

func (*Bitset256) IsEmpty 

func (p *Bitset256) IsEmpty() bool

func (*Bitset256) IsNotEmpty 

func (p *Bitset256) IsNotEmpty() bool

func (*Bitset256) Set 

func (p *Bitset256) Set(bit byte)

type ByteSlice 

type ByteSlice []byte

func (ByteSlice) Len 

func (p ByteSlice) Len() int

func (ByteSlice) Less 

func (p ByteSlice) Less(i, j int) bool

func (ByteSlice) Sort 

func (p ByteSlice) Sort()

func (ByteSlice) Swap 

func (p ByteSlice) Swap(i, j int)

type CacheKey 

type CacheKey struct {
	// contains filtered or unexported fields
}

type Context 

type Context struct {
	Is  evio.InputStream
	Out [8192]byte
}

type Dict16 

type Dict16 struct {
	// contains filtered or unexported fields
}

func (*Dict16) Len 

func (m *Dict16) Len() int

type Dict8 

type Dict8 struct {
	// contains filtered or unexported fields
}

func (*Dict8) Len 

func (m *Dict8) Len() int

type Filter 

type Filter struct {
	Limit int

	ShortCircuit bool

	// список полей фильтра
	Fields map[string]bool

	Len int
	// contains filtered or unexported fields
}

func MakeFilter 

func MakeFilter(params map[string]string) (*Filter, error)

func MakeFilterArgs 

func MakeFilterArgs(args *fasthttp.Args) (*Filter, error)

todo: eliminate Atoi?

func (*Filter) Index 

func (f *Filter) Index(sorted bool) ([]uint32, IndexIterator)

todo: для групп сортировать не нужно ?

func (*Filter) Process 

func (p *Filter) Process(limit int, consumer func(bool, map[string]bool, *Account, uint32))

func (*Filter) Test 

func (f *Filter) Test(id uint32, a *Account) bool

type Group 

type Group struct {
	Limit    int
	GroupBy  []string
	FilterBy map[string]bool
	Filter   Filter

	Order int8
	// contains filtered or unexported fields
}

Полей для группировки всего пять - sex, status, interests, country, city.

func (*Group) CacheKey 

func (p *Group) CacheKey() GroupCacheKey

func (*Group) Classify 

func (g *Group) Classify(accounts *[]Account, updateCache bool) []GroupItem

func (*Group) ClassifyAccount 

func (g *Group) ClassifyAccount(id uint32, collect func(uint64))

* передается функция для учета ключа классификации

func (*Group) DebugWriteJsonGroupItem 

func (g *Group) DebugWriteJsonGroupItem(w io.Writer, groupKey uint64, count int32)

func (*Group) FromArgs 

func (p *Group) FromArgs(args *fasthttp.Args) error

func (*Group) FromParams 

func (g *Group) FromParams(params map[string]string) error

func (*Group) Sort 

func (g *Group) Sort(counters map[uint64]int32, limit int) []GroupItem

func (*Group) WriteJsonGroupItemOut 

func (g *Group) WriteJsonGroupItemOut(out []byte, groupKey uint64, count int32) []byte

type GroupCacheKey 

type GroupCacheKey struct {
	// contains filtered or unexported fields
}

type GroupItem 

type GroupItem struct {
	Key   uint64
	Count int32
}

type IdArray 

type IdArray []byte

func (IdArray) Contains 

func (p IdArray) Contains(value uint32) bool

func (IdArray) Get 

func (p IdArray) Get(i int) uint32

func (IdArray) Len 

func (p IdArray) Len() int

func (IdArray) Put 

func (p IdArray) Put(i int, v uint32)

type IndexAndIterator 

type IndexAndIterator struct {
	// contains filtered or unexported fields
}

func (*IndexAndIterator) Len 

func (p *IndexAndIterator) Len() (l int)

func (*IndexAndIterator) Next 

func (p *IndexAndIterator) Next() uint32

todo: брать самый короткий индекс и пятится по нему назад

func (*IndexAndIterator) Prepare 

func (p *IndexAndIterator) Prepare() *IndexAndIterator

func (*IndexAndIterator) ToSingle 

func (p *IndexAndIterator) ToSingle() (index []uint32)

type IndexIterator 

type IndexIterator interface {
	Next() uint32
	// возвращает индекс если он один или nil если индексов нет или их несколько
	ToSingle() []uint32
	Len() int
}

type IndexOrIterator 

type IndexOrIterator struct {
	// contains filtered or unexported fields
}

func (*IndexOrIterator) Len 

func (p *IndexOrIterator) Len() (l int)

func (*IndexOrIterator) Next 

func (p *IndexOrIterator) Next() uint32

func (*IndexOrIterator) ToSingle 

func (p *IndexOrIterator) ToSingle() (index []uint32)

type Like 

type Like struct {
	Id uint32 `json:"id"`
	Ts uint32 `json:"ts"`
}

func (Like) MarshalEasyJSON 

func (v Like) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (Like) MarshalJSON 

func (v Like) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*Like) UnmarshalEasyJSON 

func (v *Like) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*Like) UnmarshalJSON 

func (v *Like) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type LikeSlice 

type LikeSlice []Like

func (*LikeSlice) Len 

func (p *LikeSlice) Len() int

func (*LikeSlice) Less 

func (p *LikeSlice) Less(i, j int) bool

func (*LikeSlice) Sort 

func (p *LikeSlice) Sort()

func (*LikeSlice) Swap 

func (p *LikeSlice) Swap(i, j int)

type LikeUpdate 

type LikeUpdate struct {
	Likee uint32
	Liker uint32
	Ts    uint32
}

func ParseLikesUpdate 

func ParseLikesUpdate(bytes []byte) ([]LikeUpdate, error)

type NullOp 

type NullOp uint8
const (
	Null_empty NullOp = iota
	Null_not
	Null_yes
)

type Other 

type Other struct {
	// contains filtered or unexported fields
}

type Premium 

type Premium struct {
	Start  int32 `json:"start"`
	Finish int32 `json:"finish"`
}

func (Premium) MarshalEasyJSON 

func (v Premium) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (Premium) MarshalJSON 

func (v Premium) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*Premium) UnmarshalEasyJSON 

func (v *Premium) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*Premium) UnmarshalJSON 

func (v *Premium) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type Reply 

type Reply struct {
	Status int
	Body   []byte
}

type Request 

type Request struct {
	Proto, Method, Path, Query, Head, Body string
}

type Storage 

type Storage struct {
	Accounts  []Account
	Accounts2 []Account2

	MaxId   uint32
	Now     int32
	RunType int32
	Likes   [][]Like
	// contains filtered or unexported fields
}

func InitStore 

func InitStore(cap int) *Storage

func (*Storage) CompressToAccount 

func (p *Storage) CompressToAccount(accJson *AccountJson) error

func (*Storage) CompressUpdateToAccount 

func (p *Storage) CompressUpdateToAccount(accJson *AccountJson) error

func (*Storage) LikesSort 

func (p *Storage) LikesSort()

func (*Storage) LikesUpdate 

func (p *Storage) LikesUpdate(likes []LikeUpdate)

func (*Storage) LoadData 

func (p *Storage) LoadData(dir string) error

func (*Storage) Rebuild 

func (p *Storage) Rebuild(parallel bool)

type Uint32Slice 

type Uint32Slice []uint32

func (Uint32Slice) Len 

func (p Uint32Slice) Len() int

func (Uint32Slice) Less 

func (p Uint32Slice) Less(i, j int) bool

func (Uint32Slice) Sort 

func (p Uint32Slice) Sort()

func (Uint32Slice) Swap 

func (p Uint32Slice) Swap(i, j int)

type VectorUint32 

type VectorUint32 struct {
	// contains filtered or unexported fields
}

func (*VectorUint32) CopyTo 

func (p *VectorUint32) CopyTo(out []uint32)

func (*VectorUint32) Get 

func (p *VectorUint32) Get(index int) uint32

func (*VectorUint32) Iterate 

func (p *VectorUint32) Iterate(iter func(i int, v uint32) bool) int

* Iterate() returns last index iterated

func (*VectorUint32) Len 

func (p *VectorUint32) Len() int

возвращаем предыдущее знаение

func (*VectorUint32) Push 

func (p *VectorUint32) Push(value uint32) int

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.