master

package
v0.4.15 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 10, 2024 License: Apache-2.0 Imports: 57 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LabelFeedbackType = "feedback_type"
	LabelStep         = "step"
	LabelData         = "data"
)
View Source 
const (
	ServerNode = "Server"
	WorkerNode = "Worker"
)
View Source 
const (
	PositiveFeedbackRate = "PositiveFeedbackRate"

	TaskLoadDataset            = "Load dataset"
	TaskFindItemNeighbors      = "Find neighbors of items"
	TaskFindUserNeighbors      = "Find neighbors of users"
	TaskFitRankingModel        = "Fit collaborative filtering model"
	TaskFitClickModel          = "Fit click-through rate prediction model"
	TaskSearchRankingModel     = "Search collaborative filtering  model"
	TaskSearchClickModel       = "Search click-through rate prediction model"
	TaskCacheGarbageCollection = "Collect garbage in cache"
)

Variables

View Source 
var (
	LoadDatasetStepSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "load_dataset_step_seconds",
	}, []string{LabelStep})
	LoadDatasetTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "load_dataset_total_seconds",
	})
	FindUserNeighborsSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_seconds",
	}, []string{LabelStep})
	FindUserNeighborsTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_total_seconds",
	})
	FindItemNeighborsSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_seconds",
	}, []string{"step"})
	FindItemNeighborsTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_total_seconds",
	})
	UpdateUserNeighborsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "update_user_neighbors_total",
	})
	UpdateItemNeighborsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "update_item_neighbors_total",
	})
	CacheScannedTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_scanned_total",
	})
	CacheReclaimedTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_reclaimed_total",
	})
	CacheScannedSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_scanned_seconds",
	})

	CollaborativeFilteringFitSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_fit_seconds",
	})
	CollaborativeFilteringSearchSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_search_seconds",
	})
	CollaborativeFilteringNDCG10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_ndcg_10",
	})
	CollaborativeFilteringPrecision10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_precision_10",
	})
	CollaborativeFilteringRecall10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_recall_10",
	})
	CollaborativeFilteringSearchPrecision10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_search_precision_10",
	})
	RankingFitSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_fit_seconds",
	})
	RankingSearchSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_search_seconds",
	})
	RankingPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_precision",
	})
	RankingRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_recall",
	})
	RankingAUC = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_auc",
	})
	RankingSearchPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_search_precision",
	})
	UserNeighborIndexRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "user_neighbor_index_recall",
	})
	ItemNeighborIndexRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "item_neighbor_index_recall",
	})

	UsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "users_total",
	})
	ActiveUsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "active_users_total",
	})
	InactiveUsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "inactive_users_total",
	})
	ItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "items_total",
	})
	ActiveItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "active_items_total",
	})
	InactiveItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "inactive_items_total",
	})
	UserLabelsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "user_labels_total",
	})
	ItemLabelsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "item_labels_total",
	})
	FeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "feedbacks_total",
	})
	ImplicitFeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "implicit_feedbacks_total",
	})
	PositiveFeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "positive_feedbacks_total",
	})
	NegativeFeedbackTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "negative_feedbacks_total",
	})
	MemoryInUseBytesVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "memory_inuse_bytes",
	}, []string{LabelData})
)

Functions

This section is empty.

Types

type CacheGarbageCollectionTask added in v0.4.6 

type CacheGarbageCollectionTask struct {
	*Master
}

func NewCacheGarbageCollectionTask added in v0.4.6 

func NewCacheGarbageCollectionTask(m *Master) *CacheGarbageCollectionTask

type DualDictionaryCentroidVector added in v0.4.2 

type DualDictionaryCentroidVector struct {
	// contains filtered or unexported fields
}

func (*DualDictionaryCentroidVector) Distance added in v0.4.2 

func (d *DualDictionaryCentroidVector) Distance(vector search.Vector) float32

type DualDictionaryVector added in v0.4.2 

type DualDictionaryVector struct {
	// contains filtered or unexported fields
}

func NewDualDictionaryVector added in v0.4.2 

func NewDualDictionaryVector(
	indices1 []int32, values1 []float32,
	indices2 []int32, values2 []float32,
	terms []string, isHidden bool) *DualDictionaryVector

func (*DualDictionaryVector) Centroid added in v0.4.2 

func (v *DualDictionaryVector) Centroid(vectors []search.Vector, indices []int32) search.CentroidVector

func (*DualDictionaryVector) Distance added in v0.4.2 

func (v *DualDictionaryVector) Distance(vector search.Vector) float32

func (*DualDictionaryVector) IsHidden added in v0.4.2 

func (v *DualDictionaryVector) IsHidden() bool

func (*DualDictionaryVector) Terms added in v0.4.2 

func (v *DualDictionaryVector) Terms() []string

type DualVectors added in v0.4.2 

type DualVectors struct {
	// contains filtered or unexported fields
}

func NewDualVectors added in v0.4.2 

func NewDualVectors(first, second *Vectors) *DualVectors

func (*DualVectors) Distance added in v0.4.2 

func (v *DualVectors) Distance(i, j int) float32

func (*DualVectors) Neighbors added in v0.4.2 

func (v *DualVectors) Neighbors(i int) []int32

type Feedback 

type Feedback struct {
	FeedbackType string
	UserId       string
	Item         data.Item
	Timestamp    time.Time
	Comment      string
}

type FindItemNeighborsTask added in v0.4.6 

type FindItemNeighborsTask struct {
	*Master
	// contains filtered or unexported fields
}

FindItemNeighborsTask updates neighbors of items.

func NewFindItemNeighborsTask added in v0.4.6 

func NewFindItemNeighborsTask(m *Master) *FindItemNeighborsTask

type FindUserNeighborsTask added in v0.4.6 

type FindUserNeighborsTask struct {
	*Master
	// contains filtered or unexported fields
}

FindUserNeighborsTask updates neighbors of users.

func NewFindUserNeighborsTask added in v0.4.6 

func NewFindUserNeighborsTask(m *Master) *FindUserNeighborsTask

type FitClickModelTask added in v0.4.6 

type FitClickModelTask struct {
	*Master
	// contains filtered or unexported fields
}

FitClickModelTask fits click model using latest data. After model fitted, following states are changed: 1. Click model version are increased. 2. Click model score are updated. 3. Click model, version and score are persisted to local cache.

func NewFitClickModelTask added in v0.4.6 

func NewFitClickModelTask(m *Master) *FitClickModelTask

type FitRankingModelTask added in v0.4.6 

type FitRankingModelTask struct {
	*Master
	// contains filtered or unexported fields
}

func NewFitRankingModelTask added in v0.4.6 

func NewFitRankingModelTask(m *Master) *FitRankingModelTask

type LocalCache 

type LocalCache struct {
	RankingModelName    string
	RankingModelVersion int64
	RankingModel        ranking.MatrixFactorization
	RankingModelScore   ranking.Score
	ClickModelVersion   int64
	ClickModelScore     click.Score
	ClickModel          click.FactorizationMachine
	// contains filtered or unexported fields
}

LocalCache is local cache for the master node.

func LoadLocalCache 

func LoadLocalCache(path string) (*LocalCache, error)

LoadLocalCache loads local cache from a file. If the ranking model is invalid, RankingModel == nil. If the click model is invalid, ClickModel == nil.

func (*LocalCache) WriteLocalCache 

func (c *LocalCache) WriteLocalCache() error

WriteLocalCache writes local cache to a file.

type Master 

type Master struct {
	protocol.UnimplementedMasterServer
	server.RestServer
	// contains filtered or unexported fields
}

Master is the master node.

func NewMaster 

func NewMaster(cfg *config.Config, cacheFile string, managedMode bool) *Master

NewMaster creates a master node.

func (*Master) CreateWebService 

func (m *Master) CreateWebService()

func (*Master) GetClickModel added in v0.2.2 

func (m *Master) GetClickModel(version *protocol.VersionInfo, sender protocol.Master_GetClickModelServer) error

GetClickModel returns latest click model.

func (*Master) GetMeta 

func (m *Master) GetMeta(ctx context.Context, nodeInfo *protocol.NodeInfo) (*protocol.Meta, error)

GetMeta returns latest configuration.

func (*Master) GetRankingModel 

func (m *Master) GetRankingModel(version *protocol.VersionInfo, sender protocol.Master_GetRankingModelServer) error

GetRankingModel returns latest ranking model.

func (*Master) LoadDataFromDatabase added in v0.2.6 

func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, readTypes []string, itemTTL, positiveFeedbackTTL uint, evaluator *OnlineEvaluator) (
	rankingDataset *ranking.DataSet, clickDataset *click.Dataset, latestItems map[string][]cache.Scored, popularItems map[string][]cache.Scored, err error)

LoadDataFromDatabase loads dataset from data store.

func (*Master) LoginFilter added in v0.4.5 

func (m *Master) LoginFilter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain)

func (*Master) PushTaskInfo added in v0.4.5 

func (m *Master) PushTaskInfo(
	_ context.Context,
	in *protocol.PushTaskInfoRequest) (*protocol.PushTaskInfoResponse, error)

func (*Master) RunManagedTasksLoop added in v0.4.9 

func (m *Master) RunManagedTasksLoop()

func (*Master) RunPrivilegedTasksLoop added in v0.2.4 

func (m *Master) RunPrivilegedTasksLoop()

func (*Master) RunRagtagTasksLoop added in v0.2.4 

func (m *Master) RunRagtagTasksLoop()

RunRagtagTasksLoop searches optimal recommendation model in background. It never modifies variables other than rankingModelSearcher, clickSearchedModel and clickSearchedScore.

func (*Master) Serve 

func (m *Master) Serve()

Serve starts the master node.

func (*Master) SetOneMode added in v0.4.9 

func (m *Master) SetOneMode(workerScheduleHandler http.HandlerFunc)

func (*Master) Shutdown added in v0.4.6 

func (m *Master) Shutdown()

func (*Master) StartHttpServer 

func (m *Master) StartHttpServer()

type Node 

type Node struct {
	Name          string
	Type          string
	IP            string
	HttpPort      int64
	BinaryVersion string
}

Node could be worker node for server node.

func NewNode 

func NewNode(ctx context.Context, nodeInfo *protocol.NodeInfo) *Node

NewNode creates a node from Context and NodeInfo.

type OnlineEvaluator added in v0.4.3 

type OnlineEvaluator struct {
	ReadFeedbacks      []map[int32]*i32set.Set
	PositiveFeedbacks  map[string][]lo.Tuple3[int32, int32, time.Time]
	ReverseIndex       map[lo.Tuple2[int32, int32]]time.Time
	EvaluateDays       int
	TruncatedDateToday time.Time
}

func NewOnlineEvaluator added in v0.4.3 

func NewOnlineEvaluator() *OnlineEvaluator

func (*OnlineEvaluator) Evaluate added in v0.4.3 

func (evaluator *OnlineEvaluator) Evaluate() []server.Measurement

func (*OnlineEvaluator) Positive added in v0.4.3 

func (evaluator *OnlineEvaluator) Positive(feedbackType string, userIndex, itemIndex int32, timestamp time.Time)

func (*OnlineEvaluator) Read added in v0.4.3 

func (evaluator *OnlineEvaluator) Read(userIndex, itemIndex int32, timestamp time.Time)

type ScheduleState added in v0.4.9 

type ScheduleState struct {
	IsRunning   bool      `json:"is_running"`
	SearchModel bool      `json:"search_model"`
	StartTime   time.Time `json:"start_time"`
}

type ScoreUser added in v0.4.8 

type ScoreUser struct {
	data.User
	Score float64
}

type ScoredItem added in v0.4.8 

type ScoredItem struct {
	data.Item
	Score float64
}

type SearchClickModelTask added in v0.4.6 

type SearchClickModelTask struct {
	*Master
	// contains filtered or unexported fields
}

SearchClickModelTask searches best hyper-parameters for factorization machines. It requires read lock on the click dataset.

func NewSearchClickModelTask added in v0.4.6 

func NewSearchClickModelTask(m *Master) *SearchClickModelTask

type SearchRankingModelTask added in v0.4.6 

type SearchRankingModelTask struct {
	*Master
	// contains filtered or unexported fields
}

SearchRankingModelTask searches best hyper-parameters for ranking models. It requires read lock on the ranking dataset.

func NewSearchRankingModelTask added in v0.4.6 

func NewSearchRankingModelTask(m *Master) *SearchRankingModelTask

type SinglePageAppFileSystem 

type SinglePageAppFileSystem struct {
	// contains filtered or unexported fields
}

SinglePageAppFileSystem is the file system for single page app.

func (*SinglePageAppFileSystem) Open 

func (fs *SinglePageAppFileSystem) Open(name string) (http.File, error)

Open index.html if required file not exists.

type Status 

type Status struct {
	BinaryVersion           string
	NumServers              int
	NumWorkers              int
	NumUsers                int
	NumItems                int
	NumUserLabels           int
	NumItemLabels           int
	NumTotalPosFeedback     int
	NumValidPosFeedback     int
	NumValidNegFeedback     int
	PopularItemsUpdateTime  time.Time
	LatestItemsUpdateTime   time.Time
	MatchingModelFitTime    time.Time
	MatchingModelScore      ranking.Score
	RankingModelFitTime     time.Time
	RankingModelScore       click.Score
	UserNeighborIndexRecall float32
	ItemNeighborIndexRecall float32
	MatchingIndexRecall     float32
}

type Task added in v0.2.4 

type Task interface {
	// contains filtered or unexported methods
}

type User 

type User struct {
	data.User
	LastActiveTime time.Time
	LastUpdateTime time.Time
}

type UserIterator 

type UserIterator struct {
	Cursor string
	Users  []User
}

type Vectors added in v0.4.2 

type Vectors struct {
	// contains filtered or unexported fields
}

func NewVectors added in v0.4.2 

func NewVectors(connections, connected [][]int32, weights []float32) *Vectors

func (*Vectors) Distance added in v0.4.2 

func (v *Vectors) Distance(i, j int) float32

func (*Vectors) Neighbors added in v0.4.2 

func (v *Vectors) Neighbors(i int) []int32

type VectorsInterface added in v0.4.2 

type VectorsInterface interface {
	Distance(i, j int) float32
	Neighbors(i int) []int32
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.