search

package
v1.13.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 5, 2021 License: MIT Imports: 38 Imported by: 12

Documentation

Index

Constants

View Source 
const (
	GRAMMAR_BOOST = 100.0

	GRAMMAR_BOOST_KEYWORD = 300.0

	GRAMMAR_SUGGEST_SIZE = 30

	GRAMMAR_SEARCH_SIZE = 2000

	GRAMMAR_PERCULATE_SIZE = 5

	PERCULATE_HIGHLIGHT_SEPERATOR = '$'
)
View Source 
const (
	EVAL_SET_EXPECTATION_FIRST_COLUMN = 4
	EVAL_SET_EXPECTATION_LAST_COLUMN  = 8
)
View Source 
const (
	SQ_SERVER_ERROR   = iota
	SQ_NO_EXPECTATION = iota
	SQ_BAD_STRUCTURE  = iota
	SQ_UNKNOWN        = iota
	SQ_REGULAR        = iota
	SQ_GOOD           = iota
)

Search quality enum. Order important, the lower (higher integer) the better.

View Source 
const (
	CR_WIN            = iota
	CR_LOSS           = iota
	CR_SAME           = iota
	CR_NO_EXPECTATION = iota
	CR_ERROR          = iota
)

Compare results classification.

View Source 
const (
	ET_NOT_SET       = -1
	ET_CONTENT_UNITS = iota
	ET_COLLECTIONS   = iota
	ET_LESSONS       = iota
	ET_PROGRAMS      = iota
	ET_SOURCES       = iota
	ET_EVENTS        = iota
	ET_LANDING_PAGE  = iota
	ET_BLOG_OR_TWEET = iota
	ET_EMPTY         = iota
	ET_FAILED_PARSE  = iota
	ET_BAD_STRUCTURE = iota
	ET_FAILED_SQL    = iota
)
View Source 
const (
	FILTER_NAME_SOURCE       = "source"
	FILTER_NAME_TOPIC        = "topic"
	FILTER_NAME_CONTENT_TYPE = "contentType"
	FILTER_NAME_HOLIDAYS     = "holidays"
	PREFIX_LATEST            = "[latest]"
	BLOG_OR_TWEET_MARK       = "blog_or_tweet"
)
View Source 
const (
	// Content boost.
	TITLE_BOOST       = 2.0
	DESCRIPTION_BOOST = 1.2
	FULL_TITLE_BOOST  = 1.1
	DEFAULT_BOOST     = 1.0

	// Max slop.
	SLOP = 100

	// Following two boosts may be agregated.
	// Boost for standard anylyzer, i.e., without stemming.
	STANDARD_BOOST = 1.2
	// Boost for exact phrase match, without slop.
	EXACT_BOOST = 1.5

	SPAN_NEAR_BOOST = 0.01

	MIN_SCORE_FOR_RESULTS = 0.01

	NUM_SUGGESTS = 30
)
View Source 
const (
	GRAMMARS_INDEX_BASE_NAME = "grammars"
)
View Source 
const (
	START_YEAR = 1996
)

Variables

View Source 
var COMPARE_RESULTS_NAME = map[int]string{
	CR_WIN:   "Win",
	CR_LOSS:  "Loss",
	CR_SAME:  "Same",
	CR_ERROR: "Error",
}
View Source 
var EXPECTATIONS_FOR_EVALUATION = map[int]bool{
	ET_CONTENT_UNITS: true,
	ET_COLLECTIONS:   true,
	ET_LESSONS:       true,
	ET_PROGRAMS:      true,
	ET_SOURCES:       true,
	ET_LANDING_PAGE:  true,
	ET_BLOG_OR_TWEET: true,
	ET_EMPTY:         false,
	ET_FAILED_PARSE:  false,
	ET_BAD_STRUCTURE: true,
	ET_FAILED_SQL:    true,
}
View Source 
var EXPECTATION_HIT_TYPE = map[int]string{
	ET_CONTENT_UNITS: consts.ES_RESULT_TYPE_UNITS,
	ET_COLLECTIONS:   consts.ES_RESULT_TYPE_COLLECTIONS,
	ET_LESSONS:       consts.INTENT_HIT_TYPE_LESSONS,
	ET_PROGRAMS:      consts.INTENT_HIT_TYPE_PROGRAMS,
	ET_SOURCES:       consts.ES_RESULT_TYPE_SOURCES,
	ET_LANDING_PAGE:  consts.GRAMMAR_TYPE_LANDING_PAGE,
}
View Source 
var EXPECTATION_TO_NAME = map[int]string{
	ET_CONTENT_UNITS: "et_content_units",
	ET_COLLECTIONS:   "et_collections",
	ET_LESSONS:       "et_lessons",
	ET_PROGRAMS:      "et_programs",
	ET_SOURCES:       "et_sources",
	ET_BLOG_OR_TWEET: "et_blog_or_tweet",
	ET_LANDING_PAGE:  "et_landing_page",
	ET_EMPTY:         "et_empty",
	ET_FAILED_PARSE:  "et_failed_parse",
	ET_BAD_STRUCTURE: "et_bad_structure",
	ET_FAILED_SQL:    "et_failed_sql",
}
View Source 
var EXPECTATION_URL_PATH = map[int]string{
	ET_CONTENT_UNITS: "cu",
	ET_COLLECTIONS:   "c",
	ET_LESSONS:       "lessons",
	ET_PROGRAMS:      "programs",
	ET_SOURCES:       "sources",
	ET_EVENTS:        "events",
}
View Source 
var FLAT_REPORT_HEADERS = []string{
	"Language", "Query", "Weight", "Bucket", "Comment",
	"Expectation", "Parsed", "SearchQuality", "Rank"}
View Source 
var LANDING_PAGES = map[string]string{
	"lessons":                   consts.GRAMMAR_INTENT_LANDING_PAGE_LESSONS,
	"lessons/daily":             consts.GRAMMAR_INTENT_LANDING_PAGE_LESSONS,
	"lessons/virtual":           consts.GRAMMAR_INTENT_LANDING_PAGE_VIRTUAL_LESSONS,
	"lessons/lectures":          consts.GRAMMAR_INTENT_LANDING_PAGE_LECTURES,
	"lessons/women":             consts.GRAMMAR_INTENT_LANDING_PAGE_WOMEN_LESSONS,
	"lessons/rabash":            consts.GRAMMAR_INTENT_LANDING_PAGE_RABASH_LESSONS,
	"lessons/series":            consts.GRAMMAR_INTENT_LANDING_PAGE_LESSON_SERIES,
	"programs/main":             consts.GRAMMAR_INTENT_LANDING_PAGE_PRORGRAMS,
	"programs/clips":            consts.GRAMMAR_INTENT_LANDING_PAGE_CLIPS,
	"sources":                   consts.GRAMMAR_INTENT_LANDING_PAGE_LIBRARY,
	"events":                    consts.GRAMMAR_INTENT_LANDING_PAGE_CONVENTIONS,
	"events/conventions":        consts.GRAMMAR_INTENT_LANDING_PAGE_CONVENTIONS,
	"events/holidays":           consts.GRAMMAR_INTENT_LANDING_PAGE_HOLIDAYS,
	"events/unity-days":         consts.GRAMMAR_INTENT_LANDING_PAGE_UNITY_DAYS,
	"events/friends-gatherings": consts.GRAMMAR_INTENT_LANDING_PAGE_FRIENDS_GATHERINGS,
	"events/meals":              consts.GRAMMAR_INTENT_LANDING_PAGE_MEALS,
	"topics":                    consts.GRAMMAR_INTENT_LANDING_PAGE_TOPICS,
	"publications/blog":         consts.GRAMMAR_INTENT_LANDING_PAGE_BLOG,
	"publications/twitter":      consts.GRAMMAR_INTENT_LANDING_PAGE_TWITTER,
	"publications/articles":     consts.GRAMMAR_INTENT_LANDING_PAGE_ARTICLES,
	"simple-mode":               consts.GRAMMAR_INTENT_LANDING_PAGE_DOWNLOADS,
	"help":                      consts.GRAMMAR_INTENT_LANDING_PAGE_HELP,
}
View Source 
var SEARCH_QUALITY_BY_NAME = map[string]int{
	"Good":          SQ_GOOD,
	"Regular":       SQ_REGULAR,
	"Unknown":       SQ_UNKNOWN,
	"BadStructure":  SQ_BAD_STRUCTURE,
	"NoExpectation": SQ_NO_EXPECTATION,
	"ServerError":   SQ_SERVER_ERROR,
}
View Source 
var SEARCH_QUALITY_NAME = map[int]string{
	SQ_GOOD:           "Good",
	SQ_REGULAR:        "Regular",
	SQ_UNKNOWN:        "Unknown",
	SQ_BAD_STRUCTURE:  "BadStructure",
	SQ_NO_EXPECTATION: "NoExpectation",
	SQ_SERVER_ERROR:   "ServerError",
}
View Source 
var SINGLE_QUOTE_RUNES = map[rune]bool{
	rune(39):   true,
	rune(145):  true,
	rune(146):  true,
	rune(8216): true,
	rune(8217): true,
	rune(8219): true,
	rune(1523): true,
	rune(1436): true,
	rune(1437): true,
}

Functions

func CompareResults added in v0.8.0 

func CompareResults(base int, exp int) int

Returns compare results classification constant.

func CreateGrammarIndex added in v1.6.6 

func CreateGrammarIndex(esc *elastic.Client, indexDate string) error

func CsvToString added in v1.1.8 

func CsvToString(records [][]string) (error, string)

func DeleteGrammarIndex added in v1.6.6 

func DeleteGrammarIndex(esc *elastic.Client, indexDate string) error

func EvalResultDiffHtml added in v1.6.3 

func EvalResultDiffHtml(resultDiff ResultDiff) (string, error)

func EvalResultsDiffsCount added in v1.6.3 

func EvalResultsDiffsCount(evalSet []EvalQuery, expResults []EvalResult, baseResults []EvalResult) (int, error)

Assuming |expResults| and |baseResults| are ordered by |Order| field.

func EvalResultsDiffsHtml added in v1.6.3 

func EvalResultsDiffsHtml(resultsDiffs ResultsDiffs) (string, error)

func EvalScrapeStreaming added in v1.6.3 

func EvalScrapeStreaming(in chan EvalQuery, out chan EvalResult, serverUrl string, skipExpectations bool) error

In case of error, this function will close the |out| channel. In case of |in| closed, |out| will be closed after everything scraped.

func ExpectationToString added in v0.9.5 

func ExpectationToString(e Expectation) string

func FilterValueToUid added in v0.9.5 

func FilterValueToUid(value string) string

func FoldGrammars added in v1.4.4 

func FoldGrammars(first Grammars, second Grammars)

func FoldGrammarsV2 added in v1.6.6 

func FoldGrammarsV2(first GrammarsV2, second GrammarsV2)

func GoodExpectations added in v0.9.5 

func GoodExpectations(expectations []Expectation) int

func GrammarFilterVariablesMatch added in v1.6.6 

func GrammarFilterVariablesMatch(intent string, variablesByPhrase VariablesByPhrase, cm cache.CacheManager)

func GrammarIndexName added in v1.6.6 

func GrammarIndexName(lang string, indexDate string) string

func GrammarIndexNameForServing added in v1.6.9 

func GrammarIndexNameForServing(lang string) string

func GrammarIndexNameFunc added in v1.6.9 

func GrammarIndexNameFunc(indexDate string) es.IndexNameByLang

func GrammarVariablesMatch added in v1.6.6 

func GrammarVariablesMatch(intent string, vMap map[string][]string, cm cache.CacheManager) bool

func HitMatchesExpectation added in v0.9.5 

func HitMatchesExpectation(hit *elastic.SearchHit, hitSource HitSource, e Expectation) bool

func HitSourcesEqual added in v1.6.3 

func HitSourcesEqual(a, b HitSource) bool

func IndexGrammars added in v1.6.6 

func IndexGrammars(esc *elastic.Client, indexDate string, grammars GrammarsV2, variables VariablesV2, cm cache.CacheManager) error

func LoadVariableTranslations added in v1.6.6 

func LoadVariableTranslations(variableFile string, variableName string, esc *elastic.Client, tc *TokensCache) (map[string]map[string][][]*TokenNode, error)

func MakeYearVariablesV2 added in v1.6.6 

func MakeYearVariablesV2() map[string][]string

func NewFilteredResultsSearchRequest added in v1.10.0 

func NewFilteredResultsSearchRequest(text string, filters map[string][]string, contentType string, programCollection string, sources []string, from int, size int, sortBy string, resultTypes []string, language string, preference string, deb bool) ([]*elastic.SearchRequest, error)

func NewGammarPerculateRequest added in v1.10.0 

func NewGammarPerculateRequest(query *Query, language string, preference string) *elastic.SearchRequest

func NewResultsSearchRequest added in v1.0.5 

func NewResultsSearchRequest(options SearchRequestOptions) (*elastic.SearchRequest, error)

func NewResultsSearchRequests added in v1.0.5 

func NewResultsSearchRequests(options SearchRequestOptions) ([]*elastic.SearchRequest, error)

func NewResultsSuggestGrammarV2CompletionRequest added in v1.6.6 

func NewResultsSuggestGrammarV2CompletionRequest(query *Query, language string, preference string) *elastic.SearchRequest

func NewResultsSuggestRequest added in v1.0.5 

func NewResultsSuggestRequest(resultTypes []string, index string, query Query, preference string) *elastic.SearchRequest

func NewResultsSuggestRequests added in v1.0.5 

func NewResultsSuggestRequests(resultTypes []string, query Query, preference string) []*elastic.SearchRequest

func NewSuggestGammarV2Request added in v1.6.6 

func NewSuggestGammarV2Request(query *Query, language string, preference string, hitType *string) *elastic.SearchRequest

func PrintManyTokens added in v1.6.6 

func PrintManyTokens(tokens [][]*TokenNode, prefix string, variables map[string]*Variable)

func PrintOriginalPhrase added in v1.6.6 

func PrintOriginalPhrase(t []*TokenNode, variables map[string]*Variable)

func PrintToken added in v1.6.6 

func PrintToken(token *TokenNode, variables map[string]*Variable)

func PrintTokens added in v1.6.6 

func PrintTokens(tokens []*TokenNode, prefix string, variables map[string]*Variable)

func ReadEvalSets added in v1.6.3 

func ReadEvalSets(glob string) (map[string][]EvalQuery, error)

Return map from filename to query set.

func ResultsByExpectation added in v1.1.8 

func ResultsByExpectation(queries []EvalQuery, results EvalResults) [][]string

func SplitPatterns added in v1.6.6 

func SplitPatterns(patterns []*TokenNode, variables map[string]*Variable) ([]*TokenNode, []*TokenNode)

func SuggestionHasOptions added in v0.7.1 

func SuggestionHasOptions(ss elastic.SearchSuggest) bool

func TokenNodesToString added in v1.5.2 

func TokenNodesToString(root []*TokenNode, variables map[string]*Variable) string

func TokensMatch added in v1.5.2 

func TokensMatch(tokens []*TokenNode, patterns [][]*TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error)

func TokensSingleMatch added in v1.5.4 

func TokensSingleMatch(tokens []*TokenNode, patterns []*TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error)

func VariablesAsString added in v1.6.6 

func VariablesAsString(vars []string) string

Note the sort. It might be the case where order of vars important later on. In those cases we will not be able to sort the keys. We will have to keep order and distinguish between two different order cases.

func VariablesFromString added in v1.6.6 

func VariablesFromString(vars string) []string

func VariablesMapToString added in v1.6.6 

func VariablesMapToString(variablesMap map[string][]string) string

func WriteResults added in v0.9.5 

func WriteResults(path string, queries []EvalQuery, results EvalResults) error

func WriteResultsByExpectation added in v0.9.5 

func WriteResultsByExpectation(path string, queries []EvalQuery, results EvalResults) ([][]string, error)

func WriteToCsv added in v0.9.5 

func WriteToCsv(path string, records [][]string) error

func WriteVsGoldenHTML added in v1.5.0 

func WriteVsGoldenHTML(vsGoldenHtml string, records [][]string, goldenRecords [][]string, bottomPart string) error

func YearScorePenalty added in v1.6.6 

func YearScorePenalty(vMap map[string][]string) float64

Types

type ClassificationIntent added in v1.4.0 

type ClassificationIntent struct {
	// Fields from result.
	ResultType string `json:"result_type"`
	MDB_UID    string `json:"mdb_uid"`
	Title      string `json:"title"`
	FullTitle  string `json:"full_title"`

	// Intent fields.
	ContentType    string                    `json:"content_type"`
	Exist          bool                      `json:"exist"`
	Score          *float64                  `json:"score,omitempty"`
	Explanation    elastic.SearchExplanation `json:"explanation,omitempty"`
	MaxScore       *float64                  `json:"max_score,omitempty"`
	MaxExplanation elastic.SearchExplanation `json:"max_explanation,omitempty"`
}

type CreatedSearchClicks added in v0.8.3 

type CreatedSearchClicks []SearchClick

func (CreatedSearchClicks) Len added in v0.8.3 

func (csc CreatedSearchClicks) Len() int

func (CreatedSearchClicks) Less added in v0.8.3 

func (csc CreatedSearchClicks) Less(i, j int) bool

func (CreatedSearchClicks) Swap added in v0.8.3 

func (csc CreatedSearchClicks) Swap(i, j int)

type CreatedSearchLogs added in v0.8.3 

type CreatedSearchLogs []SearchLog

func (CreatedSearchLogs) Len added in v0.8.3 

func (csl CreatedSearchLogs) Len() int

func (CreatedSearchLogs) Less added in v0.8.3 

func (csl CreatedSearchLogs) Less(i, j int) bool

func (CreatedSearchLogs) Swap added in v0.8.3 

func (csl CreatedSearchLogs) Swap(i, j int)

type CrossIter added in v1.6.6 

type CrossIter struct {
	// contains filtered or unexported fields
}

func CreateCrossIter added in v1.6.6 

func CreateCrossIter(values [][]string) *CrossIter

func (*CrossIter) Next added in v1.6.6 

func (ci *CrossIter) Next() bool

func (*CrossIter) Values added in v1.6.6 

func (ci *CrossIter) Values() []string

type ESEngine 

type ESEngine struct {
	ExecutionTimeLog *TimeLogMap

	TokensCache *TokensCache
	// contains filtered or unexported fields
}

func NewESEngine 

func NewESEngine(esc *elastic.Client, db *sql.DB, cache cache.CacheManager, tc *TokensCache, variables VariablesV2) *ESEngine

func (*ESEngine) AddIntentSecondRound added in v1.0.5 

func (e *ESEngine) AddIntentSecondRound(h *elastic.SearchHit, intent Intent, query Query) (error, *Intent, *Query)

func (*ESEngine) AddIntents added in v0.8.9 

func (e *ESEngine) AddIntents(query *Query, preference string, sortBy string, searchTags bool, searchSources bool, filterIntents []Intent) ([]Intent, error)

func (*ESEngine) CombineResultsToSingleHit added in v1.6.3 

func (e *ESEngine) CombineResultsToSingleHit(resultsByLang map[string]*elastic.SearchResult, hitType string) (map[string]*elastic.SearchResult, error)

func (*ESEngine) DoSearch 

func (e *ESEngine) DoSearch(ctx context.Context, query Query, sortBy string, from int, size int, preference string, checkTypo bool, timeoutForHighlight time.Duration) (*QueryResult, error)

func (*ESEngine) GetSuggestions 

func (e *ESEngine) GetSuggestions(ctx context.Context, query Query, preference string) (interface{}, error)

func (*ESEngine) GetTypoSuggest added in v1.6.3 

func (e *ESEngine) GetTypoSuggest(query Query, filterIntents []Intent) (null.String, error)

func (*ESEngine) IntentsToResults added in v0.9.5 

func (e *ESEngine) IntentsToResults(query *Query) (error, map[string]*elastic.SearchResult)

func (*ESEngine) NativizeTweetsHitForClient added in v1.6.3 

func (e *ESEngine) NativizeTweetsHitForClient(hit *elastic.SearchHit, innerHitsKey string) error

Moving data from InnerHits to Source (as marshaled json) (this is for client).

func (*ESEngine) SearchByFilterIntents added in v1.10.3 

func (e *ESEngine) SearchByFilterIntents(filterIntents []Intent, filters map[string][]string, originalSearchTerm string, from int, size int, sortBy string, resultTypes []string, preference string, deb bool) (map[string][]FilteredSearchResult, error)

Search according to grammar based filter.

func (*ESEngine) SearchGrammars added in v1.4.4 

func (e *ESEngine) SearchGrammars(query *Query) ([]Intent, error)

func (*ESEngine) SearchGrammarsV2 added in v1.6.6 

func (e *ESEngine) SearchGrammarsV2(query *Query, from int, size int, sortBy string, resultTypes []string, preference string) ([]Intent, []Intent, error)

Return: single hit intents, filtering intents

func (*ESEngine) SearchTweets added in v1.6.3 

func (e *ESEngine) SearchTweets(query Query, sortBy string, from int, size int, preference string) (map[string]*elastic.SearchResult, error)

func (*ESEngine) SuggestGrammars added in v1.5.4 

func (e *ESEngine) SuggestGrammars(query *Query) (map[string][]VariablesByPhrase, error)

func (*ESEngine) SuggestGrammarsV2 added in v1.6.6 

func (e *ESEngine) SuggestGrammarsV2(query *Query, preference string) (map[string][]VariablesByPhrase, error)

func (*ESEngine) VariableMapToFilterValues added in v1.6.6 

func (e *ESEngine) VariableMapToFilterValues(vMap map[string][]string, language string) []FilterValue

type ESManager added in v1.3.9 

type ESManager struct {
	// contains filtered or unexported fields
}

func MakeESManager added in v1.3.9 

func MakeESManager(url string) *ESManager

func (*ESManager) GetClient added in v1.3.9 

func (esManager *ESManager) GetClient() (*elastic.Client, error)

func (*ESManager) Stop added in v1.3.9 

func (esManager *ESManager) Stop()

type Engine 

type Engine interface {
	GetSuggestions(ctx context.Context, query Query) (interface{}, error)
	DoSearch(ctx context.Context, query Query, from int, size int, preference string) (interface{}, error)
}

type EvalQuery added in v0.7.8 

type EvalQuery struct {
	Language     string        `json:"language"`
	Query        string        `json:"query"`
	Weight       float64       `json:"weight,omitempty"`
	Bucket       string        `json:"bucket,omitempty"`
	Expectations []Expectation `json:"expectations"`
	Comment      string        `json:"comment,omitempty"`
}

func InitAndReadEvalSet added in v1.1.8 

func InitAndReadEvalSet(evalSetPath string) ([]EvalQuery, error)

func ReadEvalDiffSet added in v1.6.3 

func ReadEvalDiffSet(evalSetPath string) ([]EvalQuery, error)

func ReadEvalSet added in v0.7.8 

func ReadEvalSet(reader io.Reader, db *sql.DB) ([]EvalQuery, error)

type EvalResult added in v0.7.8 

type EvalResult struct {
	SearchQuality []int `json:"search_quality"`
	Rank          []int `json:"rank"`

	QueryResult QueryResult `json:"query_result"`
	Order       int         `json:"order,omitempty"`
	// contains filtered or unexported fields
}

func EvalScrape added in v1.6.3 

func EvalScrape(queries []EvalQuery, serverUrl string, skipExpectations bool) ([]EvalResult, error)

func EvaluateQuery added in v0.7.8 

func EvaluateQuery(q EvalQuery, serverUrl string, skipExpectations bool) EvalResult

type EvalResults added in v0.7.8 

type EvalResults struct {
	Results       []EvalResult    `json:"results"`
	TotalUnique   uint64          `json:"total_unique"`
	TotalWeighted float64         `json:"total_weighted"`
	TotalErrors   uint64          `json:"total_errors"`
	UniqueMap     map[int]float64 `json:"unique_map"`
	WeightedMap   map[int]float64 `json:"weighted_map"`
}

func Eval added in v0.7.8 

func Eval(queries []EvalQuery, serverUrl string) (EvalResults, map[int][]Loss, error)

type Expectation added in v0.8.0 

type Expectation struct {
	Type    int      `json:"type"`
	Uid     string   `json:"uid,omitempty"`
	Filters []Filter `json:"filters,omitempty"`
	Source  string   `json:"source"`
}

func ParseExpectation added in v0.8.0 

func ParseExpectation(e string, db *sql.DB) Expectation

Parses expectation described by result URL and converts to type (collections or content_units) and uid. Examples: https://kabbalahmedia.info/he/programs/cu/AsNLozeK ==> (content_units, AsNLozeK) https://kabbalahmedia.info/he/programs/c/fLWpcUjQ ==> (collections , fLWpcUjQ) https://kabbalahmedia.info/he/lessons/series/c/XZoflItG ==> (collections , XZoflItG) https://kabbalahmedia.info/he/lessons?source=bs_L2jMWyce_kB3eD83I ==> (lessons, nil, source=bs_L2jMWyce_kB3eD83I) https://kabbalahmedia.info/he/programs?topic=g3ml0jum_1nyptSIo_RWqjxgkj ==> (programs, nil, topic=g3ml0jum_1nyptSIo_RWqjxgkj) https://kabbalahmedia.info/he/sources/kB3eD83I ==> (source, kB3eD83I) [latest]https://kabbalahmedia.info/he/lessons?source=bs_qMUUn22b_hFeGidcS ==> (content_units, SLQOALyt) [latest]https://kabbalahmedia.info/he/programs?topic=g3ml0jum_1nyptSIo_RWqjxgkj ==> (content_units, erZIsm86) [latest]https://kabbalahmedia.info/he/programs/c/zf4lLwyI ==> (content_units, orMKRcNk) All events sub pages and years: https://kabbalahmedia.info/he/events/meals https://kabbalahmedia.info/he/events/friends-gatherings https://kabbalahmedia.info/he/events?year=2013

type FileVariable added in v1.6.6 

type FileVariable struct {
	// contains filtered or unexported fields
}

func (FileVariable) Match added in v1.6.6 

func (fv FileVariable) Match(token *TokenNode, variableToken *TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error)

func (FileVariable) Name added in v1.6.6 

func (fv FileVariable) Name() string

func (FileVariable) VariableToPhrases added in v1.6.6 

func (fv FileVariable) VariableToPhrases(prefix, suffix string, variables map[string]*Variable) []*PhrasesWithOrigin

type Filter added in v0.8.9 

type Filter struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type FilterValue added in v1.6.6 

type FilterValue struct {
	Name       string `json:"name,omitempty"`
	Value      string `json:"value,omitempty"`
	Origin     string `json:"origin,omitempty"`
	OriginFull string `json:"origin_full,omitempty"`
}

func VariableValuesToFilterValues added in v1.6.6 

func VariableValuesToFilterValues(values []VariableValue) []FilterValue

type FilteredSearchResult added in v1.10.0 

type FilteredSearchResult struct {
	Term                     string
	PreserveTermForHighlight bool //Use the term as highlight term even if we have same hit result from regular search
	HitIdsMap                map[string]bool
	Results                  []*elastic.SearchResult
	MaxScore                 *float64
	ProgramCollection        *string
}

type Grammar added in v1.4.4 

type Grammar struct {
	HitType   string
	Language  string
	Intent    string
	Patterns  [][]*TokenNode
	Filters   map[string][]string
	Esc       *elastic.Client
	Variables VariablesByName
}

func (*Grammar) SearchGrammar added in v1.4.4 

func (g *Grammar) SearchGrammar(query *Query, tc *TokensCache, cm cache.CacheManager) (*Intent, error)

func (*Grammar) SuggestGrammar added in v1.5.4 

func (g *Grammar) SuggestGrammar(query *Query, tc *TokensCache, cm cache.CacheManager) (VariablesByPhrase, error)

type GrammarIntent added in v1.4.4 

type GrammarIntent struct {
	LandingPage     string                     `json:"landing_page,omitempty"`
	FilterValues    []FilterValue              `json:"filter_values,omitempty"`
	Score           float64                    `json:"score,omitempty"`
	Explanation     *elastic.SearchExplanation `json:"explanation,omitempty"`
	SingleHit       *elastic.SearchHit         `json:"single_hit,omitempty"`
	SingleHitMdbUid *string                    `json:"single_hit_mdb_uid,omitempty"`
}

type GrammarRule added in v1.6.6 

type GrammarRule struct {
	HitType      string          `json:"hit_type"`
	Intent       string          `json:"intent"`
	Variables    []string        `json:"variables,omitempty"`
	Values       []string        `json:"values,omitempty"`
	Rules        []string        `json:"rules"`
	RulesSuggest es.SuggestField `json:"rules_suggest"`
}

type GrammarRuleWithPercolatorQuery added in v1.10.0 

type GrammarRuleWithPercolatorQuery struct {
	GrammarRule GrammarRule `json:"grammar_rule"`
	Query       interface{} `json:"query"`
	SearchText  string      `json:"search_text"`
}

type GrammarV2 added in v1.6.6 

type GrammarV2 struct {
	HitType  string
	Language string
	Intent   string
	Filters  map[string][]string
	// Map from variable set (as string) to list of rules.
	Patterns map[string][]string
}

type Grammars added in v1.4.4 

type Grammars = map[string]map[string]*Grammar

func MakeGrammars added in v1.4.4 

func MakeGrammars(grammarsDir string, esc *elastic.Client, tc *TokensCache, variables VariablesByLang) (Grammars, error)

func ReadGrammarFile added in v1.4.4 

func ReadGrammarFile(grammarFile string, esc *elastic.Client, tc *TokensCache, variables VariablesByLang) (Grammars, error)

type GrammarsV2 added in v1.6.6 

type GrammarsV2 = map[string]map[string]*GrammarV2

Map from lang => intent => Grammar

func MakeGrammarsV2 added in v1.6.6 

func MakeGrammarsV2(grammarsDir string) (GrammarsV2, error)

func ReadGrammarFileV2 added in v1.6.6 

func ReadGrammarFileV2(grammarFile string) (GrammarsV2, error)

type HitDiff added in v1.6.3 

type HitDiff struct {
	Rank          int       `json:"rank"`
	ExpHitSource  HitSource `json:"exp_hit_source"`
	BaseHitSource HitSource `json:"base_hit_source"`
}

type HitSource added in v0.9.5 

type HitSource struct {
	MdbUid              string      `json:"mdb_uid"`
	ResultType          string      `json:"result_type"`
	LandingPage         string      `json:"landing_page"`
	Title               string      `json:"title,omitempty"`
	Content             string      `json:"content,omitempty"`
	CarrouselHitSources []HitSource `json:"carrousel,omitempty"`
	ContentType         string      `json:"content_type,omitempty"`
	Score               float64     `json:"score,omitempty"`
}

type Intent added in v0.8.9 

type Intent struct {
	Type     string      `json:"type"`
	Language string      `json:"language"`
	Value    interface{} `json:"value,omitempty"`
}

type Loss added in v0.8.9 

type Loss struct {
	Expectation Expectation `json:"expectation,omitempty"`
	Query       EvalQuery   `json:"query,omitempty"`
	Unique      float64     `json:"unique,omitempty"`
	Weighted    float64     `json:"weighted,omitempty"`
}

type OriginalTokenNode added in v1.5.4 

type OriginalTokenNode struct {
	OriginalWholePhrase *string
	SkippedPrefix       *Span  // Stopwords before this token that were skipped.
	OriginalPhrase      string // Original string that was tokenized.
	SkippedSuffix       *Span  // Will be set only for IsEnd = true nodes.

	TokenNode *TokenNode
	Parents   []*OriginalTokenNode
	Children  []*OriginalTokenNode
}

func (*OriginalTokenNode) OriginalFullPhraseToString added in v1.5.4 

func (otn *OriginalTokenNode) OriginalFullPhraseToString() string

func (*OriginalTokenNode) OriginalVariablePrefixToString added in v1.6.6 

func (otn *OriginalTokenNode) OriginalVariablePrefixToString() string

func (*OriginalTokenNode) OriginalVariableToString added in v1.6.6 

func (otn *OriginalTokenNode) OriginalVariableToString() string

func (*OriginalTokenNode) SkippedPrefixToString added in v1.5.4 

func (otn *OriginalTokenNode) SkippedPrefixToString() string

func (*OriginalTokenNode) SkippedSuffixToString added in v1.5.4 

func (otn *OriginalTokenNode) SkippedSuffixToString() string

type PhrasesByVariables added in v1.6.6 

type PhrasesByVariables struct {
	P []*PhrasesWithOrigin
	M map[string][]string
}

type PhrasesWithOrigin added in v1.5.4 

type PhrasesWithOrigin struct {
	VariableValues []VariableValue
	// contains filtered or unexported fields
}

func CopyPhrasesWithOrigin added in v1.6.6 

func CopyPhrasesWithOrigin(p []*PhrasesWithOrigin) []*PhrasesWithOrigin

func MakePhrasesWithOrigin added in v1.6.6 

func MakePhrasesWithOrigin(prefix, phrase, suffix, token string) *PhrasesWithOrigin

func MakeVariablePhraseWithOrigin added in v1.6.6 

func MakeVariablePhraseWithOrigin(variable, prefix, suffix string) *PhrasesWithOrigin

func OriginalTokenNodesToPhrases added in v1.5.4 

func OriginalTokenNodesToPhrases(otns []*OriginalTokenNode, variables map[string]*Variable, reduceVariables bool) []*PhrasesWithOrigin

func TokenNodesToPhrases added in v1.5.2 

func TokenNodesToPhrases(root []*TokenNode, variables map[string]*Variable, reduceVariables bool) []*PhrasesWithOrigin

func (*PhrasesWithOrigin) Invalidate added in v1.6.6 

func (p *PhrasesWithOrigin) Invalidate()

func (*PhrasesWithOrigin) Join added in v1.5.4 

func (p *PhrasesWithOrigin) Join(s string) string

func (*PhrasesWithOrigin) OriginalJoin added in v1.5.4 

func (p *PhrasesWithOrigin) OriginalJoin() string

func (*PhrasesWithOrigin) Reduce added in v1.6.6 

func (p *PhrasesWithOrigin) Reduce(variableName, variableValue, prefix, suffix string)

func (*PhrasesWithOrigin) ToString added in v1.5.4 

func (p *PhrasesWithOrigin) ToString() string

func (*PhrasesWithOrigin) VariableValuesJoin added in v1.6.6 

func (p *PhrasesWithOrigin) VariableValuesJoin() string

func (*PhrasesWithOrigin) VariablesMap added in v1.6.6 

func (pwo *PhrasesWithOrigin) VariablesMap() (map[string][]string, string)

type Query 

type Query struct {
	Term          string              `json:"term,omitempty"`
	ExactTerms    []string            `json:"exact_terms,omitempty"`
	Original      string              `json:"original,omitempty"`
	Filters       map[string][]string `json:"filters,omitempty"`
	LanguageOrder []string            `json:"language_order,omitempty"`
	Deb           bool                `json:"deb,omitempty"`
	Intents       []Intent            `json:"intents,omitempty"`
}

func ParseQuery added in v1.4.4 

func ParseQuery(q string) Query

Parses query and extracts terms and filters.

func (*Query) ToFullSimpleString added in v1.1.9 

func (query *Query) ToFullSimpleString(sortBy string, from int, size int) string

func (*Query) ToSimpleString added in v1.1.9 

func (query *Query) ToSimpleString() string

func (*Query) ToString added in v1.0.5 

func (query *Query) ToString() string

type QueryResult added in v0.8.9 

type QueryResult struct {
	SearchResult *elastic.SearchResult `json:"search_result,omitempty"`
	TypoSuggest  null.String           `json:"typo_suggest"`
	Language     string                `json:"language"`
}

type ResultDiff added in v1.6.3 

type ResultDiff struct {
	ErrorStr  string    `json:"error_str"`
	Query     string    `json:"query"`
	HitsDiffs []HitDiff `json:"hits_diffs"`
}

func EvalResultDiff added in v1.6.3 

func EvalResultDiff(evalQuery EvalQuery, expResult EvalResult, baseResult EvalResult) (ResultDiff, error)

type ResultsDiffs added in v1.6.3 

type ResultsDiffs struct {
	ErrorStr     string       `json:"error_str"`
	ResultsDiffs []ResultDiff `json:"results_diffs"`
	Diffs        int          `json:"diffs"`
	Scraped      int          `json:"scraped"`
	DiffsWeight  float64      `json:"diffs_weight"`
	TotalWeight  float64      `json:"total_weight"`
}

func EvalQuerySetDiff added in v1.6.3 

func EvalQuerySetDiff(evalSet []EvalQuery, baseServerUrl, expServerUrl string, diffsLimit int32) (ResultsDiffs, error)

If |diffsLimit| > 0 will limit the diff to constant number of queries.

func EvalResultsDiffs added in v1.6.3 

func EvalResultsDiffs(evalSet []EvalQuery, expResults []EvalResult, baseResults []EvalResult) (ResultsDiffs, error)

func EvalSearchDataQuerySetsDiff added in v1.6.3 

func EvalSearchDataQuerySetsDiff(language, baseServerUrl, expServerUrl string, diffsLimit int32) ([]ResultsDiffs, error)

type SearchClick added in v0.8.0 

type SearchClick struct {
	SearchId   string    `json:"search_id"`
	Created    time.Time `json:"created"`
	LogType    string    `json:"log_type"`
	MdbUid     string    `json:"mdb_uid",omitempty`
	Index      string    `json:"index",omitempty`
	ResultType string    `json:"result_type",omitempty`
	Rank       uint32    `json:"rank",omitempty`
}

type SearchLog added in v0.7.7 

type SearchLog struct {
	SearchId         string      `json:"search_id"`
	Created          time.Time   `json:"created"`
	LogType          string      `json:"log_type"`
	Query            Query       `json:"query"`
	QueryResult      interface{} `json:"query_result,omitempty"`
	Error            interface{} `json:"error,omitempty"`
	SortBy           string      `json:"sort_by,omitempty"`
	From             uint64      `json:"from,omitempty"`
	Size             uint64      `json:"size,omitempty"`
	Suggestion       string      `json:"suggestion,omitempty"`
	ExecutionTimeLog []TimeLog   `json:"execution_time_log,omitempty"`
}

type SearchLogger added in v0.7.7 

type SearchLogger struct {
	// contains filtered or unexported fields
}

func MakeSearchLogger added in v0.7.7 

func MakeSearchLogger(esManager *ESManager) *SearchLogger

func (*SearchLogger) GetAllClicks added in v0.8.3 

func (searchLogger *SearchLogger) GetAllClicks() ([]SearchClick, error)

func (*SearchLogger) GetAllQueries added in v0.7.8 

func (searchLogger *SearchLogger) GetAllQueries(s *elastic.SliceQuery) ([]SearchLog, error)

func (*SearchLogger) GetLattestQueries added in v1.6.3 

func (searchLogger *SearchLogger) GetLattestQueries(s *elastic.SliceQuery, gte null.String, ascending null.Bool) ([]SearchLog, error)

func (*SearchLogger) LogClick added in v0.8.0 

func (searchLogger *SearchLogger) LogClick(mdbUid string, index string, resultType string, rank int, searchId string) error

func (*SearchLogger) LogSearch added in v0.7.7 

func (searchLogger *SearchLogger) LogSearch(query Query, sortBy string, from int, size int, searchId string, suggestion string, res *QueryResult, executionTimeLog *TimeLogMap) error

func (*SearchLogger) LogSearchError added in v0.7.7 

func (searchLogger *SearchLogger) LogSearchError(query Query, sortBy string, from int, size int, searchId string, suggestion string, searchErr interface{}, executionTimeLog *TimeLogMap) error

type SearchRequestOptions added in v1.0.5 

type SearchRequestOptions struct {

	// If not nil, set how long a search is allowed to take, e.g. "1s" or "500ms". Note: Not always respected by ES.
	Timeout *string
	// contains filtered or unexported fields
}

type Span added in v1.5.4 

type Span struct {
	Start int
	End   int
}

func MakeSpan added in v1.5.4 

func MakeSpan(start, end int) *Span

type TimeLog added in v1.2.1 

type TimeLog struct {
	Operation string `json:"operation"`
	Time      int64  `json:"time"`
}

type TimeLogMap added in v1.3.9 

type TimeLogMap struct {
	// contains filtered or unexported fields
}

func NewTimeLogMap added in v1.3.9 

func NewTimeLogMap() *TimeLogMap

func (*TimeLogMap) Load added in v1.3.9 

func (c *TimeLogMap) Load(key string) (time.Duration, bool)

func (*TimeLogMap) Store added in v1.3.9 

func (c *TimeLogMap) Store(key string, value time.Duration)

func (*TimeLogMap) ToMap added in v1.3.9 

func (c *TimeLogMap) ToMap() map[string]time.Duration

type Token added in v1.5.2 

type Token struct {
	Token          string `json:"token"`
	StartOffset    int    `json:"start_offset"`
	EndOffset      int    `json:"end_offset"`
	Type           string `json:"type"`
	Position       int    `json:"position"`
	PositionLength int    `json:"positionLength"`
}

type TokenNode added in v1.5.2 

type TokenNode struct {
	Token    Token
	IsEnd    bool
	Parents  []*TokenNode
	Children []*TokenNode
	// May be several original phrases per token from several sources that were merged.
	OriginalTokenNodes []*OriginalTokenNode

	// Caches phrases with origin.
	Phrases []*PhrasesWithOrigin
}

func MakeTokensFromPhrase added in v1.5.2 

func MakeTokensFromPhrase(phrase string, lang string, esc *elastic.Client, tc *TokensCache) ([]*TokenNode, error)

func MakeTokensFromPhraseIndex added in v1.5.2 

func MakeTokensFromPhraseIndex(phrase string, lang string, esc *elastic.Client, index string, ctx context.Context) ([]*TokenNode, error)

type TokensCache added in v1.5.4 

type TokensCache struct {
	// contains filtered or unexported fields
}

func MakeTokensCache added in v1.5.4 

func MakeTokensCache(size int) *TokensCache

func (*TokensCache) Get added in v1.5.4 

func (tc *TokensCache) Get(phrase string, lang string) []*TokenNode

func (*TokensCache) Has added in v1.5.4 

func (tc *TokensCache) Has(phrase string, lang string) bool

func (*TokensCache) Set added in v1.5.4 

func (tc *TokensCache) Set(phrase string, lang string, tokens []*TokenNode)

type TokensCacheElement added in v1.5.4 

type TokensCacheElement struct {
	Phrase string
	Lang   string
	Tokens []*TokenNode
}

type Translations added in v1.6.6 

type Translations = map[string]map[string]map[string][][]*TokenNode

Map from variable => language => value => Tokens

func LoadVariablesTranslations added in v1.6.6 

func LoadVariablesTranslations(variablesDir string, esc *elastic.Client, tc *TokensCache) (Translations, error)

type TranslationsV2 added in v1.6.6 

type TranslationsV2 = map[string]map[string][]string

Translations language => value => phrases

func GeneratePositionVariables added in v1.12.0 

func GeneratePositionVariables(db *sql.DB) (TranslationsV2, error)

func LoadHolidayTranslationsFromDB added in v1.7.5 

func LoadHolidayTranslationsFromDB(db *sql.DB) (TranslationsV2, error)

func LoadProgramNameTranslationsFromDB added in v1.12.0 

func LoadProgramNameTranslationsFromDB(db *sql.DB) (TranslationsV2, error)

func LoadSourceNameTranslationsFromDB added in v1.10.4 

func LoadSourceNameTranslationsFromDB(db *sql.DB) (TranslationsV2, error)

func LoadVariableTranslationsFromFile added in v1.7.5 

func LoadVariableTranslationsFromFile(variableFile string, variableName string) (TranslationsV2, error)

type Variable added in v1.6.6 

type Variable interface {
	Name() string
	Match(token *TokenNode, variableToken *TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error)
	VariableToPhrases(prefix, suffix string, variables map[string]*Variable) []*PhrasesWithOrigin
}

func MakeFileVariable added in v1.6.6 

func MakeFileVariable(name string, lang string, translations Translations) Variable

func MakeYearVariable added in v1.6.6 

func MakeYearVariable() Variable

type VariableValue added in v1.6.6 

type VariableValue struct {
	Name       string   // Variable name.
	Value      string   // Variable value.
	Tokenized  []string // Tokenized phrase.
	Origin     string   // Original phrase.
	OriginFull string   // Original phrase with prefix and suffix.
}

func MakeTextVariableValue added in v1.6.6 

func MakeTextVariableValue(prefix, phrase, suffix, token string) VariableValue

func MakeVariableValue added in v1.6.6 

func MakeVariableValue(variable, prefix, phrase, suffix, token, value string) VariableValue

type VariablesByLang added in v1.6.6 

type VariablesByLang = map[string]VariablesByName

func MakeVariables added in v1.6.6 

func MakeVariables(variablesDir string, esc *elastic.Client, tc *TokensCache) (VariablesByLang, error)

type VariablesByName added in v1.6.6 

type VariablesByName = map[string]*Variable

Map from language => name => variable.

type VariablesByPhrase added in v1.6.6 

type VariablesByPhrase = map[string]map[string][]string

Map from Original Full Phrase => $Var => values

func TokensSearch added in v1.5.4 

func TokensSearch(a []*TokenNode, b [][]*TokenNode, variables map[string]*Variable) (VariablesByPhrase, error)

Searches tokens |a| inside tokens |b|, returns the matching part. Can be optimized? Current complexity is O(|a|^2 * |b|^2), where |a| is nubmer of tokens in the whole graph of |a|.

func TokensSingleSearch added in v1.5.4 

func TokensSingleSearch(a []*TokenNode, b []*TokenNode, variables map[string]*Variable) (VariablesByPhrase, error)

func TokensSingleSearchTest added in v1.6.6 

func TokensSingleSearchTest(a []*TokenNode, b []*TokenNode, variables map[string]*Variable) (VariablesByPhrase, error)

type VariablesV2 added in v1.6.6 

type VariablesV2 = map[string]TranslationsV2

Map from variable => language => value => phrases

func LoadVariablesTranslationsV2 added in v1.6.6 

func LoadVariablesTranslationsV2(variablesDir string) (VariablesV2, error)

func MakeVariablesV2 added in v1.6.6 

func MakeVariablesV2(variablesDir string) (VariablesV2, error)

type YearVariable added in v1.6.6 

type YearVariable struct {
	// contains filtered or unexported fields
}

func (YearVariable) Match added in v1.6.6 

func (yv YearVariable) Match(token *TokenNode, variableToken *TokenNode, matchPrefixes bool, variables map[string]*Variable) (bool, []VariableValue, []*TokenNode, error)

func (YearVariable) Name added in v1.6.6 

func (yv YearVariable) Name() string

func (YearVariable) VariableToPhrases added in v1.6.6 

func (yv YearVariable) VariableToPhrases(prefix, suffix string, variables map[string]*Variable) []*PhrasesWithOrigin

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.