config

Documentation

Index

• type Config
  • func Load() (*Config, error)
  • func (c *Config) CoreAddr() string
  • func (c *Config) ListenAddr() string
  • func (c *Config) ModelsDir() string
  • func (c *Config) RequireAbsModelsDir() (string, error)
  • func (c *Config) UnloadAfterRPC() bool
• type HostPort

Types

type Config

type Config struct {
	Listen                         HostPort `yaml:"listen"`
	Core                           HostPort `yaml:"core"`
	RegistrationToken              string   `yaml:"registration_token"`
	LogLevel                       string   `yaml:"log_level"`
	ModelPath                      string   `yaml:"model_path"`
	DefaultModel                   string   `yaml:"default_model"`
	MaxContextTokens               int      `yaml:"max_context_tokens"`
	LLMHistoryMaxMessages          int      `yaml:"llm_history_max_messages"`
	LLMHistorySummarizeDropped     bool     `yaml:"llm_history_summarize_dropped"`
	LLMHistorySummaryMaxInputRunes int      `yaml:"llm_history_summary_max_input_runes"`
	LLMHistorySummaryModel         string   `yaml:"llm_history_summary_model"`
	LLMHistorySummaryRunnerListen  string   `yaml:"llm_history_summary_runner_listen"`
	LLMHistorySummaryCacheEntries  int      `yaml:"llm_history_summary_cache_entries"`
	MaxToolInvocationRounds        int      `yaml:"max_tool_invocation_rounds"`
	MaxConcurrentGenerations       int      `yaml:"max_concurrent_generations"`
	ModelRetention                 string   `yaml:"model_retention"`
	GpuLayers                      int      `yaml:"gpu_layers"`
	MLock                          bool     `yaml:"mlock"`
	MMap                           *bool    `yaml:"mmap"`
	MainGPU                        string   `yaml:"main_gpu"`
	TensorSplit                    string   `yaml:"tensor_split"`
	SilentLoading                  bool     `yaml:"silent_loading"`
	ProgressCallback               bool     `yaml:"progress_callback"`
	Threads                        int      `yaml:"threads"`
	ThreadsBatch                   int      `yaml:"threads_batch"`
	BatchSize                      int      `yaml:"batch_size"`
	F16Memory                      bool     `yaml:"f16_memory"`
	KVCacheType                    string   `yaml:"kv_cache_type"`
	FlashAttn                      string   `yaml:"flash_attn"`
	PrefixCaching                  *bool    `yaml:"prefix_caching"`
	Parallel                       int      `yaml:"parallel"`
	TopNSigma                      *float32 `yaml:"top_n_sigma"`
	FrequencyPenalty               *float32 `yaml:"frequency_penalty"`
	PresencePenalty                *float32 `yaml:"presence_penalty"`
	IgnoreEOS                      *bool    `yaml:"ignore_eos"`
	DRYMultiplier                  *float32 `yaml:"dry_multiplier"`
	DRYBase                        *float32 `yaml:"dry_base"`
	DRYAllowedLength               *int     `yaml:"dry_allowed_length"`
	DRYPenaltyLastN                *int     `yaml:"dry_penalty_last_n"`
	DRYSequenceBreakers            []string `yaml:"dry_sequence_breakers"`
	XTCProbability                 *float32 `yaml:"xtc_probability"`
	XTCThreshold                   *float32 `yaml:"xtc_threshold"`
	Mirostat                       *int     `yaml:"mirostat"`
	MirostatTau                    *float32 `yaml:"mirostat_tau"`
	MirostatEta                    *float32 `yaml:"mirostat_eta"`
	TypicalP                       *float32 `yaml:"typical_p"`
	MinKeep                        *int     `yaml:"min_keep"`
	DynamicTemperatureRange        *float32 `yaml:"dynamic_temperature_range"`
	DynamicTemperatureExponent     *float32 `yaml:"dynamic_temperature_exponent"`
	NPrev                          *int     `yaml:"n_prev"`
	NProbs                         *int     `yaml:"n_probs"`
	DebugGeneration                bool     `yaml:"debug_generation"`
	SpeculativeEnabled             bool     `yaml:"speculative_enabled"`
	SpeculativeDraftModel          string   `yaml:"speculative_draft_model"`
	SpeculativeDraftTokens         int      `yaml:"speculative_draft_tokens"`
	TokenPipelineEnabled           bool     `yaml:"token_pipeline_enabled"`
	ChatAPIEnabled                 bool     `yaml:"chat_api_enabled"`
	ChatStreamBufferSize           int      `yaml:"chat_stream_buffer_size"`
	ChatReasoningFormat            string   `yaml:"chat_reasoning_format"`
	ChatEnableThinking             *bool    `yaml:"chat_enable_thinking"`
	ChatReasoningBudget            *int     `yaml:"chat_reasoning_budget"`
	ReinitLlamaLogging             bool     `yaml:"reinit_llama_logging"`
	LogModelStats                  bool     `yaml:"log_model_stats"`
}

func Load

func Load() (*Config, error)

func (*Config) CoreAddr

func (c *Config) CoreAddr() string

func (*Config) ListenAddr

func (c *Config) ListenAddr() string

func (*Config) ModelsDir

func (c *Config) ModelsDir() string

func (*Config) RequireAbsModelsDir

func (c *Config) RequireAbsModelsDir() (string, error)

func (*Config) UnloadAfterRPC

func (c *Config) UnloadAfterRPC() bool

type HostPort

type HostPort struct {
	Host string `yaml:"host"`
	Port int    `yaml:"port"`
}