psqlfront

package module
v0.0.12 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 3, 2022 License: MIT Imports: 36 Imported by: 0

README

psql-front

Documentation Latest GitHub release Github Actions test License

NOTICE: Currently psql-front is an experimental application

psql-front is a cache server that implements the PostgreSQL wire protocol v3. It temporarily stores data retrieved from a configured Origin in a PostgreSQL cache database so that PostgresSQL clients can reference the data with SQL.

Usage

For example, if you set up config.yaml

required_version: ">=v0.0.0"

cache_database:
  host: "{{ env `POSTGRES_HOST` `localhost` }}"
  username: "{{ env `PSOTGRES_USER` `postgres` }}"
  password: "{{ env `PSOTGRES_PASSWORD` `postgres` }}"
  port: 5432
  database: "postgres"

default_ttl: 24h

certificates:
  - cert: server.crt
    key: server.key

origins:
  - id: open_data
    type: HTTP
    schema: public
    tables:
      - name: syukujitsu
        url: https://www8.cao.go.jp/chosei/shukujitsu/syukujitsu.csv
        format: csv
        ignore_lines: 1
        text_encoding: Shift_JIS
        columns:
          - name: ymd
            data_type: DATE
            constraint: NOT NULL
          - name: name
            data_type: VARCHAR
            length: 64
            constraint: NOT NULL

Start with this config.yaml as follows

$ docker compose up -d
[+] Running 2/2
 ⠿ Network psql-front_default       Created                 0.1s
 ⠿ Container psql-front-postgres-1  Started                 0.9s

$ psql-front --config config.yaml
2022/08/02 12:28:20 [info] load origin type: HTTP
2022/08/02 12:28:20 [info] load origin type: Static
2022/08/02 12:28:20 [info] use TLS
2022/08/02 12:28:20 [info] need execute migration
-- Apply --
CREATE SCHEMA IF NOT EXISTS "psqlfront";
CREATE SCHEMA IF NOT EXISTS "public";

CREATE TABLE "psqlfront"."cache" (
    schema_name VARCHAR(255) NOT NULL,
    table_name VARCHAR(255) NOT NULL,
    origin_id VARCHAR(255) NOT NULL,
    cached_at TIMESTAMP NOT NULL,
    expired_at TIMESTAMP NOT NULL,
    PRIMARY KEY(schema_name,table_name)
);
CREATE TABLE "public"."syukujitsu" (
    "ymd" DATE ,
    "name" VARCHAR(64) 
);
2022/08/02 12:28:20 [info][-] execute: ANALYZE "public"."syukujitsu";
2022/08/02 12:28:20 [info] PostgreSQL server is up and running at [[::]:5434]

Access with psql client in this state.

$ psql -h localhost -U postgres -p 5434
Password for user postgres: 
psql (14.2, server 14.4)
SSL connection (protocol: TLSv1.3, cipher: TLS_AES_128_GCM_SHA256, bits: 128, compression: off)
Type "help" for help.

postgres=# select * from public.syukujitsu;
postgres=# select * from public.syukujitsu LIMIT 1;
NOTICE:  cache hit: ["public"."syukujitsu"]
    ymd     | name 
------------+------
 1955-01-01 | 元日
(1 row)

postgres=#
Install
Binary packages

Releases

Options
$ psql-front -h    
Usage of psql-front:
Version: v0.0.0
  -config string
        psql-front config
  -log-level string
        log level (default "info")
  -port uint
        psql-front port (default 5434)

LICENSE

MIT License

Copyright (c) 2022 IKEDA Masashi

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func GetRemoteAddr 

func GetRemoteAddr(ctx context.Context) string

func RegisterOriginType 

func RegisterOriginType(typeName string, originConfigConstructor func() OriginConfig)

func UnregisterOriginType 

func UnregisterOriginType(typeName string)

func WithProxyConnOnQueryReceived 

func WithProxyConnOnQueryReceived(handler func(ctx context.Context, query string, isPreparedStmt bool, notifier Notifier) error) func(opts *ProxyConnOptions)

func WithProxyConnTLS 

func WithProxyConnTLS(tlsConfig *tls.Config) func(opts *ProxyConnOptions)

Types

type CacheDatabaseConfig 

type CacheDatabaseConfig struct {
	Host     string `yaml:"host,omitempty"`
	Username string `yaml:"username,omitempty"`
	Password string `yaml:"password,omitempty"`
	Port     int    `yaml:"port,omitempty"`
	Database string `yaml:"database,omitempty"`
	SSLMode  string `yaml:"ssl_mode,omitempty"`
}

func (*CacheDatabaseConfig) DSN 

func (cfg *CacheDatabaseConfig) DSN() string

type CacheInfo 

type CacheInfo struct {
	SchemaName, TableName, OriginID string
	CachedAt, ExpiredAt             time.Time
}

type CacheWriter 

type CacheWriter interface {
	AppendRows(context.Context, [][]interface{}) error
}

type CertificateConfig 

type CertificateConfig struct {
	Cert string `yaml:"cert,omitempty"`
	Key  string `yaml:"key,omitempty"`
	// contains filtered or unexported fields
}

type Column 

type Column struct {
	Name      string
	DataType  string
	Length    *int
	Contraint string
}

type CommonOriginConfig 

type CommonOriginConfig struct {
	ID   string         `yaml:"id"`
	Type string         `yaml:"type"`
	TTL  *time.Duration `yaml:"ttl"`

	OriginConfig OriginConfig `yaml:"-"`
}

func (*CommonOriginConfig) NewOrigin 

func (cfg *CommonOriginConfig) NewOrigin() (Origin, error)

func (*CommonOriginConfig) Ristrict 

func (cfg *CommonOriginConfig) Ristrict() error

func (*CommonOriginConfig) UnmarshalYAML 

func (cfg *CommonOriginConfig) UnmarshalYAML(unmarshal func(interface{}) error) error

type Config 

type Config struct {
	RequiredVersion string `yaml:"required_version,omitempty"`

	CacheDatabase *CacheDatabaseConfig  `yaml:"cache_database,omitempty"`
	Certificates  []*CertificateConfig  `yaml:"certificates,omitempty"`
	DefaultTTL    time.Duration         `yaml:"default_ttl,omitempty"`
	Origins       []*CommonOriginConfig `yaml:"origins,omitempty"`
	// contains filtered or unexported fields
}

func DefaultConfig 

func DefaultConfig() *Config

func (*Config) Load 

func (cfg *Config) Load(path string) error

Load loads configuration file from file paths.

func (*Config) Restrict 

func (cfg *Config) Restrict() error

func (*Config) ValidateVersion 

func (cfg *Config) ValidateVersion(version string) error

ValidateVersion validates a version satisfies required_version.

type Migrator 

type Migrator struct {
	// contains filtered or unexported fields
}

func NewMigrator 

func NewMigrator(cfg *CacheDatabaseConfig) *Migrator

func (*Migrator) ExecuteMigration 

func (m *Migrator) ExecuteMigration(tables []*Table) error

func (*Migrator) GenerateCreateSchemaStmt 

func (m *Migrator) GenerateCreateSchemaStmt(tables []*Table) string

func (*Migrator) GenerateDesiredDDLs 

func (m *Migrator) GenerateDesiredDDLs(tables []*Table) (string, error)

type Notifier 

type Notifier interface {
	Notify(ctx context.Context, resp *pgproto3.NoticeResponse) error
}

type Origin 

type Origin interface {
	ID() string
	GetTables(ctx context.Context) ([]*Table, error)
	GetRows(context.Context, CacheWriter, *Table) error
}

type OriginConfig 

type OriginConfig interface {
	Type() string
	Restrict() error
	NewOrigin(id string) (Origin, error)
}

func GetOriginConfig 

func GetOriginConfig(typeName string) (OriginConfig, bool)

type OriginNotFoundError 

type OriginNotFoundError struct {
	// contains filtered or unexported fields
}

func WrapOriginNotFoundError 

func WrapOriginNotFoundError(err error) *OriginNotFoundError

func (*OriginNotFoundError) Error 

func (onfe *OriginNotFoundError) Error() string

func (*OriginNotFoundError) Unwrap 

func (onfe *OriginNotFoundError) Unwrap() error

type ProxyConn 

type ProxyConn struct {
	// contains filtered or unexported fields
}

func NewProxyConn 

func NewProxyConn(client net.Conn, upstream net.Conn, optFns ...func(opts *ProxyConnOptions)) (*ProxyConn, error)

func (*ProxyConn) Run 

func (conn *ProxyConn) Run(ctx context.Context) error

type ProxyConnOptions 

type ProxyConnOptions struct {
	// contains filtered or unexported fields
}

type ProxyConnection 

type ProxyConnection struct {
	*pgx.Conn
	// contains filtered or unexported fields
}

func (*ProxyConnection) Close 

func (conn *ProxyConnection) Close(ctx context.Context) error

func (*ProxyConnection) ID 

func (conn *ProxyConnection) ID() string

type Server 

type Server struct {
	// contains filtered or unexported fields
}

func New 

func New(ctx context.Context, cfg *Config) (*Server, error)

func (*Server) RunWithContext 

func (server *Server) RunWithContext(ctx context.Context, address string) error

func (*Server) RunWithContextAndListener 

func (server *Server) RunWithContextAndListener(ctx context.Context, listener net.Listener) error

type Table 

type Table struct {
	SchemaName string
	RelName    string

	Columns     []*Column
	Constraints []string
}

func AnalyzeQuery 

func AnalyzeQuery(query string) ([]*Table, error)

func (*Table) GenerateDDL 

func (t *Table) GenerateDDL() (string, error)

func (*Table) GoString 

func (t *Table) GoString() string

func (*Table) String 

func (t *Table) String() string

Source Files

View all Source files

Directories

Path Synopsis
cmd
psql-front
origin
http
static

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.