remote

package
v0.11.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 28, 2026 License: MIT Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ProtocolVersion = "v1"

	ContractPath = "/v1/contract"
	HealthPath   = "/health"

	AuthPublic    = "public"
	AuthReader    = "reader"
	AuthPublisher = "publisher"
)
View Source 
const (
	ModeLocal     = "local"
	ModeGit       = "git"
	ModeCloud     = "cloud"
	ModeHybrid    = "hybrid"
	ModePublisher = "publisher"

	DefaultTokenEnv = "CRAWL_REMOTE_TOKEN"
)
View Source 
const (
	SQLiteGzipChunkedBundleFormat = "sqlite-gzip-chunked-v1"
	SQLiteGzipCompression         = "gzip"
	DefaultSQLiteBundleChunkSize  = int64(256 * 1024 * 1024)
)

Variables

View Source 
var ErrMissingToken = errors.New("remote token is missing")

Functions

func LoginPollSecretHash 

func LoginPollSecretHash(secret string) string

func NewLoginPollSecret 

func NewLoginPollSecret() (string, error)

func SQLiteBundleManifestKey added in v0.11.0 

func SQLiteBundleManifestKey(app, archive string) string

func SQLiteBundlePartKey added in v0.11.0 

func SQLiteBundlePartKey(app, archive string, index int) string

func SQLiteCompressedObjectKey added in v0.11.0 

func SQLiteCompressedObjectKey(app, archive string) string

func SQLiteObjectKey added in v0.11.0 

func SQLiteObjectKey(app, archive string) string

Types

type AppSpec added in v0.10.0 

type AppSpec struct {
	App          string            `json:"app"`
	Queries      []QuerySpec       `json:"queries,omitempty"`
	IngestTables []IngestTableSpec `json:"ingest_tables,omitempty"`
	Capabilities []string          `json:"capabilities,omitempty"`
}

type Archive 

type Archive struct {
	ID            string   `json:"id"`
	App           string   `json:"app"`
	Slug          string   `json:"slug"`
	SchemaName    string   `json:"schema_name,omitempty"`
	SchemaVersion int      `json:"schema_version,omitempty"`
	SchemaHash    string   `json:"schema_hash,omitempty"`
	Capabilities  []string `json:"capabilities,omitempty"`
	LastIngestAt  string   `json:"last_ingest_at,omitempty"`
	LastSyncAt    string   `json:"last_sync_at,omitempty"`
}

type AuthConfig 

type AuthConfig struct {
	TokenSource    string `toml:"token_source" json:"token_source"`
	KeyringService string `toml:"keyring_service" json:"keyring_service"`
	KeyringAccount string `toml:"keyring_account" json:"keyring_account"`
}

type AuthSpec added in v0.10.0 

type AuthSpec struct {
	Name        string `json:"name"`
	Description string `json:"description,omitempty"`
}

type ChainTokenProvider 

type ChainTokenProvider []TokenProvider

func (ChainTokenProvider) Token 

func (p ChainTokenProvider) Token(ctx context.Context) (string, error)

type Client 

type Client struct {
	// contains filtered or unexported fields
}

func NewClient 

func NewClient(opts Options) (*Client, error)

func NewClientFromConfig 

func NewClientFromConfig(cfg Config, opts Options) (*Client, error)

func (*Client) Archives 

func (c *Client) Archives(ctx context.Context) ([]Archive, error)

func (*Client) BatchRead 

func (c *Client) BatchRead(ctx context.Context, app, archive string, requests []QueryRequest) ([]QueryResult, error)

func (*Client) Contract added in v0.10.0 

func (c *Client) Contract(ctx context.Context) (Contract, error)

func (*Client) Ingest 

func (c *Client) Ingest(ctx context.Context, app, archive string, req IngestRequest) (IngestResult, error)

func (*Client) LoginWithGitHubToken 

func (c *Client) LoginWithGitHubToken(ctx context.Context, token string) (LoginPollResult, error)

func (*Client) PollGitHubLogin 

func (c *Client) PollGitHubLogin(ctx context.Context, loginID, pollSecret string) (LoginPollResult, error)

func (*Client) Query 

func (c *Client) Query(ctx context.Context, app, archive string, req QueryRequest) (QueryResult, error)

func (*Client) StartGitHubLogin 

func (c *Client) StartGitHubLogin(ctx context.Context, pollSecretHash string) (LoginStartResult, error)

func (*Client) Status 

func (c *Client) Status(ctx context.Context, app, archive string) (Status, error)

func (*Client) UploadSQLite added in v0.10.0 

func (c *Client) UploadSQLite(ctx context.Context, app, archive string, upload SQLiteUploadRequest) (SQLiteUploadResult, error)

func (*Client) UploadSQLiteBundleFiles added in v0.11.0 

func (c *Client) UploadSQLiteBundleFiles(ctx context.Context, app, archive string, manifest SQLiteBundleManifest, parts []SQLiteBundlePartFile) (SQLiteBundleUploadResult, error)

func (*Client) UploadSQLiteBundleManifest added in v0.11.0 

func (c *Client) UploadSQLiteBundleManifest(ctx context.Context, app, archive string, manifest SQLiteBundleManifest) (SQLiteBundleUploadResult, error)

func (*Client) UploadSQLiteBundlePart added in v0.11.0 

func (c *Client) UploadSQLiteBundlePart(ctx context.Context, app, archive string, part SQLiteBundlePartUpload) (SQLiteUploadResult, error)

func (*Client) Whoami 

func (c *Client) Whoami(ctx context.Context) (Identity, error)

type Config 

type Config struct {
	Mode       string     `toml:"mode" json:"mode"`
	Endpoint   string     `toml:"endpoint" json:"endpoint"`
	Archive    string     `toml:"archive" json:"archive"`
	TokenEnv   string     `toml:"token_env" json:"token_env"`
	StaleAfter string     `toml:"stale_after" json:"stale_after"`
	Auth       AuthConfig `toml:"auth" json:"auth"`
}

func (Config) Enabled 

func (c Config) Enabled() bool

func (*Config) Normalize 

func (c *Config) Normalize()

type Contract added in v0.10.0 

type Contract struct {
	ProtocolVersion string      `json:"protocol_version"`
	Service         string      `json:"service,omitempty"`
	Routes          []RouteSpec `json:"routes"`
	Apps            []AppSpec   `json:"apps"`
	Auth            []AuthSpec  `json:"auth,omitempty"`
	Notes           []string    `json:"notes,omitempty"`
}

func BaseContract added in v0.10.0 

func BaseContract() Contract

func (Contract) Validate added in v0.10.0 

func (c Contract) Validate() error

type EnvTokenProvider 

type EnvTokenProvider struct {
	Name string
}

func (EnvTokenProvider) Token 

func (p EnvTokenProvider) Token(context.Context) (string, error)

type Error 

type Error struct {
	Status  int    `json:"status"`
	Code    string `json:"code,omitempty"`
	Message string `json:"message,omitempty"`
}

func (*Error) Error 

func (e *Error) Error() string

type GitHubTokenLoginRequest 

type GitHubTokenLoginRequest struct {
	Token string `json:"token"`
}

type Identity 

type Identity struct {
	Owner string   `json:"owner"`
	Org   string   `json:"org"`
	Login string   `json:"login,omitempty"`
	Auth  string   `json:"auth,omitempty"`
	Roles []string `json:"roles,omitempty"`
}

type IngestManifest 

type IngestManifest struct {
	App           string `json:"app"`
	Archive       string `json:"archive"`
	SchemaName    string `json:"schema_name,omitempty"`
	SchemaVersion int    `json:"schema_version"`
	SchemaHash    string `json:"schema_hash"`
	Mode          string `json:"mode,omitempty"`
	Source        string `json:"source,omitempty"`
	SourceSyncAt  string `json:"source_sync_at,omitempty"`
}

type IngestRequest 

type IngestRequest struct {
	Manifest IngestManifest `json:"manifest"`
	Table    string         `json:"table"`
	Columns  []string       `json:"columns"`
	Rows     [][]any        `json:"rows"`
	Cursor   string         `json:"cursor,omitempty"`
	Final    bool           `json:"final,omitempty"`
}

type IngestResult 

type IngestResult struct {
	RunID        string `json:"run_id,omitempty"`
	Table        string `json:"table,omitempty"`
	RowsAccepted int64  `json:"rows_accepted,omitempty"`
	Cursor       string `json:"cursor,omitempty"`
	Complete     bool   `json:"complete,omitempty"`
}

type IngestTableSpec added in v0.10.0 

type IngestTableSpec struct {
	Name           string   `json:"name"`
	Columns        []string `json:"columns"`
	PrivacyFilters []string `json:"privacy_filters,omitempty"`
}

type LoginPollRequest 

type LoginPollRequest struct {
	LoginID    string `json:"loginID"`
	PollSecret string `json:"pollSecret"`
}

type LoginPollResult 

type LoginPollResult struct {
	Status string `json:"status"`
	Token  string `json:"token,omitempty"`
	Owner  string `json:"owner,omitempty"`
	Org    string `json:"org,omitempty"`
	Login  string `json:"login,omitempty"`
	Error  string `json:"error,omitempty"`
}

type LoginStartRequest 

type LoginStartRequest struct {
	PollSecretHash string `json:"pollSecretHash"`
}

type LoginStartResult 

type LoginStartResult struct {
	LoginID   string `json:"loginID"`
	URL       string `json:"url"`
	ExpiresAt string `json:"expiresAt,omitempty"`
}

type Options 

type Options struct {
	Endpoint      string
	HTTPClient    *http.Client
	TokenProvider TokenProvider
	UserAgent     string
}

type QueryRequest 

type QueryRequest struct {
	App     string         `json:"app,omitempty"`
	Archive string         `json:"archive,omitempty"`
	Name    string         `json:"name"`
	Args    map[string]any `json:"args,omitempty"`
	Limit   int            `json:"limit,omitempty"`
	Cursor  string         `json:"cursor,omitempty"`
}

type QueryResult 

type QueryResult struct {
	Columns    []string         `json:"columns"`
	Rows       [][]any          `json:"rows"`
	Values     []map[string]any `json:"values,omitempty"`
	Cursor     string           `json:"cursor,omitempty"`
	Stats      QueryStats       `json:"stats,omitempty"`
	SchemaHash string           `json:"schema_hash,omitempty"`
}

type QuerySpec added in v0.10.0 

type QuerySpec struct {
	Name string   `json:"name"`
	Args []string `json:"args,omitempty"`
}

type QueryStats 

type QueryStats struct {
	RowsRead    int64  `json:"rows_read,omitempty"`
	RowsWritten int64  `json:"rows_written,omitempty"`
	DurationMS  int64  `json:"duration_ms,omitempty"`
	ServedBy    string `json:"served_by,omitempty"`
}

type RouteSpec added in v0.10.0 

type RouteSpec struct {
	Method string `json:"method"`
	Path   string `json:"path"`
	Auth   string `json:"auth"`
}

type SQLiteBundle added in v0.11.0 

type SQLiteBundle struct {
	Key         string                `json:"key,omitempty"`
	Size        int64                 `json:"size,omitempty"`
	ETag        string                `json:"etag,omitempty"`
	UploadedAt  string                `json:"uploaded_at,omitempty"`
	ContentType string                `json:"content_type,omitempty"`
	Manifest    *SQLiteBundleManifest `json:"manifest,omitempty"`
}

type SQLiteBundleBuild added in v0.11.0 

type SQLiteBundleBuild struct {
	Manifest       SQLiteBundleManifest
	CompressedPath string
	Parts          []SQLiteBundlePartFile
	Cleanup        func()
}

func BuildGzipSQLiteBundle added in v0.11.0 

func BuildGzipSQLiteBundle(ctx context.Context, opts SQLiteBundleBuildOptions) (SQLiteBundleBuild, error)

type SQLiteBundleBuildOptions added in v0.11.0 

type SQLiteBundleBuildOptions struct {
	App              string
	Archive          string
	SourcePath       string
	WorkDir          string
	ChunkSize        int64
	CompressionLevel int
	GeneratedAt      time.Time
	ContentType      string
	Privacy          map[string]any
	Counts           map[string]int64
}

type SQLiteBundleCompression added in v0.11.0 

type SQLiteBundleCompression struct {
	Algorithm string `json:"algorithm,omitempty"`
}

type SQLiteBundleManifest added in v0.11.0 

type SQLiteBundleManifest struct {
	Format           string                  `json:"format"`
	App              string                  `json:"app"`
	Archive          string                  `json:"archive"`
	GeneratedAt      string                  `json:"generated_at,omitempty"`
	ContentType      string                  `json:"content_type,omitempty"`
	Compression      SQLiteBundleCompression `json:"compression,omitempty"`
	Privacy          map[string]any          `json:"privacy,omitempty"`
	Object           SQLiteBundleObject      `json:"object"`
	CompressedObject SQLiteBundleObject      `json:"compressed_object"`
	Reconstruct      string                  `json:"reconstruct,omitempty"`
	Counts           map[string]int64        `json:"counts,omitempty"`
	Parts            []SQLiteBundlePart      `json:"parts"`
}

type SQLiteBundleObject added in v0.11.0 

type SQLiteBundleObject struct {
	Key    string `json:"key,omitempty"`
	Size   int64  `json:"size,omitempty"`
	SHA256 string `json:"sha256,omitempty"`
}

type SQLiteBundlePart added in v0.11.0 

type SQLiteBundlePart struct {
	Index  int    `json:"index"`
	Key    string `json:"key,omitempty"`
	Size   int64  `json:"size,omitempty"`
	SHA256 string `json:"sha256,omitempty"`
}

type SQLiteBundlePartFile added in v0.11.0 

type SQLiteBundlePartFile struct {
	SQLiteBundlePart
	Path string
}

type SQLiteBundlePartUpload added in v0.11.0 

type SQLiteBundlePartUpload struct {
	Index       int
	Body        io.Reader
	Size        int64
	SHA256      string
	Compression string
}

type SQLiteBundleUploadResult added in v0.11.0 

type SQLiteBundleUploadResult struct {
	App      string        `json:"app,omitempty"`
	Archive  string        `json:"archive,omitempty"`
	Complete bool          `json:"complete,omitempty"`
	Bundle   *SQLiteBundle `json:"bundle,omitempty"`
}

type SQLiteObject added in v0.10.0 

type SQLiteObject struct {
	Key         string `json:"key,omitempty"`
	Size        int64  `json:"size,omitempty"`
	ETag        string `json:"etag,omitempty"`
	UploadedAt  string `json:"uploaded_at,omitempty"`
	ContentType string `json:"content_type,omitempty"`
	SHA256      string `json:"sha256,omitempty"`
}

type SQLiteUploadRequest added in v0.10.0 

type SQLiteUploadRequest struct {
	Body          io.Reader
	Size          int64
	ContentSHA256 string
	SchemaName    string
	SchemaVersion int
	SchemaHash    string
	SourceSyncAt  string
}

type SQLiteUploadResult added in v0.10.0 

type SQLiteUploadResult struct {
	App      string        `json:"app,omitempty"`
	Archive  string        `json:"archive,omitempty"`
	Complete bool          `json:"complete,omitempty"`
	Object   *SQLiteObject `json:"object,omitempty"`
}

type StaticToken 

type StaticToken string

func (StaticToken) Token 

func (t StaticToken) Token(context.Context) (string, error)

type Status 

type Status struct {
	App          string          `json:"app"`
	Archive      string          `json:"archive"`
	Mode         string          `json:"mode,omitempty"`
	GeneratedAt  string          `json:"generated_at,omitempty"`
	LastSyncAt   string          `json:"last_sync_at,omitempty"`
	LastIngestAt string          `json:"last_ingest_at,omitempty"`
	Counts       []control.Count `json:"counts,omitempty"`
	Capabilities []string        `json:"capabilities,omitempty"`
	SQLiteObject *SQLiteObject   `json:"sqlite_object,omitempty"`
	SQLiteBundle *SQLiteBundle   `json:"sqlite_bundle,omitempty"`
	Warnings     []string        `json:"warnings,omitempty"`
}

type TokenProvider 

type TokenProvider interface {
	Token(context.Context) (string, error)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.