kloakd

package module
v0.3.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 27, 2026 License: MIT Imports: 12 Imported by: 0

README

kloakd-go

Official Go SDK for the KLOAKD API — the anti-bot intelligence platform.

Requirements

  • Go 1.21+
  • KLOAKD API key and organization ID

Installation

go get github.com/kloakd/kloakd-go

Quickstart

package main

import (
    "context"
    "fmt"
    "log"

    kloakd "github.com/kloakd/kloakd-go"
)

func main() {
    client, err := kloakd.New(kloakd.Config{
        APIKey:         "sk-live-...",
        OrganizationID: "your-org-uuid",
    })
    if err != nil {
        log.Fatal(err)
    }

    ctx := context.Background()

    // Anti-bot fetch
    fetch, err := client.Evadr.Fetch(ctx, "https://books.toscrape.com", nil)
    if err != nil {
        log.Fatal(err)
    }
    fmt.Printf("Fetched via tier %d\n", fetch.TierUsed)

    // Site crawl (waits for the crawl, then returns every page)
    crawl, err := client.Webgrph.Crawl(ctx, "https://books.toscrape.com", &kloakd.CrawlOptions{
        MaxDepth: 2,
        MaxPages: 50,
    })
    fmt.Printf("Found %d pages\n", crawl.TotalPages)

    // Extract structured data
    data, err := client.Kolektr.Extract(ctx, "https://books.toscrape.com", &kloakd.ExtractOptions{
        SchemaSpec: map[string]interface{}{
            "title": "css:h3 a",
            "price": "css:p.price_color",
        },
    })
    fmt.Printf("Extracted %d records\n", data.TotalRecords)
}

Configuration

client, err := kloakd.New(kloakd.Config{
    APIKey:         "sk-live-...",   // required
    OrganizationID: "org-uuid",      // required
    BaseURL:        "https://api.kloakd.dev", // optional, default shown
    Timeout:        30 * time.Second,         // optional, default 30s
    MaxRetries:     3,                        // optional, default 3
    HTTPClient:     customHTTPClient,         // optional, for testing/proxies
})

Modules

client.Evadr — Anti-Bot Intelligence
// Fetch a URL through anti-bot bypass
result, err := client.Evadr.Fetch(ctx, url, nil)

// Fetch answers within about 85 s. A slower site is handed off as a job
// (HTTP 202); Fetch follows it to the result. Tune the wait:
fetch, err = client.Evadr.Fetch(ctx, url, &kloakd.FetchOptions{
    PollInterval: 3 * time.Second, PollTimeout: 15 * time.Minute,
})

// Async job
jobID, err := client.Evadr.FetchAsync(ctx, url, nil)

// Stream a fetch's progress (starts an async job, streams its events)
ch, err := client.Evadr.FetchStream(ctx, url, nil)
for event := range ch {
    fmt.Println(event.Type, event.Tier)
}

// Analyze anti-bot defenses
analysis, err := client.Evadr.Analyze(ctx, url, nil)
client.Webgrph — Site Mapping
// Crawl a site: starts the crawl, waits for it, returns every page
crawl, err := client.Webgrph.Crawl(ctx, url, &kloakd.CrawlOptions{MaxDepth: 3, MaxPages: 100})

// Or start it and follow it yourself
started, err := client.Webgrph.StartCrawl(ctx, url, nil)
status, err := client.Webgrph.GetCrawlStatus(ctx, started.CrawlID)

// Stream a crawl's events
ch, err := client.Webgrph.CrawlStream(ctx, url, nil)
for event := range ch {
    fmt.Println(event.Type, event.URL)
}
client.Skanyr — Site Knowledge
site, err := client.Skanyr.CreateSiteFromCrawl(ctx, crawl.CrawlID)
sync, err := client.Skanyr.Sync(ctx, siteID, &kloakd.SyncOptions{Policy: "changed"})
hits, err := client.Skanyr.Search(ctx, "pricing", &kloakd.SearchOptions{SiteID: siteID})

Detected APIs, the reader view and discovery sessions are on client.Webgrph.

client.Nexus — Strategy Engine
page, _       := client.Evadr.Fetch(ctx, url, nil)
perception, _ := client.Nexus.Analyze(ctx, url, *page.HTML, nil)
strategy, _   := client.Nexus.Synthesize(ctx, url, *page.HTML, "product prices")
verified, _   := client.Nexus.Verify(ctx, strategy.StrategyID)
result, _     := client.Nexus.Execute(ctx, strategy.StrategyID, url, nil)
client.Parlyr — Conversational NLP
parse, err := client.Parlyr.Parse(ctx, "scrape example.com", nil)

// Blocking chat (collects SSE internally)
turn, err := client.Parlyr.Chat(ctx, sessionID, "scrape example.com")

// SSE streaming
ch, err := client.Parlyr.ChatStream(ctx, sessionID, message)
for event := range ch {
    fmt.Println(event.Event, event.Data)
}
client.Fetchyr — RPA & Authentication
_, err := client.Fetchyr.StoreCredentials(ctx, "my-login", map[string]interface{}{"username": "u", "password": "p"})
session, err := client.Fetchyr.Authenticate(ctx, "my-login", loginURL)
page, err    := client.Evadr.Fetch(ctx, protectedURL, &kloakd.FetchOptions{SessionArtifactID: *session.ArtifactID})
client.Kolektr — Data Extraction
data, err := client.Kolektr.Extract(ctx, url, &kloakd.ExtractOptions{
    SchemaSpec: map[string]interface{}{"title": "css:h1", "price": "css:.price"},
})

// From raw HTML
data, err = client.Kolektr.ExtractHTML(ctx, html, url, nil)

Error Handling

import "errors"

result, err := client.Evadr.Fetch(ctx, url, nil)
if err != nil {
    var rlErr *kloakd.RateLimitError
    var quotaErr *kloakd.QuotaExceededError
    var payErr *kloakd.PaymentRequiredError
    var authErr *kloakd.AuthenticationError
    switch {
    case errors.As(err, &quotaErr):
        log.Printf("GB used up (%.1f of %.1f): buy a top-up", quotaErr.GBUsed, quotaErr.GBLimit)
    case errors.As(err, &payErr):
        log.Printf("payment failed: update it at %s", payErr.UpgradeURL)
    case errors.As(err, &rlErr):
        time.Sleep(time.Duration(rlErr.RetryAfter) * time.Second)
    case errors.As(err, &authErr):
        log.Fatal("invalid API key")
    default:
        log.Printf("error: %v", err)
    }
}

Error types: KloakdError (with Code, the API's error word), AuthenticationError (401), PaymentRequiredError (402), NotEntitledError (403), QuotaExceededError (429 gb_exhausted), RateLimitError (429), UpstreamError (502), BillingUnavailableError (503 billing_unavailable), ApiError (other 4xx/5xx).

Retry Policy

  • Retryable status codes: 429, 500, 502, 503, 504
  • Strategy: exponential backoff — 1s × 2^attempt, capped at 60s
  • 429 and 503: respect the Retry-After header / retry_after field
  • Not retried: 400, 401, 402, 403, 404, and 429 gb_exhausted (waiting does not clear it)
  • Default max retries: 3 (override via Config.MaxRetries; negative for none)

Running the Quickstart

export KLOAKD_API_KEY=sk-live-...
export KLOAKD_ORG_ID=your-org-uuid
go run ./examples/quickstart/main.go

Running Tests

go test . -v
go test . -cover

License

MIT

Documentation

Overview

Package kloakd provides the Go SDK for the KLOAKD API. Error hierarchy mirrors the design document §6.

Index

Constants

View Source 
const (
	DefaultFetchPollInterval = 3 * time.Second
	DefaultFetchPollTimeout  = 15 * time.Minute
)

Defaults for following a fetch that the API hands off as a job.

View Source 
const (
	DefaultCrawlPollInterval = 3 * time.Second
	DefaultCrawlPollTimeout  = 60 * time.Minute
)

Defaults for waiting on a crawl (crawls run asynchronously on the crawlers).

Variables

This section is empty.

Functions

This section is empty.

Types

type AnalyzeOptions 

type AnalyzeOptions struct {
	StatusCode  int
	Headers     map[string]string
	BodySnippet string
}

AnalyzeOptions configures optional parameters for Evadr.Analyze.

type AnalyzeResult 

type AnalyzeResult struct {
	Blocked            bool
	Vendor             *string
	Confidence         float64
	RecommendedActions []string
}

AnalyzeResult is returned by Evadr.Analyze.

type ApiError 

type ApiError struct {
	KloakdError
}

ApiError is returned for any other 4xx/5xx response.

func (*ApiError) Error 

func (e *ApiError) Error() string

type AuthenticationError 

type AuthenticationError struct {
	KloakdError
}

AuthenticationError is returned for HTTP 401 — invalid or expired API key.

func (*AuthenticationError) Error 

func (e *AuthenticationError) Error() string

type BillingUnavailableError added in v0.3.0 

type BillingUnavailableError struct {
	KloakdError
	RetryAfter int
}

BillingUnavailableError is returned for HTTP 503 "billing_unavailable": usage could not be checked, so the request was not started. Retry after RetryAfter seconds.

func (*BillingUnavailableError) Error added in v0.3.0 

func (e *BillingUnavailableError) Error() string

type ChatEvent 

type ChatEvent struct {
	Event string
	Data  map[string]interface{}
}

ChatEvent is yielded by Parlyr.ChatStream.

type ChatTurn 

type ChatTurn struct {
	SessionID           string
	Intent              string
	Confidence          float64
	Tier                int
	Response            string
	Entities            map[string]interface{}
	RequiresAction      bool
	ClarificationNeeded *string
}

ChatTurn is returned by Parlyr.Chat.

type CheckDuplicateOptions added in v0.3.0 

type CheckDuplicateOptions struct {
	ContentHash string
	SessionID   string
}

CheckDuplicateOptions configures optional parameters for CheckDuplicate.

type Client 

type Client struct {
	Evadr   *EvadrNamespace
	Webgrph *WebgrphNamespace
	Skanyr  *SkanyrNamespace
	Nexus   *NexusNamespace
	Parlyr  *ParlyrNamespace
	Fetchyr *FetchyrNamespace
	Kolektr *KolektrNamespace
	// contains filtered or unexported fields
}

Client is the KLOAKD SDK client. Create one with New(). 

client := kloakd.New(kloakd.Config{
    APIKey:         "sk-live-...",
    OrganizationID: "your-org-uuid",
})

func MustNew 

func MustNew(cfg Config) *Client

MustNew is like New but panics on configuration error.

func New 

func New(cfg Config) (*Client, error)

New creates a new KLOAKD client with the given configuration. Returns an error if APIKey or OrganizationID are missing.

func (*Client) Crawl added in v0.3.0 

func (c *Client) Crawl(ctx context.Context, targetURL string, opts *CrawlOrchestratorOptions) (*SiteCrawlResult, error)

Crawl is the high-level orchestrator that discovers, fetches, and optionally extracts structured data from all pages on a site in a single call.

It internally:

  1. Discovers pages via Webgrph BFS crawl
  2. Fetches each page through the 4-tier anti-bot engine (Evadr)
  3. Extracts structured data if ExtractSchema is provided (Kolektr)

Per-page failures are caught and marked Success=false — the crawl never aborts on a single page error.

func (*Client) CrawlStream added in v0.3.0 

func (c *Client) CrawlStream(ctx context.Context, targetURL string, opts *CrawlOrchestratorOptions) (<-chan CrawlProgressEvent, error)

CrawlStream is the streaming version of Crawl. It sends CrawlProgressEvent values on the returned channel as pages are discovered and fetched.

type Config 

type Config struct {
	// APIKey is the KLOAKD API key (required). Starts with "sk-".
	APIKey string

	// OrganizationID is the UUID of the organization (required).
	OrganizationID string

	// BaseURL overrides the API base URL. Defaults to https://api.kloakd.dev.
	BaseURL string

	// Timeout is the per-request timeout. Defaults to 30s.
	Timeout time.Duration

	// MaxRetries sets the number of retry attempts for retryable errors.
	// 0 (unset) means the default, 3; a negative value means no retries.
	MaxRetries int

	// HTTPClient allows injecting a custom *http.Client (e.g. for testing).
	HTTPClient *http.Client
}

Config holds all configuration for the KLOAKD client.

type CrawlEvent 

type CrawlEvent struct {
	Type       string
	URL        *string
	Depth      *int
	PagesFound *int
	Metadata   map[string]interface{}
}

CrawlEvent is yielded by Webgrph.CrawlStream.

type CrawlOptions 

type CrawlOptions struct {
	MaxDepth             int
	MaxPages             int
	IncludeExternalLinks bool
	SessionArtifactID    string
	FastMode             bool
	CaptureMode          string
	// PollInterval and PollTimeout govern waiting for the crawl to finish
	// (defaults 3 s and 60 min; ctx cancellation also stops the wait).
	PollInterval time.Duration
	PollTimeout  time.Duration
}

CrawlOptions configures optional parameters for Webgrph.Crawl.

type CrawlOrchestratorOptions added in v0.3.0 

type CrawlOrchestratorOptions struct {
	MaxDepth             int
	MaxPages             int
	ExtractSchema        map[string]string
	OutputFormat         string
	IncludeExternalLinks bool
	SessionArtifactID    string
}

CrawlOrchestratorOptions configures the high-level Crawl() and CrawlStream() methods.

type CrawlPage added in v0.3.0 

type CrawlPage struct {
	Success        bool
	URL            string
	StatusCode     int
	TierUsed       int
	HTML           *string
	StructuredData map[string]interface{}
	ArtifactID     *string
	Error          *string
}

CrawlPage represents a single page result from the high-level Crawl() orchestrator.

func (CrawlPage) Ok added in v0.3.0 

func (p CrawlPage) Ok() bool

Ok returns true if the page was fetched successfully.

type CrawlProgressEvent added in v0.3.0 

type CrawlProgressEvent struct {
	Type     string
	URL      *string
	Page     *int
	Total    *int
	Success  *bool
	Error    *string
	Metadata map[string]interface{}
}

CrawlProgressEvent is yielded by Client.CrawlStream.

type CrawlResult 

type CrawlResult struct {
	Success         bool
	CrawlID         string
	URL             string
	Status          string
	TotalPages      int
	MaxDepthReached int
	Pages           []PageNode
	Total           int
	ArtifactID      *string
	Error           *string
}

CrawlResult is returned by Webgrph.Crawl.

type CrawlStreamOptions 

type CrawlStreamOptions struct {
	MaxDepth          int
	MaxPages          int
	SessionArtifactID string
}

CrawlStreamOptions configures optional parameters for Webgrph.CrawlStream.

type CreateWorkflowOptions added in v0.3.0 

type CreateWorkflowOptions struct {
	Description  string
	WorkflowType string
}

CreateWorkflowOptions configures optional parameters for CreateWorkflow.

type DeduplicationResult 

type DeduplicationResult struct {
	IsDuplicate     bool
	OriginalURL     *string
	SimilarityScore float64
	MatchType       *string
}

DeduplicationResult is returned by Fetchyr.CheckDuplicate.

type EvadrNamespace 

type EvadrNamespace struct {
	// contains filtered or unexported fields
}

EvadrNamespace exposes the Anti-Bot Intelligence module. Access via client.Evadr.

func (*EvadrNamespace) Analyze 

func (e *EvadrNamespace) Analyze(ctx context.Context, targetURL string, opts *AnalyzeOptions) (*AnalyzeResult, error)

Analyze inspects the anti-bot defenses of the given URL.

func (*EvadrNamespace) DeleteProxy added in v0.2.0 

func (e *EvadrNamespace) DeleteProxy(ctx context.Context, name string) error

DeleteProxy deletes a stored proxy configuration by name.

func (*EvadrNamespace) Fetch 

func (e *EvadrNamespace) Fetch(ctx context.Context, targetURL string, opts *FetchOptions) (*FetchResult, error)

Fetch performs an anti-bot fetch of the given URL.

func (*EvadrNamespace) FetchAsync 

func (e *EvadrNamespace) FetchAsync(ctx context.Context, targetURL string, opts *FetchAsyncOptions) (string, error)

FetchAsync enqueues an anti-bot fetch job and returns the job ID.

func (*EvadrNamespace) FetchStream 

func (e *EvadrNamespace) FetchStream(ctx context.Context, targetURL string, opts *FetchStreamOptions) (<-chan FetchEvent, error)

FetchStream starts an asynchronous fetch and streams its progress events (GET evadr/jobs/{job_id}/events). The channel closes when the job ends or ctx is cancelled. There is no POST stream route; this is the live route.

func (*EvadrNamespace) GetJob added in v0.2.0 

func (e *EvadrNamespace) GetJob(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJob polls an async fetch job by ID.

func (*EvadrNamespace) JobEvents added in v0.3.0 

func (e *EvadrNamespace) JobEvents(ctx context.Context, jobID string) (<-chan map[string]interface{}, error)

JobEvents streams an async fetch job's progress events (SSE). The channel closes when the job ends or ctx is cancelled.

func (*EvadrNamespace) ListProfiles added in v0.2.0 

func (e *EvadrNamespace) ListProfiles(ctx context.Context) (map[string]interface{}, error)

ListProfiles lists available evasion profiles.

func (*EvadrNamespace) ListProxies added in v0.2.0 

func (e *EvadrNamespace) ListProxies(ctx context.Context) (map[string]interface{}, error)

ListProxies lists stored proxy configurations (names only, no secrets).

func (*EvadrNamespace) ListVendors added in v0.2.0 

func (e *EvadrNamespace) ListVendors(ctx context.Context) (map[string]interface{}, error)

ListVendors lists all known anti-bot detection vendors.

func (*EvadrNamespace) Profile added in v0.3.0 

func (e *EvadrNamespace) Profile(ctx context.Context, targetURL string) (map[string]interface{}, error)

Profile analyses a URL and returns its canonical site profile.

func (*EvadrNamespace) Scan added in v0.2.0 

func (e *EvadrNamespace) Scan(ctx context.Context, targetURL string) (map[string]interface{}, error)

Scan fetches a URL once (plain HTTP) and reports its anti-bot infrastructure, vendors and challenges.

func (*EvadrNamespace) StoreProxy 

func (e *EvadrNamespace) StoreProxy(ctx context.Context, name, proxyURL string) error

StoreProxy registers a named proxy in the organization's proxy pool.

type ExecuteWorkflowOptions added in v0.3.0 

type ExecuteWorkflowOptions struct {
	SessionArtifactID string
	InputData         map[string]interface{}
}

ExecuteWorkflowOptions configures optional parameters for ExecuteWorkflow.

type ExecutionResult 

type ExecutionResult struct {
	ExecutionID string
	WorkflowID  string
	Status      string
	StartedAt   *string
	CompletedAt *string
	Result      map[string]interface{}
	Error       *string
}

ExecutionResult is returned by Fetchyr.ExecuteWorkflow / GetExecution.

type ExtractHTMLOptions 

type ExtractHTMLOptions struct {
	SchemaSpec map[string]interface{}
}

ExtractHTMLOptions configures optional parameters for Kolektr.ExtractHTML.

type ExtractOptions added in v0.3.0 

type ExtractOptions struct {
	// SchemaSpec describes the fields to extract (sent as schema_spec).
	SchemaSpec        map[string]interface{}
	Level             string
	FetchArtifactID   string
	SessionArtifactID string
	ApiMapArtifactID  string
	Options           map[string]interface{}
}

ExtractOptions configures optional parameters for Kolektr.Extract.

type ExtractionResult 

type ExtractionResult struct {
	Success      bool
	URL          string
	Method       string
	Records      []map[string]interface{}
	TotalRecords int
	PagesScraped int
	ArtifactID   *string
	JobID        *string
	Error        *string
}

ExtractionResult is returned by Kolektr.Extract and Kolektr.ExtractHTML.

type FetchAsyncOptions 

type FetchAsyncOptions struct {
	ForceBrowser      bool
	UseProxy          bool
	SessionArtifactID string
	WebhookURL        string
}

FetchAsyncOptions configures optional parameters for Evadr.FetchAsync.

type FetchEvent 

type FetchEvent struct {
	Type     string
	Tier     *int
	Vendor   *string
	Metadata map[string]interface{}
}

FetchEvent is yielded by Evadr.FetchStream.

type FetchOptions 

type FetchOptions struct {
	ForceBrowser      bool
	UseProxy          bool
	SessionArtifactID string
	// PollInterval and PollTimeout govern following a fetch the API hands
	// off as a job (HTTP 202, after about 85 s). Defaults 3 s and 15 min;
	// ctx cancellation also stops the wait.
	PollInterval time.Duration
	PollTimeout  time.Duration
}

FetchOptions configures optional parameters for Evadr.Fetch.

type FetchResult 

type FetchResult struct {
	Success         bool
	URL             string
	StatusCode      int
	TierUsed        int
	HTML            *string
	VendorDetected  *string
	AntiBotBypassed bool
	ArtifactID      *string
	Error           *string
}

FetchResult is returned by Evadr.Fetch.

type FetchStreamOptions 

type FetchStreamOptions struct {
	ForceBrowser      bool
	UseProxy          bool
	SessionArtifactID string
}

FetchStreamOptions configures optional parameters for Evadr.FetchStream.

type FetchyrNamespace 

type FetchyrNamespace struct {
	// contains filtered or unexported fields
}

FetchyrNamespace exposes the RPA & Authentication module. Access via client.Fetchyr.

func (*FetchyrNamespace) Authenticate added in v0.3.0 

func (f *FetchyrNamespace) Authenticate(ctx context.Context, credentialName, targetURL string) (*SessionResult, error)

Authenticate logs in to a site with credentials stored under credentialName (StoreCredentials) and returns the authenticated session. It replaces Login, whose route was removed: the API no longer takes selectors or a password per call.

func (*FetchyrNamespace) CheckDuplicate added in v0.3.0 

func (f *FetchyrNamespace) CheckDuplicate(ctx context.Context, targetURL string, opts *CheckDuplicateOptions) (*DeduplicationResult, error)

CheckDuplicate reports whether a URL (or its content hash) was already seen, within a deduplication session if given.

func (*FetchyrNamespace) CreateDedupSession added in v0.2.0 

func (f *FetchyrNamespace) CreateDedupSession(ctx context.Context, config map[string]interface{}) (map[string]interface{}, error)

CreateDedupSession creates a deduplication session.

func (*FetchyrNamespace) CreateMultiSiteWorkflow added in v0.2.0 

func (f *FetchyrNamespace) CreateMultiSiteWorkflow(ctx context.Context, name string, sites []map[string]interface{}) (map[string]interface{}, error)

CreateMultiSiteWorkflow creates a multi-site orchestration workflow.

func (*FetchyrNamespace) CreateWorkflow 

func (f *FetchyrNamespace) CreateWorkflow(ctx context.Context, name string, steps []map[string]interface{}, opts *CreateWorkflowOptions) (*WorkflowResult, error)

CreateWorkflow creates a new RPA workflow.

func (*FetchyrNamespace) DeleteCredentials added in v0.2.0 

func (f *FetchyrNamespace) DeleteCredentials(ctx context.Context, name string) error

DeleteCredentials deletes stored credentials by name.

func (*FetchyrNamespace) DeleteWorkflow added in v0.2.0 

func (f *FetchyrNamespace) DeleteWorkflow(ctx context.Context, workflowID string) error

DeleteWorkflow deletes a workflow definition.

func (*FetchyrNamespace) DetectForms 

func (f *FetchyrNamespace) DetectForms(ctx context.Context, targetURL string, sessionArtifactID string) (*FormDetectionResult, error)

DetectForms detects forms on a page.

func (*FetchyrNamespace) DetectMFA 

func (f *FetchyrNamespace) DetectMFA(ctx context.Context, targetURL string, sessionArtifactID string) (*MfaDetectionResult, error)

DetectMFA detects MFA challenges on a page.

func (*FetchyrNamespace) DiscoverAuthChain added in v0.3.0 

func (f *FetchyrNamespace) DiscoverAuthChain(ctx context.Context, targetURL string) (map[string]interface{}, error)

DiscoverAuthChain starts discovering how a site authenticates.

func (*FetchyrNamespace) ExecuteWorkflow 

func (f *FetchyrNamespace) ExecuteWorkflow(ctx context.Context, workflowID, targetURL string, opts *ExecuteWorkflowOptions) (*ExecutionResult, error)

ExecuteWorkflow runs a workflow against targetURL.

func (*FetchyrNamespace) FillForm added in v0.2.0 

func (f *FetchyrNamespace) FillForm(ctx context.Context, targetURL string, formData map[string]string, sessionArtifactID string) (map[string]interface{}, error)

FillForm fills and submits a form on a page.

func (*FetchyrNamespace) GetAuthChainStatus added in v0.3.0 

func (f *FetchyrNamespace) GetAuthChainStatus(ctx context.Context, discoveryID string) (map[string]interface{}, error)

GetAuthChainStatus gets an auth-chain discovery's status.

func (*FetchyrNamespace) GetDedupDomainStatistics added in v0.2.0 

func (f *FetchyrNamespace) GetDedupDomainStatistics(ctx context.Context, domain string) (map[string]interface{}, error)

GetDedupDomainStatistics gets deduplication statistics for a domain.

func (*FetchyrNamespace) GetDedupSession added in v0.2.0 

func (f *FetchyrNamespace) GetDedupSession(ctx context.Context, sessionID string) (map[string]interface{}, error)

GetDedupSession gets a deduplication session by ID.

func (*FetchyrNamespace) GetDedupSessionStatistics added in v0.2.0 

func (f *FetchyrNamespace) GetDedupSessionStatistics(ctx context.Context, sessionID string) (map[string]interface{}, error)

GetDedupSessionStatistics gets statistics for a deduplication session.

func (*FetchyrNamespace) GetExecution 

func (f *FetchyrNamespace) GetExecution(ctx context.Context, workflowID, executionID string) (*ExecutionResult, error)

GetExecution retrieves the result of a workflow execution.

func (*FetchyrNamespace) GetMFAChallenge added in v0.2.0 

func (f *FetchyrNamespace) GetMFAChallenge(ctx context.Context, challengeID string) (map[string]interface{}, error)

GetMFAChallenge gets a specific MFA challenge by ID.

func (*FetchyrNamespace) GetMFAStatistics added in v0.2.0 

func (f *FetchyrNamespace) GetMFAStatistics(ctx context.Context, domain string) (map[string]interface{}, error)

GetMFAStatistics gets MFA statistics for a domain.

func (*FetchyrNamespace) GetWorkflow added in v0.2.0 

func (f *FetchyrNamespace) GetWorkflow(ctx context.Context, workflowID string) (map[string]interface{}, error)

GetWorkflow gets a workflow definition by ID.

func (*FetchyrNamespace) GetWorkflowStatistics added in v0.2.0 

func (f *FetchyrNamespace) GetWorkflowStatistics(ctx context.Context, workflowID string) (map[string]interface{}, error)

GetWorkflowStatistics gets execution statistics for a workflow.

func (*FetchyrNamespace) ListCredentials added in v0.2.0 

func (f *FetchyrNamespace) ListCredentials(ctx context.Context) (map[string]interface{}, error)

ListCredentials lists stored credential names (no secret values).

func (*FetchyrNamespace) ListDedupSessions added in v0.2.0 

func (f *FetchyrNamespace) ListDedupSessions(ctx context.Context) (map[string]interface{}, error)

ListDedupSessions lists active deduplication sessions.

func (*FetchyrNamespace) ListMFAChallenges added in v0.2.0 

func (f *FetchyrNamespace) ListMFAChallenges(ctx context.Context) (map[string]interface{}, error)

ListMFAChallenges lists pending MFA challenges.

func (*FetchyrNamespace) ListSessions added in v0.2.0 

func (f *FetchyrNamespace) ListSessions(ctx context.Context) (map[string]interface{}, error)

ListSessions lists active browser sessions.

func (*FetchyrNamespace) ListWorkflows added in v0.2.0 

func (f *FetchyrNamespace) ListWorkflows(ctx context.Context) (map[string]interface{}, error)

ListWorkflows lists all RPA workflow definitions.

func (*FetchyrNamespace) ResolveMFA added in v0.3.0 

func (f *FetchyrNamespace) ResolveMFA(ctx context.Context, challengeID, resolution string) (map[string]interface{}, error)

ResolveMFA resolves a queued MFA challenge (the code, or "approved"). It replaces SubmitMFA, whose route was removed.

func (*FetchyrNamespace) ResumeExecution added in v0.3.0 

func (f *FetchyrNamespace) ResumeExecution(ctx context.Context, workflowID, executionID string, resume map[string]interface{}) (*ExecutionResult, error)

ResumeExecution resumes a paused execution (after MFA).

func (*FetchyrNamespace) StoreCredentials added in v0.2.0 

func (f *FetchyrNamespace) StoreCredentials(ctx context.Context, name string, credentials map[string]interface{}) (map[string]interface{}, error)

StoreCredentials stores login credentials securely.

func (*FetchyrNamespace) TerminateSession added in v0.2.0 

func (f *FetchyrNamespace) TerminateSession(ctx context.Context, artifactID string) error

TerminateSession terminates a session and releases Chrome slot.

func (*FetchyrNamespace) UpdateWorkflow added in v0.2.0 

func (f *FetchyrNamespace) UpdateWorkflow(ctx context.Context, workflowID string, updates map[string]interface{}) (map[string]interface{}, error)

UpdateWorkflow updates an existing workflow.

type FormDetectionResult 

type FormDetectionResult struct {
	URL               string
	FormsFound        int
	LoginFormDetected bool
	Fields            []map[string]interface{}
}

FormDetectionResult is returned by Fetchyr.DetectForms.

type HandedOff added in v0.3.0 

type HandedOff struct {
	JobID         string
	Reason        string
	WaitedSeconds float64
	PollURL       string
}

HandedOff describes a 202 from the synchronous fetch: still running, continuing as the job JobID.

type KloakdError 

type KloakdError struct {
	StatusCode int
	Message    string
	// Code is the API's error word (detail.error), e.g. "gb_exhausted".
	Code string
}

KloakdError is the base error type for all SDK errors. Every error carries StatusCode and Message.

func (*KloakdError) Error 

func (e *KloakdError) Error() string

type KolektrNamespace 

type KolektrNamespace struct {
	// contains filtered or unexported fields
}

KolektrNamespace exposes the Data Extraction module. Access via client.Kolektr.

func (*KolektrNamespace) CreateJob added in v0.2.0 

func (k *KolektrNamespace) CreateJob(ctx context.Context, config map[string]interface{}) (map[string]interface{}, error)

CreateJob creates an extraction job.

func (*KolektrNamespace) CreateScraper added in v0.2.0 

func (k *KolektrNamespace) CreateScraper(ctx context.Context, config map[string]interface{}) (map[string]interface{}, error)

CreateScraper creates a scraper configuration.

func (*KolektrNamespace) DeleteContent added in v0.2.0 

func (k *KolektrNamespace) DeleteContent(ctx context.Context, itemID string) error

DeleteContent deletes a content item.

func (*KolektrNamespace) DeleteScraper added in v0.2.0 

func (k *KolektrNamespace) DeleteScraper(ctx context.Context, scraperID string) error

DeleteScraper deletes a scraper configuration.

func (*KolektrNamespace) Extract added in v0.3.0 

func (k *KolektrNamespace) Extract(ctx context.Context, targetURL string, opts *ExtractOptions) (*ExtractionResult, error)

Extract extracts structured data from a URL with the full Kolektr pipeline (POST kolektr/extract).

func (*KolektrNamespace) ExtractAllAPIData added in v0.2.0 

func (k *KolektrNamespace) ExtractAllAPIData(ctx context.Context, apiEndpoint string) (map[string]interface{}, error)

ExtractAllAPIData extracts all data from a discovered API endpoint.

func (*KolektrNamespace) ExtractHTML 

func (k *KolektrNamespace) ExtractHTML(ctx context.Context, html, targetURL string, opts *ExtractHTMLOptions) (*ExtractionResult, error)

ExtractHTML extracts structured data from raw HTML without making a network request.

func (*KolektrNamespace) GetAPIData added in v0.2.0 

func (k *KolektrNamespace) GetAPIData(ctx context.Context, apiEndpoint string) (map[string]interface{}, error)

GetAPIData gets all extracted data for a discovered API endpoint.

func (*KolektrNamespace) GetAPIDataPaginated added in v0.2.0 

func (k *KolektrNamespace) GetAPIDataPaginated(ctx context.Context, apiEndpoint string, offset, limit int) (map[string]interface{}, error)

GetAPIDataPaginated gets API data with pagination.

func (*KolektrNamespace) GetContent added in v0.2.0 

func (k *KolektrNamespace) GetContent(ctx context.Context, itemID string) (map[string]interface{}, error)

GetContent gets a content item by ID.

func (*KolektrNamespace) GetJob added in v0.2.0 

func (k *KolektrNamespace) GetJob(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJob gets an extraction job by ID.

func (*KolektrNamespace) GetJobProgress added in v0.2.0 

func (k *KolektrNamespace) GetJobProgress(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJobProgress gets job progress.

func (*KolektrNamespace) GetJobProgressEvents added in v0.2.0 

func (k *KolektrNamespace) GetJobProgressEvents(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJobProgressEvents gets job progress events.

func (*KolektrNamespace) GetJobProgressLatest added in v0.2.0 

func (k *KolektrNamespace) GetJobProgressLatest(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJobProgressLatest gets latest progress event for a job.

func (*KolektrNamespace) GetJobProgressSummary added in v0.2.0 

func (k *KolektrNamespace) GetJobProgressSummary(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJobProgressSummary gets job progress summary.

func (*KolektrNamespace) GetJobStatus added in v0.2.0 

func (k *KolektrNamespace) GetJobStatus(ctx context.Context, jobID string) (map[string]interface{}, error)

GetJobStatus gets extraction job status.

func (*KolektrNamespace) GetPipelineEvents added in v0.2.0 

func (k *KolektrNamespace) GetPipelineEvents(ctx context.Context, pipelineID string) (map[string]interface{}, error)

GetPipelineEvents gets events for a pipeline run.

func (*KolektrNamespace) GetPipelineStream added in v0.2.0 

func (k *KolektrNamespace) GetPipelineStream(ctx context.Context, pipelineID string) (map[string]interface{}, error)

GetPipelineStream streams pipeline data.

func (*KolektrNamespace) GetProgressPhase added in v0.2.0 

func (k *KolektrNamespace) GetProgressPhase(ctx context.Context, phaseName string) (map[string]interface{}, error)

GetProgressPhase gets info for a specific progress phase.

func (*KolektrNamespace) GetProgressPhaseSteps added in v0.2.0 

func (k *KolektrNamespace) GetProgressPhaseSteps(ctx context.Context, phaseName string) (map[string]interface{}, error)

GetProgressPhaseSteps gets steps for a progress phase.

func (*KolektrNamespace) GetProgressSummary added in v0.2.0 

func (k *KolektrNamespace) GetProgressSummary(ctx context.Context) (map[string]interface{}, error)

GetProgressSummary gets overall progress summary.

func (*KolektrNamespace) GetScraper added in v0.2.0 

func (k *KolektrNamespace) GetScraper(ctx context.Context, scraperID string) (map[string]interface{}, error)

GetScraper gets a scraper configuration by ID.

func (*KolektrNamespace) ListContent added in v0.2.0 

func (k *KolektrNamespace) ListContent(ctx context.Context) (map[string]interface{}, error)

ListContent lists content items.

func (*KolektrNamespace) ListJobs added in v0.2.0 

func (k *KolektrNamespace) ListJobs(ctx context.Context) (map[string]interface{}, error)

ListJobs lists extraction jobs.

func (*KolektrNamespace) ListProgressPhases added in v0.2.0 

func (k *KolektrNamespace) ListProgressPhases(ctx context.Context) (map[string]interface{}, error)

ListProgressPhases lists all progress phases.

func (*KolektrNamespace) ListScrapers added in v0.2.0 

func (k *KolektrNamespace) ListScrapers(ctx context.Context) (map[string]interface{}, error)

ListScrapers lists scraper configurations.

func (*KolektrNamespace) Page deprecated 

func (k *KolektrNamespace) Page(ctx context.Context, targetURL string, opts *ExtractOptions) (*ExtractionResult, error)

Page is the 0.2 name of Extract.

Deprecated: use Extract.

func (*KolektrNamespace) UpdateScraper added in v0.2.0 

func (k *KolektrNamespace) UpdateScraper(ctx context.Context, scraperID string, updates map[string]interface{}) (map[string]interface{}, error)

UpdateScraper updates a scraper configuration.

type MfaDetectionResult 

type MfaDetectionResult struct {
	MfaDetected bool
	ChallengeID *string
	MfaType     *string
	Message     *string
}

MfaDetectionResult is returned by Fetchyr.DetectMFA.

type NexusAnalyzeOptions 

type NexusAnalyzeOptions struct {
	Constraints map[string]interface{}
}

NexusAnalyzeOptions configures optional parameters for Nexus.Analyze.

type NexusAnalyzeResult 

type NexusAnalyzeResult struct {
	URL          string
	PerceptionID string
	ReasoningID  string
	Strategy     map[string]interface{}
	PageType     string
	Confidence   float64
	ArtifactID   *string
}

NexusAnalyzeResult is returned by Nexus.Analyze.

type NexusExecuteResult 

type NexusExecuteResult struct {
	StrategyID string
	Success    bool
	Records    []map[string]interface{}
	ArtifactID *string
	DurationMs int
	Error      *string
}

NexusExecuteResult is returned by Nexus.Execute.

type NexusKnowledgeResult 

type NexusKnowledgeResult struct {
	LearnedConcepts []map[string]interface{}
	Patterns        []map[string]interface{}
	TotalPatterns   int
	DurationMs      int
}

NexusKnowledgeResult is returned by Nexus.Knowledge.

type NexusNamespace 

type NexusNamespace struct {
	// contains filtered or unexported fields
}

NexusNamespace exposes the Strategy Engine module (5-layer cognitive pipeline). Access via client.Nexus.

func (*NexusNamespace) Analyze 

func (n *NexusNamespace) Analyze(ctx context.Context, targetURL, html string, opts *NexusAnalyzeOptions) (*NexusAnalyzeResult, error)

Analyze classifies a page and proposes an extraction strategy. The API needs the page's HTML (fetch it first, e.g. with Evadr.Fetch).

func (*NexusNamespace) CleanupCache added in v0.2.0 

func (n *NexusNamespace) CleanupCache(ctx context.Context) (map[string]interface{}, error)

CleanupCache cleans up recommendation cache.

func (*NexusNamespace) CreatePreference added in v0.2.0 

func (n *NexusNamespace) CreatePreference(ctx context.Context, preference map[string]interface{}) (map[string]interface{}, error)

CreatePreference creates a recommendation preference.

func (*NexusNamespace) DeletePreference added in v0.2.0 

func (n *NexusNamespace) DeletePreference(ctx context.Context, preferenceID string) error

DeletePreference deletes a recommendation preference.

func (*NexusNamespace) DisableHook added in v0.2.0 

func (n *NexusNamespace) DisableHook(ctx context.Context, hookName string) (map[string]interface{}, error)

DisableHook disables a recommendation hook.

func (*NexusNamespace) EnableHook added in v0.2.0 

func (n *NexusNamespace) EnableHook(ctx context.Context, hookName string) (map[string]interface{}, error)

EnableHook enables a recommendation hook.

func (*NexusNamespace) Execute 

func (n *NexusNamespace) Execute(ctx context.Context, strategyID, targetURL string, options map[string]interface{}) (*NexusExecuteResult, error)

Execute runs a verified strategy against a URL.

func (*NexusNamespace) GetCacheStatistics added in v0.2.0 

func (n *NexusNamespace) GetCacheStatistics(ctx context.Context) (map[string]interface{}, error)

GetCacheStatistics gets recommendation cache statistics.

func (*NexusNamespace) GetHooksStatus added in v0.2.0 

func (n *NexusNamespace) GetHooksStatus(ctx context.Context) (map[string]interface{}, error)

GetHooksStatus gets hooks status.

func (*NexusNamespace) GetPreferences added in v0.2.0 

func (n *NexusNamespace) GetPreferences(ctx context.Context, userID string) (map[string]interface{}, error)

GetPreferences gets user preferences.

func (*NexusNamespace) GetRecommendationStatistics added in v0.2.0 

func (n *NexusNamespace) GetRecommendationStatistics(ctx context.Context) (map[string]interface{}, error)

GetRecommendationStatistics gets recommendation statistics.

func (*NexusNamespace) InvalidateCache added in v0.2.0 

func (n *NexusNamespace) InvalidateCache(ctx context.Context) (map[string]interface{}, error)

InvalidateCache invalidates recommendation cache.

func (*NexusNamespace) Knowledge 

func (n *NexusNamespace) Knowledge(ctx context.Context, executionResultID string) (*NexusKnowledgeResult, error)

Knowledge stores learned concepts and patterns from an execution result.

func (*NexusNamespace) ListRecommendationApplications added in v0.2.0 

func (n *NexusNamespace) ListRecommendationApplications(ctx context.Context) (map[string]interface{}, error)

ListRecommendationApplications lists recommendation applications.

func (*NexusNamespace) Reason added in v0.2.0 

func (n *NexusNamespace) Reason(ctx context.Context, context_ map[string]interface{}) (map[string]interface{}, error)

Reason performs logical reasoning via the Strategy Engine.

func (*NexusNamespace) RecommendAnalyze added in v0.2.0 

func (n *NexusNamespace) RecommendAnalyze(ctx context.Context, data map[string]interface{}) (map[string]interface{}, error)

RecommendAnalyze analyzes and gets recommendations.

func (*NexusNamespace) Synthesize 

func (n *NexusNamespace) Synthesize(ctx context.Context, targetURL, html, goal string) (*NexusSynthesisResult, error)

Synthesize generates an extraction program for a page and a goal.

func (*NexusNamespace) UpdatePreference added in v0.2.0 

func (n *NexusNamespace) UpdatePreference(ctx context.Context, preferenceID string, data map[string]interface{}) (map[string]interface{}, error)

UpdatePreference updates a recommendation preference.

func (*NexusNamespace) Verify 

func (n *NexusNamespace) Verify(ctx context.Context, strategyID string) (*NexusVerifyResult, error)

Verify verifies the safety of a strategy before execution.

type NexusSynthesisResult 

type NexusSynthesisResult struct {
	StrategyID      string
	StrategyName    string
	StrategyType    string
	Program         string
	Confidence      float64
	Steps           []map[string]interface{}
	SynthesisTimeMs int
}

NexusSynthesisResult is returned by Nexus.Synthesize.

type NexusVerifyResult 

type NexusVerifyResult struct {
	StrategyID  string
	Verified    bool
	SafetyScore float64
	Violations  []string
}

NexusVerifyResult is returned by Nexus.Verify.

type NotEntitledError 

type NotEntitledError struct {
	KloakdError
	Module     string
	UpgradeURL string
}

NotEntitledError is returned for HTTP 403 — tenant not on required plan.

func (*NotEntitledError) Error 

func (e *NotEntitledError) Error() string

type PageNode 

type PageNode struct {
	ID        string
	URL       string
	ParentURL *string
	Depth     int
	Category  string
	Title     *string
	Children  []string
	Metadata  map[string]interface{}
}

PageNode represents a single page in the site hierarchy.

type PageOptions deprecated 

type PageOptions = ExtractOptions

PageOptions is the 0.2 name of ExtractOptions.

Deprecated: use ExtractOptions.

type ParlyrNamespace 

type ParlyrNamespace struct {
	// contains filtered or unexported fields
}

ParlyrNamespace exposes the Conversational NLP module. Access via client.Parlyr.

func (*ParlyrNamespace) Chat 

func (p *ParlyrNamespace) Chat(ctx context.Context, sessionID, message string) (*ChatTurn, error)

Chat sends a message to a conversation session and blocks until the full response is collected from the SSE stream.

func (*ParlyrNamespace) ChatStream 

func (p *ParlyrNamespace) ChatStream(ctx context.Context, sessionID, message string) (<-chan ChatEvent, error)

ChatStream opens an SSE stream for a conversation turn. Returns a read-only channel of ChatEvent values.

func (*ParlyrNamespace) DeleteSession 

func (p *ParlyrNamespace) DeleteSession(ctx context.Context, sessionID string) error

DeleteSession removes a conversation session.

func (*ParlyrNamespace) Parse 

func (p *ParlyrNamespace) Parse(ctx context.Context, message string, opts *ParseOptions) (*ParseResult, error)

Parse interprets a natural language message.

type ParseOptions 

type ParseOptions struct {
	SessionID string
}

ParseOptions configures optional parameters for Parlyr.Parse.

type ParseResult 

type ParseResult struct {
	Intent              string
	Confidence          float64
	Tier                int
	Source              string
	Entities            map[string]interface{}
	RequiresAction      bool
	ClarificationNeeded *string
	Reasoning           *string
	DetectedURL         *string
}

ParseResult is returned by Parlyr.Parse.

type PaymentRequiredError added in v0.3.0 

type PaymentRequiredError struct {
	KloakdError
	UpgradeURL string
}

PaymentRequiredError is returned for HTTP 402: the organisation's paid plan has a failed payment; new billable work is refused until it is paid.

func (*PaymentRequiredError) Error added in v0.3.0 

func (e *PaymentRequiredError) Error() string

type QuotaExceededError added in v0.3.0 

type QuotaExceededError struct {
	KloakdError
	UpgradeURL         string
	GBUsed             float64
	GBLimit            float64
	PurchasedGBBalance float64
}

QuotaExceededError is returned for HTTP 429 "gb_exhausted": the plan's GB allocation and this period's top-ups are used up. Retrying does not help; buy a top-up or upgrade.

func (*QuotaExceededError) Error added in v0.3.0 

func (e *QuotaExceededError) Error() string

type RateLimitError 

type RateLimitError struct {
	KloakdError
	RetryAfter int    // seconds
	ResetAt    string // ISO timestamp
}

RateLimitError is returned for HTTP 429 — quota exceeded.

func (*RateLimitError) Error 

func (e *RateLimitError) Error() string

type SearchOptions added in v0.3.0 

type SearchOptions struct {
	SiteID   string
	MimeType string
	Limit    int
	Offset   int
}

SearchOptions configures optional parameters for Skanyr.Search.

type SessionResult 

type SessionResult struct {
	Success      bool
	SessionID    string
	ArtifactID   *string
	TargetDomain *string
	AuthMethod   *string
	Error        *string
}

SessionResult is returned by Fetchyr.Authenticate.

type SiteCrawlResult added in v0.3.0 

type SiteCrawlResult struct {
	Success              bool
	URL                  string
	TotalPagesDiscovered int
	PagesFetched         int
	PagesFailed          int
	Pages                []CrawlPage
	CrawlArtifactID      *string
	Error                *string
}

SiteCrawlResult is returned by Client.Crawl.

type SkanyrNamespace 

type SkanyrNamespace struct {
	// contains filtered or unexported fields
}

SkanyrNamespace exposes the Site Knowledge module: synced sites, their resources, extraction and search.

In 0.3.0 the discovery methods of 0.2 (Discover, DiscoverStream, AnalyzeBundle, Hierarchy, ExpandNode, Retry, Health, GetApiMap, GetJob, GetDiscovery*) are removed: the API no longer serves those routes. The session, detected-API and reader-view methods moved to Webgrph. Access via client.Skanyr.

func (*SkanyrNamespace) CreateSiteFromCrawl added in v0.3.0 

func (s *SkanyrNamespace) CreateSiteFromCrawl(ctx context.Context, crawlID string) (map[string]interface{}, error)

CreateSiteFromCrawl registers a site from a finished Webgrph crawl.

func (*SkanyrNamespace) DeleteSite added in v0.3.0 

func (s *SkanyrNamespace) DeleteSite(ctx context.Context, siteID string) error

DeleteSite deletes a synced site.

func (*SkanyrNamespace) Extract added in v0.3.0 

func (s *SkanyrNamespace) Extract(ctx context.Context, contentHash, mimeType string, metadata map[string]interface{}) (map[string]interface{}, error)

Extract extracts text and structure from stored content.

func (*SkanyrNamespace) GetSite added in v0.3.0 

func (s *SkanyrNamespace) GetSite(ctx context.Context, siteID string) (map[string]interface{}, error)

GetSite gets a synced site.

func (*SkanyrNamespace) ListSites added in v0.3.0 

func (s *SkanyrNamespace) ListSites(ctx context.Context) (map[string]interface{}, error)

ListSites lists the organisation's synced sites (under "items").

func (*SkanyrNamespace) Search added in v0.3.0 

func (s *SkanyrNamespace) Search(ctx context.Context, query string, opts *SearchOptions) (map[string]interface{}, error)

Search searches extracted content.

func (*SkanyrNamespace) SetSitePolicy added in v0.3.0 

func (s *SkanyrNamespace) SetSitePolicy(ctx context.Context, siteID, policy string) (map[string]interface{}, error)

SetSitePolicy sets a site's sync policy.

func (*SkanyrNamespace) SiteAPIs added in v0.3.0 

func (s *SkanyrNamespace) SiteAPIs(ctx context.Context, siteID string) (map[string]interface{}, error)

SiteAPIs lists the APIs found on a site.

func (*SkanyrNamespace) SiteResources added in v0.3.0 

func (s *SkanyrNamespace) SiteResources(ctx context.Context, siteID string) (map[string]interface{}, error)

SiteResources lists a site's resources.

func (*SkanyrNamespace) Sync added in v0.3.0 

func (s *SkanyrNamespace) Sync(ctx context.Context, siteID string, opts *SyncOptions) (map[string]interface{}, error)

Sync re-fetches a site's changed resources.

type SyncOptions added in v0.3.0 

type SyncOptions struct {
	URLs   []string
	Policy string
}

SyncOptions configures optional parameters for Skanyr.Sync.

type UpstreamError 

type UpstreamError struct {
	KloakdError
}

UpstreamError is returned for HTTP 502 — upstream site fetch failed.

func (*UpstreamError) Error 

func (e *UpstreamError) Error() string

type WebgrphNamespace 

type WebgrphNamespace struct {
	// contains filtered or unexported fields
}

WebgrphNamespace exposes the Site Mapping & Discovery module. Access via client.Webgrph.

func (*WebgrphNamespace) CancelCrawl added in v0.3.0 

func (w *WebgrphNamespace) CancelCrawl(ctx context.Context, crawlID string) (map[string]interface{}, error)

CancelCrawl cancels a running crawl.

func (*WebgrphNamespace) Crawl 

func (w *WebgrphNamespace) Crawl(ctx context.Context, targetURL string, opts *CrawlOptions) (*CrawlResult, error)

Crawl starts a site crawl, waits for it to finish, and returns every discovered page (GET webgrph/crawl/{id}/pages, followed to the end).

func (*WebgrphNamespace) CrawlAll 

func (w *WebgrphNamespace) CrawlAll(ctx context.Context, targetURL string, opts *CrawlOptions) ([]PageNode, error)

CrawlAll is Crawl, returning only the pages.

func (*WebgrphNamespace) CrawlEvents added in v0.3.0 

func (w *WebgrphNamespace) CrawlEvents(ctx context.Context, crawlID string) (<-chan map[string]interface{}, error)

CrawlEvents streams a crawl's events (SSE).

func (*WebgrphNamespace) CrawlStream 

func (w *WebgrphNamespace) CrawlStream(ctx context.Context, targetURL string, opts *CrawlStreamOptions) (<-chan CrawlEvent, error)

CrawlStream starts a crawl and streams its events (GET webgrph/crawl/{crawl_id}/events). The channel closes when the crawl ends or ctx is cancelled.

func (*WebgrphNamespace) DeleteSession added in v0.3.0 

func (w *WebgrphNamespace) DeleteSession(ctx context.Context, sessionID string) error

DeleteSession deletes a discovery session.

func (*WebgrphNamespace) DetectedAPIs added in v0.3.0 

func (w *WebgrphNamespace) DetectedAPIs(ctx context.Context, pageURL string) (map[string]interface{}, error)

DetectedAPIs lists the APIs detected on a page.

func (*WebgrphNamespace) EndSession added in v0.3.0 

func (w *WebgrphNamespace) EndSession(ctx context.Context, sessionID string) (map[string]interface{}, error)

EndSession ends a discovery session.

func (*WebgrphNamespace) GetCrawlPages added in v0.2.0 

func (w *WebgrphNamespace) GetCrawlPages(ctx context.Context, crawlID, cursor string, limit int) (map[string]interface{}, error)

GetCrawlPages gets one page of discovered pages (cursor "" for the first).

func (*WebgrphNamespace) GetCrawlStatus added in v0.2.0 

func (w *WebgrphNamespace) GetCrawlStatus(ctx context.Context, crawlID string) (map[string]interface{}, error)

GetCrawlStatus polls crawl status and stats.

func (*WebgrphNamespace) GetDashboardSummary added in v0.2.0 

func (w *WebgrphNamespace) GetDashboardSummary(ctx context.Context) (map[string]interface{}, error)

GetDashboardSummary gets analytics dashboard summary.

func (*WebgrphNamespace) GetDiscoveryPatterns added in v0.2.0 

func (w *WebgrphNamespace) GetDiscoveryPatterns(ctx context.Context) (map[string]interface{}, error)

GetDiscoveryPatterns gets content discovery pattern analytics.

func (*WebgrphNamespace) GetEfficiencyMetrics added in v0.2.0 

func (w *WebgrphNamespace) GetEfficiencyMetrics(ctx context.Context) (map[string]interface{}, error)

GetEfficiencyMetrics gets scraping efficiency metrics.

func (*WebgrphNamespace) GetErrorSummary added in v0.2.0 

func (w *WebgrphNamespace) GetErrorSummary(ctx context.Context) (map[string]interface{}, error)

GetErrorSummary gets error summary analytics.

func (*WebgrphNamespace) GetJobTrends added in v0.2.0 

func (w *WebgrphNamespace) GetJobTrends(ctx context.Context) (map[string]interface{}, error)

GetJobTrends gets job trends analytics.

func (*WebgrphNamespace) GetSession added in v0.3.0 

func (w *WebgrphNamespace) GetSession(ctx context.Context, sessionID string) (map[string]interface{}, error)

GetSession gets a discovery session.

func (*WebgrphNamespace) GetSiteMappingTrends added in v0.2.0 

func (w *WebgrphNamespace) GetSiteMappingTrends(ctx context.Context) (map[string]interface{}, error)

GetSiteMappingTrends gets site mapping trends.

func (*WebgrphNamespace) GetUserBehaviorInsights added in v0.2.0 

func (w *WebgrphNamespace) GetUserBehaviorInsights(ctx context.Context) (map[string]interface{}, error)

GetUserBehaviorInsights gets user behavior insights.

func (*WebgrphNamespace) ListSessions added in v0.3.0 

func (w *WebgrphNamespace) ListSessions(ctx context.Context, activeOnly bool) (map[string]interface{}, error)

ListSessions lists discovery sessions (items under "items").

func (*WebgrphNamespace) ReaderView added in v0.3.0 

func (w *WebgrphNamespace) ReaderView(ctx context.Context, targetURL, crawlID string) (map[string]interface{}, error)

ReaderView returns a page's readable content (from a crawl's cache when crawlID is given).

func (*WebgrphNamespace) SaveSession added in v0.3.0 

func (w *WebgrphNamespace) SaveSession(ctx context.Context, session map[string]interface{}) (map[string]interface{}, error)

SaveSession creates or updates a discovery session ("url" is required).

func (*WebgrphNamespace) StartCrawl added in v0.3.0 

func (w *WebgrphNamespace) StartCrawl(ctx context.Context, targetURL string, opts *CrawlOptions) (*CrawlResult, error)

StartCrawl starts a site crawl and returns at once (HTTP 202) with its crawl ID; follow it with GetCrawlStatus, CrawlEvents or Crawl.

func (*WebgrphNamespace) UpdateSessionJob added in v0.3.0 

func (w *WebgrphNamespace) UpdateSessionJob(ctx context.Context, sessionID, jobID string) (map[string]interface{}, error)

UpdateSessionJob links a job to a discovery session.

type WorkflowResult 

type WorkflowResult struct {
	WorkflowID   string
	Name         string
	Description  *string
	WorkflowType string
	Steps        []map[string]interface{}
	IsActive     bool
	CreatedAt    string
	Error        *string
}

WorkflowResult is returned by Fetchyr.CreateWorkflow.

Source Files

View all Source files

Directories

Path Synopsis
examples
quickstart command
Quickstart: Discover and extract from a site in ~10 lines.
 Quickstart: Discover and extract from a site in ~10 lines.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.