Documentation
¶
Overview ¶
Example (Err_not_ready_basic_handling) ¶
Example_err_not_ready_basic_handling demonstrates basic handling of ErrNotReady
ctx := context.Background()
// Create a server that's slow to respond
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
time.Sleep(2 * time.Second)
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}))
defer srv.Close()
cl := httprc.NewClient()
ctrl, err := cl.Start(ctx)
if err != nil {
fmt.Println("Failed to start client:", err)
return
}
defer ctrl.Shutdown(time.Second)
resource, err := httprc.NewResource[map[string]string](
srv.URL,
httprc.JSONTransformer[map[string]string](),
)
if err != nil {
fmt.Println("Failed to create resource:", err)
return
}
// Add with timeout - will return ErrNotReady
addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
defer cancel()
err = ctrl.Add(addCtx, resource)
if err != nil {
if errors.Is(err, httprc.ErrNotReady()) {
// Resource registered, will fetch in background
fmt.Println("Resource registered but not ready yet")
fmt.Println("Safe to continue with application startup")
return
}
// Registration failed
fmt.Println("Failed to register resource:", err)
return
}
// Resource registered AND ready with data
fmt.Println("Resource ready")
Output: Resource registered but not ready yet Safe to continue with application startup
Example (Err_not_ready_checking_underlying_error) ¶
Example_err_not_ready_checking_underlying_error demonstrates how to check the underlying error wrapped by ErrNotReady
ctx := context.Background()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
time.Sleep(2 * time.Second)
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}))
defer srv.Close()
cl := httprc.NewClient()
ctrl, err := cl.Start(ctx)
if err != nil {
fmt.Println("Failed to start client:", err)
return
}
defer ctrl.Shutdown(time.Second)
resource, err := httprc.NewResource[map[string]string](
srv.URL,
httprc.JSONTransformer[map[string]string](),
)
if err != nil {
fmt.Println("Failed to create resource:", err)
return
}
// Add with timeout
addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
defer cancel()
err = ctrl.Add(addCtx, resource)
if err != nil {
if errors.Is(err, httprc.ErrNotReady()) {
// Resource registered, check why it's not ready
// errors.Is() automatically unwraps the error chain
if errors.Is(err, context.DeadlineExceeded) {
fmt.Println("Resource registered but timed out waiting for data")
fmt.Println("Will continue fetching in background")
} else {
fmt.Printf("Resource registered but not ready: %v\n", err)
}
return
}
// Registration failed
fmt.Println("Registration failed:", err)
return
}
fmt.Println("Resource ready")
Output: Resource registered but timed out waiting for data Will continue fetching in background
Example (Err_not_ready_retry_logic) ¶
Example_err_not_ready_retry_logic demonstrates proper retry logic that distinguishes between registration failures and ErrNotReady
ctx := context.Background()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
time.Sleep(2 * time.Second)
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
}))
defer srv.Close()
cl := httprc.NewClient()
ctrl, err := cl.Start(ctx)
if err != nil {
fmt.Println("Failed to start client:", err)
return
}
defer ctrl.Shutdown(time.Second)
var resource httprc.Resource
url := srv.URL
// Retry logic: only retry registration failures
for attempt := 1; attempt <= 3; attempt++ {
resource, err = httprc.NewResource[map[string]string](
url,
httprc.JSONTransformer[map[string]string](),
)
if err != nil {
fmt.Printf("Attempt %d: failed to create resource: %v\n", attempt, err)
continue
}
addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
err = ctrl.Add(addCtx, resource)
cancel()
if err == nil {
// Success - registered and ready
fmt.Println("Resource registered and ready")
return
}
if errors.Is(err, httprc.ErrNotReady()) {
// Registered successfully, just not ready yet
// Don't retry Add() - it would fail with duplicate URL
fmt.Printf("Attempt %d: Resource registered, not ready yet\n", attempt)
fmt.Println("Resource will fetch in background, continuing...")
return
}
// Registration failed - retry
fmt.Printf("Attempt %d: Registration failed: %v\n", attempt, err)
if attempt < 3 {
time.Sleep(time.Second * time.Duration(attempt))
}
}
Output: Attempt 1: Resource registered, not ready yet Resource will fetch in background, continuing...
Index ¶
- Constants
- func ErrAlreadyRunning() error
- func ErrBlockedByWhitelist() error
- func ErrNotReady() error
- func ErrRecoveredFromPanic() error
- func ErrResourceAlreadyExists() error
- func ErrResourceNotFound() error
- func ErrTransformerFailed() error
- func ErrTransformerRequired() error
- func ErrURLCannotBeEmpty() error
- func ErrUnexpectedStatusCode() error
- type AddOption
- type BlockAllWhitelist
- type Client
- type Controller
- type ErrorSink
- type HTTPClient
- type InsecureWhitelist
- type MapWhitelist
- type NewClientOption
- type NewClientResourceOption
- type NewResourceOption
- type RegexpWhitelist
- type Resource
- type ResourceBase
- func (r *ResourceBase[T]) ConstantInterval() time.Duration
- func (r *ResourceBase[T]) Get(dst any) error
- func (r *ResourceBase[T]) IsBusy() bool
- func (r *ResourceBase[T]) MaxInterval() time.Duration
- func (r *ResourceBase[T]) MinInterval() time.Duration
- func (r *ResourceBase[T]) Next() time.Time
- func (r *ResourceBase[T]) Ready(ctx context.Context) error
- func (r *ResourceBase[T]) Resource() T
- func (r *ResourceBase[T]) SetBusy(v bool)
- func (r *ResourceBase[T]) SetMaxInterval(v time.Duration)
- func (r *ResourceBase[T]) SetMinInterval(v time.Duration)
- func (r *ResourceBase[T]) SetNext(v time.Time)
- func (r *ResourceBase[T]) Sync(ctx context.Context) error
- func (r *ResourceBase[T]) URL() string
- type TraceSink
- type TransformFunc
- type Transformer
- type Whitelist
- type WhitelistFunc
Examples ¶
Constants ¶
const ( // ReadBufferSize is the default buffer size for reading HTTP responses (10MB) ReadBufferSize = 1024 * 1024 * 10 // MaxBufferSize is the maximum allowed buffer size (1GB) MaxBufferSize = 1024 * 1024 * 1000 )
Buffer size constants
const ( // DefaultMaxInterval is the default maximum interval between fetches (30 days) DefaultMaxInterval = 24 * time.Hour * 30 // DefaultMinInterval is the default minimum interval between fetches (15 minutes) DefaultMinInterval = 15 * time.Minute )
Interval constants
const (
// DefaultWorkers is the default number of worker goroutines
DefaultWorkers = 5
)
Client worker constants
Variables ¶
This section is empty.
Functions ¶
func ErrAlreadyRunning ¶
func ErrAlreadyRunning() error
func ErrBlockedByWhitelist ¶
func ErrBlockedByWhitelist() error
func ErrNotReady ¶ added in v3.0.3
func ErrNotReady() error
ErrNotReady returns a sentinel error indicating that the resource was successfully registered with the backend and is being actively managed, but the first fetch and transformation has not completed successfully yet.
This error is returned by Add() when:
- The resource was successfully added to the backend (registration succeeded)
- WithWaitReady(true) was specified (the default)
- The Ready() call failed (timeout, transform error, context cancelled, etc.)
When Add() returns this error, the resource IS in the backend's resource map and will continue to be fetched periodically in the background according to the refresh interval. The application can safely proceed - the resource data may become available later when a fetch succeeds.
IMPORTANT: "Not ready" means the first fetch and transformation has not completed successfully. The resource may eventually become ready (if the transformation succeeds on a subsequent retry), or it may never become ready (if the data is permanently invalid or the server is unreachable). The backend will continue retrying according to the configured refresh interval.
The underlying error (context deadline, transform failure, etc.) is wrapped using Go 1.20+ multiple error wrapping and can be examined with errors.Is() or errors.As(). You do not need to manually unwrap the error.
Example:
err := ctrl.Add(ctx, resource)
if err != nil {
if errors.Is(err, httprc.ErrNotReady()) {
// Resource registered, will fetch in background
log.Print("Resource not ready yet, continuing startup")
// Can also check the underlying cause
if errors.Is(err, context.DeadlineExceeded) {
log.Print("Timed out waiting for first fetch")
}
return nil
}
// Registration failed
return fmt.Errorf("failed to register resource: %w", err)
}
// Resource registered AND ready with data
func ErrRecoveredFromPanic ¶
func ErrRecoveredFromPanic() error
func ErrResourceAlreadyExists ¶
func ErrResourceAlreadyExists() error
func ErrResourceNotFound ¶
func ErrResourceNotFound() error
func ErrTransformerFailed ¶
func ErrTransformerFailed() error
func ErrTransformerRequired ¶
func ErrTransformerRequired() error
func ErrURLCannotBeEmpty ¶
func ErrURLCannotBeEmpty() error
func ErrUnexpectedStatusCode ¶
func ErrUnexpectedStatusCode() error
Types ¶
type AddOption ¶
func WithWaitReady ¶
WithWaitReady specifies whether the client should wait for the resource to be ready before returning from the Add method.
By default, the client will wait for the resource to be ready before returning. If you specify this option with a value of false, the client will not wait for the resource to be fully registered, which is usually not what you want. This option exists to accommodate for cases where you for some reason want to add a resource to the controller, but want to do something else before you wait for it. Make sure to call `r.Ready()` later on to ensure that the resource is ready before you try to access it.
type BlockAllWhitelist ¶
type BlockAllWhitelist struct{}
BlockAllWhitelist is a Whitelist implementation that blocks all URLs.
func NewBlockAllWhitelist ¶
func NewBlockAllWhitelist() BlockAllWhitelist
NewBlockAllWhitelist creates a new BlockAllWhitelist instance. It is safe to use the zero value of this type; this constructor is provided for consistency.
func (BlockAllWhitelist) IsAllowed ¶
func (BlockAllWhitelist) IsAllowed(_ string) bool
type Client ¶
type Client struct {
// contains filtered or unexported fields
}
Client is the main entry point for the httprc package.
Example ¶
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
type HelloWorld struct {
Hello string `json:"hello"`
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
json.NewEncoder(w).Encode(map[string]string{"hello": "world"})
}))
options := []httprc.NewClientOption{
// By default the client will allow all URLs (which is what the option
// below is explicitly specifying). If you want to restrict what URLs
// are allowed, you can specify another whitelist.
//
// httprc.WithWhitelist(httprc.NewInsecureWhitelist()),
}
// If you would like to handle errors from asynchronous workers, you can specify a error sink.
// This is disabled in this example because the trace logs are dynamic
// and thus would interfere with the runnable example test.
// options = append(options, httprc.WithErrorSink(errsink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))))
// If you would like to see the trace logs, you can specify a trace sink.
// This is disabled in this example because the trace logs are dynamic
// and thus would interfere with the runnable example test.
// options = append(options, httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))))
// Create a new client
cl := httprc.NewClient(options...)
// Start the client, and obtain a Controller object
ctrl, err := cl.Start(ctx)
if err != nil {
fmt.Println(err.Error())
return
}
// The following is required if you want to make sure that there are no
// dangling goroutines hanging around when you exit. For example, if you
// are running tests to check for goroutine leaks, you should call this
// function before the end of your test.
defer ctrl.Shutdown(time.Second)
// Create a new resource that is synchronized every so often
//
// By default the client will attempt to fetch the resource once
// as soon as it can, and then if no other metadata is provided,
// it will fetch the resource every 15 minutes.
//
// If the resource responds with a Cache-Control/Expires header,
// the client will attempt to respect that, and will try to fetch
// the resource again based on the values obatained from the headers.
r, err := httprc.NewResource[HelloWorld](srv.URL, httprc.JSONTransformer[HelloWorld]())
if err != nil {
fmt.Println(err.Error())
return
}
// Add the resource to the controller, so that it starts fetching.
// By default, a call to `Add()` will block until the first fetch
// succeeds, via an implicit call to `r.Ready()`
// You can change this behavior if you specify the `WithWaitReady(false)`
// option.
ctrl.Add(ctx, r)
// if you specified `httprc.WithWaitReady(false)` option, the fetch will happen
// "soon", but you're not guaranteed that it will happen before the next
// call to `Lookup()`. If you want to make sure that the resource is ready,
// you can call `Ready()` like so:
/*
{
tctx, tcancel := context.WithTimeout(ctx, time.Second)
defer tcancel()
if err := r.Ready(tctx); err != nil {
fmt.Println(err.Error())
return
}
}
*/
m := r.Resource()
fmt.Println(m.Hello)
Output: world
func NewClient ¶
func NewClient(options ...NewClientOption) *Client
NewClient creates a new `httprc.Client` object.
By default ALL urls are allowed. This may not be suitable for you if are using this in a production environment. You are encouraged to specify a whitelist using the `WithWhitelist` option.
NOTE: In future versions, this function signature should be changed to return an error to properly handle option parsing failures.
func (*Client) Start ¶
func (c *Client) Start(octx context.Context) (Controller, error)
Start sets the client into motion. It will start a number of worker goroutines, and return a Controller object that you can use to control the execution of the client.
If you attempt to call Start more than once, it will return an error.
type Controller ¶
type Controller interface {
// Add adds a new `http.Resource` to the controller. If the resource already exists,
// it will return an error.
Add(context.Context, Resource, ...AddOption) error
// Lookup a `httprc.Resource` by its URL. If the resource does not exist, it
// will return an error.
Lookup(context.Context, string) (Resource, error)
// Remove a `httprc.Resource` from the controller by its URL. If the resource does
// not exist, it will return an error.
Remove(context.Context, string) error
// Refresh forces a resource to be refreshed immediately. If the resource does
// not exist, or if the refresh fails, it will return an error.
Refresh(context.Context, string) error
ShutdownContext(context.Context) error
Shutdown(time.Duration) error
}
type HTTPClient ¶
HTTPClient is an interface that abstracts a "net/http".Client, so that users can provide their own implementation of the HTTP client, if need be.
type InsecureWhitelist ¶
type InsecureWhitelist struct{}
InsecureWhitelist is a Whitelist implementation that allows all URLs. Be careful when using this in your production code: make sure you do not blindly register URLs from untrusted sources.
func NewInsecureWhitelist ¶
func NewInsecureWhitelist() InsecureWhitelist
NewInsecureWhitelist creates a new InsecureWhitelist instance. It is safe to use the zero value of this type; this constructor is provided for consistency.
func (InsecureWhitelist) IsAllowed ¶
func (InsecureWhitelist) IsAllowed(_ string) bool
type MapWhitelist ¶
type MapWhitelist interface {
Whitelist
Add(string) MapWhitelist
}
MapWhitelist is a jwk.Whitelist object comprised of a map of strings. If the URL exists in the map, then the URL is allowed to be fetched.
func NewMapWhitelist ¶
func NewMapWhitelist() MapWhitelist
type NewClientOption ¶
func WithErrorSink ¶
func WithErrorSink(sink ErrorSink) NewClientOption
WithErrorSink specifies the error sink to use for the client. If not specified, the client will use a NopErrorSink.
func WithTraceSink ¶
func WithTraceSink(sink TraceSink) NewClientOption
WithTraceSink specifies the trace sink to use for the client. If not specified, the client will use a NopTraceSink.
func WithWhitelist ¶
func WithWhitelist(wl Whitelist) NewClientOption
WithWhitelist specifies the whitelist to use for the client. If not specified, the client will use a BlockAllWhitelist.
func WithWorkers ¶
func WithWorkers(n int) NewClientOption
WithWorkers specifies the number of concurrent workers to use for the client. If n is less than or equal to 0, the client will use a single worker.
type NewClientResourceOption ¶
type NewClientResourceOption interface {
option.Interface
// contains filtered or unexported methods
}
func WithHTTPClient ¶
func WithHTTPClient(cl HTTPClient) NewClientResourceOption
WithHTTPClient specifies the HTTP client to use for the client. If not specified, the client will use http.DefaultClient.
This option can be passed to NewClient or NewResource.
type NewResourceOption ¶
func WithConstantInterval ¶
func WithConstantInterval(d time.Duration) NewResourceOption
WithConstantInterval specifies the interval between fetches. When you specify this option, the client will fetch the resource at the specified intervals, regardless of the response's Cache-Control or Expires headers.
By default this option is disabled.
func WithMaxInterval ¶
func WithMaxInterval(d time.Duration) NewResourceOption
WithMaxInterval specifies the maximum interval between fetches.
This option affects the dynamic calculation of the interval between fetches. If the value calculated from the http.Response is greater than the this value, the client will use this value instead.
func WithMinInterval ¶
func WithMinInterval(d time.Duration) NewResourceOption
WithMinInterval specifies the minimum interval between fetches.
This option affects the dynamic calculation of the interval between fetches. If the value calculated from the http.Response is less than the this value, the client will use this value instead.
type RegexpWhitelist ¶
type RegexpWhitelist struct {
// contains filtered or unexported fields
}
RegexpWhitelist is a jwk.Whitelist object comprised of a list of *regexp.Regexp objects. All entries in the list are tried until one matches. If none of the *regexp.Regexp objects match, then the URL is deemed unallowed.
func NewRegexpWhitelist ¶
func NewRegexpWhitelist() *RegexpWhitelist
NewRegexpWhitelist creates a new RegexpWhitelist instance. It is safe to use the zero value of this type; this constructor is provided for consistency.
func (*RegexpWhitelist) Add ¶
func (w *RegexpWhitelist) Add(pat *regexp.Regexp) *RegexpWhitelist
Add adds a new regular expression to the list of expressions to match against.
func (*RegexpWhitelist) IsAllowed ¶
func (w *RegexpWhitelist) IsAllowed(u string) bool
IsAllowed returns true if any of the patterns in the whitelist returns true.
type Resource ¶
type Resource interface {
Get(any) error
Next() time.Time
SetNext(time.Time)
URL() string
Sync(context.Context) error
ConstantInterval() time.Duration
MaxInterval() time.Duration
SetMaxInterval(time.Duration)
MinInterval() time.Duration
SetMinInterval(time.Duration)
IsBusy() bool
SetBusy(bool)
Ready(context.Context) error
}
Resource is a single resource that can be retrieved via HTTP, and (possibly) transformed into an arbitrary object type.
Realistically, there is no need for third-parties to implement this interface. This exists to provide a way to aggregate `httprc.ResourceBase` objects with different specialized types into a single collection.
See ResourceBase for details
type ResourceBase ¶
type ResourceBase[T any] struct { // contains filtered or unexported fields }
ResourceBase is a generic Resource type
func NewResource ¶
func NewResource[T any](s string, transformer Transformer[T], options ...NewResourceOption) (*ResourceBase[T], error)
NewResource creates a new Resource object which after fetching the resource from the URL, will transform the response body using the provided Transformer to an object of type T.
This function will return an error if the URL is not a valid URL (i.e. it cannot be parsed by url.Parse), or if the transformer is nil.
func (*ResourceBase[T]) ConstantInterval ¶
func (r *ResourceBase[T]) ConstantInterval() time.Duration
func (*ResourceBase[T]) Get ¶
func (r *ResourceBase[T]) Get(dst any) error
Get assigns the value of the resource to the provided pointer. If using the `httprc.ResourceBase[T]` type directly, you can use the `Resource()` method to get the resource directly.
This method exists because parametric types cannot be assigned to a single object type that return different return values of the specialized type. i.e. for resources `ResourceBase[A]` and `ResourceBase[B]`, we cannot have a single interface that can be assigned to the same interface type `X` that expects a `Resource()` method that returns `A` or `B` depending on the type of the resource. When accessing the resource through the `httprc.Resource` interface, use this method to obtain the stored value.
func (*ResourceBase[T]) IsBusy ¶
func (r *ResourceBase[T]) IsBusy() bool
func (*ResourceBase[T]) MaxInterval ¶
func (r *ResourceBase[T]) MaxInterval() time.Duration
func (*ResourceBase[T]) MinInterval ¶
func (r *ResourceBase[T]) MinInterval() time.Duration
func (*ResourceBase[T]) Next ¶
func (r *ResourceBase[T]) Next() time.Time
func (*ResourceBase[T]) Ready ¶
func (r *ResourceBase[T]) Ready(ctx context.Context) error
Ready returns an empty error when the resource is ready. If the context is canceled before the resource is ready, it will return the error from the context.
func (*ResourceBase[T]) Resource ¶
func (r *ResourceBase[T]) Resource() T
Resource returns the last fetched resource. If the resource has not been fetched yet, this will return the zero value of type T.
If you would rather wait until the resource is fetched, you can use the `Ready()` method to wait until the resource is ready (i.e. fetched at least once).
func (*ResourceBase[T]) SetBusy ¶
func (r *ResourceBase[T]) SetBusy(v bool)
func (*ResourceBase[T]) SetMaxInterval ¶
func (r *ResourceBase[T]) SetMaxInterval(v time.Duration)
func (*ResourceBase[T]) SetMinInterval ¶
func (r *ResourceBase[T]) SetMinInterval(v time.Duration)
func (*ResourceBase[T]) SetNext ¶
func (r *ResourceBase[T]) SetNext(v time.Time)
func (*ResourceBase[T]) URL ¶
func (r *ResourceBase[T]) URL() string
URL returns the URL of the resource.
type TransformFunc ¶
TransformFunc is a function type that implements the Transformer interface.
type Transformer ¶
Transformer is used to convert the body of an HTTP response into an appropriate object of type T.
func BytesTransformer ¶
func BytesTransformer() Transformer[[]byte]
BytesTransformer returns a Transformer that reads the entire response body as a byte slice. This is the default Transformer used by httprc.Client
func JSONTransformer ¶
func JSONTransformer[T any]() Transformer[T]
JSONTransformer returns a Transformer that decodes the response body as JSON into the provided type T.
type Whitelist ¶
Whitelist is an interface that allows you to determine if a given URL is allowed or not. Implementations of this interface can be used to restrict the URLs that the client can access.
By default all URLs are allowed, but this may not be ideal in production environments for security reasons.
This exists because you might use this module to store resources provided by user of your application, in which case you cannot necessarily trust that the URLs are safe.
You will HAVE to provide some sort of whitelist.
type WhitelistFunc ¶
WhitelistFunc is a function type that implements the Whitelist interface.
func (WhitelistFunc) IsAllowed ¶
func (f WhitelistFunc) IsAllowed(u string) bool