fscache

README

fscache

Usage

Streaming File Cache for #golang

fscache allows multiple readers to read from a cache while its being written to. blog post

A Caching Middle-ware:

package main

import(
	"net/http"
	"time"

	"github.com/djherbis/fscache"
)

func main(){
	c, err := fscache.New("./cache", 0700, 0)
	if err != nil {
		log.Fatal(err.Error())
	}

	handler := func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintf(w, "%v: %s", time.Now(), "hello world")
	}

	http.ListenAndServe(":8080", fscache.Handler(c, http.HandlerFunc(handler)))
}

Using the Cache directly:

package main

import (
	"io"
	"log"
	"os"

	"github.com/djherbis/fscache"
)

func main() {

	// create the cache, keys expire after 1 hour.
	c, err := fscache.New("./cache", 0755, 1)
	if err != nil {
		log.Fatal(err.Error())
	}
	
	// wipe the cache when done
	defer c.Clean()

	// Get() and it's streams can be called concurrently but just for example:
	for i := 0; i < 3; i++ {
		r, w, err := c.Get("stream")
		if err != nil {
			log.Fatal(err.Error())
		}

		if w != nil { // a new stream, write to it.
			go func(){
				w.Write([]byte("hello world\n"))
				w.Close()
			}()
		}

		// the stream has started, read from it
		io.Copy(os.Stdout, r)
		r.Close()
	}
}

Installation

go get github.com/djherbis/fscache

Documentation

Overview

Example

// create the cache, keys expire after 1 hour.
c, err := New("./cache", 0755, 1)
if err != nil {
	log.Fatal(err.Error())
}

// wipe the cache when done
defer c.Clean()

// Get() and it's streams can be called concurrently but just for example:
for i := 0; i < 3; i++ {
	r, w, err := c.Get("stream")
	if err != nil {
		log.Fatal(err.Error())
	}

	if w != nil { // a new stream, write to it.
		go func() {
			w.Write([]byte("hello world\n"))
			w.Close()
		}()
	}

	// the stream has started, read from it
	io.Copy(os.Stdout, r)
	r.Close()
}

Output:

hello world
hello world
hello world

Index

• func Handler(c Cache, h http.Handler) http.Handler
• type Cache
  • func New(dir string, perms os.FileMode, expiry time.Duration) (Cache, error)
  • func NewCache(fs FileSystem, grim Reaper) (Cache, error)
• type File
• type FileSystem
  • func NewFs(dir string, mode os.FileMode) (FileSystem, error)
  • func NewMemFs() FileSystem
• type Reaper
  • func NewReaper(expiry, period time.Duration) Reaper

Examples

• Package
• Handler

Functions

func Handler

func Handler(c Cache, h http.Handler) http.Handler

Handler is a caching middle-ware for http Handlers. It responds to http requests via the passed http.Handler, and caches the response using the passed cache. The cache key for the request is the req.URL.String().

Example

c, err := New("./server", 0700, 0)
if err != nil {
	log.Fatal(err.Error())
}
defer c.Clean()

handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "Hello Client")
})

ts := httptest.NewServer(Handler(c, handler))
defer ts.Close()

resp, err := http.Get(ts.URL)
if err != nil {
	log.Fatal(err.Error())
}
io.Copy(os.Stdout, resp.Body)
resp.Body.Close()

Output:

Hello Client

Types

type Cache

type Cache interface {

	// Get manages access to the streams in the cache.
	// If the key does not exist, w != nil and you can start writing to the stream.
	// If the key does exist, w == nil.
	// r will always be non-nil as long as err == nil and you must close r when you're done reading.
	// Get can be called concurrently, and writing and reading is concurrent safe.
	Get(key string) (io.ReadCloser, io.WriteCloser, error)

	// Remove deletes the stream from the cache, blocking until the underlying
	// file can be deleted (all active streams finish with it).
	// It is safe to call Remove concurrently with Get.
	Remove(key string) error

	// Exists checks if a key is in the cache.
	// It is safe to call Exists concurrently with Get.
	Exists(key string) bool

	// Clean will empty the cache and delete the cache folder.
	// Clean is not safe to call while streams are being read/written.
	Clean() error
}

Cache works like a concurrent-safe map for streams.

func New

func New(dir string, perms os.FileMode, expiry time.Duration) (Cache, error)

New creates a new Cache using NewFs(dir, perms). expiry is the duration after which an un-accessed key will be removed from the cache, a zero value expiro means never expire.

func NewCache

func NewCache(fs FileSystem, grim Reaper) (Cache, error)

NewCache creates a new Cache based on FileSystem fs. fs.Files() are loaded using the name they were created with as a key. Reaper is used to determine when files expire, nil means never expire.

type File

type File interface {
	Name() string
	io.WriteCloser
}

File wraps the underlying WriteCloser source.

type FileSystem

type FileSystem interface {
	// Reload should look through the FileSystem and call the suplied fn
	// with the key/filename pairs that are found.
	Reload(func(key, name string)) error

	// Create should return a new File using a name generated from the passed key.
	Create(key string) (File, error)

	// Open takes a File.Name() and returns a concurrent-safe reader to the file.
	// The reader may return io.EOF when reaches the end of written content, but
	// if more content is written and Read is called again, it must continue
	// reading the data.
	Open(name string) (io.ReadCloser, error)

	// Remove takes a File.Name() and deletes the underlying file.
	// It does not have to worry about concurrent use, the Cache will wait
	// for all activity in the file to cease.
	Remove(name string) error

	// RemoveAll should empty the FileSystem of all files.
	RemoveAll() error

	// AccessTimes takes a File.Name() and returns the last time the file was read,
	// and the last time it was written to.
	// It will be used to check expiry of a file, and must be concurrent safe
	// with modifications to the FileSystem (writes, reads etc.)
	AccessTimes(name string) (rt, wt time.Time, err error)
}

FileSystem is used as the source for a Cache.

func NewFs

func NewFs(dir string, mode os.FileMode) (FileSystem, error)

NewFs returns a FileSystem rooted at directory dir. Dir is created with perms if it doesn't exist.

func NewMemFs

func NewMemFs() FileSystem

NewMemFs creates an in-memory FileSystem. It does not support persistence (Reload is a nop).

type Reaper

type Reaper interface {
	// Returns the amount of time to wait before the next scheduled Reaping.
	Next() time.Duration

	// Given a key and the last r/w times of a file, return true
	// to remove the file from the cache, false to keep it.
	Reap(key string, lastRead, lastWrite time.Time) bool
}

Reaper is used to control when streams expire from the cache. It is called once right after loading, and then it is run again after every Next() period of time.

func NewReaper

func NewReaper(expiry, period time.Duration) Reaper

NewReaper returns a simple reaper which runs every "period" and reaps files which are older than "expiry".