yadal

package module
v0.0.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 9, 2022 License: Apache-2.0 Imports: 2 Imported by: 1

README

A project of SENROK Open Source

YaDAL

GoDoc

Yet Another Data Access Layer: Access data freely, efficiently, without the tears 😢

inspired by Databend's OpenDAL

Table of contents

Features

Freely

  • Access different storage services in the same way
  • Behavior tests for all services
    • S3 and S3 compatible services
    • fs: POSIX compatible filesystem

Without the tears 😢

  • Powerful Layer Middlewares
    • Auto Retry (Backoff)
    • Logging Layer
    • Tracing Layer
    • Metrics Layer
  • Compress/Decompress
  • Service-side encryption

Efficiently

  • Zero cost: mapping to underlying API calls directly
  • Auto metadata reuse: avoid extra metadata calls

Installation

go get -u github.com/senrok/yadal

Get started

import (
	"context"
	"fmt"
	"github.com/senrok/yadal"
	"github.com/senrok/yadal/providers/s3"
	"os"
)

func main() {
	acc, _ := s3.NewDriver(context.Background(), s3.Options{
		Bucket:    os.Getenv("Bucket"),
		Endpoint:  os.Getenv("Endpoint"),
		Root:      os.Getenv("Root"),
		Region:    os.Getenv("Region"),
		AccessKey: os.Getenv("AccessKey"),
		SecretKey: os.Getenv("SecretKey"),
	})
	op := yadal.NewOperatorFromAccessor(acc)
	// Create object handler
	o := op.Object("test_file")

	// Write data
	if err := o.Write(context.Background(), []byte("Hello,World!")); err != nil {
		return
	}

	// Read data
	bs, _ := o.Read(context.Background())
	fmt.Println(bs)
	name := o.Name()
	fmt.Println(name)
	path := o.Path()
	fmt.Println(path)
	meta, _ := o.Metadata(context.Background())

	_ = meta.ETag()
	_ = meta.ContentLength()
	_ = meta.ContentMD5()
	_ = meta.Mode()

	// Delete
	_ = o.Delete(context.Background())

	// Read Dir
	ds := op.Object("test-dir/")
	iter, _ := ds.List(context.Background())
	for iter.HasNext() {
		entry, _ := iter.Next(context.Background())
		entry.Path()
		entry.Metadata()
	}
}

See the Documentation or explore more examples

Documentation

GoDoc
Object
Handler
func ExampleOperator_Object() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	// it returns a object.Object handler 
	object := op.Object("test")
	fmt.Println(object.ID())
	fmt.Println(object.Path())
	fmt.Println(object.Name())
}
Create a dir or a object

It creates an empty object, like using the following linux commands:

  • touch path/to/file
  • mkdir path/to/dir/

The behaviors:

  • create on existing dir will succeed.
  • create on existing file will overwrite and truncate it.

a dir:

func ExampleOperator_Object_create() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test/")
	_ = object.Create(context.TODO())
}

a object:

func ExampleOperator_Object_create() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	_ = object.Create(context.TODO())
}
IsExist
func ExampleOperator_Object_isExist() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	fmt.Println(object.IsExist(context.TODO()))
}
Metadata
func ExampleOperator_Object_metadata() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	meta, err := object.Metadata(context.TODO())
	if err == errors.ErrNotFound {
		fmt.Println("not found")
		return
	}
	fmt.Println(meta.LastModified())
	fmt.Println(meta.ETag())
	fmt.Println(meta.ContentLength())
	fmt.Println(meta.ContentMD5())
	fmt.Println(meta.Mode())
}
Read

It returns a io.ReadCloser holds the whole object.

func ExampleOperator_Object_read() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	_ = object.Write(context.TODO(), []byte("Hello,World!"))
	reader, _ := object.Read(context.TODO())

	_, _ = io.ReadAll(reader)
}
Range Read

It returns a io.ReadCloser holds specified range of object .

func ExampleOperator_Object_rangeRead() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	_ = object.Write(context.TODO(), []byte("Hello,World!"))
	reader, _ := object.RangeRead(context.TODO(), options.NewRangeBounds(options.Range(0, 11)))
	_, _ = object.RangeRead(context.TODO(), options.NewRangeBounds(options.Start(2)))
	_, _ = object.RangeRead(context.TODO(), options.NewRangeBounds(options.End(11)))

	_, _ = io.ReadAll(reader)
}
Write

It writes bytes into object.

func ExampleOperator_Object_write() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	_ = object.Write(context.TODO(), []byte("Hello,World!"))
}
Delete

It deletes object.

func ExampleOperator_Object_delete() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("test")
	_ = object.Delete(context.TODO())
}
List current directory

It returns a interfaces.ObjectStream.

func ExampleOperator_Object_list() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)
	object := op.Object("dir/")
	stream, _ := object.List(context.TODO())
	for stream.HasNext() {
		entry, _ := stream.Next(context.TODO())
		if entry != nil {
			fmt.Println(entry.Path())
			fmt.Println(entry.Metadata().LastModified())
			fmt.Println(entry.Metadata().ContentLength())
		}
	}
}
Layers
Retry
func ExampleOperator_Layer_retry() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)

	seed := time.Now().UnixNano()
	random := rand.New(rand.NewSource(seed))

	// retry layer
	retryLayer := layers.NewRetryLayer(
		layers.SetStrategy(
			strategy.Limit(5),
			strategy.BackoffWithJitter(
				backoff.BinaryExponential(10*time.Millisecond),
				jitter.Deviation(random, 0.5),
			),
		),
	)

	op.Layer(retryLayer)
}

See more backoff strategies

Logging
func ExampleOperator_Layer_logging() {
	acc, _ := newS3Accessor()
	op := NewOperatorFromAccessor(acc)

	// logger
	logger, _ := zap.NewProduction()
	s := logger.Sugar()
	
	// logging layer
	loggingLayer := layers.NewLoggingLayer(
		layers.SetLogger(
			layers.NewLo
			ggerAdapter(s.Info, s.Infof),
		),
	)

	op.Layer(loggingLayer)
}

License

The Project is licensed under the Apache License, Version 2.0.

Documentation

Index

Examples

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Operator 

type Operator struct {
	// contains filtered or unexported fields
}

func NewOperatorFromAccessor 

func NewOperatorFromAccessor(acc interfaces.Accessor) Operator

NewOperatorFromAccessor returns the Operator from the interfaces.Accessor

Example 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
// Create object handler
o := op.Object("test_file")

// Write data
if err := o.Write(context.Background(), []byte("Hello,World!")); err != nil {
	return
}

// Read data
bs, _ := o.Read(context.Background())
bytes, _ := io.ReadAll(bs)
fmt.Println(string(bytes))
name := o.Name()
fmt.Println(name)
path := o.Path()
fmt.Println(path)
meta, _ := o.Metadata(context.Background())

_ = meta.ETag()
_ = meta.ContentLength()
_ = meta.ContentMD5()
_ = meta.Mode()

// Delete
_ = o.Delete(context.Background())

dirFile := op.Object("test-dir/test")
dirFile.Create(context.TODO())

// Read Dir
ds := op.Object("test-dir/")
iter, _ := ds.List(context.Background())
for iter.HasNext() {
	entry, _ := iter.Next(context.Background())
	if entry != nil {
		fmt.Println(entry.Path())
		fmt.Println(*entry.Metadata().ContentLength())
	}
}

Output:

Hello,World!
test_file
test_file
test-dir/test
0

func (*Operator) Layer 

func (o *Operator) Layer(layer interfaces.Layer) *Operator

Layer appends a layers.Layer

Example (Logging) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)

// logger
logger, _ := zap.NewProduction()
s := logger.Sugar()

// logging layer
loggingLayer := layers.NewLoggingLayer(
	layers.SetLogger(
		layers.NewLoggerAdapter(s.Info, s.Infof),
	),
)

op.Layer(loggingLayer)
Example (Retry) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)

seed := time.Now().UnixNano()
random := rand.New(rand.NewSource(seed))

// retry layer
// see strategies: https://github.com/Rican7/retry
retryLayer := layers.NewRetryLayer(
	layers.SetStrategy(
		strategy.Limit(5),
		strategy.BackoffWithJitter(
			backoff.BinaryExponential(10*time.Millisecond),
			jitter.Deviation(random, 0.5),
		),
	),
)

op.Layer(retryLayer)

func (*Operator) Metadata added in v0.0.2 

func (o *Operator) Metadata() interfaces.Metadata

func (*Operator) Object 

func (o *Operator) Object(path string) object.Object

Object returns an object.Object handler

Example 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("dir/test")
fmt.Println(object.ID())
fmt.Println(object.Path())
fmt.Println(object.Name())

Output:

/tmp/dir/test
dir/test
test
Example (Create) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
_ = object.Create(context.TODO())
fmt.Println(object.ID())
fmt.Println(object.Path())

Output:

/tmp/test
test
Example (Delete) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
_ = object.Delete(context.TODO())
Example (IsExist) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
exist, _ := object.IsExist(context.TODO())
fmt.Println(exist)

Output:

false
Example (List) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
o := op.Object("dir/test")
_ = o.Create(context.TODO())
object := op.Object("dir/")
stream, _ := object.List(context.TODO())
for stream.HasNext() {
	entry, _ := stream.Next(context.TODO())
	if entry != nil {
		fmt.Println(entry.Path())
		//fmt.Println(entry.Metadata().LastModified())
		fmt.Println(*entry.Metadata().ContentLength())
	}
}

Output:

dir/test
0
Example (Metadata) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
_ = object.Create(context.TODO())
meta, err := object.Metadata(context.TODO())
if err == errors.ErrNotFound {
	fmt.Println("not found")
	return
}
//fmt.Println(meta.LastModified())
fmt.Println(*meta.ETag())
fmt.Println(*meta.ContentLength())
fmt.Println(*meta.ContentMD5())
fmt.Println(meta.Mode())
fmt.Println(object.Path())

Output:

0

file
test
Example (RangeRead) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
_ = object.Write(context.TODO(), []byte("Hello,World!"))
reader, _ := object.RangeRead(context.TODO(), options.NewRangeBounds(options.Range(3, 8)))
// object.RangeRead(context.TODO(), options.NewRangeBounds(options.Start(2)))
// object.RangeRead(context.TODO(), options.NewRangeBounds(options.End(11)))

bytes, _ := io.ReadAll(reader)
fmt.Println(string(bytes))

Output:

lo,Wo
Example (Read) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
_ = object.Write(context.TODO(), []byte("Hello,World!"))
reader, _ := object.Read(context.TODO())

bytes, _ := io.ReadAll(reader)
fmt.Println(string(bytes))

Output:

Hello,World!
Example (Write) 
acc, _ := newFsAccessor()
op := NewOperatorFromAccessor(acc)
object := op.Object("test")
_ = object.Write(context.TODO(), []byte("Hello,World!"))
reader, _ := object.Read(context.TODO())
bytes, _ := io.ReadAll(reader)
fmt.Println(string(bytes))

Output:

Hello,World!

Source Files

View all Source files

Directories

Path Synopsis
fs
s3

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.