opendal

package module
v0.1.14 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 2, 2025 License: Apache-2.0 Imports: 9 Imported by: 0

README

Apache OpenDAL™ Go Binding

Go Reference

opendal-go is a Native support Go binding without CGO enabled and is built on top of opendal-c.

go get github.com/apache/opendal/bindings/go@latest

opendal-go requires libffi to be installed.

Basic Usage

package main

import (
	"fmt"
	"github.com/apache/opendal-go-services/memory"
	opendal "github.com/apache/opendal/bindings/go"
)

func main() {
	// Initialize a new in-memory operator
	op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{})
	if err != nil {
		panic(err)
	}
	defer op.Close()

	// Write data to a file named "test"
	err = op.Write("test", []byte("Hello opendal go binding!"))
	if err != nil {
		panic(err)
	}

	// Read data from the file "test"
	data, err := op.Read("test")
	if err != nil {
		panic(err)
	}
	fmt.Printf("Read content: %s\n", data)

	// List all entries under the root directory "/"
	lister, err := op.List("/")
	if err != nil {
		panic(err)
	}
	defer lister.Close()

	// Iterate through all entries
	for lister.Next() {
		entry := lister.Entry()

		// Get entry name (not used in this example)
		_ = entry.Name()

		// Get metadata for the current entry
		meta, _ := op.Stat(entry.Path())

		// Print file size
		fmt.Printf("Size: %d bytes\n", meta.ContentLength())

		// Print last modified time
		fmt.Printf("Last modified: %s\n", meta.LastModified())

		// Check if the entry is a directory or a file
		fmt.Printf("Is directory: %v, Is file: %v\n", meta.IsDir(), meta.IsFile())
	}

	// Check for any errors that occurred during iteration
	if err := lister.Error(); err != nil {
		panic(err)
	}

	// Copy a file
	op.Copy("test", "test_copy")

	// Rename a file
	op.Rename("test", "test_rename")

	// Delete a file
	op.Delete("test_rename")
}

Run Tests

Behavior Tests
cd tests/behavior_tests
# Test a specific backend
export OPENDAL_TEST=memory
# Run all tests
CGO_ENABLE=0 go test -v -run TestBehavior
# Run specific test
CGO_ENABLE=0 go test -v -run TestBehavior/Write
# Run synchronously
CGO_ENABLE=0 GOMAXPROCS=1 go test -v -run TestBehavior
Benchmark
cd tests/behavior_tests
# Benchmark a specific backend
export OPENDAL_TEST=memory

go test -bench .
A benchmark between OpenDAL Go binding and aws-sdk-go on minio S3 compatible storage
goos: linux
goarch: amd64
pkg: opendal_test
cpu: AMD EPYC 7763 64-Core Processor                
BenchmarkWrite
BenchmarkWrite/4KiB/OpenDAL
BenchmarkWrite/4KiB/OpenDAL-4         	     405	   3430652 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4KiB/OpenDAL-4         	     313	   3781825 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4KiB/OpenDAL-4         	     346	   3354802 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4KiB/OpenDAL-4         	     397	   3374467 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4KiB/OpenDAL-4         	     364	   6171386 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4KiB/OpenDAL-4         	     388	   3789795 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4KiB/AWS_S3
BenchmarkWrite/4KiB/AWS_S3-4          	     384	   3716060 ns/op	   62929 B/op	     312 allocs/op
BenchmarkWrite/4KiB/AWS_S3-4          	     402	   3145797 ns/op	   62077 B/op	     312 allocs/op
BenchmarkWrite/4KiB/AWS_S3-4          	     372	   3050911 ns/op	   61902 B/op	     312 allocs/op
BenchmarkWrite/4KiB/AWS_S3-4          	     400	   3081028 ns/op	   61556 B/op	     312 allocs/op
BenchmarkWrite/4KiB/AWS_S3-4          	     342	   3111741 ns/op	   61970 B/op	     312 allocs/op
BenchmarkWrite/4KiB/AWS_S3-4          	     464	   2933844 ns/op	   61617 B/op	     312 allocs/op
BenchmarkWrite/256KiB/OpenDAL
BenchmarkWrite/256KiB/OpenDAL-4       	     228	   6253966 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/256KiB/OpenDAL-4       	     190	   5859882 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/256KiB/OpenDAL-4       	     216	   6008253 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/256KiB/OpenDAL-4       	     200	   5958440 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/256KiB/OpenDAL-4       	     193	   5658798 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/256KiB/OpenDAL-4       	     210	   6250594 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/256KiB/AWS_S3
BenchmarkWrite/256KiB/AWS_S3-4        	     208	   5665223 ns/op	   90323 B/op	     312 allocs/op
BenchmarkWrite/256KiB/AWS_S3-4        	     196	   5368602 ns/op	   88719 B/op	     312 allocs/op
BenchmarkWrite/256KiB/AWS_S3-4        	     218	   6108315 ns/op	   90784 B/op	     312 allocs/op
BenchmarkWrite/256KiB/AWS_S3-4        	     217	   5828966 ns/op	   90379 B/op	     312 allocs/op
BenchmarkWrite/256KiB/AWS_S3-4        	     210	   6447429 ns/op	   90981 B/op	     312 allocs/op
BenchmarkWrite/256KiB/AWS_S3-4        	     177	   6323915 ns/op	   91548 B/op	     313 allocs/op
BenchmarkWrite/4MiB/OpenDAL
BenchmarkWrite/4MiB/OpenDAL-4         	      62	  19049500 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4MiB/OpenDAL-4         	      54	  19271521 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4MiB/OpenDAL-4         	      69	  17849900 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4MiB/OpenDAL-4         	      73	  18117693 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4MiB/OpenDAL-4         	      74	  18249816 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4MiB/OpenDAL-4         	      62	  19556730 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/4MiB/AWS_S3
BenchmarkWrite/4MiB/AWS_S3-4          	      43	  29514084 ns/op	   92211 B/op	     320 allocs/op
BenchmarkWrite/4MiB/AWS_S3-4          	      39	  29297735 ns/op	   92514 B/op	     320 allocs/op
BenchmarkWrite/4MiB/AWS_S3-4          	      42	  28956593 ns/op	   92282 B/op	     320 allocs/op
BenchmarkWrite/4MiB/AWS_S3-4          	      42	  29218821 ns/op	   92282 B/op	     320 allocs/op
BenchmarkWrite/4MiB/AWS_S3-4          	      40	  28988214 ns/op	   91391 B/op	     320 allocs/op
BenchmarkWrite/4MiB/AWS_S3-4          	      43	  28668319 ns/op	   91242 B/op	     320 allocs/op
BenchmarkWrite/16MiB/OpenDAL
BenchmarkWrite/16MiB/OpenDAL-4        	      21	  53528117 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/16MiB/OpenDAL-4        	      21	  55328986 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/16MiB/OpenDAL-4        	      21	  54221620 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/16MiB/OpenDAL-4        	      20	  54044030 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/16MiB/OpenDAL-4        	      21	  53692610 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/16MiB/OpenDAL-4        	      21	  53288370 ns/op	     384 B/op	      10 allocs/op
BenchmarkWrite/16MiB/AWS_S3
BenchmarkWrite/16MiB/AWS_S3-4         	      12	  99294840 ns/op	   99850 B/op	     322 allocs/op
BenchmarkWrite/16MiB/AWS_S3-4         	      12	  97405067 ns/op	   99850 B/op	     322 allocs/op
BenchmarkWrite/16MiB/AWS_S3-4         	      12	  97906212 ns/op	   99848 B/op	     322 allocs/op
BenchmarkWrite/16MiB/AWS_S3-4         	      12	  98766864 ns/op	   96378 B/op	     322 allocs/op
BenchmarkWrite/16MiB/AWS_S3-4         	      12	  97967605 ns/op	   99850 B/op	     322 allocs/op
BenchmarkWrite/16MiB/AWS_S3-4         	      12	  97842268 ns/op	   96376 B/op	     322 allocs/op
BenchmarkRead
BenchmarkRead/4KiB/OpenDAL
BenchmarkRead/4KiB/OpenDAL-4          	     972	   1217197 ns/op	    4760 B/op	      15 allocs/op
BenchmarkRead/4KiB/OpenDAL-4          	     985	   1226752 ns/op	    4760 B/op	      15 allocs/op
BenchmarkRead/4KiB/OpenDAL-4          	     982	   1216827 ns/op	    4760 B/op	      15 allocs/op
BenchmarkRead/4KiB/OpenDAL-4          	     987	   1227682 ns/op	    4760 B/op	      15 allocs/op
BenchmarkRead/4KiB/OpenDAL-4          	     987	   1214420 ns/op	    4760 B/op	      15 allocs/op
BenchmarkRead/4KiB/OpenDAL-4          	     987	   1215074 ns/op	    4760 B/op	      15 allocs/op
BenchmarkRead/4KiB/AWS_S3
BenchmarkRead/4KiB/AWS_S3-4           	    1594	    759864 ns/op	   34988 B/op	     344 allocs/op
BenchmarkRead/4KiB/AWS_S3-4           	    1605	    751628 ns/op	   34990 B/op	     344 allocs/op
BenchmarkRead/4KiB/AWS_S3-4           	    1620	    744649 ns/op	   34991 B/op	     344 allocs/op
BenchmarkRead/4KiB/AWS_S3-4           	    1623	    752885 ns/op	   34994 B/op	     344 allocs/op
BenchmarkRead/4KiB/AWS_S3-4           	    1629	    742307 ns/op	   34991 B/op	     344 allocs/op
BenchmarkRead/4KiB/AWS_S3-4           	    1622	    750455 ns/op	   34992 B/op	     344 allocs/op
BenchmarkRead/256KiB/OpenDAL
BenchmarkRead/256KiB/OpenDAL-4        	     741	   1612259 ns/op	  262812 B/op	      15 allocs/op
BenchmarkRead/256KiB/OpenDAL-4        	     727	   1633753 ns/op	  262812 B/op	      15 allocs/op
BenchmarkRead/256KiB/OpenDAL-4        	     751	   1619770 ns/op	  262812 B/op	      15 allocs/op
BenchmarkRead/256KiB/OpenDAL-4        	     756	   1619435 ns/op	  262812 B/op	      15 allocs/op
BenchmarkRead/256KiB/OpenDAL-4        	     750	   1629211 ns/op	  262812 B/op	      15 allocs/op
BenchmarkRead/256KiB/OpenDAL-4        	     757	   1611646 ns/op	  262812 B/op	      15 allocs/op
BenchmarkRead/256KiB/AWS_S3
BenchmarkRead/256KiB/AWS_S3-4         	     736	   1655133 ns/op	 1209847 B/op	     365 allocs/op
BenchmarkRead/256KiB/AWS_S3-4         	     735	   1673714 ns/op	 1209874 B/op	     365 allocs/op
BenchmarkRead/256KiB/AWS_S3-4         	     735	   1653903 ns/op	 1209852 B/op	     365 allocs/op
BenchmarkRead/256KiB/AWS_S3-4         	     728	   1643222 ns/op	 1209855 B/op	     365 allocs/op
BenchmarkRead/256KiB/AWS_S3-4         	     726	   1674202 ns/op	 1209842 B/op	     365 allocs/op
BenchmarkRead/256KiB/AWS_S3-4         	     717	   1656092 ns/op	 1209876 B/op	     365 allocs/op
BenchmarkRead/4MiB/OpenDAL
BenchmarkRead/4MiB/OpenDAL-4          	     250	   4933426 ns/op	 4194991 B/op	      15 allocs/op
BenchmarkRead/4MiB/OpenDAL-4          	     255	   4874283 ns/op	 4194991 B/op	      15 allocs/op
BenchmarkRead/4MiB/OpenDAL-4          	     252	   4769956 ns/op	 4194992 B/op	      16 allocs/op
BenchmarkRead/4MiB/OpenDAL-4          	     244	   4800674 ns/op	 4194992 B/op	      16 allocs/op
BenchmarkRead/4MiB/OpenDAL-4          	     242	   4826974 ns/op	 4194992 B/op	      15 allocs/op
BenchmarkRead/4MiB/OpenDAL-4          	     254	   4882666 ns/op	 4194996 B/op	      16 allocs/op
BenchmarkRead/4MiB/AWS_S3
BenchmarkRead/4MiB/AWS_S3-4           	     122	  10379507 ns/op	21133764 B/op	     388 allocs/op
BenchmarkRead/4MiB/AWS_S3-4           	     100	  10838399 ns/op	21133778 B/op	     389 allocs/op
BenchmarkRead/4MiB/AWS_S3-4           	      92	  11831700 ns/op	21133738 B/op	     388 allocs/op
BenchmarkRead/4MiB/AWS_S3-4           	     100	  10002235 ns/op	21133764 B/op	     388 allocs/op
BenchmarkRead/4MiB/AWS_S3-4           	     100	  10416467 ns/op	21133731 B/op	     388 allocs/op
BenchmarkRead/4MiB/AWS_S3-4           	     100	  10452328 ns/op	21133719 B/op	     388 allocs/op
BenchmarkRead/16MiB/OpenDAL
BenchmarkRead/16MiB/OpenDAL-4         	      81	  14654347 ns/op	16777903 B/op	      15 allocs/op
BenchmarkRead/16MiB/OpenDAL-4         	      78	  14977822 ns/op	16777908 B/op	      16 allocs/op
BenchmarkRead/16MiB/OpenDAL-4         	      81	  14833921 ns/op	16777906 B/op	      16 allocs/op
BenchmarkRead/16MiB/OpenDAL-4         	      76	  14663855 ns/op	16777906 B/op	      16 allocs/op
BenchmarkRead/16MiB/OpenDAL-4         	      75	  14545209 ns/op	16777906 B/op	      16 allocs/op
BenchmarkRead/16MiB/OpenDAL-4         	      80	  14585250 ns/op	16777905 B/op	      16 allocs/op
BenchmarkRead/16MiB/AWS_S3
BenchmarkRead/16MiB/AWS_S3-4          	      52	  24826707 ns/op	102660545 B/op	     398 allocs/op
BenchmarkRead/16MiB/AWS_S3-4          	      46	  24588321 ns/op	102660529 B/op	     397 allocs/op
BenchmarkRead/16MiB/AWS_S3-4          	      44	  26526367 ns/op	102660527 B/op	     397 allocs/op
BenchmarkRead/16MiB/AWS_S3-4          	      44	  26996660 ns/op	102660522 B/op	     397 allocs/op
BenchmarkRead/16MiB/AWS_S3-4          	      42	  26319939 ns/op	102660552 B/op	     398 allocs/op
BenchmarkRead/16MiB/AWS_S3-4          	      45	  25286454 ns/op	102660542 B/op	     398 allocs/op
PASS
ok  	opendal_test	147.053s

Diff with benchstat

benchstat aws.txt opendal.txt
goos: linux
goarch: amd64
pkg: opendal_test
cpu: AMD EPYC 7763 64-Core Processor                
             │    aws.txt    │             opendal.txt              │
             │    sec/op     │    sec/op      vs base               │
Write/4KiB      3.096m ± 20%    3.606m ± 71%  +16.47% (p=0.015 n=6)
Write/256KiB    5.969m ± 10%    5.983m ±  5%        ~ (p=1.000 n=6)
Write/4MiB      29.10m ±  1%    18.65m ±  5%  -35.92% (p=0.002 n=6)
Write/16MiB     97.94m ±  1%    53.87m ±  3%  -45.00% (p=0.002 n=6)
Read/4KiB       751.0µ ±  1%   1217.0µ ±  1%  +62.04% (p=0.002 n=6)
Read/256KiB     1.656m ±  1%    1.620m ±  1%   -2.18% (p=0.002 n=6)
Read/4MiB      10.434m ± 13%    4.851m ±  2%  -53.51% (p=0.002 n=6)
Read/16MiB      25.80m ±  5%    14.66m ±  2%  -43.19% (p=0.002 n=6)
geomean         8.050m          6.461m        -19.73%

             │    aws.txt    │             opendal.txt             │
             │     B/op      │     B/op      vs base               │
Write/4KiB      61936.0 ± 2%     384.0 ± 0%  -99.38% (p=0.002 n=6)
Write/256KiB    90581.5 ± 2%     384.0 ± 0%  -99.58% (p=0.002 n=6)
Write/4MiB      92246.5 ± 1%     384.0 ± 0%  -99.58% (p=0.002 n=6)
Write/16MiB     99849.0 ± 3%     384.0 ± 0%  -99.62% (p=0.002 n=6)
Read/4KiB      34.171Ki ± 0%   4.648Ki ± 0%  -86.40% (p=0.002 n=6)
Read/256KiB    1181.5Ki ± 0%   256.7Ki ± 0%  -78.28% (p=0.002 n=6)
Read/4MiB      20.155Mi ± 0%   4.001Mi ± 0%  -80.15% (p=0.002 n=6)
Read/16MiB      97.90Mi ± 0%   16.00Mi ± 0%  -83.66% (p=0.002 n=6)
geomean         500.3Ki        14.12Ki       -97.18%

             │   aws.txt   │            opendal.txt            │
             │  allocs/op  │ allocs/op   vs base               │
Write/4KiB     312.00 ± 0%   10.00 ± 0%  -96.79% (p=0.002 n=6)
Write/256KiB   312.00 ± 0%   10.00 ± 0%  -96.79% (p=0.002 n=6)
Write/4MiB     320.00 ± 0%   10.00 ± 0%  -96.88% (p=0.002 n=6)
Write/16MiB    322.00 ± 0%   10.00 ± 0%  -96.89% (p=0.002 n=6)
Read/4KiB      344.00 ± 0%   15.00 ± 0%  -95.64% (p=0.002 n=6)
Read/256KiB    365.00 ± 0%   15.00 ± 0%  -95.89% (p=0.002 n=6)
Read/4MiB      388.00 ± 0%   15.50 ± 3%  -96.01% (p=0.002 n=6)
Read/16MiB     397.50 ± 0%   16.00 ± 6%  -95.97% (p=0.002 n=6)
geomean         343.6        12.40       -96.39%

Capabilities

  • OperatorInfo
  • Stat
    • Metadata
  • IsExist
  • Read
    • Read
    • Reader -- implement as io.ReadSeekCloser
  • Write
    • Write
    • Writer -- implement as io.WriteCloser
  • Delete
  • CreateDir
  • Lister
    • Entry
    • Metadata -- Need support from the C binding
  • Copy
  • Rename

Development

The guide is based on Linux and Windows. For other platforms, please adjust the commands accordingly.

To develop the Go binding, you need to have the following dependencies installed:

  • zstd
  • Rust toolchain
  • Go
  • (Required for Windows) libffi-8.dll in the root of the workspace directory

We use go workspace to manage and build the dependencies. To set up the workspace, run the following commands:

For Linux, macOS and Windows (MSVC)
mkdir opendal_workspace
cd opendal_workspace
git clone --depth 1 git@github.com:apache/opendal.git
git clone --depth 1 git@github.com:apache/opendal-go-services.git

go work init
go work use ./opendal/bindings/go
go work use ./opendal/bindings/go/tests/behavior_tests

cp opendal/bindings/go/Makefile .

cd -

To build and run tests, run the following commands:

cd opendal_workspace

# specify the backend to test
export OPENDAL_TEST=fs
export OPENDAL_FS_ROOT=/tmp/opendal

make tests

cd -

To run the benchmarks, you can use the following command:

make bench

License and Trademarks

Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

Apache OpenDAL, OpenDAL, and Apache are either registered trademarks or trademarks of the Apache Software Foundation.

Documentation

Overview

Package opendal provides a Go binding for Apache OpenDAL (Open Data Access Layer).

OpenDAL is a data access layer that allows users to easily interact with various storage services using a unified API. This Go binding enables Go developers to leverage OpenDAL's capabilities without the need for CGO.

Key features:

  • Unified interface for multiple storage backends (e.g., S3, Azure Blob, local filesystem)
  • Native Go implementation using purego and libffi
  • No CGO dependency, ensuring better portability and easier cross-compilation
  • Supports common operations like read, write, delete, list, and metadata retrieval

Basic usage:

import (
	"github.com/apache/opendal/bindings/go
	"github.com/apache/opendal-go-services/memory
)

func main() {
	op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{
		"root": "/path/to/root",
	})
	if err != nil {
		log.Fatal(err)
	}
	defer op.Close()

	// Perform operations using the operator
	err = op.Write("example.txt", []byte("Hello, OpenDAL!"))
	if err != nil {
		log.Fatal(err)
	}
}

This package aims to provide a seamless experience for Go developers working with various storage systems, combining the flexibility of OpenDAL with the performance and simplicity of native Go code.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func BytePtrFromString added in v0.1.10

func BytePtrFromString(s string) (*byte, error)

func BytePtrToString added in v0.1.10

func BytePtrToString(p *byte) string

func FreeLibrary added in v0.1.10

func FreeLibrary(handle uintptr) error

func GetProcAddress added in v0.1.10

func GetProcAddress(handle uintptr, name string) (uintptr, error)

func LoadLibrary added in v0.1.10

func LoadLibrary(path string) (uintptr, error)

Types

type Capability

type Capability struct {
	// contains filtered or unexported fields
}

Capability represents the set of operations and features supported by an Operator.

Each field indicates the support level for a specific capability:

  • bool fields: false indicates no support, true indicates support.
  • uint fields: Represent size limits or thresholds for certain operations.

This struct covers a wide range of capabilities including:

  • Basic operations: stat, read, write, delete, copy, rename, list
  • Advanced features: multipart uploads, presigned URLs, batch operations
  • Operation modifiers: cache control, content type, if-match conditions

The capability information helps in understanding the functionalities available for a specific storage backend or Operator configuration.

func (*Capability) Blocking

func (c *Capability) Blocking() bool

func (*Capability) Copy

func (c *Capability) Copy() bool

func (*Capability) CreateDir

func (c *Capability) CreateDir() bool

func (*Capability) Delete

func (c *Capability) Delete() bool

func (*Capability) List

func (c *Capability) List() bool

func (*Capability) ListWithLimit

func (c *Capability) ListWithLimit() bool

func (*Capability) ListWithRecursive

func (c *Capability) ListWithRecursive() bool

func (*Capability) ListWithStartAfter

func (c *Capability) ListWithStartAfter() bool

func (*Capability) Presign

func (c *Capability) Presign() bool

func (*Capability) PresignRead

func (c *Capability) PresignRead() bool

func (*Capability) PresignStat

func (c *Capability) PresignStat() bool

func (*Capability) PresignWrite

func (c *Capability) PresignWrite() bool

func (*Capability) Read

func (c *Capability) Read() bool

func (*Capability) ReadWithIfMatchNone

func (c *Capability) ReadWithIfMatchNone() bool

func (*Capability) ReadWithIfmatch

func (c *Capability) ReadWithIfmatch() bool

func (*Capability) ReadWithOverrideCacheControl

func (c *Capability) ReadWithOverrideCacheControl() bool

func (*Capability) ReadWithOverrideContentDisposition

func (c *Capability) ReadWithOverrideContentDisposition() bool

func (*Capability) ReadWithOverrideContentType

func (c *Capability) ReadWithOverrideContentType() bool

func (*Capability) Rename

func (c *Capability) Rename() bool

func (*Capability) Shared added in v0.1.5

func (c *Capability) Shared() bool

func (*Capability) Stat

func (c *Capability) Stat() bool

func (*Capability) StatWithIfNoneMatch

func (c *Capability) StatWithIfNoneMatch() bool

func (*Capability) StatWithIfmatch

func (c *Capability) StatWithIfmatch() bool

func (*Capability) Write

func (c *Capability) Write() bool

func (*Capability) WriteCanAppend

func (c *Capability) WriteCanAppend() bool

func (*Capability) WriteCanEmpty

func (c *Capability) WriteCanEmpty() bool

func (*Capability) WriteCanMulti

func (c *Capability) WriteCanMulti() bool

func (*Capability) WriteMultiMaxSize

func (c *Capability) WriteMultiMaxSize() uint

func (*Capability) WriteMultiMinSize

func (c *Capability) WriteMultiMinSize() uint

func (*Capability) WriteTotalMaxSize

func (c *Capability) WriteTotalMaxSize() uint

func (*Capability) WriteWithCacheControl

func (c *Capability) WriteWithCacheControl() bool

func (*Capability) WriteWithContentDisposition

func (c *Capability) WriteWithContentDisposition() bool

func (*Capability) WriteWithContentType

func (c *Capability) WriteWithContentType() bool

type Entry

type Entry struct {
	// contains filtered or unexported fields
}

Entry represents a path and its associated metadata as returned by Lister.

An Entry provides basic information about a file or directory encountered during a list operation. It contains the path of the item and minimal metadata.

Limitations

The Entry itself does not contain comprehensive metadata. For detailed metadata information, use the op.Stat() method with the Entry's path.

Usage

Entries are typically obtained through iteration of a Lister:

for lister.Next() {
	entry := lister.Entry()
	// Process the entry
	fmt.Println(entry.Name())
}

Fetching Detailed Metadata

To obtain comprehensive metadata for an Entry, use op.Stat():

meta, err := op.Stat(entry.Path())
if err != nil {
	log.Printf("Error fetching metadata: %v", err)
	return
}
fmt.Printf("Size: %d, Last Modified: %s\n", meta.ContentLength(), meta.LastModified())

Methods

Entry provides methods to access basic information:

  • Path(): Returns the full path of the entry.
  • Name(): Returns the name of the entry (last component of the path).

func (*Entry) Name

func (e *Entry) Name() string

Name returns the last component of the entry's path.

func (*Entry) Path

func (e *Entry) Path() string

Path returns the full path of the entry.

type Error

type Error struct {
	// contains filtered or unexported fields
}

func (*Error) Code

func (e *Error) Code() ErrorCode

func (*Error) Error

func (e *Error) Error() string

func (*Error) Message

func (e *Error) Message() string

type ErrorCode

type ErrorCode int32

ErrorCode is all kinds of ErrorCode of opendal

const (
	// OpenDAL don't know what happened here, and no actions other than just
	// returning it back. For example, s3 returns an internal service error.
	CodeUnexpected ErrorCode = iota
	// Underlying service doesn't support this operation.
	CodeUnsupported
	// The config for backend is invalid.
	CodeConfigInvalid
	// The given path is not found.
	CodeNotFound
	// The given path doesn't have enough permission for this operation
	CodePermissioDenied
	// The given path is a directory.
	CodeIsADirectory
	// The given path is not a directory.
	CodeNotADirectory
	// The given path already exists thus we failed to the specified operation on it.
	CodeAlreadyExists
	// Requests that sent to this path is over the limit, please slow down.
	CodeRateLimited
	// The given file paths are same.
	CodeIsSameFile
	// The condition of this operation is not match.
	//
	// The `condition` itself is context based.
	//
	// For example, in S3, the `condition` can be:
	// 1. writing a file with If-Match header but the file's ETag is not match (will get a 412 Precondition Failed).
	// 2. reading a file with If-None-Match header but the file's ETag is match (will get a 304 Not Modified).
	//
	// As OpenDAL cannot handle the `condition not match` error, it will always return this error to users.
	// So users could to handle this error by themselves.
	CodeConditionNotMatch
	// The range of the content is not satisfied.
	//
	// OpenDAL returns this error to indicate that the range of the read request is not satisfied.
	CodeRangeNotSatisfied
)

type FFI added in v0.1.13

type FFI[T any] struct {
	// contains filtered or unexported fields
}

type Lister

type Lister struct {
	// contains filtered or unexported fields
}

Lister provides an mechanism for listing entries at a specified path.

Lister is a wrapper around the C-binding function `opendal_operator_list`. It allows for efficient iteration over entries in a storage system.

Limitations

  • The current implementation does not support the `list_with` functionality.

Usage

Lister should be used in conjunction with its Next() and Entry() methods to iterate through entries. The iteration ends when there are no more entries or when an error occurs.

Behavior

  • Next() returns false when there are no more entries or if an error has occurred.
  • Entry() returns nil if there are no more entries or if an error has been encountered.

Example

lister, err := op.List("path/to/list")
if err != nil {
	log.Fatal(err)
}

for lister.Next() {
	entry := lister.Entry()
	// Process the entry
	fmt.Println(entry.Name())
}

func (*Lister) Close

func (l *Lister) Close() error

This method implements the io.Closer interface. It should be called when the Lister is no longer needed to ensure proper resource cleanup.

func (*Lister) Entry

func (l *Lister) Entry() *Entry

Entry returns the current Entry in the list. Returns nil if there are no more entries

func (*Lister) Error

func (l *Lister) Error() error

func (*Lister) Next

func (l *Lister) Next() bool

Next advances the Lister to the next entry in the list.

This method must be called before accessing the current entry. It prepares the next entry for reading and indicates whether there are more entries to process.

Returns

  • bool: true if there is another entry to process, false if the end of the list has been reached or an error occurred.

Usage

Next should be used in a loop condition to iterate through all entries:

for lister.Next() {
	entry := lister.Entry()
	// Process the entry
}

Error Handling

If an error occurs during iteration, Next will return false. The error can then be retrieved by calling the Err method on the Lister.

Example

lister, err := op.List("path/to/list")
if err != nil {
	log.Fatal(err)
}

for lister.Next() {
	entry := lister.Entry()
	fmt.Println(entry.Name())
}

type Metadata

type Metadata struct {
	// contains filtered or unexported fields
}

Metadata represents essential information about a file or directory.

This struct contains basic attributes commonly used in file systems and object storage systems.

func (*Metadata) ContentLength

func (m *Metadata) ContentLength() uint64

ContentLength returns the size of the file in bytes.

For directories, this value may not be meaningful and could be zero.

func (*Metadata) IsDir

func (m *Metadata) IsDir() bool

IsDir returns true if the metadata represents a directory, false otherwise.

func (*Metadata) IsFile

func (m *Metadata) IsFile() bool

IsFile returns true if the metadata represents a file, false otherwise.

func (*Metadata) LastModified

func (m *Metadata) LastModified() time.Time

LastModified returns the time when the file or directory was last modified.

The returned time is in UTC.

type Operator

type Operator struct {
	// contains filtered or unexported fields
}

Operator is the entry point for all public APIs in OpenDAL.

Operator provides a unified interface for interacting with various storage services. It encapsulates the underlying storage operations and presents a consistent API regardless of the storage backend.

Usage

Create an Operator using NewOperator, perform operations, and always remember to Close the operator when finished to release resources.

Example

func main() {
	// Create a new operator for the memory backend
	op, err := opendal.NewOperator(memory.Scheme, opendal.OperatorOptions{})
	if err != nil {
		log.Fatal(err)
	}
	defer op.Close() // Ensure the operator is closed when done

	// Perform operations using the operator
	err = op.Write("example.txt", []byte("Hello, OpenDAL!"))
	if err != nil {
		log.Fatal(err)
	}

	data, err := op.Read("example.txt")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(data))
}

Note: Always use defer op.Close() to ensure proper resource cleanup.

Available Operations

Operator provides methods for common storage operations including:

  • Read: Read data from a path
  • Write: Write data to a path
  • Stat: Get metadata for a path
  • Delete: Remove a file or directory
  • List: Enumerate entries in a directory
  • and more...

Refer to the individual method documentation for detailed usage information.

func NewOperator

func NewOperator(scheme Scheme, opts OperatorOptions) (op *Operator, err error)

NewOperator creates and initializes a new Operator for the specified storage scheme.

Parameters:

  • scheme: The storage scheme (e.g., "memory", "s3", "fs").
  • options: Configuration options for the operator.

Returns:

  • *Operator: A new Operator instance.
  • error: An error if initialization fails, or nil if successful.

Note: Remember to call Close() on the returned Operator when it's no longer needed.

func (*Operator) Check

func (op *Operator) Check() (err error)

Check verifies if the operator is functioning correctly.

This function performs a health check on the operator by sending a `list` request to the root path. It returns any errors encountered during this process.

Returns

  • error: An error if the check fails, or nil if the operator is working correctly.

Details

The check is performed by attempting to list the contents of the root directory. This operation tests the basic functionality of the operator, including connectivity and permissions.

Example

func exampleCheck(op *opendal.Operator) {
	err = op.Check()
	if err != nil {
		log.Printf("Operator check failed: %v", err)
	} else {
		log.Println("Operator is functioning correctly")
	}
}

Note: This example assumes proper error handling and import statements.

func (*Operator) Close

func (op *Operator) Close()

Close releases all resources associated with the Operator.

It's important to call this method when the Operator is no longer needed to ensure proper cleanup of underlying resources.

Note: It's recommended to use defer op.Close() immediately after creating an Operator.

func (*Operator) Copy

func (op *Operator) Copy(src, dest string) error

Copy duplicates a file from the source path to the destination path.

This function copies the contents of the file at 'from' to a new or existing file at 'to'.

Parameters

  • from: The source file path.
  • to: The destination file path.

Returns

  • error: An error if the copy operation fails, or nil if successful.

Behavior

  • Both 'from' and 'to' must be file paths, not directories.
  • If 'to' already exists, it will be overwritten.
  • If 'from' and 'to' are identical, an 'IsSameFile' error will be returned.
  • The copy operation is idempotent; repeated calls with the same parameters will yield the same result.

Example

func exampleCopy(op *operatorCopy) {
	err = op.Copy("path/from/file", "path/to/file")
	if err != nil {
		log.Printf("Copy operation failed: %v", err)
	} else {
		log.Println("File copied successfully")
	}
}

Note: This example assumes proper error handling and import statements.

func (*Operator) CreateDir

func (op *Operator) CreateDir(path string) error

CreateDir creates a directory at the specified path.

CreateDir is a wrapper around the C-binding function `opendal_operator_create_dir`. It provides a way to create directories in the storage system.

Parameters

  • path: The path where the directory should be created.

Returns

  • error: An error if the directory creation fails, or nil if successful.

Notes

It is mandatory to include a trailing slash (/) in the path to indicate that it is a directory. Failing to do so may result in a `CodeNotADirectory` error being returned by OpenDAL.

Behavior

  • Creating a directory that already exists will succeed without error.
  • Directory creation is always recursive, similar to the `mkdir -p` command.

Example

func exampleCreateDir(op *opendal.Operator) {
	err = op.CreateDir("test/")
	if err != nil {
		log.Fatal(err)
	}
}

Note: This example assumes proper error handling and import statements. The trailing slash in "test/" is important to indicate it's a directory.

func (*Operator) Delete

func (op *Operator) Delete(path string) error

Delete removes the file or directory at the specified path.

Parameters

  • path: The path of the file or directory to delete.

Returns

  • error: An error if the deletion fails, or nil if successful.

Note

Use with caution as this operation is irreversible.

func (*Operator) Info

func (op *Operator) Info() *OperatorInfo

Info returns metadata about the Operator.

This method provides access to essential information about the Operator, including its storage scheme, root path, name, and capabilities.

Returns:

  • *OperatorInfo: A pointer to an OperatorInfo struct containing the Operator's metadata.

func (*Operator) IsExist

func (op *Operator) IsExist(path string) (bool, error)

IsExist checks if a file or directory exists at the specified path.

This method provides a convenient way to determine the existence of a resource without fetching its full metadata.

Parameters

  • path: The path of the file or directory to check.

Returns

  • bool: true if the resource exists, false otherwise.
  • error: An error if the check operation fails, or nil if the check is successful. Note that a false return value with a nil error indicates that the resource does not exist.

Example

exists, err := op.IsExist("path/to/file")
if err != nil {
	log.Fatalf("Error checking existence: %v", err)
}
if exists {
	fmt.Println("The file exists")
} else {
	fmt.Println("The file does not exist")
}

func (*Operator) List

func (op *Operator) List(path string) (*Lister, error)

List returns a Lister to iterate over entries that start with the given path in the parent directory.

This function creates a new Lister to enumerate entries in the specified path.

Parameters

  • path: The starting path for listing entries.

Returns

  • *Lister: A new Lister instance for iterating over entries.
  • error: An error if the listing operation fails, or nil if successful.

Notes

  1. List is a wrapper around the C-binding function `opendal_operator_list`. Recursive listing is not currently supported.
  2. Returned entries do not include metadata information. Use op.Stat to fetch metadata for individual entries.

Example

func exampleList(op *opendal.Operator) {
	lister, err := op.List("test")
	if err != nil {
		log.Fatal(err)
	}

	for lister.Next() {
		entry := lister.Entry()

		meta, err := op.Stat(entry.Path())
		if err != nil {
			log.Printf("Error fetching metadata for %s: %v", entry.Path(), err)
			continue
		}

		fmt.Printf("Name: %s\n", entry.Name())
		fmt.Printf("Length: %d\n", meta.ContentLength())
		fmt.Printf("Last Modified: %s\n", meta.LastModified())
		fmt.Printf("Is Directory: %v, Is File: %v\n", meta.IsDir(), meta.IsFile())
		fmt.Println("---")
	}
	if err := lister.Err(); err != nil {
		log.Printf("Error during listing: %v", err)
	}
}

Note: Always check lister.Err() after the loop to catch any errors that occurred during iteration.

func (*Operator) Read

func (op *Operator) Read(path string) ([]byte, error)

Read reads the entire contents of the file at the specified path into a byte slice.

This function is a wrapper around the C-binding function `opendal_operator_read`.

Parameters

  • path: The path of the file to read.

Returns

  • []byte: The contents of the file as a byte slice.
  • error: An error if the read operation fails, or nil if successful.

Notes

  • This implementation does not support the `read_with` functionality.
  • Read allocates a new byte slice internally. For more precise memory control or lazy reading, consider using the Reader() method instead.

Example

func exampleRead(op *opendal.Operator) {
	data, err := op.Read("test")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("Read: %s\n", data)
}

Note: This example assumes proper error handling and import statements.

func (*Operator) Reader

func (op *Operator) Reader(path string) (*Reader, error)

Reader creates a new Reader for reading the contents of a file at the specified path.

This function is a wrapper around the C-binding function `opendal_operator_reader`.

Parameters

  • path: The path of the file to read.

Returns

  • *Reader: A reader for accessing the file's contents. It implements `io.ReadCloser`.
  • error: An error if the reader creation fails, or nil if successful.

Notes

  • This implementation does not support the `reader_with` functionality.
  • The returned reader allows for more controlled and efficient reading of large files.

Example

func exampleReader(op *opendal.Operator) {
	r, err := op.Reader("path/to/file")
	if err != nil {
		log.Fatal(err)
	}
	defer r.Close()

	size := 1024 // Read 1KB at a time
	buffer := make([]byte, size)

	for {
		n, err := r.Read(buffer)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Read %d bytes: %s\n", n, buffer[:n])
	}
}

Note: This example assumes proper error handling and import statements.

func (*Operator) Rename

func (op *Operator) Rename(src, dest string) error

Rename changes the name or location of a file from the source path to the destination path.

This function moves a file from 'from' to 'to', effectively renaming or relocating it.

Parameters

  • from: The current file path.
  • to: The new file path.

Returns

  • error: An error if the rename operation fails, or nil if successful.

Behavior

  • Both 'from' and 'to' must be file paths, not directories.
  • If 'to' already exists, it will be overwritten.
  • If 'from' and 'to' are identical, an 'IsSameFile' error will be returned.

Example

func exampleRename(op *opendal.Operator) {
	err = op.Rename("path/from/file", "path/to/file")
	if err != nil {
		log.Printf("Rename operation failed: %v", err)
	} else {
		log.Println("File renamed successfully")
	}
}

Note: This example assumes proper error handling and import statements.

func (*Operator) Stat

func (op *Operator) Stat(path string) (*Metadata, error)

Stat retrieves metadata for the specified path.

This function is a wrapper around the C-binding function `opendal_operator_stat`.

Parameters

  • path: The path of the file or directory to get metadata for.

Returns

  • *Metadata: Metadata of the specified path.
  • error: An error if the operation fails, or nil if successful.

Notes

  • The current implementation does not support `stat_with` functionality.
  • If the path does not exist, an error with code opendal.CodeNotFound will be returned.

Example

func exampleStat(op *opendal.Operator) {
	meta, err := op.Stat("/path/to/file")
	if err != nil {
		if e, ok := err.(*opendal.Error); ok && e.Code() == opendal.CodeNotFound {
			fmt.Println("File not found")
			return
		}
		log.Fatalf("Stat operation failed: %v", err)
	}
	fmt.Printf("File size: %d bytes\n", meta.ContentLength())
	fmt.Printf("Last modified: %v\n", meta.LastModified())
}

Note: This example assumes proper error handling and import statements.

func (*Operator) Write

func (op *Operator) Write(path string, data []byte) error

Write writes the given bytes to the specified path.

Write is a wrapper around the C-binding function `opendal_operator_write`. It provides a simplified interface for writing data to the storage. Currently, this implementation does not support the `Operator::write_with` method from the original Rust library, nor does it support streaming writes or multipart uploads.

Parameters

  • path: The destination path where the bytes will be written.
  • data: The byte slice containing the data to be written.

Returns

  • error: An error if the write operation fails, or nil if successful.

Example

func exampleWrite(op *opendal.Operator) {
	err = op.Write("test", []byte("Hello opendal go binding!"))
	if err != nil {
		log.Fatal(err)
	}
}

Note: This example assumes proper error handling and import statements.

func (*Operator) Writer added in v0.1.4

func (op *Operator) Writer(path string) (*Writer, error)

Writer returns a new Writer for the specified path.

Writer is a wrapper around the C-binding function `opendal_operator_writer`. It provides a way to obtain a writer for writing data to the storage system.

Parameters

  • path: The destination path where data will be written.

Returns

  • *Writer: A pointer to a Writer instance, or an error if the operation fails.

Example

func exampleWriter(op *opendal.Operator) {
	writer, err := op.Writer("test/")
	if err != nil {
		log.Fatal(err)
	}
	defer writer.Close()
	_, err = writer.Write([]byte("Hello opendal writer!"))
	if err != nil {
		log.Fatal(err)
	}
}

Note: This example assumes proper error handling and import statements.

type OperatorInfo

type OperatorInfo struct {
	// contains filtered or unexported fields
}

OperatorInfo provides metadata about an Operator instance.

This struct contains essential information about the storage backend and its capabilities, allowing users to query details about the Operator they are working with.

func (*OperatorInfo) GetFullCapability

func (i *OperatorInfo) GetFullCapability() *Capability

func (*OperatorInfo) GetName

func (i *OperatorInfo) GetName() string

func (*OperatorInfo) GetNativeCapability

func (i *OperatorInfo) GetNativeCapability() *Capability

func (*OperatorInfo) GetRoot

func (i *OperatorInfo) GetRoot() string

func (*OperatorInfo) GetScheme

func (i *OperatorInfo) GetScheme() string

type OperatorOptions

type OperatorOptions map[string]string

OperatorOptions contains configuration parameters for creating an Operator.

This struct allows users to specify various settings and credentials required for connecting to and interacting with different storage backends.

Fields in this struct vary depending on the storage scheme being used. Refer to the documentation of specific storage backends for details on required and optional fields.

Example usage:

options := opendal.OperatorOptions{
	"root": "/path/to/root",
	"endpoint": "https://example.com",
	"access_key_id": "your_access_key",
	"secret_access_key": "your_secret_key",
}

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

func (*Reader) Close

func (r *Reader) Close() error

Close releases resources associated with the OperatorReader.

func (*Reader) Read

func (r *Reader) Read(buf []byte) (int, error)

Read reads data from the underlying storage into the provided buffer.

This method implements the io.Reader interface for OperatorReader.

Parameters

  • buf: A pre-allocated byte slice where the read data will be stored. The length of buf determines the maximum number of bytes to read.

Returns

  • int: The number of bytes read. Returns 0 if no data is available or the end of the file is reached.
  • error: An error if the read operation fails, or nil if successful. Note that this method does not return io.EOF; it returns nil at the end of the file.

Notes

  • The caller is responsible for pre-allocating the buffer and determining its size.

Example

reader, err := op.Reader("path/to/file")
if err != nil {
	log.Fatal(err)
}
defer reader.Close()

buf := make([]byte, 1024)
for {
	n, err := reader.Read(buf)
	if err != nil {
		log.Fatal(err)
	}
	if n == 0 {
		break // End of file
	}
	// Process buf[:n]
}

Note: Always check the number of bytes read (n) as it may be less than len(buf).

func (*Reader) Seek added in v0.1.11

func (r *Reader) Seek(offset int64, whence int) (int64, error)

Seek sets the offset for the next Read operation on the reader.

This method implements the io.Seeker interface for Reader.

Parameters

  • offset: The offset from the origin (specified by whence).
  • whence: The reference point for offset. Can be:
  • io.SeekStart (0): Relative to the start of the file
  • io.SeekCurrent (1): Relative to the current position
  • io.SeekEnd (2): Relative to the end of the file

Returns

  • int64: The new absolute position in the file after the seek operation.
  • error: An error if the seek operation fails, or nil if successful.

Example

reader, err := op.Reader("path/to/file")
if err != nil {
	log.Fatal(err)
}
defer reader.Close()

// Seek to the middle of the file
pos, err := reader.Seek(1000, io.SeekStart)
if err != nil {
	log.Fatal(err)
}
fmt.Printf("New position: %d\n", pos)

// Seek relative to current position
pos, err = reader.Seek(100, io.SeekCurrent)
if err != nil {
	log.Fatal(err)
}
fmt.Printf("New position: %d\n", pos)

Note: The actual new position may differ from the requested position if the underlying storage system has restrictions on seeking.

type Scheme

type Scheme interface {
	// Name returns the unique identifier of the scheme.
	Name() string

	// Path returns the filesystem path where the scheme's shared library (.so) is located.
	Path() string

	// LoadOnce initializes the scheme. It ensures that initialization occurs only once,
	// even if called multiple times. Subsequent calls after the first should be no-ops.
	//
	// Returns an error if initialization fails.
	LoadOnce() error
}

Scheme defines the interface for storage scheme implementations.

A Scheme represents a specific storage backend (e.g., S3, filesystem, memory) and provides methods to identify and initialize the scheme.

Implementations of this interface should be thread-safe, especially the LoadOnce method.

type Writer added in v0.1.4

type Writer struct {
	// contains filtered or unexported fields
}

func (*Writer) Close added in v0.1.4

func (w *Writer) Close() error

Close finishes the write and releases the resources associated with the Writer. It is important to call Close after writing all the data to ensure that the data is properly flushed and written to the storage. Otherwise, the data may be lost.

func (*Writer) Write added in v0.1.4

func (w *Writer) Write(p []byte) (n int, err error)

Write writes the given bytes to the specified path.

Write is a wrapper around the C-binding function `opendal_operator_write`. It provides a simplified interface for writing data to the storage. Write can be called multiple times to write additional data to the same path.

The maximum size of the data that can be written in a single call is 256KB.

Parameters

  • path: The destination path where the bytes will be written.
  • data: The byte slice containing the data to be written.

Returns

  • error: An error if the write operation fails, or nil if successful.

Example

func exampleWrite(op *opendal.Operator) {
	err = op.Write("test", []byte("Hello opendal go binding!"))
	if err != nil {
		log.Fatal(err)
	}
}

Note: This example assumes proper error handling and import statements.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL