s3

Documentation

Index

• Variables
• func ListFiles(pth string, accessKey, secretKey string) ([]stat.Stats, error)
• func Stat(pth string, accessKey, secretKey string) (stat.Stats, error)
• type Options
  • func NewOptions() *Options
• type Reader
  • func NewReader(pth string, accessKey, secretKey string) (*Reader, error)
  • func (r *Reader) Close() (err error)
  • func (r *Reader) Read(p []byte) (n int, err error)
  • func (r *Reader) ReadLine() (ln []byte, err error)
  • func (r *Reader) Stats() stat.Stats
• type Writer
  • func NewWriter(pth string, accessKey, secretKey string, opt *Options) (*Writer, error)
  • func (w *Writer) Abort() (err error)
  • func (w *Writer) Close() error
  • func (w *Writer) Stats() stat.Stats
  • func (w *Writer) Write(p []byte) (n int, err error)
  • func (w *Writer) WriteLine(ln []byte) (err error)

Examples

• NewReader
• NewWriter
• Reader.Close
• Reader.Read
• Reader.ReadLine
• Reader.Stats
• Writer.Abort
• Writer.Close
• Writer.Stats
• Writer.Write
• Writer.WriteLine

Variables

var (
	// domain of s3 compatible api
	StoreHost = "s3.amazonaws.com"
)

Functions

func ListFiles

func ListFiles(pth string, accessKey, secretKey string) ([]stat.Stats, error)

ListFiles will list all file objects in the provided pth directory. pth is assumed to be a directory and so a trailing "/" is appended if one does not already exist.

func Stat

func Stat(pth string, accessKey, secretKey string) (stat.Stats, error)

Stat a s3 directory or file for additional information

Types

type Options

type Options struct {
	*buf.Options
}

func NewOptions

func NewOptions() *Options

type Reader

type Reader struct {
	// contains filtered or unexported fields
}

Reader will read in streamed bytes from the s3 object.NewS3Client

func NewReader

func NewReader(pth string, accessKey, secretKey string) (*Reader, error)

Example

pth := fmt.Sprintf("s3://%v/read/test.txt", testBucket)
r, err := NewReader(pth, testAccessKey, testSecretKey)
if r == nil {
	return
}

fmt.Println(err)        // output: <nil>
fmt.Println(r.sts.Path) // output: s3://task-tools-s3test/read/test.txt
fmt.Println(r.sts.Size) // output: 20

Output:

<nil>
s3://task-tools-s3test/read/test.txt
20

func (*Reader) Close

func (r *Reader) Close() (err error)

Example

pth := fmt.Sprintf("s3://%v/read/test.txt", testBucket)
r, _ := NewReader(pth, testAccessKey, testSecretKey)
if r == nil {
	return
}

r.ReadLine()
r.ReadLine()
r.ReadLine()
err := r.Close()
sts := r.Stats()

fmt.Println(err)          // output: <nil>
fmt.Println(sts.ByteCnt)  // output: 20
fmt.Println(sts.LineCnt)  // output: 2
fmt.Println(sts.Checksum) // output: 54f30d75cf7374c7e524a4530dbc93c2

Output:

<nil>
20
2
54f30d75cf7374c7e524a4530dbc93c2

func (*Reader) Read

func (r *Reader) Read(p []byte) (n int, err error)

Example

pth := fmt.Sprintf("s3://%v/read/test.txt", testBucket)
r, err := NewReader(pth, testAccessKey, testSecretKey)
if r == nil {
	return
}

b := make([]byte, 20)
n, err := r.Read(b)

fmt.Println(n)             // output: 20
fmt.Println(err)           // output: <nil>
fmt.Print(string(b))       // output: test line, test line
fmt.Println(r.sts.ByteCnt) // output: 20
fmt.Println(r.sts.LineCnt) // output: 0

Output:

20
<nil>
test line
test line
20
0

func (*Reader) ReadLine

func (r *Reader) ReadLine() (ln []byte, err error)

Example

pth := fmt.Sprintf("s3://%v/read/test.txt", testBucket)
r, _ := NewReader(pth, testAccessKey, testSecretKey)
if r == nil {
	return
}

ln1, err1 := r.ReadLine()
ln2, err2 := r.ReadLine()

fmt.Println(string(ln1))   // output: test line
fmt.Println(err1)          // output: <nil>
fmt.Println(string(ln2))   // output: test line
fmt.Println(err2)          // output: <nil>
fmt.Println(r.sts.ByteCnt) // output: 20
fmt.Println(r.sts.LineCnt) // output: 2

Output:

test line
<nil>
test line
<nil>
20
2

func (*Reader) Stats

func (r *Reader) Stats() stat.Stats

Example

pth := fmt.Sprintf("s3://%v/read/test.txt", testBucket)
r, _ := NewReader(pth, testAccessKey, testSecretKey)
if r == nil {
	return
}

r.ReadLine()
sts := r.Stats()
fmt.Println(sts.ByteCnt) // output: 10
fmt.Println(sts.LineCnt) // output: 1

Output:

10
1

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer will write to local buffer first and will copy all the written contents to the S3 destination after calling Close(). Close() must be called in order for the written contents to be written to S3.

Calling Abort() before Close() will cleanup the buffer. Calling Close() after Abort() will not result in any writing to S3.

Calling Abort() after Close() will do nothing.

func NewWriter

func NewWriter(pth string, accessKey, secretKey string, opt *Options) (*Writer, error)

Example

pth := fmt.Sprintf("s3://%v/write/test.txt", testBucket)
w, err := NewWriter(pth, testAccessKey, testSecretKey, nil)
if w == nil {
	return
}

fmt.Println(err)               // output: <nil>
fmt.Println(w.sts.Path)        // output: s3://task-tools-s3test/write/test.txt
fmt.Println(w.s3Client != nil) // output: true
fmt.Println(w.bfr != nil)      // output: true
fmt.Println(w.bucket)          // output: task-tools-s3test
fmt.Println(w.objPth)          // output: write/test.txt
fmt.Println(w.tmpPth == "")    // output: true

Output:

<nil>
s3://task-tools-s3test/write/test.txt
true
true
task-tools-s3test
write/test.txt
true

func (*Writer) Abort

func (w *Writer) Abort() (err error)

Abort will: - clear and close buffer

Calling Close after Abort will do nothing. Writing after calling Abort has undefined behavior.

Example

pth := fmt.Sprintf("s3://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testAccessKey, testSecretKey, nil)
if w == nil {
	return
}

w.WriteLine([]byte("test line"))

fmt.Println(w.done) // output: false
err := w.Abort()

fmt.Println(err)    // output: <nil>
fmt.Println(w.done) // output: true

Output:

false
<nil>
true

func (*Writer) Close

func (w *Writer) Close() error

Close will: - calculate final checksum - copy (mv) buffer to pth file - clear and close buffer - report any errors

If an error is returned it should be assumed that S3 object writing failed.

Calling Abort after Close will do nothing. Writing after calling Close has undefined behavior.

Example

pth := fmt.Sprintf("s3://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testAccessKey, testSecretKey, nil)
if w == nil {
	return
}

w.WriteLine([]byte("test line"))
w.WriteLine([]byte("test line"))
err := w.Close()

fmt.Println(err)                     // output: <nil>
fmt.Println(w.done)                  // output: true
fmt.Println(w.objSts.Checksum != "") // output: true

// cleanup
rmTestFile(pth)

Output:

<nil>
true
true

func (*Writer) Stats

func (w *Writer) Stats() stat.Stats

Example

pth := fmt.Sprintf("s3://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testAccessKey, testSecretKey, nil)
if w == nil {
	return
}

w.WriteLine([]byte("test line"))
w.WriteLine([]byte("test line"))
sts := w.Stats()

fmt.Println(sts.Path)          // output: s3://task-tools-s3test/write/test.txt
fmt.Println(sts.ByteCnt)       // output: 20
fmt.Println(sts.LineCnt)       // output: 2
fmt.Println(sts.Size)          // output: 0
fmt.Println(sts.Checksum)      // output:
fmt.Println(sts.Created == "") // output: true

Output:

s3://task-tools-s3test/write/test.txt
20
2
0

true

func (*Writer) Write

func (w *Writer) Write(p []byte) (n int, err error)

Example

pth := fmt.Sprintf("s3://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testAccessKey, testSecretKey, nil)
if w == nil {
	return
}

n, err := w.Write([]byte("test line"))

fmt.Println(n)   // output: 9
fmt.Println(err) // output: <nil>

Output:

9
<nil>

func (*Writer) WriteLine

func (w *Writer) WriteLine(ln []byte) (err error)

Example

pth := fmt.Sprintf("s3://%v/write/test.txt", testBucket)
w, _ := NewWriter(pth, testAccessKey, testSecretKey, nil)
if w == nil {
	return
}

err := w.WriteLine([]byte("test line"))

fmt.Println(err) // output: <nil>

Output:

<nil>