datapackage

Documentation

Index

• type Package
  • func FromReader(r io.Reader, basePath string, loaders ...validator.RegistryLoader) (*Package, error)
  • func FromString(in string, basePath string, loaders ...validator.RegistryLoader) (*Package, error)
  • func Load(path string, loaders ...validator.RegistryLoader) (*Package, error)
  • func New(descriptor map[string]interface{}, basePath string, ...) (*Package, error)
  • func (p *Package) AddResource(d map[string]interface{}) error
  • func (p *Package) Descriptor() map[string]interface{}
  • func (p *Package) GetResource(name string) *Resource
  • func (p *Package) RemoveResource(name string)
  • func (p *Package) ResourceNames() []string
  • func (p *Package) Resources() []*Resource
  • func (p *Package) SaveDescriptor(path string) error
  • func (p *Package) Update(newDescriptor map[string]interface{}, loaders ...validator.RegistryLoader) error
  • func (p *Package) Zip(path string) error
• type Resource
  • func NewResource(d map[string]interface{}, registry validator.Registry) (*Resource, error)
  • func NewResourceFromString(res string, registry validator.Registry) (*Resource, error)
  • func NewResourceWithDefaultRegistry(d map[string]interface{}) (*Resource, error)
  • func NewUncheckedResource(d map[string]interface{}) *Resource
  • func (r *Resource) Cast(out interface{}, opts ...csv.CreationOpts) error
  • func (r *Resource) CastColumn(name string, out interface{}, opts ...csv.CreationOpts) error
  • func (r *Resource) Descriptor() map[string]interface{}
  • func (r *Resource) GetSchema() (schema.Schema, error)
  • func (r *Resource) GetTable(opts ...csv.CreationOpts) (table.Table, error)
  • func (r *Resource) Iter(opts ...csv.CreationOpts) (table.Iterator, error)
  • func (r *Resource) Name() string
  • func (r *Resource) RawRead() (io.ReadCloser, error)
  • func (r *Resource) ReadAll(opts ...csv.CreationOpts) ([][]string, error)
  • func (r *Resource) Tabular() bool
  • func (r *Resource) Update(d map[string]interface{}, loaders ...validator.RegistryLoader) error

Examples

• Load (Cast)
• Load (ReadAll)
• Load (ReadAllRemote)
• Load (ReadRaw)
• Resource.CastColumn

Types

type Package

type Package struct {
	// contains filtered or unexported fields
}

Package represents a https://specs.frictionlessdata.io/data-package/

func FromReader

func FromReader(r io.Reader, basePath string, loaders ...validator.RegistryLoader) (*Package, error)

FromReader creates a data package from an io.Reader.

func FromString

func FromString(in string, basePath string, loaders ...validator.RegistryLoader) (*Package, error)

FromString creates a data package from a string representation of the package descriptor.

func Load

func Load(path string, loaders ...validator.RegistryLoader) (*Package, error)

Load the data package descriptor from the specified URL or file path. If path has the ".zip" extension, it will be saved in local filesystem and decompressed before loading.

Example (Cast)

dir, _ := ioutil.TempDir("", "datapackage_exampleload")
defer os.RemoveAll(dir)
descriptorPath := filepath.Join(dir, "pkg.json")
descriptorContents := `{"resources": [{ 
		  "name": "res1",
		  "path": "data.csv",
		  "profile": "tabular-data-resource",
		  "schema": {"fields": [{"name":"name", "type":"string"}]}
		}]}`
ioutil.WriteFile(descriptorPath, []byte(descriptorContents), 0666)

resPath := filepath.Join(dir, "data.csv")
resContent := []byte("foo\nbar")
ioutil.WriteFile(resPath, resContent, 0666)

pkg, _ := Load(descriptorPath, validator.InMemoryLoader())
res := pkg.GetResource("res1")
people := []struct {
	Name string `tableheader:"name"`
}{}
res.Cast(&people)
fmt.Printf("%+v", people)

Output:

[{Name:foo} {Name:bar}]

Example (ReadAll)

dir, _ := ioutil.TempDir("", "datapackage_exampleload")
dir = filepath.Clean(dir) // removes possible trailing slashes.
defer os.RemoveAll(dir)
descriptorPath := filepath.Join(dir, "pkg.json")
descriptorContents := `{"resources": [{ 
		  "name": "res1",
		  "path": "data.csv",
		  "profile": "tabular-data-resource",
		  "schema": {"fields": [{"name":"name", "type":"string"}]}
		}]}`
ioutil.WriteFile(descriptorPath, []byte(descriptorContents), 0666)

resPath := filepath.Join(dir, "data.csv")
resContent := []byte("foo\nbar")
ioutil.WriteFile(resPath, resContent, 0666)

pkg, _ := Load(descriptorPath, validator.InMemoryLoader())
contents, _ := pkg.GetResource("res1").ReadAll()
fmt.Println(contents)

Output:

[[foo] [bar]]

Example (ReadAllRemote)

ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
	// If the request is for data, returns the content.
	switch {
	case r.RequestURI == "/data.csv":
		fmt.Fprintf(w, "foo\nbar")
	default:
		fmt.Fprintf(w, `{"resources": [{ 
				"name": "res1",
				"path": "data.csv",
				"profile": "tabular-data-resource",
				"schema": {"fields": [{"name":"name", "type":"string"}]}
			  }]}`)
	}
}))
defer ts.Close()
pkg, _ := Load(ts.URL, validator.InMemoryLoader())
contents, _ := pkg.GetResource("res1").ReadAll()
fmt.Println(contents)

Output:

[[foo] [bar]]

Example (ReadRaw)

dir, _ := ioutil.TempDir("", "datapackage_exampleload")
dir = filepath.Clean(dir) // removes possible trailing slashes.
defer os.RemoveAll(dir)
descriptorPath := filepath.Join(dir, "pkg.json")
descriptorContents := `{"resources": [{ 
		  "name": "res1",
		  "path": "schemaorg.json",
		  "format": "application/ld+json",
		  "profile": "data-resource"
		}]}`
ioutil.WriteFile(descriptorPath, []byte(descriptorContents), 0666)

resPath := filepath.Join(dir, "schemaorg.json")
resContent := []byte(`{"@context": {"@vocab": "http://schema.org/"}}`)
ioutil.WriteFile(resPath, resContent, 0666)

pkg, _ := Load(descriptorPath, validator.InMemoryLoader())
rc, _ := pkg.GetResource("res1").RawRead()
defer rc.Close()
contents, _ := ioutil.ReadAll(rc)
fmt.Println(string(contents))

Output:

{"@context": {"@vocab": "http://schema.org/"}}

func New

func New(descriptor map[string]interface{}, basePath string, loaders ...validator.RegistryLoader) (*Package, error)

New creates a new data package based on the descriptor.

func (*Package) AddResource

func (p *Package) AddResource(d map[string]interface{}) error

AddResource adds a new resource to the package, updating its descriptor accordingly.

func (*Package) Descriptor

func (p *Package) Descriptor() map[string]interface{}

Descriptor returns a deep copy of the underlying descriptor which describes the package.

func (*Package) GetResource

func (p *Package) GetResource(name string) *Resource

GetResource return the resource which the passed-in name or nil if the resource is not part of the package.

func (*Package) RemoveResource

func (p *Package) RemoveResource(name string)

RemoveResource removes the resource from the package, updating its descriptor accordingly.

func (*Package) ResourceNames

func (p *Package) ResourceNames() []string

ResourceNames return a slice containing the name of the resources.

func (*Package) Resources

func (p *Package) Resources() []*Resource

Resources returns a copy of data package resources.

func (*Package) SaveDescriptor

func (p *Package) SaveDescriptor(path string) error

SaveDescriptor saves the data package descriptor to the passed-in file path. It create creates the named file with mode 0666 (before umask), truncating it if it already exists.

func (*Package) Update

func (p *Package) Update(newDescriptor map[string]interface{}, loaders ...validator.RegistryLoader) error

Update the package with the passed-in descriptor. The package will only be updated if the the new descriptor is valid, otherwise the error will be returned.

func (*Package) Zip

func (p *Package) Zip(path string) error

Zip saves a zip-compressed file containing the package descriptor and all resource data. It create creates the named file with mode 0666 (before umask), truncating it if it already exists.

type Resource

type Resource struct {
	// contains filtered or unexported fields
}

Resource describes a data resource such as an individual file or table.

func NewResource

func NewResource(d map[string]interface{}, registry validator.Registry) (*Resource, error)

NewResource creates a new Resource from the passed-in descriptor, if valid. The passed-in validator.Registry will be the source of profiles used in the validation.

func NewResourceFromString

func NewResourceFromString(res string, registry validator.Registry) (*Resource, error)

NewResourceFromString creates a new Resource from the passed-in JSON descriptor, if valid. The passed-in validator.Registry will be the source of profiles used in the validation.

func NewResourceWithDefaultRegistry

func NewResourceWithDefaultRegistry(d map[string]interface{}) (*Resource, error)

NewResourceWithDefaultRegistry creates a new Resource from the passed-in descriptor. It uses the default registry to validate the resource descriptor.

func NewUncheckedResource

func NewUncheckedResource(d map[string]interface{}) *Resource

NewUncheckedResource returns an Resource instance based on the descriptor without any verification. The returned Resource might not be valid.

func (*Resource) Cast

func (r *Resource) Cast(out interface{}, opts ...csv.CreationOpts) error

Cast resource contents. The result argument must necessarily be the address for a slice. The slice may be nil or previously allocated.

func (*Resource) CastColumn

func (r *Resource) CastColumn(name string, out interface{}, opts ...csv.CreationOpts) error

CastColumn casts a column from tabular resource contents. The out argument must necessarily be the address for a slice. The slice may be nil or previously allocated.

Example

resStr := `
	{
		"name":    "col",
		"data":    "name,age\nfoo,42\nbar,84",
		"format":  "csv",
		"profile": "tabular-data-resource",
		"schema": {"fields": [{"name": "name", "type": "string"},{"name": "age", "type": "integer"}]}
	}`
res, _ := NewResourceFromString(resStr, validator.MustInMemoryRegistry())
var ages []float64
res.CastColumn("age", &ages, csv.LoadHeaders())
fmt.Println(ages)

Output:

[42 84]

func (*Resource) Descriptor

func (r *Resource) Descriptor() map[string]interface{}

Descriptor returns a copy of the underlying descriptor which describes the resource.

func (*Resource) GetSchema

func (r *Resource) GetSchema() (schema.Schema, error)

GetSchema returns the schema associated to the resource, if present. The returned schema is based on a copy of the descriptor. Changes to it won't affect the data package descriptor structure.

func (*Resource) GetTable

func (r *Resource) GetTable(opts ...csv.CreationOpts) (table.Table, error)

GetTable returns a table object to access the data. Returns an error if the resource is not tabular.

func (*Resource) Iter

func (r *Resource) Iter(opts ...csv.CreationOpts) (table.Iterator, error)

Iter returns an Iterator to read the tabular resource. Iter returns an error if the table physical source can not be iterated. The iteration process always start at the beginning of the table.

func (*Resource) Name

func (r *Resource) Name() string

Name returns the resource name.

func (*Resource) RawRead

func (r *Resource) RawRead() (io.ReadCloser, error)

RawRead returns an io.ReaderCloser associated to the resource contents. It can be used to access the content of non-tabular resources.

func (*Resource) ReadAll

func (r *Resource) ReadAll(opts ...csv.CreationOpts) ([][]string, error)

ReadAll reads all rows from the table and return it as strings.

func (*Resource) Tabular

func (r *Resource) Tabular() bool

Tabular checks whether the resource is tabular.

func (*Resource) Update

func (r *Resource) Update(d map[string]interface{}, loaders ...validator.RegistryLoader) error

Update the resource with the passed-in descriptor. The resource will only be updated if the the new descriptor is valid, otherwise the error will be returned.