soda

package module
Version: v1.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 23, 2020 License: MIT Imports: 12 Imported by: 1

README

GoDoc Build Status codecov Go Report Card

go-soda

Socrata Open Data API (SODA) GET client for Golang

SODAGopherLove

For SODA docs see http://dev.socrata.com

Features

This is a simple client for get requests only. The client provides basic structs for querying and filtering the database. Although all operations are supported most are just provided as a string to allow for all kind of complex queries.

Install

Just go get it

go get -u github.com/SebastiaanKlippert/go-soda

GetRequest

The default GetRequest struct is not safe for use in multiple goroutines, create one for each goroutine or use the OffsetGetRequest.

OffsetGetRequest

The OffsetGetRequest is a wrapper around the GetRequest and provides an easy offset counter to get loads of data. It can be shared by multiple goroutines to get your data a lot faster.

Metadata

For each GetRequest you can request metadata (using a separate API call). The metadata contains info about the dataset like creation and update times, licensing info and advanced column info.

sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
metadata, err := sodareq.Metadata.Get()

GetRequest sample

See the test file for more examples.

func QuerySample() {

	sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")

	//count all records
	count, err := sodareq.Count()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(count)
	
	//get dataset last updated time
	modified, err := sodareq.Modified()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(modified)	

	//list all fields/columns
	fields, err := sodareq.Fields()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(fields)

	//get some JSON data using a complex query
	sodareq.Format = "json"
	sodareq.Query.Select = []string{"farm_name", "category", "item", "zipcode"}
	sodareq.Query.Where = `lower(farm_name) like '%sun%farm%' AND (item in('Radishes', 
	  'Cucumbers') OR lower(item) like '%flower%')`
	sodareq.Query.Limit = 1000
	sodareq.Query.AddOrder("farm_name", soda.DirAsc)
	sodareq.Query.AddOrder("category", soda.DirDesc)

	//count this result first
	querycount, err := sodareq.Count()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(querycount)

	//get the results
	resp, err := sodareq.Get()
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	rawresp, _ := ioutil.ReadAll(resp.Body)
	fmt.Println(string(rawresp))
}

func JSONSample() {

	sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")

	//get some JSON data
	sodareq.Format = "json"
	sodareq.Filters["item"] = "Radishes"
	sodareq.Query.Limit = 10

	resp, err := sodareq.Get()
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	results := make([]map[string]interface{}, 0)
	err = json.NewDecoder(resp.Body).Decode(&results)
	if err != nil {
		log.Fatal(err)
	}

	//Process data here
	for _, r := range results {
		fmt.Println(r["farm_name"], r["item"])
	}
}

func CSVSample() {
	sodareq := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
	sodareq.Format = "csv"
	sodareq.Filters["item"] = "Radishes"
	sodareq.Query.Limit = 10

	resp, err := sodareq.Get()
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	//Process data here
	csvreader := csv.NewReader(resp.Body)
	for {
		record, err := csvreader.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}
		fmt.Println(record)
	}
}

OffsetGetRequest sample

Get all data in batches of 2000 rows using 4 goroutines

func GetAllData() error {

	gr := soda.NewGetRequest("https://data.ct.gov/resource/y6p2-px98", "")
	gr.Format = "json"
	gr.Query.AddOrder("zipcode", soda.DirAsc)

	ogr, err := soda.NewOffsetGetRequest(gr)
	if err != nil {
		return err
	}

	for i := 0; i < 4; i++ {

		ogr.Add(1)
		go func() {
			defer ogr.Done()

			for {
				resp, err := ogr.Next(2000)
				if err == soda.ErrDone {
					break
				}
				if err != nil {
					log.Fatal(err)
				}

				results := make([]map[string]interface{}, 0)
				err = json.NewDecoder(resp.Body).Decode(&results)
				resp.Body.Close()
				if err != nil {
					log.Fatal(err)
				}
				//Process your data
			}
		}()

	}
	ogr.Wait()

	return nil
}

Documentation

Overview

Package soda provides HTTP GET tools for SODA (Socrata Open Data API) webservices, see http://dev.socrata.com/

Index

Constants

This section is empty.

Variables

View Source
var ErrDone = errors.New("Done")

ErrDone is returned by OffsetGetRequest.Next when done

Functions

This section is empty.

Types

type Column

type Column struct {
	DataTypeName   string `json:"dataTypeName"`
	FieldName      string `json:"fieldName"`
	Format         Format `json:"format"`
	ID             int    `json:"id"`
	Name           string `json:"name"`
	Position       int    `json:"position"`
	RenderTypeName string `json:"renderTypeName"`
	TableColumnID  int    `json:"tableColumnId"`
	Width          int    `json:"width"`
}

Column describes one data column

type Direction

type Direction bool

Direction is used to set the sort direction to ascending or descending

const (
	// DirAsc is used to set ascending sort order
	DirAsc Direction = false

	// DirDesc is used to set descending sort order
	DirDesc Direction = true
)

type Format

type Format struct {
	PrecisionStyle string `json:"precisionStyle"`
	Align          string `json:"align"`
	NoCommas       string `json:"noCommas"`
}

Format describes column formats

type GetRequest

type GetRequest struct {
	Format     string //json, csv etc
	Filters    SimpleFilters
	Query      SoSQL
	Metadata   metadata
	HTTPClient *http.Client //For clients who need a custom HTTP client
	// contains filtered or unexported fields
}

GetRequest is a wrapper/container for SODA requests. This is NOT safe for use by multiple goroutines as Format, Filters and Query will be overwritten. Create a new GetRequest in each goroutine you use or use an OffsetGetRequest

func NewGetRequest

func NewGetRequest(endpoint, apptoken string) *GetRequest

NewGetRequest creates a new GET request, the endpoint must be specified without the format. For example https://data.ct.gov/resource/hma6-9xbg

func (*GetRequest) Count

func (r *GetRequest) Count() (uint, error)

Count gets the total number of records in the dataset by executing a SODA request

func (*GetRequest) Fields

func (r *GetRequest) Fields() ([]string, error)

Fields returns all the fields present in the dataset (ignores select fields). Spaces in fieldnames are replaced by underscores.

func (*GetRequest) Get

func (r *GetRequest) Get() (*http.Response, error)

Get executes the HTTP GET request

func (*GetRequest) GetEndpoint

func (r *GetRequest) GetEndpoint() string

GetEndpoint returns the complete SODA URL with format

func (*GetRequest) Modified

func (r *GetRequest) Modified() (time.Time, error)

Modified returns when the dataset was last updated

func (*GetRequest) URLValues

func (r *GetRequest) URLValues() url.Values

URLValues returns the url.Values for the GetRequest

type Metadata

type Metadata struct {
	AverageRating int       `json:"averageRating"`
	Category      string    `json:"category"`
	Columns       []Column  `json:"columns"`
	CreatedAt     Timestamp `json:"createdAt"`
	DisplayType   string    `json:"displayType"`
	DownloadCount int       `json:"downloadCount"`
	Flags         []string  `json:"flags"`
	Grants        []struct {
		Flags     []string `json:"flags"`
		Inherited bool     `json:"inherited"`
		Type      string   `json:"type"`
	} `json:"grants"`
	ID             string    `json:"id"`
	IndexUpdatedAt Timestamp `json:"indexUpdatedAt"`
	License        struct {
		Name string `json:"name"`
	} `json:"license"`
	LicenseID string `json:"licenseId"`
	Metadata  struct {
		AvailableDisplayTypes []string `json:"availableDisplayTypes"`
		CustomFields          struct {
			Licentie struct {
				Licentie string `json:"Licentie"`
			} `json:"Licentie"`
		} `json:"custom_fields"`
		RdfSubject       string `json:"rdfSubject"`
		RenderTypeConfig struct {
			Visible struct {
				Table bool `json:"table"`
			} `json:"visible"`
		} `json:"renderTypeConfig"`
		RowLabel string `json:"rowLabel"`
	} `json:"metadata"`
	Name             string `json:"name"`
	NewBackend       bool   `json:"newBackend"`
	NumberOfComments int    `json:"numberOfComments"`
	Oid              int    `json:"oid"`
	Owner            struct {
		DisplayName string   `json:"displayName"`
		ID          string   `json:"id"`
		Rights      []string `json:"rights"`
		RoleName    string   `json:"roleName"`
		ScreenName  string   `json:"screenName"`
	} `json:"owner"`
	PublicationAppendEnabled bool      `json:"publicationAppendEnabled"`
	PublicationDate          Timestamp `json:"publicationDate"`
	PublicationGroup         int       `json:"publicationGroup"`
	PublicationStage         string    `json:"publicationStage"`
	Query                    struct{}  `json:"-"` //TODO
	Ratings                  struct {
		Rating int `json:"rating"`
	} `json:"ratings"`
	Rights        []string  `json:"rights"`
	RowsUpdatedAt Timestamp `json:"rowsUpdatedAt"`
	RowsUpdatedBy string    `json:"rowsUpdatedBy"`
	TableAuthor   struct {
		DisplayName string   `json:"displayName"`
		ID          string   `json:"id"`
		Rights      []string `json:"rights"`
		RoleName    string   `json:"roleName"`
		ScreenName  string   `json:"screenName"`
	} `json:"tableAuthor"`
	TableID          int       `json:"tableId"`
	Tags             []string  `json:"tags"`
	TotalTimesRated  int       `json:"totalTimesRated"`
	ViewCount        int       `json:"viewCount"`
	ViewLastModified Timestamp `json:"viewLastModified"`
	ViewType         string    `json:"viewType"`
}

Metadata contains the resource metadata

type OffsetGetRequest

type OffsetGetRequest struct {
	sync.WaitGroup
	// contains filtered or unexported fields
}

OffsetGetRequest is a request getter that gets all the records using the filters and limits from gr and is safe to use by multiple goroutines, use Next(number) to get the next number of records. A sync.WaitGroup is embedded for easy concurrency.

func NewOffsetGetRequest

func NewOffsetGetRequest(gr *GetRequest) (*OffsetGetRequest, error)

NewOffsetGetRequest creates a new OffsetGetRequest from gr and does a count request to determine the number of records to get

func (*OffsetGetRequest) Count

func (o *OffsetGetRequest) Count() uint

Count returns the number of records from memory

func (*OffsetGetRequest) IsDone

func (o *OffsetGetRequest) IsDone() bool

IsDone returns if we have gotten all records

func (*OffsetGetRequest) Next

func (o *OffsetGetRequest) Next(number uint) (*http.Response, error)

Next gets the next number of records

type SimpleFilters

type SimpleFilters map[string]string

SimpleFilters is the easiest way to filter columns for equality. Add the column to filter on a map key and the filter value as map value. If you include multiple filters, the filters will be combined using a boolean AND. See http://dev.socrata.com/docs/filtering.html

func (SimpleFilters) URLValues

func (sf SimpleFilters) URLValues() url.Values

URLValues returns the url.Values for the SimpleFilters

type SoSQL

type SoSQL struct {
	Select []string //The set of columns to be returned. Default: All columns, equivalent to $select=*
	Where  string   //Filters the rows to be returned. Default: No filter, and returning a max of $limit values
	Order  []struct {
		Column string //Column name
		Desc   bool   //Descending. Default: false = Ascending
	} //Specifies the order of results. Default: Unspecified order, but it will be consistent across paging
	Group  string //Column to group results on, similar to SQL Grouping. Default: No grouping
	Limit  uint   //Maximum number of results to return. Default: 1000 (with a maximum of 50,000)
	Offset uint   //Offset count into the results to start at, used for paging. Default: 0
	Q      string //Performs a full text search for a value. Default: No search

}

SoSQL implements the Socrata Query Language and is used to build more complex queries. See http://dev.socrata.com/docs/queries.html

func (*SoSQL) AddOrder

func (sq *SoSQL) AddOrder(column string, dir Direction)

AddOrder can be called for each field you want to sort the result on. If parameter descending is true, the column will be sorted descending, or ascending if false.

func (*SoSQL) ClearOrder

func (sq *SoSQL) ClearOrder()

ClearOrder removes all order fields

func (*SoSQL) URLValues

func (sq *SoSQL) URLValues() url.Values

URLValues returns the url.Values for the SoSQL query

type Timestamp

type Timestamp time.Time

Timestamp is a time.Time struct unmarshalled from a unix epoch time

func (Timestamp) Time

func (t Timestamp) Time() time.Time

Time returns t as time.Time

func (*Timestamp) UnmarshalJSON

func (t *Timestamp) UnmarshalJSON(b []byte) error

UnmarshalJSON sets t from a timestamp

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL