billtitles

package module
v0.0.0-...-ab470a8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 26, 2021 License: MIT Imports: 20 Imported by: 0

README 

# billtitles

An application to load bill title data into a sql repository. This repository works in conjunction with `github.com/aih/bills`, a Go repository to process bill metadata files to create an index of `title: bills`.

This repository provides a module that converts the json output from `bills` into sqlite3. It also provides a (crude) Go API to query for bills and titles.

To test and build, install `cmake` and run `make`. This will initially take some time to load data, then run the tests and build three executables into the `cmd/bin` directory:

billquery:: a Go http service to retrieve bill metadata by bill number, and retrieve related bills (superceded by the API at `billtitles-py`) 

related:: collects the 'esSimilarCategory.json' files in the directory provided at `-p`, and adds them to the sql database

titlesjson:: loads title data from files `data/titles.json` and `data/maintitles.json` into a sql repository (`billtitles.db`)

Documentation

Index

Constants

View Source 
const BILLSRELATED_DB = "billsrelated.db"
View Source 
const BILLTITLES_DB = "billtitles.db"

Variables

View Source 
var (
	TitleIndex              = "titles.json"
	MainTitleIndex          = "maintitles.json"
	SampleTitleIndex        = "sampletitles.json"
	PathToDataDir           = "data"
	TitlesPath              = path.Join(PathToDataDir, TitleIndex)
	MainTitlesPath          = path.Join(PathToDataDir, MainTitleIndex)
	SampleTitlesPath        = path.Join(PathToDataDir, SampleTitleIndex)
	BillnumberRegexCompiled = regexp.MustCompile(`(?P<congress>[1-9][0-9]*)(?P<stage>[a-z]{1,8})(?P<billnumber>[1-9][0-9]*)(?P<version>[a-z]+)?`)
	BillVersionsOrdered     = billVersions{"ih": 0, "rh": 1, "rfs": 2, "eh": 3, "es": 4, "enr": 5}
	ZLogLevels              = LogLevels{"Debug": zerolog.DebugLevel, "Info": zerolog.InfoLevel, "Error": zerolog.ErrorLevel}
)

Constants for this package

Functions

func AddBillNumbersToTitle 

func AddBillNumbersToTitle(titleMap *sync.Map, title string, billnumbers []string) (*sync.Map, error)

func AddBillStructDb 

func AddBillStructDb(db *gorm.DB, billStruct *Bill)

func AddBillnumberversionsDb 

func AddBillnumberversionsDb(db *gorm.DB, billnumberversions []string)

func AddTitle 

func AddTitle(titleMap *sync.Map, title string) (*sync.Map, error)

func AddTitleDb 

func AddTitleDb(db *gorm.DB, title string) *gorm.DB

func AddTitleStructDb 

func AddTitleStructDb(db *gorm.DB, title *Title)

func BillNumberVersionToBillNumber 

func BillNumberVersionToBillNumber(billNumberVersion string) string

func BillNumberVersionsToBillNumbers 

func BillNumberVersionsToBillNumbers(billNumberVersions []string) (billNumbers []string)

func GetBillnumbersByTitle 

func GetBillnumbersByTitle(titleMap *sync.Map, title string) (billnumbers []string, err error)

func GetDb 

func GetDb(dbname string) *gorm.DB

func GetDb_old 

func GetDb_old(dbname string) *gorm.DB

func GetRelatedDb 

func GetRelatedDb(dbname string) *gorm.DB

func GetSyncMapKeys 

func GetSyncMapKeys(m *sync.Map) (s string)

Gets keys of a sync.Map

func LoadBillTitleToDBFromJson 

func LoadBillTitleToDBFromJson(db *gorm.DB, jsonPath string, isForWhole bool)

func LoadBillsRelatedToDBFromJson 

func LoadBillsRelatedToDBFromJson(db *gorm.DB, parentPath string)

jsonPath := RelatedJsonPath db := GetDb(BILLSRELATED_DB)

func LoadEnv 

func LoadEnv() (err error)

func LoadTitlesMap 

func LoadTitlesMap(titlePath string) (*sync.Map, error)

func LoadTitlesToDBFromJson 

func LoadTitlesToDBFromJson(db *gorm.DB, jsonPath string)

jsonPath := TitlesPath db := GetDb(BILLTITLES_DB) NOTE: This adds all bills at the 'ih' version, since we currently only have the json map with billnumbers, not billnumberversions

func MakeSampleTitlesFile 

func MakeSampleTitlesFile(titleMap *sync.Map)

func MarshalJSONStringArray 

func MarshalJSONStringArray(m *sync.Map) ([]byte, error)

Marshals a sync.Map object of the type map[string][]string see https://stackoverflow.com/a/46390611/628748 and https://stackoverflow.com/a/65442862/628748

func RemoveDuplicates 

func RemoveDuplicates(elements []string) []string

Removes duplicates in a list of strings Returns the deduplicated list Trims leading and trailing space for each element

func RemoveTitle 

func RemoveTitle(titleMap *sync.Map, title string) (*sync.Map, error)

func RemoveTitleDb 

func RemoveTitleDb(db *gorm.DB, title string)

func SaveTitlesMap 

func SaveTitlesMap(titleMap *sync.Map, titlePath string) (err error)

func UnmarshalJson 

func UnmarshalJson(data []byte) (*sync.Map, error)

Unmarshals from JSON to a syncMap See https://stackoverflow.com/a/65442862/628748

func UnmarshalJsonFile 

func UnmarshalJsonFile(jpath string) (*sync.Map, error)

func UnmarshalTitlesJson 

func UnmarshalTitlesJson(data []byte) (*sync.Map, error)

Unmarshals from JSON in the form of map[string][]string to a syncMap See https://stackoverflow.com/a/65442862/628748

func UnmarshalTitlesJsonFile 

func UnmarshalTitlesJsonFile(jpath string) (*sync.Map, error)

Types

type Bill 

type Bill struct {
	gorm.Model
	Billnumber        string   `gorm:"not null" json:"billnumber"`
	Billnumberversion string   `gorm:"index:,unique" json:"billnumberversion"`
	Titles            []*Title `gorm:"many2many:bill_titles;" json:"titles"`
	TitlesWhole       []*Title `gorm:"many2many:bill_titleswhole" json:"titles_whole"`
}

func GetBillsByTitleDb 

func GetBillsByTitleDb(db *gorm.DB, title string) (bills []*Bill)

func GetBillsWithSameTitleDb 

func GetBillsWithSameTitleDb(db *gorm.DB, billnumber string) (bills, bills_whole []*Bill, err error)

type BillTitle 

type BillTitle struct {
	gorm.Model
	Billnumber     string `gorm:"index:,not null" json:"billnumber"`
	Title          string `gorm:"index:,not null" json:"title"`
	IsForWholeBill bool   `gorm:"not null" json:"is_for_whole_bill"`
}

type BillToBill 

type BillToBill struct {
	gorm.Model
	Billnumber    string  `gorm:"UNIQUE_INDEX:compositeindex;index:,not null" json:"billnumber"`
	Billnumber_to string  `gorm:"UNIQUE_INDEX:compositeindex;index:,not null" json:"billnumber_to"`
	Reason        string  `gorm:"UNIQUE_INDEX:compositeindex;not null" json:"reason"`
	Score         float64 `json:"score"`
	ScoreOther    float64 `json:"score_other"` // score for other bill
	Identified_by string  `gorm:"index:,not null" json:"identified_by"`
}

type LogLevel 

type LogLevel int8

type LogLevels 

type LogLevels map[string]zerolog.Level

type RelatedBillItem 

type RelatedBillItem struct {
	BillId                        string   `json:"bill_id"`
	IdentifiedBy                  string   `json:"identified_by"`
	Reason                        string   `json:"reason"`
	Type                          string   `json:"type"`
	BillCongressTypeNumber        string   `json:"bill_congress_type_number"`
	BillCongressTypeNumberVersion string   `json:"bill_congress_type_number_version"`
	Titles                        []string `json:"titles"`
	TitlesWholeBill               []string `json:"titles_whole_bill"`
}

type RelatedBillMap 

type RelatedBillMap map[string]RelatedBillItem

func GetRelatedBills 

func GetRelatedBills(titleMap *sync.Map, titleWholeBillMap *sync.Map, relatedBillItem RelatedBillItem) (relatedBills RelatedBillMap, err error)

type Title 

type Title struct {
	gorm.Model
	Title string  `gorm:"index:,unique"`
	Bills []*Bill `gorm:"many2many:bill_titles;"`
}

func GetTitlesByBillnumberDb 

func GetTitlesByBillnumberDb(db *gorm.DB, billnumber string) (titles []*Title)

func GetTitlesByBillnumberVersionDb 

func GetTitlesByBillnumberVersionDb(db *gorm.DB, billnumberversion string) (titles []*Title)

func GetTitlesWholeByBillnumberDb 

func GetTitlesWholeByBillnumberDb(db *gorm.DB, billnumber string) (titles []*Title)

func GetTitlesWholeByBillnumberVersionDb 

func GetTitlesWholeByBillnumberVersionDb(db *gorm.DB, billnumberversion string) (titles []*Title)

Source Files

View all Source files

Directories

Path Synopsis
cmd
billquery
related
titlesjson
internal
projectpath
testutils

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.