ldp

Documentation

Index

• Constants
• Variables
• func InitTableSchemas(dbtype DBType, tx *sql.Tx, spec *LoadSpec, lg *Logging) error
• func Load(params *LoadParams) error
• type Column
• type DBType
  • func SelectDBType(dbtype string) (DBType, error)
• type LoadParams
  • func (o *LoadParams) Init() (*LoadParams, error)
  • func (o *LoadParams) Validate() error
• type LoadSchema
• type LoadSpec
• type Logging
  • func NewLogging(logger *log.Logger, verbose bool, debug bool) *Logging
  • func (l *Logging) Logf(level int, format string, v ...interface{})
• type Postgres
  • func (p Postgres) EncodeStringConst(s string) string
  • func (p Postgres) JSONType() string
• type Redshift
  • func (r Redshift) EncodeStringConst(s string) string
  • func (r Redshift) JSONType() string

Constants

const (
	Warning = 0
	Verbose = 1
	Debug   = 2
)

Variables

var DefaultLoadSpec = &LoadSpec{
	[]*LoadSchema{
		&LoadSchema{
			Module: "mod-users",
			Path:   "/groups",
			Table:  "groups",
			TempColumns: []*Column{
				&Column{
					Name: "group",
					Type: "",
				},
				&Column{
					Name: "desc",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/locations",
			Table:  "locations",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "isActive",
					Type: "boolean",
				},
				&Column{
					Name: "description",
					Type: "",
				},
				&Column{
					Name: "discoveryDisplayName",
					Type: "",
				},
				&Column{
					Name: "institutionId",
					Type: "",
				},
				&Column{
					Name: "campusId",
					Type: "",
				},
				&Column{
					Name: "libraryId",
					Type: "",
				},
				&Column{
					Name: "primaryServicePoint",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-users",
			Path:   "/users",
			Table:  "users",
			TempColumns: []*Column{
				&Column{
					Name: "username",
					Type: "",
				},
				&Column{
					Name: "barcode",
					Type: "",
				},
				&Column{
					Name: "active",
					Type: "boolean",
				},
				&Column{
					Name: "type",
					Type: "",
				},
				&Column{
					Name: "patronGroup",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-circulation-storage",
			Path:   "/loan-storage/loans",
			Table:  "loans",
			TempColumns: []*Column{
				&Column{
					Name: "userId",
					Type: "",
				},
				&Column{
					Name: "proxyUserId",
					Type: "",
				},
				&Column{
					Name: "itemId",
					Type: "",
				},
				&Column{
					Name: "loanDate",
					Type: "timestamptz",
				},
				&Column{
					Name: "dueDate",
					Type: "timestamptz",
				},
				&Column{
					Name: "returnDate",
					Type: "timestamptz",
				},
				&Column{
					Name: "systemReturnDate",
					Type: "timestamptz",
				},
				&Column{
					Name: "action",
					Type: "",
				},
				&Column{
					Name: "actionComment",
					Type: "",
				},
				&Column{
					Name: "itemStatus",
					Type: "",
				},
				&Column{
					Name: "renewalCount",
					Type: "bigint",
				},
				&Column{
					Name: "loanPolicyId",
					Type: "",
				},
				&Column{
					Name: "checkoutServicePointId",
					Type: "",
				},
				&Column{
					Name: "checkinServicePointId",
					Type: "",
				},
				&Column{
					Name: "patronGroupIdAtCheckout",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/location-units/institutions",
			Table:  "institutions",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/location-units/campuses",
			Table:  "campuses",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "institutionId",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/location-units/libraries",
			Table:  "libraries",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "campusId",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/service-points",
			Table:  "service_points",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "discoveryDisplayName",
					Type: "",
				},
				&Column{
					Name: "description",
					Type: "",
				},
				&Column{
					Name: "shelvingLagTime",
					Type: "bigint",
				},
				&Column{
					Name: "pickupLocation",
					Type: "boolean",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/material-types",
			Table:  "material_types",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "source",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/instance-types",
			Table:  "instance_types",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "source",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/instance-storage/instances",
			Table:  "instances",
			TempColumns: []*Column{
				&Column{
					Name: "hrid",
					Type: "",
				},
				&Column{
					Name: "title",
					Type: "",
				},
				&Column{
					Name: "indexTitle",
					Type: "",
				},
				&Column{
					Name: "source",
					Type: "",
				},
				&Column{
					Name: "instanceTypeId",
					Type: "",
				},
				&Column{
					Name: "modeOfIssuanceId",
					Type: "",
				},
				&Column{
					Name: "catalogedDate",
					Type: "",
				},
				&Column{
					Name: "previouslyHeld",
					Type: "boolean",
				},
				&Column{
					Name: "staffSuppress",
					Type: "boolean",
				},
				&Column{
					Name: "discoverySuppress",
					Type: "boolean",
				},
				&Column{
					Name: "statusId",
					Type: "",
				},
				&Column{
					Name: "statusUpdatedDate",
					Type: "timestamptz",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/holdings-storage/holdings",
			Table:  "holdings",
			TempColumns: []*Column{
				&Column{
					Name: "hrid",
					Type: "",
				},
				&Column{
					Name: "holdingsTypeId",
					Type: "",
				},
				&Column{
					Name: "instanceId",
					Type: "",
				},
				&Column{
					Name: "permanentLocationId",
					Type: "",
				},
				&Column{
					Name: "temporaryLocationId",
					Type: "",
				},
				&Column{
					Name: "callNumberTypeId",
					Type: "",
				},
				&Column{
					Name: "callNumberPrefix",
					Type: "",
				},
				&Column{
					Name: "callNumber",
					Type: "",
				},
				&Column{
					Name: "callNumberSuffix",
					Type: "",
				},
				&Column{
					Name: "shelvingTitle",
					Type: "",
				},
				&Column{
					Name: "acquisitionFormat",
					Type: "",
				},
				&Column{
					Name: "acquisitionMethod",
					Type: "",
				},
				&Column{
					Name: "receiptStatus",
					Type: "",
				},
				&Column{
					Name: "illPolicyId",
					Type: "",
				},
				&Column{
					Name: "retentionPolicy",
					Type: "",
				},
				&Column{
					Name: "digitizationPolicy",
					Type: "",
				},
				&Column{
					Name: "copyNumber",
					Type: "",
				},
				&Column{
					Name: "numberOfItems",
					Type: "",
				},
				&Column{
					Name: "discoverySuppress",
					Type: "boolean",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/item-storage/items",
			Table:  "items",
			TempColumns: []*Column{
				&Column{
					Name: "hrid",
					Type: "",
				},
				&Column{
					Name: "holdingsRecordId",
					Type: "",
				},
				&Column{
					Name: "discoverySuppress",
					Type: "",
				},
				&Column{
					Name: "accessionNumber",
					Type: "",
				},
				&Column{
					Name: "barcode",
					Type: "",
				},
				&Column{
					Name: "volume",
					Type: "",
				},
				&Column{
					Name: "enumeration",
					Type: "",
				},
				&Column{
					Name: "chronology",
					Type: "",
				},
				&Column{
					Name: "itemLevelCallNumber",
					Type: "",
				},
				&Column{
					Name: "itemLevelCallNumberPrefix",
					Type: "",
				},
				&Column{
					Name: "itemLevelCallNumberSuffix",
					Type: "",
				},
				&Column{
					Name: "itemLevelCallNumberTypeId",
					Type: "",
				},
				&Column{
					Name: "itemIdentifier",
					Type: "",
				},
				&Column{
					Name: "numberOfPieces",
					Type: "",
				},
				&Column{
					Name: "descriptionOfPieces",
					Type: "",
				},
				&Column{
					Name: "numberOfMissingPieces",
					Type: "",
				},
				&Column{
					Name: "missingPieces",
					Type: "",
				},
				&Column{
					Name: "itemDamagedStatusId",
					Type: "",
				},
				&Column{
					Name: "permanentLoanTypeId",
					Type: "",
				},
				&Column{
					Name: "temporaryLoanTypeId",
					Type: "",
				},
				&Column{
					Name: "permanentLocationId",
					Type: "",
				},
				&Column{
					Name: "temporaryLocationId",
					Type: "",
				},
				&Column{
					Name: "purchaseOrderLineIdentifier",
					Type: "",
				},
				&Column{
					Name: "materialTypeID",
					Type: "",
				},
				&Column{
					Name: "inTransitDestinationServicePointId",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/shelf-locations",
			Table:  "shelf_locations",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/loan-types",
			Table:  "loan_types",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-finance-storage",
			Path:   "/finance-storage/budgets",
			Table:  "budgets",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "budgetStatus",
					Type: "",
				},
				&Column{
					Name: "allowableEncumbrance",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "allowableExpenditure",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "allocated",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "allocation",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "awaitingPayment",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "available",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "encumbered",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "expenditures",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "unavailable",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "overEncumbrance",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "overExpended",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "fundId",
					Type: "",
				},
				&Column{
					Name: "fiscalYearId",
					Type: "",
				},
				&Column{
					Name: "limitEncPercent",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "limitExpPercent",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "code",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-finance-storage",
			Path:   "/finance-storage/transactions",
			Table:  "transactions",
			TempColumns: []*Column{
				&Column{
					Name: "amount",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "currency",
					Type: "",
				},
				&Column{
					Name: "description",
					Type: "",
				},
				&Column{
					Name: "fiscalYearId",
					Type: "",
				},
				&Column{
					Name: "fromFundId",
					Type: "",
				},
				&Column{
					Name: "paymentEncumbranceId",
					Type: "",
				},
				&Column{
					Name: "source",
					Type: "",
				},
				&Column{
					Name: "sourceFiscalYearId",
					Type: "",
				},
				&Column{
					Name: "sourceInvoiceId",
					Type: "",
				},
				&Column{
					Name: "toFundId",
					Type: "",
				},
				&Column{
					Name: "transactionType",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-finance-storage",
			Path:   "/finance-storage/funds",
			Table:  "funds",
			TempColumns: []*Column{
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "description",
					Type: "",
				},
				&Column{
					Name: "externalAccountNo",
					Type: "",
				},
				&Column{
					Name: "fundTypeId",
					Type: "",
				},
				&Column{
					Name: "ledgerId",
					Type: "",
				},
				&Column{
					Name: "name",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-finance-storage",
			Path:   "/finance-storage/ledgers",
			Table:  "ledgers",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "description",
					Type: "",
				},
				&Column{
					Name: "ledgerStatus",
					Type: "",
				},
				&Column{
					Name: "allocated",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "available",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "unavailable",
					Type: "numeric(12,2)",
				},
				&Column{
					Name: "currency",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/instance-formats",
			Table:  "instance_formats",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "source",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-circulation",
			Path:   "/circulation/loans",
			Table:  "temp_loans",
			TempColumns: []*Column{
				&Column{
					Name: "temp_location",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-finance-storage",
			Path:   "/finance-storage/fiscal-years",
			Table:  "fiscal_years",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
				&Column{
					Name: "code",
					Type: "",
				},
				&Column{
					Name: "currency",
					Type: "",
				},
				&Column{
					Name: "description",
					Type: "",
				},
				&Column{
					Name: "periodStart",
					Type: "timestamptz",
				},
				&Column{
					Name: "periodEnd",
					Type: "timestamptz",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/identifier-types",
			Table:  "identifier_types",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
			},
		},
		&LoadSchema{
			Module: "mod-inventory-storage",
			Path:   "/call-number-types",
			Table:  "call_number_types",
			TempColumns: []*Column{
				&Column{
					Name: "name",
					Type: "",
				},
			},
		},
	},
}

Functions

func InitTableSchemas

func InitTableSchemas(dbtype DBType, tx *sql.Tx, spec *LoadSpec,
	lg *Logging) error

func Load

func Load(params *LoadParams) error

Types

type Column

type Column struct {
	Name string
	Type string
}

type DBType

type DBType interface {
	JSONType() string
	EncodeStringConst(string) string
}

func SelectDBType

func SelectDBType(dbtype string) (DBType, error)

type LoadParams

type LoadParams struct {

	// DatabaseType specifies which database system is being used, either
	// "postgresql" or "redshift".  (Required)
	DatabaseType string

	// DatabaseHost is the TCP host address that the database listens on.
	// (Required)
	DatabaseHost string

	// DatabasePort is the TCP port that the database listens on.
	// (Required)
	DatabasePort string

	// DatabaseUser is the user name for connecting to the database.
	// (Required)
	DatabaseUser string

	// DatabasePassword is the password for connecting to the database.
	// (Required)
	DatabasePassword string

	// DatabaseName is the name of the database to load data into.
	// (Required)
	DatabaseName string

	// DatabaseSSLMode specifies the SSL mode when connecting to the
	// database.  This should normally be "require" (the default, if this
	// value is set to ""), or "disable" if there is a need to disable SSL
	// See the LDP documentation for warnings about disabling SSL.
	// (Optional)
	DatabaseSSLMode string

	// LoadDir specifies a directory containing source data in JSON files
	// to be loaded into the database.  If LoadDir != "", then the
	// extraction-related parameters below will be ignored.
	LoadDir string

	// OkapiURL specifies the base URL for the Okapi service which will be
	// used to extract data.  (Required)
	OkapiURL string

	// OkapiTenant is the Okapi tenant to extract data from.  (Required)
	OkapiTenant string

	// OkapiUser is the user name for logging into Okapi.  (Required)
	OkapiUser string

	// OkapiPassword is the password for logging into Okapi.  (Required)
	OkapiPassword string

	// ExtractDir specifies a directory to use as temporary file space
	// when extracting data.  (Required)
	ExtractDir string

	// DisableAnonymization specifies whether anonymization of personal
	// data should be anonymized when they are extracted.  The default
	// (false) is to anonymize data.  See the LDP documentation for
	// warnings about setting this value to true.  (Optional)
	DisableAnonymization bool

	// Logger optionally specifies a logger for the server.  If nil,
	// the log package's standard logger is used.  (Optional)
	Logger *log.Logger

	// SaveTemps prevents clean up (deletion) of temporary files used for
	// data extraction.  (Optional)
	SaveTemps bool

	// Verbose enables verbose progress output.  (Optional)
	Verbose bool

	// Debug enables detailed debugging output.  (Optional)
	Debug bool
}

LoadParams is the set of parameters accepted by the Load() function.

func (*LoadParams) Init

func (o *LoadParams) Init() (*LoadParams, error)

Init returns a copy of opts, with default values filled in where needed.

func (*LoadParams) Validate

func (o *LoadParams) Validate() error

Validate examines the parameter values and ensures that they are specified correctly, e.g. that required values are present.

type LoadSchema

type LoadSchema struct {
	Module      string
	Path        string
	Table       string
	TempColumns []*Column
}

type LoadSpec

type LoadSpec struct {
	Schema []*LoadSchema
}

type Logging

type Logging struct {
	Logger *log.Logger
	Level  int
}

func NewLogging

func NewLogging(logger *log.Logger, verbose bool, debug bool) *Logging

func (*Logging) Logf

func (l *Logging) Logf(level int, format string, v ...interface{})

Logf is similar to log.Printf, but it formats multiline logs so that they are prefixed with ">" and indented.

type Postgres

type Postgres struct {
}

func (Postgres) EncodeStringConst

func (p Postgres) EncodeStringConst(s string) string

func (Postgres) JSONType

func (p Postgres) JSONType() string

type Redshift

type Redshift struct {
}

func (Redshift) EncodeStringConst

func (r Redshift) EncodeStringConst(s string) string

func (Redshift) JSONType

func (r Redshift) JSONType() string