storage

Documentation

Index

• func OrderPerception(enabled bool) func(db *gorm.DB) *gorm.DB
• type ConsoleLog
• type Db
  • func NewDb() *Db
  • func (db *Db) Get() (*gorm.DB, error)
• type Event
• type Header
• type NetworkLog
• type RequestType
• type S3Dialector
  • func (s S3Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{})
  • func (s S3Dialector) DataTypeOf(field *schema.Field) string
  • func (s S3Dialector) DefaultValueOf(field *schema.Field) clause.Expression
  • func (s S3Dialector) Explain(sql string, vars ...interface{}) string
  • func (s S3Dialector) Initialize(db *gorm.DB) error
  • func (s S3Dialector) Migrator(db *gorm.DB) gorm.Migrator
  • func (s S3Dialector) Name() string
  • func (s S3Dialector) QuoteTo(writer clause.Writer, s2 string)
• type TLS
• type TLSCertificate
  • func (tlsCert *TLSCertificate) AddDNSName(name string)
• type TLSCertificateDNSName
• type Technologie
• type URL
  • func (url *URL) AddHeader(key string, value string)
  • func (url *URL) AddTechnologie(value string)
  • func (url *URL) MarshallCSV() (res []string)
  • func (url *URL) MarshallJSON() ([]byte, error)

Functions

func OrderPerception

func OrderPerception(enabled bool) func(db *gorm.DB) *gorm.DB

OrderPerception orders by perception hash if enabled

Types

type ConsoleLog

type ConsoleLog struct {
	gorm.Model `json:"-"`

	URLID uint `json:"-"`

	Time  time.Time `json:"time"`
	Type  string    `json:"type"`
	Value string    `json:"value"`
}

ConsoleLog contains the console log, and exceptions emitted

type Db

type Db struct {
	Location      string
	SkipMigration bool

	// cli flags
	Disabled bool
	S3       bool
	Debug    bool
}

Db is the SQLite3 db handler ype

func NewDb

func NewDb() *Db

NewDb sets up a new DB

func (*Db) Get

func (db *Db) Get() (*gorm.DB, error)

Get gets a db handle

type Event

type Event struct {
	gorm.Model `json:"-"`

	Type     string `json:"type"`
	Name     string `json:"name"`
	Function string `json:"function"`
}

type Header

type Header struct {
	gorm.Model `json:"-"`

	URLID uint `json:"-"`

	Key   string `json:"key"`
	Value string `json:"value"`
}

Header contains an HTTP header

type NetworkLog

type NetworkLog struct {
	gorm.Model `json:"-"`

	URLID uint `json:"-"`

	RequestID   string      `json:"request_id"`
	RequestType RequestType `json:"request_type"`
	StatusCode  int64       `json:"status_code"`
	URL         string      `json:"url"`
	FinalURL    string      `json:"final_url"`
	IP          string      `json:"ip"`
	Time        time.Time   `json:"time"`
	Error       string      `json:"error"`
}

NetworkLog contains Chrome networks events that were emitted

type RequestType

type RequestType int

RequestType are network log types

const (
	HTTP RequestType = 0
	WS
)

type S3Dialector

type S3Dialector struct {
}

func (S3Dialector) BindVarTo

func (s S3Dialector) BindVarTo(writer clause.Writer, stmt *gorm.Statement, v interface{})

func (S3Dialector) DataTypeOf

func (s S3Dialector) DataTypeOf(field *schema.Field) string

func (S3Dialector) DefaultValueOf

func (s S3Dialector) DefaultValueOf(field *schema.Field) clause.Expression

func (S3Dialector) Explain

func (s S3Dialector) Explain(sql string, vars ...interface{}) string

func (S3Dialector) Initialize

func (s S3Dialector) Initialize(db *gorm.DB) error

func (S3Dialector) Migrator

func (s S3Dialector) Migrator(db *gorm.DB) gorm.Migrator

func (S3Dialector) Name

func (s S3Dialector) Name() string

func (S3Dialector) QuoteTo

func (s S3Dialector) QuoteTo(writer clause.Writer, s2 string)

type TLS

type TLS struct {
	gorm.Model `json:"-"`

	URLID uint `json:"-"`

	Version         uint16           `json:"version"`
	ServerName      string           `json:"server_name"`
	TLSCertificates []TLSCertificate `json:"certificates"`
}

TLS contains TLS information for a URL

type TLSCertificate

type TLSCertificate struct {
	gorm.Model `json:"-"`

	TLSID uint `json:"-"`

	Raw                []byte                  `json:"-"`
	DNSNames           []TLSCertificateDNSName `json:"dns_names"`
	SubjectCommonName  string                  `json:"subject_common_name"`
	IssuerCommonName   string                  `json:"issuer_common_name"`
	SignatureAlgorithm string                  `json:"signature_algorithm"`
	PubkeyAlgorithm    string                  `json:"pubkey_algorithm"`
}

TLSCertificate contain TLS Certificate information

func (*TLSCertificate) AddDNSName

func (tlsCert *TLSCertificate) AddDNSName(name string)

AddDNSName adds a new DNS Name to a Certificate

type TLSCertificateDNSName

type TLSCertificateDNSName struct {
	gorm.Model `json:"-"`

	TLSCertificateID uint   `json:"-"`
	Name             string `json:"name"`
}

TLSCertificateDNSName has DNS names for a TLS certificate

type Technologie

type Technologie struct {
	gorm.Model `json:"-"`

	URLID uint `json:"-"`

	Value string `json:"value"`
}

Technologie contains a technologie

type URL

type URL struct {
	gorm.Model `json:"-"`

	URL            string `json:"url"`
	FinalURL       string `json:"final_url"`
	ResponseCode   int    `json:"response_code"`
	ResponseReason string `json:"response_reason"`
	Proto          string `json:"proto"`
	ContentLength  int64  `json:"content_length"`
	Title          string `json:"title"`
	Filename       string `json:"-"`
	IsPDF          bool   `json:"-"`
	PerceptionHash string `json:"-"`
	DOM            string `json:"dom"`
	Screenshot     string `json:"-"`

	TLS TLS `json:"tls"`

	Headers       []Header      `json:"headers"`
	Technologies  []Technologie `json:"technologies"`
	Console       []ConsoleLog  `json:"console"`
	Network       []NetworkLog  `json:"network"`
	Events        []Event       `json:"events" gorm:"-"`
	ScreenshotUrl string        `json:"screenshot_url" gorm:"-"`
}

URL contains information about a URL

func (*URL) AddHeader

func (url *URL) AddHeader(key string, value string)

AddHeader adds a new header to a URL

func (*URL) AddTechnologie

func (url *URL) AddTechnologie(value string)

AddTechnlogies adds a new technologies to a URL

func (*URL) MarshallCSV

func (url *URL) MarshallCSV() (res []string)

MarshallCSV returns values as a slice

func (*URL) MarshallJSON

func (url *URL) MarshallJSON() ([]byte, error)

MarshallJSON returns values as a slice