database

package
v5.0.8 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 14, 2024 License: GPL-3.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var AnalysisSnapshotAnalyzedAtTTLs = []string{"threat_mixtape"}
View Source 
var AnalysisSnapshotHourTTLs = []string{"big_ol_histogram", "tls_proto", "http_proto", "exploded_dns", "rare_signatures", "port_info"}
View Source 
var ErrDatabaseNameEmpty = errors.New("database name cannot be empty")
View Source 
var ErrDatabaseNotFound = errors.New("database does not exist")
View Source 
var ErrImportTwiceNonRolling = errors.New("cannot import more than once to a non-rolling database")
View Source 
var ErrInvalidDatabaseConnection = fmt.Errorf("database connection is nil")
View Source 
var ErrInvalidMinMaxTimestamp = fmt.Errorf("invalid min or max timestamp")
View Source 
var ErrMissingConfig = errors.New("config cannot be nil")
View Source 
var ErrNoMetaDBImportRecordForDatabase = errors.New("no import record found for database")
View Source 
var LogTableTTLs = []string{"conn", "http", "ssl", "dns", "pdns_raw"}
View Source 
var LogTableViewsDayTTLs = []string{"pdns"}
View Source 
var LogTableViewsHourTTLs = []string{"usni", "udns", "uconn", "mime_type_uris"}
View Source 
var MetaDatabaseTTLs = []string{"historical_first_seen", "files"}
View Source 
var MetaDatabaseYearTTLS = []string{"imports"}

Functions

func DatabaseExists 

func DatabaseExists(ctx context.Context, conn driver.Conn, dbName string) (bool, error)

func GetFlatDatabaseList 

func GetFlatDatabaseList(dbs []ImportDatabase) []string

GetFlatDatabaseList returns a list of database names from a list of ImportDatabase structs

func GetRollingStatus 

func GetRollingStatus(dbCtx context.Context, conn driver.Conn, dbName string) (bool, error)

GetRollingStatus gets the rolling status of a database

func SensorDatabaseExists 

func SensorDatabaseExists(ctx context.Context, conn driver.Conn, dbName string) (bool, error)

Types

type BulkWriter 

type BulkWriter struct {
	WriteChannel chan Data
	ProgChannel  chan int
	WriteWg      *errgroup.Group // wait for writing to finish
	// contains filtered or unexported fields
}

func NewBulkWriter 

func NewBulkWriter(db Database, conf *config.Config, numWorkers int, database string, writerName string, query string, limiter *rate.Limiter, withProgress bool) *BulkWriter

NewBulkWriter creates a new writer object to write output data to collections

func (*BulkWriter) Close 

func (w *BulkWriter) Close()

Close waits for the write threads to finish

func (*BulkWriter) Start 

func (w *BulkWriter) Start(id int)

Start kicks off a new write thread

type DB 

type DB struct {
	Conn driver.Conn

	Rolling bool

	ImportStartedAt time.Time
	// contains filtered or unexported fields
}

DB is the workhorse container for messing with the database

func ConnectToDB 

func ConnectToDB(ctx context.Context, db string, cfg *config.Config, cancel context.CancelFunc) (*DB, error)

ConnectToDB sets up a new connection to the specified database

func SetUpNewImport 

func SetUpNewImport(afs afero.Fs, cfg *config.Config, dbName string, rollingFlag bool, rebuildFlag bool) (*DB, error)

SetUpNewImport creates the database requested for this import and returns a new DB struct for connection to said database

func (*DB) AddImportFinishedRecordToMetaDB 

func (db *DB) AddImportFinishedRecordToMetaDB(importID util.FixedString, minTS, maxTS time.Time) error

AddImportFinishedRecordToMetaDB inserts a record into the metadatabase.imports table to mark that an import has finished

func (*DB) AddImportStartRecordToMetaDB 

func (db *DB) AddImportStartRecordToMetaDB(importID util.FixedString) error

AddImportStartRecordToMetaDB inserts a record into the metadatabase.imports table to mark that an import has started

func (*DB) CheckIfFilesWereAlreadyImported 

func (db *DB) CheckIfFilesWereAlreadyImported(fileMap map[string][]string) (int, error)

CheckIfFilesWereAlreadyImported calls checkFileHashes for each log type

func (*DB) GetBeaconMinMaxTimestamps 

func (db *DB) GetBeaconMinMaxTimestamps() (time.Time, time.Time, bool, error)

func (*DB) GetContext 

func (db *DB) GetContext() context.Context

GetContext returns the context for the database connection

func (*DB) GetNetworkSize 

func (db *DB) GetNetworkSize(minTS time.Time) (uint64, error)

GetNetworkSize returns the number of distinct internal hosts for the past 24 hours, which is used to determine prevalence

func (*DB) GetSelectedDB 

func (db *DB) GetSelectedDB() string

GetSelectedDB returns the name of the target database of db connection

func (*DB) GetTrueMinMaxTimestamps 

func (db *DB) GetTrueMinMaxTimestamps() (time.Time, time.Time, bool, bool, error)

func (*DB) MarkFileImportedInMetaDB 

func (db *DB) MarkFileImportedInMetaDB(hash util.FixedString, importID util.FixedString, path string) error

MarkFileImportedInMetaDB adds the given path to the metadatabase.files table to mark it as being used

func (*DB) QueryParameters 

func (db *DB) QueryParameters(params clickhouse.Parameters) context.Context

QueryParameters generates ClickHouse query parameters by creating a context with the specified parameters in it

func (*DB) ResetTemporaryTables 

func (db *DB) ResetTemporaryTables() error

ResetTemporaryTables clears out data in tmp tables (if they exist) from the previous import

func (*DB) TruncateTmpLinkTables 

func (db *DB) TruncateTmpLinkTables() error

TruncateTmpLinkTables truncates the tables that are used to link zeek uids. This should be called after each import so that these tmp tables don't take up unnecessary disk space.

type Data 

type Data any

type Database 

type Database interface {
	GetContext() context.Context
	QueryParameters(clickhouse.Parameters) context.Context
	// contains filtered or unexported methods
}

Interface to allow creating a BulkWriter from a DB or a serverConn

type ImportDatabase 

type ImportDatabase struct {
	Name    string    `ch:"database"`
	Rolling bool      `ch:"rolling"`
	MinTS   time.Time `ch:"min_ts"`
	MaxTS   time.Time `ch:"max_ts"`
}

type MetaDBImportRecord 

type MetaDBImportRecord struct {
	ImportID         *util.FixedString `ch:"import_id"`
	Rolling          bool              `ch:"rolling"`
	Database         string            `ch:"database"`
	Rebuild          bool              `ch:"rebuild"`
	StartedAt        int64             `ch:"started_at"`
	EndedAt          time.Time         `ch:"ended_at"`
	HoursSeen        []time.Time       `ch:"hours_seen"`
	ImportVersion    string            `ch:"import_version"`
	MinTimestamp     time.Time         `ch:"min_timestamp"`
	MaxTimestamp     time.Time         `ch:"max_timestamp"`
	MinOpenTimestamp time.Time         `ch:"min_open_timestamp"`
	MaxOpenTimestamp time.Time         `ch:"max_open_timestamp"`
}

type MetaDBImportedFile 

type MetaDBImportedFile struct {
	Hash      *util.FixedString `ch:"hash"`
	ImportID  *util.FixedString `ch:"import_id"`
	Database  string            `ch:"database"`
	Timestamp time.Time         `ch:"ts"`
	Path      string            `ch:"path"`
}

type ServerConn 

type ServerConn struct {
	Conn driver.Conn
	// contains filtered or unexported fields
}

func ConnectToServer 

func ConnectToServer(ctx context.Context, cfg *config.Config) (*ServerConn, error)

ConnectToServer connects to the clickhouse server as the default user

func (*ServerConn) ClearMetaDBEntriesForDatabase 

func (server *ServerConn) ClearMetaDBEntriesForDatabase(database string) error

ClearMetaDBEntriesForDatabase deletes all file and import record entries in the metadatabase for the specified database

func (*ServerConn) CreateServerDBTables added in v5.0.7 

func (server *ServerConn) CreateServerDBTables() error

func (*ServerConn) DeleteSensorDB 

func (server *ServerConn) DeleteSensorDB(database string) error

DeleteSensorDB deletes the specified database along with its associated imported files in metadatabase.files

func (*ServerConn) DropMultipleSensorDatabases 

func (server *ServerConn) DropMultipleSensorDatabases(dbName string, wildcardStart, wildcardEnd bool) (int, error)

DropMultipleSensorDatabases drops the databases that match the specified wildcard a wildcard can be in the beginning, end, or both

func (*ServerConn) GetContext 

func (server *ServerConn) GetContext() context.Context

GetContext returns the context for the database connection

func (*ServerConn) ListImportDatabases 

func (server *ServerConn) ListImportDatabases() ([]ImportDatabase, error)

func (*ServerConn) QueryParameters 

func (server *ServerConn) QueryParameters(params clickhouse.Parameters) context.Context

QueryParameters generates ClickHouse query parameters by creating a context with the specified parameters in it

type ValidMIMEType 

type ValidMIMEType struct {
	MIMEType  string `ch:"mime_type"`
	Extension string `ch:"extension"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.