certstream

package module
v0.25.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 22, 2025 License: MIT Imports: 31 Imported by: 1

README

build coverage goreport Docs

CertStream

Small library wrapping github.com/google/certificate-transparency-go.

Requires a Postgres database to use.

func grabdata() {
	fulllimiter := bwlimit.NewLimiter()
	fulldialer := fulllimiter.Wrap(nil)
	var filllimiter *bwlimit.Limiter
	var filldialer proxy.ContextDialer

	cfg := certstream.NewConfig()
	cfg.Logger = slog.Default()
	cfg.PgUser = "username"
	cfg.PgPass = "password"
	cfg.PgName = "certstream"
	cfg.PgAddr = "127.0.0.1:5432"
	cfg.HeadDialer = fulldialer
	filllimiter = bwlimit.NewLimiter(int64(1 * 1024 * 1024))
	filldialer = filllimiter.Wrap(fulldialer)
	cfg.TailDialer = filldialer

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	var wg sync.WaitGroup
	cs, err := certstream.Start(ctx, &wg, cfg)
	defer wg.Wait()

	if err != nil {
        panic(err)
	} else {
		for le := range cs.C {
			fmt.Printf("%q %v %v\n", le.Domain, le.Historical, le.Cert().DNSNames)
		}
	}
}

Documentation

Index

Constants

View Source 
const SelectDnsnameLike = `SELECT * FROM CERTDB_domain WHERE domain LIKE $1;`
View Source 
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source 
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source 
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`

Variables

View Source 
var BatchSize = 1024
View Source 
var BulkRange = int64(4096)
View Source 
var CreateSchema string
View Source 
var DefaultTransport = &http.Transport{
	TLSHandshakeTimeout:   30 * time.Second,
	ResponseHeaderTimeout: 30 * time.Second,
	MaxIdleConnsPerHost:   2,
	DisableKeepAlives:     false,
	ExpectContinueTimeout: 1 * time.Second,
	ForceAttemptHTTP2:     true,
}
View Source 
var ErrLogIdle errLogIdle
View Source 
var FunctionFindSince string
View Source 
var FunctionOperatorID string
View Source 
var FunctionStreamID string
View Source 
var IdleCloseTime = time.Hour * 24 * 7
View Source 
var MaxErrors = 100
View Source 
var ProcAddNewEntry string
View Source 
var SelectGaps string
View Source 
var SelectIDSince string

Functions

func GetLogList 

func GetLogList(ctx context.Context, httpClient *http.Client, listUrl string) (logList *loglist3.LogList, err error)

GetLogList fetches a CT log list from the given listUrl. Usually you would pass loglist3.AllLogListURL for the listUrl.

func OperatorDomain 

func OperatorDomain(urlString string) string

OperatorDomain returns the TLD+1 given an URL.

func ScanCertificate added in v0.12.0 

func ScanCertificate(row Scanner, cert *PgCertificate) (err error)

func ScanDnsname added in v0.12.0 

func ScanDnsname(row Scanner, p *PgDnsname) error

func ScanDnsnamesView added in v0.12.0 

func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)

func ScanIdent added in v0.12.0 

func ScanIdent(row Scanner, ident *PgIdent) error

func ScanLogEntry added in v0.12.0 

func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)

Types

type CertStream 

type CertStream struct {
	Config                      // copy of config
	C          <-chan *LogEntry // log entry channel
	HeadClient *http.Client     // main HTTP client, uses Config.HeadDialer
	TailClient *http.Client     // may be nil if not backfilling
	DB         *PgDB
	// contains filtered or unexported fields
}

func Start added in v0.12.0 

func Start(ctx context.Context, wg *sync.WaitGroup, cfg *Config) (cs *CertStream, err error)

func (*CertStream) Close added in v0.19.0 

func (cs *CertStream) Close()

func (*CertStream) CountStreams added in v0.0.3 

func (cs *CertStream) CountStreams() (n int)

func (*CertStream) LogError added in v0.1.0 

func (cs *CertStream) LogError(err error, msg string, args ...any) error

func (*CertStream) LogInfo added in v0.12.0 

func (cs *CertStream) LogInfo(msg string, args ...any)

func (*CertStream) Operators added in v0.0.2 

func (cs *CertStream) Operators() (operators []*LogOperator)

type Certificate added in v0.10.0 

type Certificate struct {
	PreCert   bool
	Seen      time.Time
	Signature []byte
	*x509.Certificate
}

func (*Certificate) GetCommonName added in v0.24.29 

func (c *Certificate) GetCommonName() (s string)

type Config added in v0.12.0 

type Config struct {
	Logger     Logger              // if not nil Logger to use, no default
	HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
	TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
	PgUser     string              // PostgreSQL user, default "certstream"
	PgPass     string              // PostgreSQL password, default "certstream"
	PgName     string              // PostgreSQL db name, default "certstream"
	PgAddr     string              // PostgreSQL address, no default
	PgPrefix   string              // PostgreSQL naming prefix, default "certdb_"
	PgConns    int                 // max number of database connections, default 100
	PgMaxAge   int                 // maximum age in days to backfill
	PgNoSSL    bool                // if true, do not use SSL
}

func NewConfig added in v0.12.0 

func NewConfig() *Config

NewConfig returns a new default Config

type JsonCertificate added in v0.12.0 

type JsonCertificate struct {
	PreCert        bool         `json:",omitempty"`
	Signature      hexEncoded   `json:",omitempty"` // SHA256 signature, searchable on crt.sh
	Issuer         JsonIdentity `json:",omitempty"`
	Subject        JsonIdentity `json:",omitempty"`
	CommonName     string       `json:",omitempty"` // Subject common name
	DNSNames       []string     `json:",omitempty"`
	EmailAddresses []string     `json:",omitempty"`
	IPAddresses    []string     `json:",omitempty"`
	URIs           []string     `json:",omitempty"`
	NotBefore      time.Time    `json:",omitempty"`
	NotAfter       time.Time    `json:",omitempty"`
	Since          time.Time    `json:",omitzero"`
}

func NewJSONCertificate added in v0.12.0 

func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)

func (*JsonCertificate) SetCommonName added in v0.24.29 

func (js *JsonCertificate) SetCommonName()

type JsonIdentity added in v0.12.0 

type JsonIdentity struct {
	ID           int    `json:",omitempty"`
	Country      string `json:",omitempty"`
	Organization string `json:",omitempty"`
	Province     string `json:",omitempty"`
	CommonName   string `json:",omitempty"`
}

func (*JsonIdentity) Fill added in v0.12.0 

func (id *JsonIdentity) Fill(name *pkix.Name)

type LogEntry 

type LogEntry struct {
	*LogStream
	Err          error           // error from RawLogEntryFromLeaf or ToLogEntry, or nil
	RawLogEntry  *ct.RawLogEntry // may be nil in case of error
	*ct.LogEntry                 // may be nil in case of error
	Id           int64           // database id, if available
	Historical   bool            // true if the entry is from gap or backfilling
}

func (*LogEntry) Cert 

func (le *LogEntry) Cert() (crt *Certificate)

Cert returns the Certificate given a LogEntry or nil.

func (*LogEntry) Index 

func (le *LogEntry) Index() (index int64)

Index returns the log index or -1 if none is available.

func (*LogEntry) String 

func (le *LogEntry) String() (s string)

type LogOperator added in v0.0.3 

type LogOperator struct {
	*CertStream
	*loglist3.Operator
	Domain string       // e.g. "letsencrypt.org" or "googleapis.com"
	Count  atomic.Int64 // atomic; sum of the stream's Count
	Id     int32        // database ID, if available
	// contains filtered or unexported fields
}

func (*LogOperator) ErrorCount added in v0.22.0 

func (lo *LogOperator) ErrorCount() (n int)

func (*LogOperator) Errors added in v0.22.0 

func (lo *LogOperator) Errors() (errs []StreamError)

func (*LogOperator) StreamCount added in v0.18.0 

func (lo *LogOperator) StreamCount() (n int)

func (*LogOperator) Streams added in v0.0.3 

func (lo *LogOperator) Streams() (sl []*LogStream)

type LogStream 

type LogStream struct {
	*LogOperator
	*loglist3.Log
	HeadClient *client.LogClient
	TailClient *client.LogClient
	Count      atomic.Int64 // number of certificates sent to the channel
	MinIndex   atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
	MaxIndex   atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
	LastIndex  atomic.Int64 // atomic: highest index that is available from stream source
	InsideGaps atomic.Int64 // atomic: number of remaining entries inside gaps
	Id         int32        // database ID, if available
}

func (*LogStream) GetRawEntries added in v0.1.0 

func (ls *LogStream) GetRawEntries(ctx context.Context, start, end int64, historical bool, handleFn handleEntryFn, gapcounter *atomic.Int64) (wanted bool)

func (*LogStream) NewLastIndex added in v0.1.0 

func (ls *LogStream) NewLastIndex(ctx context.Context) (lastIndex int64, err error)

func (*LogStream) String 

func (ls *LogStream) String() string

type Logger added in v0.1.0 

type Logger interface {
	Info(msg string, args ...any)
	Error(msg string, args ...any)
}

type PgCertificate added in v0.12.0 

type PgCertificate struct {
	Id         int64
	NotBefore  time.Time
	NotAfter   time.Time
	CommonName string
	SubjectID  int
	IssuerID   int
	Sha256     []byte
	PreCert    bool
	Since      time.Time
}

type PgDB added in v0.12.0 

type PgDB struct {
	*CertStream
	*pgxpool.Pool
	Pfx     func(string) string // prefix replacer
	Workers atomic.Int32
	// contains filtered or unexported fields
}

PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database

func NewPgDB added in v0.12.0 

func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)

NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.

func (*PgDB) AverageNewEntryTime added in v0.14.0 

func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)

func (*PgDB) Close added in v0.13.0 

func (cdb *PgDB) Close()

func (*PgDB) Estimate added in v0.12.0 

func (cdb *PgDB) Estimate(table string) (f float64)

func (*PgDB) GetCertificateByHash added in v0.12.0 

func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByID added in v0.12.0 

func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateByLogEntry added in v0.12.0 

func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)

func (*PgDB) GetCertificateSince added in v0.24.19 

func (cdb *PgDB) GetCertificateSince(ctx context.Context, jcert *JsonCertificate) (since time.Time, err error)

func (*PgDB) GetCertificatesByCommonName added in v0.24.5 

func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)

func (*PgDB) QueueUsage added in v0.15.0 

func (cdb *PgDB) QueueUsage() (pct int)

type PgDnsname added in v0.12.0 

type PgDnsname struct {
	Dnsname string
	CertID  int64
}

type PgDnsnamesView added in v0.12.0 

type PgDnsnamesView struct {
	CertID    int64
	FQDN      string
	NotBefore time.Time
	Idna      bool
	Valid     bool
	PreCert   bool
	Issuer    string
	Subject   string
	Crtsh     string
	Domain    string
	Tld       string
}

type PgIdent added in v0.12.0 

type PgIdent struct {
	Id           int
	Organization string
	Province     string
	Country      string
}

type PgLogEntry added in v0.12.0 

type PgLogEntry struct {
	Seen     time.Time // CT log entry timestamp
	LogIndex int64     // CT log index for the stream
	CertID   int64     // database ID of cert
	StreamID int       // database ID of stream
}

type Scanner added in v0.12.0 

type Scanner interface {
	Scan(dest ...any) error
}

type StreamError added in v0.22.0 

type StreamError struct {
	*LogStream
	When time.Time
	Err  error
}

func (StreamError) Error added in v0.22.0 

func (ewt StreamError) Error() string

func (StreamError) Unwrap added in v0.22.0 

func (ewt StreamError) Unwrap() error

Source Files

View all Source files

Directories

Path Synopsis
cmd
certstream command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.