Documentation
¶
Index ¶
- Constants
- Variables
- func GetHTTPCalls(s string) (n int64)
- func GetHTTPCallsMap() (m map[string]int64)
- func OperatorDomain(urlString string) string
- func RenderSQL(query string, args ...any) string
- func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
- func ScanDnsname(row Scanner, p *PgDnsname) error
- func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
- func ScanIdent(row Scanner, ident *PgIdent) error
- func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
- type CertStream
- func (cs *CertStream) CountStreams() (n int)
- func (cs *CertStream) DB() (db *PgDB)
- func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
- func (cs *CertStream) LogError(err error, msg string, args ...any) error
- func (cs *CertStream) LogInfo(msg string, args ...any)
- func (cs *CertStream) Operators() (operators []*LogOperator)
- type Certificate
- type Config
- type JsonCertificate
- type JsonIdentity
- type LogEntry
- type LogOperator
- func (lo *LogOperator) CallCount() (n int64)
- func (lo *LogOperator) Email() []string
- func (lo *LogOperator) ErrorCount() (n int)
- func (lo *LogOperator) Errors() (errs []*StreamError)
- func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
- func (lo *LogOperator) Name() string
- func (lo *LogOperator) StatusCounts() (m map[int]int)
- func (lo *LogOperator) StreamCount() (n int)
- func (lo *LogOperator) Streams() (sl []*LogStream)
- type LogStream
- type Logger
- type PgCertificate
- type PgDB
- func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)
- func (cdb *PgDB) Close()
- func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) Estimate(table string) (f float64)
- func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)
- func (cdb *PgDB) GetHistoricalCertificates(ctx context.Context, expiresAfter time.Time, ...) (err error)
- func (cdb *PgDB) QueueUsage() (pct int)
- type PgDnsname
- type PgDnsnamesView
- type PgIdent
- type PgLogEntry
- type Scanner
- type StreamError
Constants ¶
View Source
const SelectBackfillIndex = `SELECT backfill_logindex FROM CERTDB_stream WHERE id = $1;`
View Source
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source
const SelectFindGap = `SELECT start_gap, end_gap FROM CERTDB_findgap($1, $2, $3);`
View Source
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndexFrom = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1 AND logindex >= $2;`
View Source
const UpdateBackfillIndex = `UPDATE CERTDB_stream SET backfill_logindex = $1 WHERE id = $2 AND backfill_logindex < $1;`
Variables ¶
View Source
var CreateSchema string
View Source
var DbIngestBatchSize = 1000 // number of entries to send to ingest at a time
View Source
var DefaultTransport = &http.Transport{ TLSHandshakeTimeout: 30 * time.Second, ResponseHeaderTimeout: 30 * time.Second, MaxIdleConnsPerHost: 2, DisableKeepAlives: false, ExpectContinueTimeout: 10 * time.Second, ForceAttemptHTTP2: true, }
View Source
var ErrHeadLogOpen = errors.New("head log open failed")
View Source
var ErrLogEntriesTooOld = errors.New("log entries are older than max age")
View Source
var ErrLogIdle errLogIdle
View Source
var ErrLogStreamRetryable = errors.New("logstream retryable")
View Source
var ErrSTHDiffTooLow = errors.New("STH diff too low")
View Source
var ErrSunlightClientMissing = errors.New("sunlight client missing")
View Source
var ErrTailLogOpen = errors.New("tail log open failed")
View Source
var FuncIngestBatch string
View Source
var FuncSetSince string
View Source
var FuncSubdomain string
View Source
var FunctionOperatorID string
View Source
var FunctionStreamID string
View Source
var HistoricalBatchSize = 1000 // number of rows to SELECT when getting historical certificates
View Source
var IdleCloseTime = time.Hour * 24 * 7
View Source
var LogBatchSize = int64(1024)
View Source
var MaxErrors = 100
Functions ¶
func GetHTTPCalls ¶ added in v0.32.7
func GetHTTPCallsMap ¶ added in v0.32.7
func OperatorDomain ¶
OperatorDomain returns the TLD+1 given an URL.
func ScanCertificate ¶ added in v0.12.0
func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
func ScanDnsname ¶ added in v0.12.0
func ScanDnsnamesView ¶ added in v0.12.0
func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
func ScanLogEntry ¶ added in v0.12.0
func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
Types ¶
type CertStream ¶
type CertStream struct {
Config // copy of config
C <-chan *LogEntry // log entry channel
HeadClient *http.Client // main HTTP client, uses Config.HeadDialer
TailClient *http.Client // may be nil if not backfilling
LogToggle atomic.Bool // if true, log stream activity
// contains filtered or unexported fields
}
func (*CertStream) CountStreams ¶ added in v0.0.3
func (cs *CertStream) CountStreams() (n int)
func (*CertStream) DB ¶ added in v0.12.0
func (cs *CertStream) DB() (db *PgDB)
func (*CertStream) GetLogStreamByID ¶ added in v0.27.0
func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
func (*CertStream) LogError ¶ added in v0.1.0
func (cs *CertStream) LogError(err error, msg string, args ...any) error
func (*CertStream) LogInfo ¶ added in v0.12.0
func (cs *CertStream) LogInfo(msg string, args ...any)
func (*CertStream) Operators ¶ added in v0.0.2
func (cs *CertStream) Operators() (operators []*LogOperator)
type Certificate ¶ added in v0.10.0
func (*Certificate) GetCommonName ¶ added in v0.24.29
func (c *Certificate) GetCommonName() (s string)
type Config ¶ added in v0.12.0
type Config struct {
Logger Logger // if not nil Logger to use, no default
HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
HeadLog string // log HTTP requests using the head dialer to this file
TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
PgUser string // PostgreSQL user, default "certstream"
PgPass string // PostgreSQL password, default "certstream"
PgName string // PostgreSQL db name, default "certstream"
PgAddr string // PostgreSQL address, no default
PgPrefix string // PostgreSQL naming prefix, default "certdb_"
PgConns int // max number of database connections, default 100
PgWorkerBits int // number of prefix bits that determine DB workers, default 5 (32 workers)
PgMaxAge int // maximum age in days to backfill
PgNoSSL bool // if true, do not use SSL
PgSyncCommit bool // if true, do not set synchronous_commit=off
Concurrency int // number of concurrent requests per stream, default is 4
DataDir string // log and cache directory; leave empty to disable
CacheMaxAge time.Duration // remove cached tile data older than this age; zero disables caching of tiles
TailLog string // log HTTP requests using the tail dialer to this file
}
type JsonCertificate ¶ added in v0.12.0
type JsonCertificate struct {
PreCert bool `json:",omitempty"`
Signature hexEncoded `json:",omitempty"` // SHA256 signature, searchable on crt.sh
Issuer JsonIdentity `json:",omitempty"`
Subject JsonIdentity `json:",omitempty"`
CommonName string `json:",omitempty"` // Subject common name
DNSNames []string `json:",omitempty"`
EmailAddresses []string `json:",omitempty"`
IPAddresses []string `json:",omitempty"`
URIs []string `json:",omitempty"`
NotBefore time.Time `json:",omitempty"`
NotAfter time.Time `json:",omitempty"`
Since time.Time `json:",omitzero"`
}
func NewJSONCertificate ¶ added in v0.12.0
func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)
func (*JsonCertificate) SetCommonName ¶ added in v0.24.29
func (js *JsonCertificate) SetCommonName()
type JsonIdentity ¶ added in v0.12.0
type LogEntry ¶
type LogEntry struct {
*LogStream
Err error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
LogIndex int64
PreCert bool
Certificate *x509.Certificate
Id int64 // database id, if available
Historical bool // true if the entry is from gap or backfilling
Signature []byte
Seen time.Time
}
func (*LogEntry) Cert ¶
func (le *LogEntry) Cert() (crt *Certificate)
Cert returns the Certificate given a LogEntry or nil.
type LogOperator ¶ added in v0.0.3
type LogOperator struct {
*CertStream
Domain string // e.g. "letsencrypt.org" or "googleapis.com"
Count atomic.Int64 // atomic; sum of the stream's Count
Status429 atomic.Int64 // atomic; number of 429 Too Many Requests
Id int32 // database ID, if available
// contains filtered or unexported fields
}
func (*LogOperator) CallCount ¶ added in v0.31.6
func (lo *LogOperator) CallCount() (n int64)
func (*LogOperator) Email ¶ added in v0.28.0
func (lo *LogOperator) Email() []string
func (*LogOperator) ErrorCount ¶ added in v0.22.0
func (lo *LogOperator) ErrorCount() (n int)
func (*LogOperator) Errors ¶ added in v0.22.0
func (lo *LogOperator) Errors() (errs []*StreamError)
func (*LogOperator) GetStreamByID ¶ added in v0.27.0
func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
func (*LogOperator) Name ¶ added in v0.28.0
func (lo *LogOperator) Name() string
func (*LogOperator) StatusCounts ¶ added in v0.30.8
func (lo *LogOperator) StatusCounts() (m map[int]int)
func (*LogOperator) StreamCount ¶ added in v0.18.0
func (lo *LogOperator) StreamCount() (n int)
func (*LogOperator) Streams ¶ added in v0.0.3
func (lo *LogOperator) Streams() (sl []*LogStream)
type LogStream ¶
type LogStream struct {
*LogOperator
Count atomic.Int64 // number of certificates sent to the channel
MinIndex atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
MaxIndex atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
LastIndex atomic.Int64 // atomic: highest index that is available from stream source
Backfill atomic.Int64 // atomic: number of remaining entries to backfill until we reach head
Logger *slog.Logger // toggled by LogToggle
Id int32 // database ID, if available
// contains filtered or unexported fields
}
type PgCertificate ¶ added in v0.12.0
type PgDB ¶ added in v0.12.0
type PgDB struct {
*CertStream
*pgxpool.Pool
Pfx func(string) string // prefix replacer
Workers atomic.Int32
// contains filtered or unexported fields
}
PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database
func NewPgDB ¶ added in v0.12.0
func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)
NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.
func (*PgDB) AverageNewEntryTime ¶ added in v0.14.0
func (*PgDB) DeleteCertificates ¶ added in v0.28.4
func (*PgDB) DeleteStream ¶ added in v0.28.4
func (*PgDB) GetCertificateByHash ¶ added in v0.12.0
func (*PgDB) GetCertificateByID ¶ added in v0.12.0
func (*PgDB) GetCertificateByLogEntry ¶ added in v0.12.0
func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
func (*PgDB) GetCertificatesByCommonName ¶ added in v0.24.5
func (*PgDB) GetHistoricalCertificates ¶ added in v0.28.7
func (*PgDB) QueueUsage ¶ added in v0.15.0
type PgDnsnamesView ¶ added in v0.12.0
type PgLogEntry ¶ added in v0.12.0
type StreamError ¶ added in v0.22.0
func (StreamError) Error ¶ added in v0.22.0
func (ewt StreamError) Error() string
func (StreamError) Unwrap ¶ added in v0.22.0
func (ewt StreamError) Unwrap() error
Source Files
¶
- atomic_minmax.go
- certificate.go
- certstream.go
- config.go
- errlogidle.go
- getloglist.go
- httpcallcounter.go
- jsoncertificate.go
- jsonidentity.go
- logentry.go
- logger.go
- logoperator.go
- logstream.go
- logstream_backoff.go
- logstream_rawentries.go
- logstream_tileentries.go
- operatordomain.go
- pgbackfill.go
- pgbatcher.go
- pgcertificate.go
- pgdb.go
- pgdnsname.go
- pgdnsnamesview.go
- pgident.go
- pglogentry.go
- pgschema.go
- requestlog.go
- streamerror.go
- sunlight_cache.go
- sunlight_client.go
- toggled_logger.go
- updatestreams.go
- wraperr.go
Click to show internal directories.
Click to hide internal directories.