Documentation
¶
Index ¶
- Constants
- Variables
- func OperatorDomain(urlString string) string
- func RenderSQL(query string, args ...any) string
- func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
- func ScanDnsname(row Scanner, p *PgDnsname) error
- func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
- func ScanIdent(row Scanner, ident *PgIdent) error
- func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
- type CertStream
- func (cs *CertStream) Close()
- func (cs *CertStream) CountStreams() (n int)
- func (cs *CertStream) DB() (db *PgDB)
- func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
- func (cs *CertStream) LogError(err error, msg string, args ...any) error
- func (cs *CertStream) LogInfo(msg string, args ...any)
- func (cs *CertStream) Operators() (operators []*LogOperator)
- type Certificate
- type Config
- type JsonCertificate
- type JsonIdentity
- type LogEntry
- type LogOperator
- func (lo *LogOperator) Email() []string
- func (lo *LogOperator) ErrorCount() (n int)
- func (lo *LogOperator) Errors() (errs []*StreamError)
- func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
- func (lo *LogOperator) Name() string
- func (lo *LogOperator) StreamCount() (n int)
- func (lo *LogOperator) Streams() (sl []*LogStream)
- type LogStream
- type Logger
- type PgCertificate
- type PgDB
- func (cdb *PgDB) AverageNewEntryTime() (d time.Duration)
- func (cdb *PgDB) Close()
- func (cdb *PgDB) DeleteCertificates(ctx context.Context, cutoff time.Time, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) DeleteStream(ctx context.Context, streamId int32, batchSize int) (rowsDeleted int64, err error)
- func (cdb *PgDB) Estimate(table string) (f float64)
- func (cdb *PgDB) GetCertificateByHash(ctx context.Context, hash []byte) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByID(ctx context.Context, id int64) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
- func (cdb *PgDB) GetCertificatesByCommonName(ctx context.Context, commonname string) (certs []*JsonCertificate, err error)
- func (cdb *PgDB) QueueUsage() (pct int)
- type PgDnsname
- type PgDnsnamesView
- type PgIdent
- type PgLogEntry
- type Scanner
- type StreamError
Constants ¶
View Source
const SelectEstimate = `SELECT reltuples AS estimate FROM pg_class WHERE relname = $1;`
View Source
const SelectMaxIndex = `SELECT MAX(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
View Source
const SelectMinIndex = `SELECT MIN(logindex) AS logindex FROM CERTDB_entry WHERE stream = $1;`
Variables ¶
View Source
var BulkRange = int64(4096)
View Source
var CreateSchema string
View Source
var DbBatchSize = 100
View Source
var DefaultTransport = &http.Transport{ TLSHandshakeTimeout: 30 * time.Second, ResponseHeaderTimeout: 30 * time.Second, MaxIdleConnsPerHost: 2, DisableKeepAlives: false, ExpectContinueTimeout: 1 * time.Second, ForceAttemptHTTP2: true, }
View Source
var ErrLogIdle errLogIdle
View Source
var ErrSunlightClientMissing = errors.New("sunlight client missing")
View Source
var FuncIngestBatch string
View Source
var FunctionOperatorID string
View Source
var FunctionStreamID string
View Source
var IdleCloseTime = time.Hour * 24 * 7
View Source
var LogBatchSize = int64(1000)
View Source
var MaxErrors = 100
View Source
var SelectAllGaps string
Functions ¶
func OperatorDomain ¶
OperatorDomain returns the TLD+1 given an URL.
func ScanCertificate ¶ added in v0.12.0
func ScanCertificate(row Scanner, cert *PgCertificate) (err error)
func ScanDnsname ¶ added in v0.12.0
func ScanDnsnamesView ¶ added in v0.12.0
func ScanDnsnamesView(row Scanner, dnsname *PgDnsnamesView) (err error)
func ScanLogEntry ¶ added in v0.12.0
func ScanLogEntry(row Scanner, entry *PgLogEntry) (err error)
Types ¶
type CertStream ¶
type CertStream struct {
Config // copy of config
C <-chan *LogEntry // log entry channel
HeadClient *http.Client // main HTTP client, uses Config.HeadDialer
TailClient *http.Client // may be nil if not backfilling
// contains filtered or unexported fields
}
func (*CertStream) Close ¶ added in v0.19.0
func (cs *CertStream) Close()
func (*CertStream) CountStreams ¶ added in v0.0.3
func (cs *CertStream) CountStreams() (n int)
func (*CertStream) DB ¶ added in v0.12.0
func (cs *CertStream) DB() (db *PgDB)
func (*CertStream) GetLogStreamByID ¶ added in v0.27.0
func (cs *CertStream) GetLogStreamByID(id int32) (ls *LogStream)
func (*CertStream) LogError ¶ added in v0.1.0
func (cs *CertStream) LogError(err error, msg string, args ...any) error
func (*CertStream) LogInfo ¶ added in v0.12.0
func (cs *CertStream) LogInfo(msg string, args ...any)
func (*CertStream) Operators ¶ added in v0.0.2
func (cs *CertStream) Operators() (operators []*LogOperator)
type Certificate ¶ added in v0.10.0
func (*Certificate) GetCommonName ¶ added in v0.24.29
func (c *Certificate) GetCommonName() (s string)
type Config ¶ added in v0.12.0
type Config struct {
Logger Logger // if not nil Logger to use, no default
HeadDialer proxy.ContextDialer // dialer for following the head, defaults to &net.Dialer{}
TailDialer proxy.ContextDialer // if not nil, backfill db using this dialer, no default
PgUser string // PostgreSQL user, default "certstream"
PgPass string // PostgreSQL password, default "certstream"
PgName string // PostgreSQL db name, default "certstream"
PgAddr string // PostgreSQL address, no default
PgPrefix string // PostgreSQL naming prefix, default "certdb_"
PgConns int // max number of database connections, default 100
PgWorkerBits int // number of prefix bits that determine DB workers, default 5 (32 workers)
PgMaxAge int // maximum age in days to backfill
PgNoSSL bool // if true, do not use SSL
GetEntriesParallelism int // number of concurrent GetRawEntries requests per range, default 8
}
type JsonCertificate ¶ added in v0.12.0
type JsonCertificate struct {
PreCert bool `json:",omitempty"`
Signature hexEncoded `json:",omitempty"` // SHA256 signature, searchable on crt.sh
Issuer JsonIdentity `json:",omitempty"`
Subject JsonIdentity `json:",omitempty"`
CommonName string `json:",omitempty"` // Subject common name
DNSNames []string `json:",omitempty"`
EmailAddresses []string `json:",omitempty"`
IPAddresses []string `json:",omitempty"`
URIs []string `json:",omitempty"`
NotBefore time.Time `json:",omitempty"`
NotAfter time.Time `json:",omitempty"`
Since time.Time `json:",omitzero"`
}
func NewJSONCertificate ¶ added in v0.12.0
func NewJSONCertificate(cert *Certificate) (jsoncert *JsonCertificate)
func (*JsonCertificate) SetCommonName ¶ added in v0.24.29
func (js *JsonCertificate) SetCommonName()
type JsonIdentity ¶ added in v0.12.0
type LogEntry ¶
type LogEntry struct {
*LogStream
Err error // error from RawLogEntryFromLeaf or ToLogEntry, or nil
LogIndex int64
PreCert bool
Certificate *x509.Certificate
Id int64 // database id, if available
Historical bool // true if the entry is from gap or backfilling
Signature []byte
Seen time.Time
}
func (*LogEntry) Cert ¶
func (le *LogEntry) Cert() (crt *Certificate)
Cert returns the Certificate given a LogEntry or nil.
type LogOperator ¶ added in v0.0.3
type LogOperator struct {
*CertStream
Domain string // e.g. "letsencrypt.org" or "googleapis.com"
Count atomic.Int64 // atomic; sum of the stream's Count
Id int32 // database ID, if available
// contains filtered or unexported fields
}
func (*LogOperator) Email ¶ added in v0.28.0
func (lo *LogOperator) Email() []string
func (*LogOperator) ErrorCount ¶ added in v0.22.0
func (lo *LogOperator) ErrorCount() (n int)
func (*LogOperator) Errors ¶ added in v0.22.0
func (lo *LogOperator) Errors() (errs []*StreamError)
func (*LogOperator) GetStreamByID ¶ added in v0.27.0
func (lo *LogOperator) GetStreamByID(id int32) (ls *LogStream)
func (*LogOperator) Name ¶ added in v0.28.0
func (lo *LogOperator) Name() string
func (*LogOperator) StreamCount ¶ added in v0.18.0
func (lo *LogOperator) StreamCount() (n int)
func (*LogOperator) Streams ¶ added in v0.0.3
func (lo *LogOperator) Streams() (sl []*LogStream)
type LogStream ¶
type LogStream struct {
*LogOperator
Count atomic.Int64 // number of certificates sent to the channel
MinIndex atomic.Int64 // atomic: lowest index seen so far, -1 if none seen yet
MaxIndex atomic.Int64 // atomic: highest index seen so far, -1 if none seen yet
LastIndex atomic.Int64 // atomic: highest index that is available from stream source
InsideGaps atomic.Int64 // atomic: number of remaining entries inside gaps
Id int32 // database ID, if available
// contains filtered or unexported fields
}
type PgCertificate ¶ added in v0.12.0
type PgDB ¶ added in v0.12.0
type PgDB struct {
*CertStream
*pgxpool.Pool
Pfx func(string) string // prefix replacer
Workers atomic.Int32
// contains filtered or unexported fields
}
PgDB integrates with sql.DB to manage certificate stream data for a PostgreSQL database
func NewPgDB ¶ added in v0.12.0
func NewPgDB(ctx context.Context, cs *CertStream) (cdb *PgDB, err error)
NewPgDB creates a PgDB and creates the needed tables and indices if they don't exist.
func (*PgDB) AverageNewEntryTime ¶ added in v0.14.0
func (*PgDB) DeleteCertificates ¶ added in v0.28.4
func (*PgDB) DeleteStream ¶ added in v0.28.4
func (*PgDB) GetCertificateByHash ¶ added in v0.12.0
func (*PgDB) GetCertificateByID ¶ added in v0.12.0
func (*PgDB) GetCertificateByLogEntry ¶ added in v0.12.0
func (cdb *PgDB) GetCertificateByLogEntry(ctx context.Context, entry *PgLogEntry) (cert *JsonCertificate, err error)
func (*PgDB) GetCertificatesByCommonName ¶ added in v0.24.5
func (*PgDB) QueueUsage ¶ added in v0.15.0
type PgDnsnamesView ¶ added in v0.12.0
type PgLogEntry ¶ added in v0.12.0
type StreamError ¶ added in v0.22.0
func (StreamError) Error ¶ added in v0.22.0
func (ewt StreamError) Error() string
func (StreamError) Unwrap ¶ added in v0.22.0
func (ewt StreamError) Unwrap() error
Source Files
¶
- certificate.go
- certstream.go
- config.go
- errlogidle.go
- getloglist.go
- jsoncertificate.go
- jsonidentity.go
- logentry.go
- logger.go
- logoperator.go
- logstream.go
- operatordomain.go
- pgbackfill.go
- pgbatcher.go
- pgcertificate.go
- pgdb.go
- pgdnsname.go
- pgdnsnamesview.go
- pgident.go
- pglogentry.go
- pgschema.go
- streamerror.go
- sunlight_client.go
- updatestreams.go
- wraperr.go
Click to show internal directories.
Click to hide internal directories.