Documentation
¶
Index ¶
- Constants
- Variables
- func Anonymize(method string, s string) string
- func AnonymizeSql(method string, sqlId, sql string) string
- func Choose(msg string, items []string) (string, error)
- func Confirm(msg string) bool
- func CountAuditlogs(ctx context.Context, db *sqlx.DB, dbname, table string, opts AuditLogScanOpts) (int, error)
- func DecodeReplaySqls(s *bufio.Scanner, dbs, users map[string]struct{}, from, to int64, ...) (map[string][]*ReplaySql, int64, int, error)
- func DetectCharset(r *bufio.Reader) (string, error)
- func EncodeReplaySql(ts, client, user, db, queryId, stmt string, durationMs int64) string
- func ExpandHome(path string) string
- func ExtractQueriesFromAuditLogs(writers []SqlWriter, auditlogPaths []string, encoding string, ...) (int, error)
- func FileGlob(paths []string) ([]string, error)
- func GetDBAuditLogs(ctx context.Context, w SqlWriter, db *sqlx.DB, dbname, table string, ...) (int, error)
- func GetEncoding(name string) (encoding.Encoding, error)
- func IsStringType(colType string) bool
- func LLMGendataConfig(ctx context.Context, apiKey, baseURL, model, prompt_ string, ...) (string, error)
- func MustJsonMarshal(v any) []byte
- func MustYamlMarshal(v any) []byte
- func NewDB(host string, port uint16, user, password, catalog, db string) (*sqlx.DB, error)
- func ParallelGroup(parallel int) *errgroup.Group
- func ReadFileOrStdin(path string) (string, error)
- func ReplaySqls(ctx context.Context, host string, port uint16, ...) error
- func RunCreateSQL(ctx context.Context, conn *sqlx.DB, db string, sqlFile string, beCount int, ...) (needDependence string, err error)
- func SanitizeLike(s string) string
- func ScpFromRemote(ctx context.Context, privKey, remoteUrl, localPath string) error
- func SetupAnonymizer(method, hashdictPath string, idMinLength int, reserveIds ...string)
- func ShowBackendCount(ctx context.Context, conn *sqlx.DB) (count int, err error)
- func ShowDatabases(ctx context.Context, conn *sqlx.DB, dbnamePrefix string) ([]string, error)
- func ShowFronendsDisksDir(ctx context.Context, conn *sqlx.DB, diskType string) (dir string, err error)
- func SshLs(_ context.Context, privKey, remoteUrl string) ([]string, error)
- func StoreMiniHashDict(method, hashdictPath string)
- func StreamLoad(ctx context.Context, ...) error
- func WriteFile(path string, content string) error
- type AuditLogScanOpts
- type AuditLogScanner
- type BytesEncoder
- type ClientSqls
- type ColumnStats
- type CreateParserListener
- type DBSchema
- type DummyEncoder
- type GenRule
- type ReplayClient
- type ReplayResult
- type ReplaySql
- type ReplaySqlMeta
- type Schema
- type SchemaType
- type SimpleAuditLogScanner
- type SqlWriter
- type TableGen
- type TableStats
- type Utf8Encoder
Constants ¶
View Source
const ( ColumnSeparator = '☆' // make me happy DefaultGenRowCount = 1000 GenDataFileFirstLinePrefix = "columns:" // optional first line prefix if stream load needs 'columns: xxx' header )
View Source
const ( ReplaySqlPrefix = `/*dodo{` ReplaySqlSuffix = `*/` ReplayResultFileExt = ".result" ReplayCustomClientPrefix = "client" )
View Source
const (
AnonymizeHashBytes = 8
)
View Source
const (
LLMOutputPrefix = "```yaml\n"
)
View Source
const (
StreamLoadMaxRetries = 3
)
Variables ¶
View Source
var (
InternalSqlComment = "/*dodo*/"
)
View Source
var (
NumberRe = regexp.MustCompile(`\d+`)
)
Functions ¶
func AnonymizeSql ¶
func CountAuditlogs ¶
func DecodeReplaySqls ¶
func EncodeReplaySql ¶
func ExpandHome ¶
func ExtractQueriesFromAuditLogs ¶
func ExtractQueriesFromAuditLogs( writers []SqlWriter, auditlogPaths []string, encoding string, opts AuditLogScanOpts, parallel int, ) (int, error)
ExtractQueriesFromAuditLog extracts the query from an audit log.
func GetDBAuditLogs ¶
func IsStringType ¶
func LLMGendataConfig ¶
func LLMGendataConfig( ctx context.Context, apiKey, baseURL, model, prompt_ string, tables, columnStats, sqls []string, ) (string, error)
Use Deepseek by default, but you can use OpenAI by setting the apiKey and baseURL
func MustJsonMarshal ¶
func MustYamlMarshal ¶
func ParallelGroup ¶
func ReadFileOrStdin ¶
func ReplaySqls ¶
func RunCreateSQL ¶
func SanitizeLike ¶
func ScpFromRemote ¶
ScpFromRemote copies a file from a remote server to the local machine using scp.
privKey is the path to the private key to use for authentication. remoteUrl is the address of file on the remote server, format user@host:port/path. localPath is the path of the local file to copy to.
func SetupAnonymizer ¶
func ShowBackendCount ¶
func ShowDatabases ¶
func ShowFronendsDisksDir ¶
func StoreMiniHashDict ¶
func StoreMiniHashDict(method, hashdictPath string)
func StreamLoad ¶
Types ¶
type AuditLogScanOpts ¶
type AuditLogScanner ¶
type AuditLogScanner interface {
Init()
ScanOne(oneLine []byte) error
Consume(w SqlWriter) (int, error)
Close()
}
Not thread safe.
func NewAuditLogScanner ¶
func NewAuditLogScanner(opts AuditLogScanOpts) AuditLogScanner
type BytesEncoder ¶
func NewBytesEncoder ¶
func NewBytesEncoder(srcEncoding encoding.Encoding) BytesEncoder
type ClientSqls ¶
type ColumnStats ¶
type CreateParserListener ¶
type CreateParserListener struct {
*parser.BaseDorisParserListener
// contains filtered or unexported fields
}
func (*CreateParserListener) ExitPropertyItem ¶
func (l *CreateParserListener) ExitPropertyItem(ctx *parser.PropertyItemContext)
Modify property value
type DBSchema ¶
type DBSchema struct {
Name string `yaml:"db"`
Schemas []*Schema `yaml:"-"`
Stats []*TableStats `yaml:"tables,omitempty"`
}
type DummyEncoder ¶
type DummyEncoder struct {
}
type ReplayClient ¶
type ReplayClient struct {
// contains filtered or unexported fields
}
func (*ReplayClient) Close ¶
func (c *ReplayClient) Close(closefile bool)
type ReplayResult ¶
type ReplayResult struct {
Ts string `json:"ts,omitempty"`
QueryId string `json:"queryId"`
ReturnRows int `json:"returnRows"`
ReturnRowsHash string `json:"returnRowsHash,omitempty"`
DurationMs int64 `json:"durationMs"`
Err string `json:"err,omitempty"`
Stmt string `json:"stmt,omitempty"`
}
func (*ReplayResult) String ¶
func (re *ReplayResult) String() string
type ReplaySql ¶
type ReplaySql struct {
ReplaySqlMeta
Stmt string
}
func (*ReplaySql) ToReplayResult ¶
func (s *ReplaySql) ToReplayResult() *ReplayResult
type ReplaySqlMeta ¶
type ReplaySqlMeta struct {
Ts_ string `json:"ts"`
Ts int64 `json:"-"`
Client string `json:"client"`
User string `json:"user"`
Db string `json:"db"`
QueryId string `json:"queryId"`
DurationMs int64 `json:"durationMs,omitempty"`
}
ReplaySqlMeta will be prepend to every sql as a comment.
e.g. "/*dodo{"ts": "2024-09-20 00:00:00", "client": "127.0.0.1:32345", "user": "root", "db": "test", "queryId": "1"}*/ <the sql>"
func (*ReplaySqlMeta) Timestamp ¶
func (m *ReplaySqlMeta) Timestamp() (ms int64, err error)
type Schema ¶
type Schema struct {
Name string `db:"TABLE_NAME"`
Type SchemaType `db:"TABLE_TYPE"`
DB string `db:"TABLE_SCHEMA"`
CreateStmt string `db:"-"`
}
func ShowCreateTables ¶
func ShowTables ¶
type SchemaType ¶
type SchemaType string
var ( SchemaTypeTable SchemaType = "TABLE" SchemaTypeView SchemaType = "VIEW" SchemaTypeMaterializedView SchemaType = "MATERIALIZED_VIEW" )
func (SchemaType) Lower ¶
func (s SchemaType) Lower() string
type SimpleAuditLogScanner ¶
type SimpleAuditLogScanner struct {
AuditLogScanOpts
// contains filtered or unexported fields
}
func NewSimpleAuditLogScanner ¶
func NewSimpleAuditLogScanner(opts AuditLogScanOpts) *SimpleAuditLogScanner
func (*SimpleAuditLogScanner) Close ¶
func (*SimpleAuditLogScanner) Close()
func (*SimpleAuditLogScanner) Consume ¶
func (s *SimpleAuditLogScanner) Consume(w SqlWriter) (int, error)
func (*SimpleAuditLogScanner) Init ¶
func (s *SimpleAuditLogScanner) Init()
func (*SimpleAuditLogScanner) ScanOne ¶
func (s *SimpleAuditLogScanner) ScanOne(oneLog []byte) error
type TableGen ¶
type TableGen struct {
Name string
Columns []string
DDLFile string
Rows int
RefToTable map[string]struct{} // ref generator to other tables
StreamloadColMapping string
// contains filtered or unexported fields
}
func NewTableGen ¶
func (*TableGen) RecordRefTables ¶
func (*TableGen) RemoveRefTable ¶
type TableStats ¶
type TableStats struct {
Name string `yaml:"name"`
RowCount int64 `yaml:"row_count"`
Columns []*ColumnStats `yaml:"columns,omitempty"`
}
type Utf8Encoder ¶
type Utf8Encoder struct {
// contains filtered or unexported fields
}
Source Files
¶
Click to show internal directories.
Click to hide internal directories.