package db import ( "context" "fmt" "strings" "github.com/ClickHouse/clickhouse-go/v2" ) const schemaSQL = ` CREATE TABLE IF NOT EXISTS proxy_logs ( request_id String, method String, path String, query String, client_ip String, request_headers String, request_body String, request_truncated Bool DEFAULT false, status_code Int32, response_headers String, response_body String, response_truncated Bool DEFAULT false, is_stream Bool DEFAULT false, latency_ms Int64, started_at DateTime64(3), finished_at DateTime64(3), error String ) ENGINE = MergeTree PARTITION BY toYYYYMM(started_at) ORDER BY (started_at, request_id) ` func migrate(ctx context.Context, conn clickhouse.Conn) error { if err := conn.Exec(ctx, schemaSQL); err != nil { return fmt.Errorf("create proxy_logs: %w", err) } rows, err := conn.Query(ctx, ` SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = 'proxy_logs' ORDER BY position`) if err != nil { return fmt.Errorf("query proxy_logs columns: %w", err) } var columns []schemaColumn for rows.Next() { var column schemaColumn if err := rows.Scan(&column.name, &column.typ); err != nil { rows.Close() return fmt.Errorf("scan proxy_logs columns: %w", err) } columns = append(columns, column) } if err := rows.Err(); err != nil { rows.Close() return fmt.Errorf("read proxy_logs columns: %w", err) } rows.Close() var table schemaTable err = conn.QueryRow(ctx, ` SELECT engine, partition_key, sorting_key FROM system.tables WHERE database = currentDatabase() AND name = 'proxy_logs'`).Scan( &table.engine, &table.partitionKey, &table.sortingKey, ) if err != nil { return fmt.Errorf("query proxy_logs table: %w", err) } if err := validateProxyLogsSchema(columns, table); err != nil { return fmt.Errorf("incompatible proxy_logs schema: %w", err) } return nil } type schemaColumn struct { name string typ string } type schemaTable struct { engine string partitionKey string sortingKey string } var proxyLogsColumns = []schemaColumn{ {"request_id", "String"}, {"method", "String"}, {"path", "String"}, {"query", "String"}, {"client_ip", "String"}, {"request_headers", "String"}, {"request_body", "String"}, {"request_truncated", "Bool"}, {"status_code", "Int32"}, {"response_headers", "String"}, {"response_body", "String"}, {"response_truncated", "Bool"}, {"is_stream", "Bool"}, {"latency_ms", "Int64"}, {"started_at", "DateTime64(3)"}, {"finished_at", "DateTime64(3)"}, {"error", "String"}, } func validateProxyLogsSchema(columns []schemaColumn, table schemaTable) error { if len(columns) != len(proxyLogsColumns) { return fmt.Errorf("got %d columns, want %d", len(columns), len(proxyLogsColumns)) } for i, want := range proxyLogsColumns { if columns[i] != want { return fmt.Errorf("column %d is %s %s, want %s %s", i+1, columns[i].name, columns[i].typ, want.name, want.typ) } } if table.engine != "MergeTree" { return fmt.Errorf("engine is %q, want MergeTree", table.engine) } if compactExpression(table.partitionKey) != "toYYYYMM(started_at)" { return fmt.Errorf("partition key is %q, want toYYYYMM(started_at)", table.partitionKey) } if compactExpression(table.sortingKey) != "started_at,request_id" { return fmt.Errorf("sorting key is %q, want started_at, request_id", table.sortingKey) } return nil } func compactExpression(value string) string { return strings.Join(strings.Fields(value), "") }