135 lines
3.4 KiB
Go
135 lines
3.4 KiB
Go
package db
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/ClickHouse/clickhouse-go/v2"
|
|
)
|
|
|
|
const schemaSQL = `
|
|
CREATE TABLE IF NOT EXISTS proxy_logs (
|
|
request_id String,
|
|
method String,
|
|
path String,
|
|
query String,
|
|
client_ip String,
|
|
request_headers String,
|
|
request_body String,
|
|
request_truncated Bool DEFAULT false,
|
|
status_code Int32,
|
|
response_headers String,
|
|
response_body String,
|
|
response_truncated Bool DEFAULT false,
|
|
is_stream Bool DEFAULT false,
|
|
latency_ms Int64,
|
|
started_at DateTime64(3),
|
|
finished_at DateTime64(3),
|
|
error String
|
|
)
|
|
ENGINE = MergeTree
|
|
PARTITION BY toYYYYMM(started_at)
|
|
ORDER BY (started_at, request_id)
|
|
`
|
|
|
|
func migrate(ctx context.Context, conn clickhouse.Conn) error {
|
|
if err := conn.Exec(ctx, schemaSQL); err != nil {
|
|
return fmt.Errorf("create proxy_logs: %w", err)
|
|
}
|
|
|
|
rows, err := conn.Query(ctx, `
|
|
SELECT name, type
|
|
FROM system.columns
|
|
WHERE database = currentDatabase() AND table = 'proxy_logs'
|
|
ORDER BY position`)
|
|
if err != nil {
|
|
return fmt.Errorf("query proxy_logs columns: %w", err)
|
|
}
|
|
var columns []schemaColumn
|
|
for rows.Next() {
|
|
var column schemaColumn
|
|
if err := rows.Scan(&column.name, &column.typ); err != nil {
|
|
rows.Close()
|
|
return fmt.Errorf("scan proxy_logs columns: %w", err)
|
|
}
|
|
columns = append(columns, column)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
rows.Close()
|
|
return fmt.Errorf("read proxy_logs columns: %w", err)
|
|
}
|
|
rows.Close()
|
|
|
|
var table schemaTable
|
|
err = conn.QueryRow(ctx, `
|
|
SELECT engine, partition_key, sorting_key
|
|
FROM system.tables
|
|
WHERE database = currentDatabase() AND name = 'proxy_logs'`).Scan(
|
|
&table.engine, &table.partitionKey, &table.sortingKey,
|
|
)
|
|
if err != nil {
|
|
return fmt.Errorf("query proxy_logs table: %w", err)
|
|
}
|
|
if err := validateProxyLogsSchema(columns, table); err != nil {
|
|
return fmt.Errorf("incompatible proxy_logs schema: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type schemaColumn struct {
|
|
name string
|
|
typ string
|
|
}
|
|
|
|
type schemaTable struct {
|
|
engine string
|
|
partitionKey string
|
|
sortingKey string
|
|
}
|
|
|
|
var proxyLogsColumns = []schemaColumn{
|
|
{"request_id", "String"},
|
|
{"method", "String"},
|
|
{"path", "String"},
|
|
{"query", "String"},
|
|
{"client_ip", "String"},
|
|
{"request_headers", "String"},
|
|
{"request_body", "String"},
|
|
{"request_truncated", "Bool"},
|
|
{"status_code", "Int32"},
|
|
{"response_headers", "String"},
|
|
{"response_body", "String"},
|
|
{"response_truncated", "Bool"},
|
|
{"is_stream", "Bool"},
|
|
{"latency_ms", "Int64"},
|
|
{"started_at", "DateTime64(3)"},
|
|
{"finished_at", "DateTime64(3)"},
|
|
{"error", "String"},
|
|
}
|
|
|
|
func validateProxyLogsSchema(columns []schemaColumn, table schemaTable) error {
|
|
if len(columns) != len(proxyLogsColumns) {
|
|
return fmt.Errorf("got %d columns, want %d", len(columns), len(proxyLogsColumns))
|
|
}
|
|
for i, want := range proxyLogsColumns {
|
|
if columns[i] != want {
|
|
return fmt.Errorf("column %d is %s %s, want %s %s", i+1, columns[i].name, columns[i].typ, want.name, want.typ)
|
|
}
|
|
}
|
|
if table.engine != "MergeTree" {
|
|
return fmt.Errorf("engine is %q, want MergeTree", table.engine)
|
|
}
|
|
if compactExpression(table.partitionKey) != "toYYYYMM(started_at)" {
|
|
return fmt.Errorf("partition key is %q, want toYYYYMM(started_at)", table.partitionKey)
|
|
}
|
|
if compactExpression(table.sortingKey) != "started_at,request_id" {
|
|
return fmt.Errorf("sorting key is %q, want started_at, request_id", table.sortingKey)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func compactExpression(value string) string {
|
|
return strings.Join(strings.Fields(value), "")
|
|
}
|