fix: restore reviewable migration evidence

2026-07-10 18:26:48 +08:00
commit d1bbb5370c
42 changed files with 6419 additions and 0 deletions
@@ -0,0 +1,245 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"net/netip"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.misaka.ren/M1saka/token_thief/db"
+)
+
+// Config 保存从环境变量加载的全部运行配置。
+type Config struct {
+	ListenAddr                    string
+	UpstreamURL                   *url.URL
+	ClickHouseURL                 string
+	MaxBodyBytes                  int64
+	LogQueueSize                  int
+	LogQueueBytes                 int64
+	LogBatchSize                  int
+	LogBatchInterval              time.Duration
+	LogWorkers                    int
+	FilterFile                    string
+	DBReconnectInterval           time.Duration
+	ReadTimeout                   time.Duration
+	WriteTimeout                  time.Duration
+	IdleTimeout                   time.Duration
+	UpstreamTimeout               time.Duration
+	UpstreamResponseTimeout       time.Duration
+	UpstreamStreamIdleTimeout     time.Duration
+	UpstreamTLSInsecureSkipVerify bool
+	TrustedProxies                []netip.Prefix
+}
+
+// Load 从进程环境读取配置；缺少必填字段时返回错误。
+func Load() (*Config, error) {
+	maxBodyBytes, err := getEnvInt64("MAX_BODY_BYTES", 1<<20)
+	if err != nil {
+		return nil, err
+	}
+	logQueueSize, err := getEnvInt("LOG_QUEUE_SIZE", 256)
+	if err != nil {
+		return nil, err
+	}
+	logQueueBytes, err := getEnvInt64("LOG_QUEUE_BYTES", 64<<20)
+	if err != nil {
+		return nil, err
+	}
+	logBatchSize, err := getEnvInt("LOG_BATCH_SIZE", 50)
+	if err != nil {
+		return nil, err
+	}
+	logBatchInterval, err := getEnvDuration("LOG_BATCH_INTERVAL", 2*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	logWorkers, err := getEnvInt("LOG_WORKERS", 2)
+	if err != nil {
+		return nil, err
+	}
+	dbReconnectInterval, err := getEnvDuration("DB_RECONNECT_INTERVAL", 10*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	readTimeout, err := getEnvDuration("READ_TIMEOUT", 30*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	writeTimeout, err := getEnvDuration("WRITE_TIMEOUT", 10*time.Minute)
+	if err != nil {
+		return nil, err
+	}
+	idleTimeout, err := getEnvDuration("IDLE_TIMEOUT", 5*time.Minute)
+	if err != nil {
+		return nil, err
+	}
+	upstreamTimeout, err := getEnvDuration("UPSTREAM_TIMEOUT", 30*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	upstreamResponseTimeout, err := getEnvDuration("UPSTREAM_RESPONSE_TIMEOUT", 30*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	upstreamStreamIdleTimeout, err := getEnvDuration("UPSTREAM_STREAM_IDLE_TIMEOUT", 2*time.Minute)
+	if err != nil {
+		return nil, err
+	}
+	upstreamTLSInsecureSkipVerify, err := getEnvBool("UPSTREAM_TLS_INSECURE_SKIP_VERIFY", false)
+	if err != nil {
+		return nil, err
+	}
+	trustedProxies, err := getEnvPrefixes("TRUSTED_PROXIES")
+	if err != nil {
+		return nil, err
+	}
+
+	cfg := &Config{
+		ListenAddr:                    getEnv("LISTEN_ADDR", ":8080"),
+		ClickHouseURL:                 os.Getenv("CLICKHOUSE_URL"),
+		MaxBodyBytes:                  maxBodyBytes,
+		LogQueueSize:                  logQueueSize,
+		LogQueueBytes:                 logQueueBytes,
+		LogBatchSize:                  logBatchSize,
+		LogBatchInterval:              logBatchInterval,
+		LogWorkers:                    logWorkers,
+		FilterFile:                    getEnv("FILTER_FILE", "./filter.yaml"),
+		DBReconnectInterval:           dbReconnectInterval,
+		ReadTimeout:                   readTimeout,
+		WriteTimeout:                  writeTimeout,
+		IdleTimeout:                   idleTimeout,
+		UpstreamTimeout:               upstreamTimeout,
+		UpstreamResponseTimeout:       upstreamResponseTimeout,
+		UpstreamStreamIdleTimeout:     upstreamStreamIdleTimeout,
+		UpstreamTLSInsecureSkipVerify: upstreamTLSInsecureSkipVerify,
+		TrustedProxies:                trustedProxies,
+	}
+
+	upstream := os.Getenv("UPSTREAM_URL")
+	if upstream == "" {
+		return nil, errors.New("UPSTREAM_URL is required")
+	}
+	u, err := url.Parse(upstream)
+	if err != nil {
+		return nil, fmt.Errorf("invalid UPSTREAM_URL: %w", err)
+	}
+	if u.Scheme == "" || u.Host == "" {
+		return nil, fmt.Errorf("invalid UPSTREAM_URL: %q", upstream)
+	}
+	cfg.UpstreamURL = u
+
+	if cfg.ClickHouseURL == "" {
+		return nil, errors.New("CLICKHOUSE_URL is required")
+	}
+	if _, err := db.ClickHouseOptions(cfg.ClickHouseURL); err != nil {
+		return nil, err
+	}
+	positiveValues := []struct {
+		key   string
+		value int64
+	}{
+		{key: "MAX_BODY_BYTES", value: cfg.MaxBodyBytes},
+		{key: "LOG_QUEUE_SIZE", value: int64(cfg.LogQueueSize)},
+		{key: "LOG_QUEUE_BYTES", value: cfg.LogQueueBytes},
+		{key: "LOG_BATCH_SIZE", value: int64(cfg.LogBatchSize)},
+		{key: "LOG_BATCH_INTERVAL", value: int64(cfg.LogBatchInterval)},
+		{key: "LOG_WORKERS", value: int64(cfg.LogWorkers)},
+		{key: "DB_RECONNECT_INTERVAL", value: int64(cfg.DBReconnectInterval)},
+		{key: "READ_TIMEOUT", value: int64(cfg.ReadTimeout)},
+		{key: "WRITE_TIMEOUT", value: int64(cfg.WriteTimeout)},
+		{key: "IDLE_TIMEOUT", value: int64(cfg.IdleTimeout)},
+		{key: "UPSTREAM_TIMEOUT", value: int64(cfg.UpstreamTimeout)},
+		{key: "UPSTREAM_RESPONSE_TIMEOUT", value: int64(cfg.UpstreamResponseTimeout)},
+		{key: "UPSTREAM_STREAM_IDLE_TIMEOUT", value: int64(cfg.UpstreamStreamIdleTimeout)},
+	}
+	for _, item := range positiveValues {
+		if item.value <= 0 {
+			return nil, fmt.Errorf("%s must be greater than zero", item.key)
+		}
+	}
+	return cfg, nil
+}
+
+func getEnv(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func getEnvInt(key string, def int) (int, error) {
+	if v := os.Getenv(key); v != "" {
+		n, err := strconv.Atoi(v)
+		if err != nil {
+			return 0, fmt.Errorf("invalid %s: %w", key, err)
+		}
+		return n, nil
+	}
+	return def, nil
+}
+
+func getEnvInt64(key string, def int64) (int64, error) {
+	if v := os.Getenv(key); v != "" {
+		n, err := strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			return 0, fmt.Errorf("invalid %s: %w", key, err)
+		}
+		return n, nil
+	}
+	return def, nil
+}
+
+func getEnvDuration(key string, def time.Duration) (time.Duration, error) {
+	if v := os.Getenv(key); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			return 0, fmt.Errorf("invalid %s: %w", key, err)
+		}
+		return d, nil
+	}
+	return def, nil
+}
+
+func getEnvBool(key string, def bool) (bool, error) {
+	if v := os.Getenv(key); v != "" {
+		b, err := strconv.ParseBool(v)
+		if err != nil {
+			return false, fmt.Errorf("invalid %s: %w", key, err)
+		}
+		return b, nil
+	}
+	return def, nil
+}
+
+func getEnvPrefixes(key string) ([]netip.Prefix, error) {
+	v := os.Getenv(key)
+	if v == "" {
+		return nil, nil
+	}
+
+	parts := strings.Split(v, ",")
+	prefixes := make([]netip.Prefix, 0, len(parts))
+	for _, part := range parts {
+		value := strings.TrimSpace(part)
+		if value == "" {
+			return nil, fmt.Errorf("invalid %s: empty IP or CIDR", key)
+		}
+
+		if addr, err := netip.ParseAddr(value); err == nil {
+			prefixes = append(prefixes, netip.PrefixFrom(addr, addr.BitLen()))
+			continue
+		}
+
+		prefix, err := netip.ParsePrefix(value)
+		if err != nil || !prefix.IsValid() || prefix != prefix.Masked() {
+			return nil, fmt.Errorf("invalid %s entry %q", key, value)
+		}
+		prefixes = append(prefixes, prefix)
+	}
+	return prefixes, nil
+}
@@ -0,0 +1,134 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// FilterMode 决定如何应用 patterns。
+type FilterMode string
+
+const (
+	FilterWhitelist FilterMode = "whitelist"
+	FilterBlacklist FilterMode = "blacklist"
+	FilterDisabled  FilterMode = "disabled"
+)
+
+// Filter 表示路径匹配规则。
+type Filter struct {
+	Mode     FilterMode `yaml:"mode"`
+	Patterns []string   `yaml:"patterns"`
+
+	compiled []*regexp.Regexp
+}
+
+// LoadFilter 从 yaml 文件加载过滤配置；文件不存在则默认全部记录。
+//
+// Pattern 语法（glob 风格）：
+//   - `*`  匹配单个路径段内除 `/` 之外的任意字符（包括零个）。
+//   - `**` 匹配任意字符，含 `/`，可跨段。
+//   - `?`  匹配单个非 `/` 字符。
+//   - 其它字符按字面匹配。
+//
+// 示例：
+//   - `/v1/audio/*`              匹配 /v1/audio/speech、/v1/audio/transcriptions
+//   - `/v1/videos/**`            匹配 /v1/videos/任意子路径
+//   - `/v1beta/models/*:generateContent` 匹配 Gemini 风格端点
+func LoadFilter(filePath string) (*Filter, error) {
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return &Filter{Mode: FilterDisabled}, nil
+		}
+		return nil, err
+	}
+	var f Filter
+	if err := yaml.Unmarshal(data, &f); err != nil {
+		return nil, fmt.Errorf("parse filter yaml: %w", err)
+	}
+	switch f.Mode {
+	case FilterWhitelist, FilterBlacklist, FilterDisabled:
+	case "":
+		f.Mode = FilterDisabled
+	default:
+		return nil, fmt.Errorf("unknown filter mode: %q", f.Mode)
+	}
+	if err := f.compile(); err != nil {
+		return nil, err
+	}
+	return &f, nil
+}
+
+// NewFilter 程序化构造一个 Filter（主要供测试使用）。
+func NewFilter(mode FilterMode, patterns []string) (*Filter, error) {
+	f := &Filter{Mode: mode, Patterns: append([]string(nil), patterns...)}
+	if err := f.compile(); err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+func (f *Filter) compile() error {
+	f.compiled = make([]*regexp.Regexp, 0, len(f.Patterns))
+	for _, p := range f.Patterns {
+		re, err := CompileGlob(p)
+		if err != nil {
+			return fmt.Errorf("invalid pattern %q: %w", p, err)
+		}
+		f.compiled = append(f.compiled, re)
+	}
+	return nil
+}
+
+// ShouldLog 决定一个请求 path 是否需要被记录。
+func (f *Filter) ShouldLog(reqPath string) bool {
+	if f == nil || f.Mode == FilterDisabled {
+		return true
+	}
+	matched := false
+	for _, re := range f.compiled {
+		if re.MatchString(reqPath) {
+			matched = true
+			break
+		}
+	}
+	switch f.Mode {
+	case FilterWhitelist:
+		return matched
+	case FilterBlacklist:
+		return !matched
+	default:
+		return true
+	}
+}
+
+// CompileGlob 将 glob 风格 pattern 转换为 anchored 正则表达式。
+func CompileGlob(pattern string) (*regexp.Regexp, error) {
+	var sb strings.Builder
+	sb.WriteString("^")
+	for i := 0; i < len(pattern); i++ {
+		c := pattern[i]
+		switch c {
+		case '*':
+			if i+1 < len(pattern) && pattern[i+1] == '*' {
+				sb.WriteString(".*")
+				i++
+			} else {
+				sb.WriteString("[^/]*")
+			}
+		case '?':
+			sb.WriteString("[^/]")
+		case '.', '+', '(', ')', '|', '^', '$', '{', '}', '[', ']', '\\':
+			sb.WriteByte('\\')
+			sb.WriteByte(c)
+		default:
+			sb.WriteByte(c)
+		}
+	}
+	sb.WriteString("$")
+	return regexp.Compile(sb.String())
+}