m1saka/token_thief
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore reviewable migration evidence

Browse Source
This commit is contained in:
MiMoCode
2026-07-10 18:26:48 +08:00
commit d1bbb5370c
42 changed files with 6419 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/scripts/dbcheck/main.go
+109
View File
@@ -0,0 +1,109 @@
// Standalone helper for smoke.ps1. It reads CHECK_RIDS (JSON array) and verifies rows in proxy_logs.
package main
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"git.misaka.ren/M1saka/token_thief/db"
)
type row struct {
RequestID string
Method string
Path string
StatusCode int
RequestTruncated bool
ResponseTruncated bool
IsStream bool
LatencyMS int64
ReqBodyLen int64
RespBodyLen int64
ReqHeaders string
ErrorMsg string
}
func main() {
log.SetFlags(0)
dsn := os.Getenv("CLICKHOUSE_URL")
if dsn == "" {
log.Fatalf("CLICKHOUSE_URL not set")
}
ridsRaw := os.Getenv("CHECK_RIDS")
if ridsRaw == "" {
log.Fatalf("CHECK_RIDS not set")
}
var rids []string
if err := json.Unmarshal([]byte(ridsRaw), &rids); err != nil {
log.Fatalf("parse CHECK_RIDS: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
opts, err := db.ClickHouseOptions(dsn)
if err != nil {
log.Fatalf("connect: %v", err)
}
conn, err := clickhouse.Open(opts)
if err != nil {
log.Fatalf("connect: %v", err)
}
defer conn.Close()
const q = `
SELECT request_id, method, path, status_code,
request_truncated, response_truncated, is_stream, latency_ms,
toInt64(length(request_body)),
toInt64(length(response_body)),
request_headers,
error
FROM proxy_logs WHERE request_id = ?
ORDER BY started_at DESC LIMIT 1`
ok := 0
for _, rid := range rids {
var r row
err := conn.QueryRow(ctx, q, rid).Scan(
&r.RequestID, &r.Method, &r.Path, &r.StatusCode,
&r.RequestTruncated, &r.ResponseTruncated, &r.IsStream, &r.LatencyMS,
&r.ReqBodyLen, &r.RespBodyLen, &r.ReqHeaders, &r.ErrorMsg,
)
if err != nil {
fmt.Printf(" FAIL rid=%s: not found in db (%v)\n", rid, err)
continue
}
ok++
fmt.Printf(" OK rid=%s\n", rid)
fmt.Printf(" method=%s path=%s status=%d latency_ms=%d\n", r.Method, r.Path, r.StatusCode, r.LatencyMS)
fmt.Printf(" req_body=%d B (truncated=%v) resp_body=%d B (truncated=%v) is_stream=%v\n",
r.ReqBodyLen, r.RequestTruncated, r.RespBodyLen, r.ResponseTruncated, r.IsStream)
hasAuth := false
var hm map[string][]string
if err := json.Unmarshal([]byte(r.ReqHeaders), &hm); err == nil {
_, hasAuth = hm["Authorization"]
}
fmt.Printf(" headers has Authorization=%v\n", hasAuth)
if r.ErrorMsg != "" {
fmt.Printf(" error=%s\n", trunc(r.ErrorMsg, 200))
}
}
fmt.Printf("\n found %d / %d\n", ok, len(rids))
if ok != len(rids) {
os.Exit(1)
}
}
func trunc(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "..."
}
tests/scripts/dump-stream-body.ps1
+54
View File
@@ -0,0 +1,54 @@
param([string]$RequestID)
if (-not $RequestID) { throw "RequestID is required" }
. .\tests\scripts\load-env.ps1 | Out-Null
$env:DUMP_RID = $RequestID
@'
package main
import (
"context"
"fmt"
"log"
"os"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"git.misaka.ren/M1saka/token_thief/db"
)
func main() {
log.SetFlags(0)
dsn := os.Getenv("CLICKHOUSE_URL")
if dsn == "" {
log.Fatal("CLICKHOUSE_URL not set")
}
rid := os.Getenv("DUMP_RID")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
opts, err := db.ClickHouseOptions(dsn)
if err != nil {
log.Fatal(err)
}
conn, err := clickhouse.Open(opts)
if err != nil {
log.Fatal(err)
}
defer conn.Close()
var body string
if err := conn.QueryRow(ctx,
`SELECT response_body FROM proxy_logs WHERE request_id = ? ORDER BY started_at DESC LIMIT 1`,
rid,
).Scan(&body); err != nil {
log.Fatal(err)
}
fmt.Println(body)
}
'@ | Set-Content -Path tmp_dump.go -Encoding UTF8 -NoNewline
go run tmp_dump.go
Remove-Item tmp_dump.go -Force
Remove-Item Env:DUMP_RID -ErrorAction SilentlyContinue
tests/scripts/load-env.ps1
+25
View File
@@ -0,0 +1,25 @@
# Load KEY=VALUE pairs from .env into the current PowerShell process.
# Usage: . .\tests\scripts\load-env.ps1
param(
[string]$Path = ".env"
)
if (-not (Test-Path $Path)) {
Write-Error "env file not found: $Path"
return
}
Get-Content $Path | ForEach-Object {
$line = $_.Trim()
if ($line -eq "" -or $line.StartsWith("#")) { return }
$idx = $line.IndexOf("=")
if ($idx -lt 1) { return }
$key = $line.Substring(0, $idx).Trim()
$val = $line.Substring($idx + 1).Trim()
if (($val.StartsWith('"') -and $val.EndsWith('"')) -or
($val.StartsWith("'") -and $val.EndsWith("'"))) {
$val = $val.Substring(1, $val.Length - 2)
}
[Environment]::SetEnvironmentVariable($key, $val, "Process")
Write-Host " loaded $key"
}
tests/scripts/smoke.ps1
+135
View File
@@ -0,0 +1,135 @@
# End-to-end smoke test:
# start proxy -> run chat cases -> wait batch flush -> verify ClickHouse -> stop proxy.
param(
[string]$Model = "gpt-5.4-mini",
[string]$ApiKey = $env:NEWAPI_KEY,
[string]$ProxyBase = "http://127.0.0.1:8080"
)
if (-not $ApiKey) { throw "NEWAPI_KEY not set" }
$ErrorActionPreference = "Stop"
function Section($name) {
Write-Host ""
Write-Host ("=" * 70) -ForegroundColor Cyan
Write-Host $name -ForegroundColor Cyan
Write-Host ("=" * 70) -ForegroundColor Cyan
}
Section "Start proxy"
if (Test-Path proxy.log) { Remove-Item proxy.log -Force }
if (Test-Path proxy.err.log) { Remove-Item proxy.err.log -Force }
$proxy = Start-Process -FilePath .\TokenThief.exe -PassThru -RedirectStandardOutput proxy.log -RedirectStandardError proxy.err.log -WindowStyle Hidden
Write-Host "proxy pid=$($proxy.Id)"
Start-Sleep -Seconds 2
$health = & curl.exe -s -o NUL -w "%{http_code}" "$ProxyBase/healthz"
Write-Host "healthz: $health"
if ($health -ne "200") {
Get-Content proxy.log -Tail 30 | Write-Host
Get-Content proxy.err.log -Tail 30 | Write-Host
throw "proxy did not start"
}
$results = @{}
function CallChat {
param([string]$Url, [string]$JsonBody)
$bodyTmp = New-TemporaryFile
$headTmp = New-TemporaryFile
$outTmp = New-TemporaryFile
# PowerShell 5.1 Set-Content -Encoding UTF8 writes a BOM; newapi rejects BOM-prefixed JSON.
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
[System.IO.File]::WriteAllText($bodyTmp.FullName, $JsonBody, $utf8NoBom)
$code = & curl.exe -s -X POST $Url `
-H "Content-Type: application/json" `
-H "Authorization: Bearer $ApiKey" `
-D $headTmp.FullName `
-o $outTmp.FullName `
--data-binary "@$($bodyTmp.FullName)" `
-w "%{http_code}"
$rid = ""
foreach ($line in Get-Content $headTmp.FullName) {
if ($line -match '^X-Request-Id:\s*(.+)$') {
$rid = $matches[1].Trim()
break
}
}
$body = Get-Content $outTmp.FullName -Raw -ErrorAction SilentlyContinue
if (-not $body) { $body = "" }
Remove-Item $bodyTmp.FullName, $headTmp.FullName, $outTmp.FullName -Force -ErrorAction SilentlyContinue
return @{ StatusCode = $code; RequestID = $rid; Body = $body }
}
try {
Section "T1: non-stream chat completions"
$t1Body = '{"model":"' + $Model + '","messages":[{"role":"user","content":"Say hello in one short sentence."}],"stream":false}'
$r = CallChat "$ProxyBase/v1/chat/completions" $t1Body
Write-Host " status=$($r.StatusCode) rid=$($r.RequestID)"
$preview = if ($r.Body.Length -gt 200) { $r.Body.Substring(0, 200) } else { $r.Body }
Write-Host " body(first 200): $preview"
$results.T1 = $r
Section "T2: stream chat completions"
$t2Body = '{"model":"' + $Model + '","messages":[{"role":"user","content":"Say exactly: one two three four five"}],"stream":true}'
$r = CallChat "$ProxyBase/v1/chat/completions" $t2Body
Write-Host " status=$($r.StatusCode) rid=$($r.RequestID)"
$chunkCount = ([regex]::Matches($r.Body, "^data:", "Multiline")).Count
$hasDone = $r.Body.Contains("[DONE]")
Write-Host " SSE chunk lines=$chunkCount contains [DONE]=$hasDone body_len=$($r.Body.Length)"
$results.T2 = $r
Section "T3: large body (request_truncated should be true)"
$bigContent = "x" * (12 * 1024 * 1024)
$t3Body = '{"model":"' + $Model + '","messages":[{"role":"user","content":"' + $bigContent + '"}],"stream":false,"max_tokens":5}'
Write-Host " request body size: $($t3Body.Length) bytes"
$r = CallChat "$ProxyBase/v1/chat/completions" $t3Body
Write-Host " status=$($r.StatusCode) rid=$($r.RequestID)"
$preview = if ($r.Body.Length -gt 200) { $r.Body.Substring(0, 200) } else { $r.Body }
Write-Host " body(first 200): $preview"
$results.T3 = $r
Section "T4: nonexistent model (upstream error response should be logged)"
$t4Body = '{"model":"definitely-not-a-real-model-xyz","messages":[{"role":"user","content":"hi"}]}'
$r = CallChat "$ProxyBase/v1/chat/completions" $t4Body
Write-Host " status=$($r.StatusCode) rid=$($r.RequestID)"
$preview = if ($r.Body.Length -gt 200) { $r.Body.Substring(0, 200) } else { $r.Body }
Write-Host " body(first 200): $preview"
$results.T4 = $r
Section "T5: healthz (should not be logged)"
$code = & curl.exe -s -o NUL -w "%{http_code}" "$ProxyBase/healthz"
Write-Host " /healthz status=$code"
Section "Wait batch flush (4s)"
Start-Sleep -Seconds 4
Section "T6: ClickHouse verification"
$rids = @()
foreach ($k in @("T1","T2","T3","T4")) {
if ($results[$k].RequestID) { $rids += $results[$k].RequestID }
}
Write-Host " request_ids: $($rids -join ', ')"
$env:CHECK_RIDS = ($rids | ConvertTo-Json -Compress)
if ($rids.Count -eq 1) { $env:CHECK_RIDS = "[`"$($rids[0])`"]" }
& go run .\tests\scripts\dbcheck\main.go
Remove-Item Env:CHECK_RIDS -ErrorAction SilentlyContinue
} finally {
Section "Stop proxy"
Stop-Process -Id $proxy.Id -Force
Write-Host "tail proxy.log:"
Get-Content proxy.log -Tail 30 | Write-Host
if (Test-Path proxy.err.log) {
$err = Get-Content proxy.err.log -ErrorAction SilentlyContinue
if ($err) {
Write-Host "proxy.err.log:"
$err | Write-Host
}
}
}