137 lines
6.7 KiB
Go
137 lines
6.7 KiB
Go
package protocol
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const reasoningJSON = `{"type":"reasoning","id":"r","content":[{"type":"reasoning_text","text":"SECRET"}],"summary":[{"type":"summary_text","text":"SUMMARY"}],"encrypted_content":"CIPHER"}`
|
|
const visibleJSON = `{"type":"message","id":"m","role":"assistant","phase":"final_answer","content":[{"type":"output_text","text":"OK","logprobs":[]}]}`
|
|
|
|
func reasoningSSE(events ...string) string {
|
|
sep := string([]byte{10, 10})
|
|
return "data: " + strings.Join(events, sep+"data: ") + sep
|
|
}
|
|
func TestReasoningConversions(t *testing.T) {
|
|
for _, tool := range []bool{false, true} {
|
|
for _, incomplete := range []bool{false, true} {
|
|
visible, initial, delta := visibleJSON, `{"type":"message","content":[]}`, `{"type":"response.output_text.delta","output_index":1,"delta":"OK"}`
|
|
if tool {
|
|
visible = `{"type":"function_call","call_id":"c","name":"lookup","arguments":"{}"}`
|
|
initial = `{"type":"function_call","call_id":"c","name":"lookup","arguments":""}`
|
|
delta = `{"type":"response.function_call_arguments.delta","output_index":1,"delta":"{}"}`
|
|
}
|
|
status, details := "completed", ""
|
|
if incomplete {
|
|
status = "incomplete"
|
|
details = `,"incomplete_details":{"reason":"max_output_tokens"}`
|
|
}
|
|
body := `{"id":"res","model":"m","status":"` + status + `","output":[` + reasoningJSON + `,` + visible + `],"usage":{"input_tokens":9,"output_tokens":7,"output_tokens_details":{"reasoning_tokens":5}}` + details + `}`
|
|
src := reasoningSSE(`{"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"r"}}`,
|
|
`{"type":"response.content_part.added","output_index":0,"item_id":"r","content_index":0,"part":{"type":"reasoning_text","text":""}}`,
|
|
`{"type":"response.reasoning_text.delta","output_index":0,"item_id":"r","content_index":0,"delta":"SECRET"}`,
|
|
`{"type":"response.reasoning_text.done","output_index":0,"item_id":"r","content_index":0,"text":"SECRET"}`,
|
|
`{"type":"response.content_part.done","output_index":0,"item_id":"r","content_index":0,"part":{"type":"reasoning_text","text":"SECRET"}}`,
|
|
`{"type":"response.reasoning_summary_part.added","output_index":0,"item_id":"r","summary_index":0,"part":{"type":"summary_text","text":""}}`,
|
|
`{"type":"response.reasoning_summary_text.delta","output_index":0,"item_id":"r","summary_index":0,"delta":"SUMMARY"}`,
|
|
`{"type":"response.reasoning_summary_text.done","output_index":0,"item_id":"r","summary_index":0,"text":"SUMMARY"}`,
|
|
`{"type":"response.reasoning_summary_part.done","output_index":0,"item_id":"r","summary_index":0,"part":{"type":"summary_text","text":"SUMMARY"}}`,
|
|
`{"type":"response.output_item.done","output_index":0,"item":`+reasoningJSON+`}`,
|
|
`{"type":"response.output_item.added","output_index":1,"item":`+initial+`}`, delta,
|
|
`{"type":"response.output_item.done","output_index":1,"item":`+visible+`}`,
|
|
`{"type":"response.`+status+`","response":`+body+`}`)
|
|
for _, target := range []Kind{Chat, Anthropic, Responses} {
|
|
out, err := ConvertResponse(Responses, target, []byte(body))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var b bytes.Buffer
|
|
u, err := Stream(context.Background(), &b, strings.NewReader(src), Responses, target, "m")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !u.Known || u.Input != 9 || u.Output != 7 || !bytes.Contains(u.Raw, []byte("reasoning_tokens")) {
|
|
t.Fatalf("usage: %+v", u)
|
|
}
|
|
if target == Responses {
|
|
if string(out) != body || b.String() != src {
|
|
t.Fatal("changed same-protocol bytes")
|
|
}
|
|
continue
|
|
}
|
|
for _, result := range []string{string(out), b.String()} {
|
|
for _, secret := range []string{"SECRET", "SUMMARY", "CIPHER", "signature", "thinking", "phase"} {
|
|
if strings.Contains(result, secret) {
|
|
t.Fatal("leaked " + secret)
|
|
}
|
|
}
|
|
want := "OK"
|
|
if tool {
|
|
want = "lookup"
|
|
}
|
|
if !strings.Contains(result, want) {
|
|
t.Fatal("lost visible content")
|
|
}
|
|
finish := "stop"
|
|
if target == Anthropic {
|
|
finish = "end_turn"
|
|
}
|
|
if tool {
|
|
finish = "tool_calls"
|
|
if target == Anthropic {
|
|
finish = "tool_use"
|
|
}
|
|
}
|
|
if incomplete {
|
|
finish = "length"
|
|
if target == Anthropic {
|
|
finish = "max_tokens"
|
|
}
|
|
}
|
|
if !strings.Contains(result, finish) {
|
|
t.Fatal("lost finish " + finish)
|
|
}
|
|
}
|
|
if target == Chat && !strings.Contains(b.String(), "[DONE]") {
|
|
t.Fatal("missing terminal")
|
|
}
|
|
if target == Anthropic && (!strings.Contains(b.String(), "message_stop") || !strings.Contains(b.String(), `"index":0`)) {
|
|
t.Fatal("terminal/index")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReasoningRejectsInvalidStreams(t *testing.T) {
|
|
start := `{"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"r"}}`
|
|
part := `{"type":"response.reasoning_summary_part.added","output_index":0,"item_id":"r","summary_index":0,"part":{"type":"summary_text","text":""}}`
|
|
done := `{"type":"response.output_item.done","output_index":0,"item":` + reasoningJSON + `}`
|
|
for name, events := range map[string][]string{
|
|
"duplicate": {start, start}, "part before item": {part}, "open part": {start, part, done}, "closed item": {start, done, part},
|
|
"wrong id": {start, strings.Replace(part, `"item_id":"r"`, `"item_id":"x"`, 1)},
|
|
"unknown": {start, `{"type":"response.reasoning_unknown.delta","output_index":0}`},
|
|
"text collision": {start, `{"type":"response.output_text.delta","output_index":0,"item_id":"r","delta":"lost"}`},
|
|
"tool collision": {start, `{"type":"response.function_call_arguments.delta","output_index":0,"item_id":"r","delta":"{}"}`},
|
|
"unknown part": {start, strings.Replace(part, "summary_text", "output_text", 1)},
|
|
"negative": {strings.Replace(start, `"output_index":0`, `"output_index":-1`, 1)},
|
|
"fractional": {strings.Replace(start, `"output_index":0`, `"output_index":0.5`, 1)},
|
|
"final missing": {start, done, `{"type":"response.completed","response":{"status":"completed","output":[]}}`},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
var b bytes.Buffer
|
|
_, err := Stream(context.Background(), &b, strings.NewReader(reasoningSSE(events...)), Responses, Chat, "")
|
|
if err == nil || strings.Contains(err.Error(), "truncat") {
|
|
t.Fatalf("expected validation error: %v", err)
|
|
}
|
|
})
|
|
}
|
|
for _, bad := range []string{strings.Replace(reasoningJSON, "reasoning_text", "output_text", 1), strings.Replace(reasoningJSON, "summary_text", "unknown", 1), `{"type":"reasoning","summary":{}}`, strings.Replace(visibleJSON, `"logprobs":[]`, `"logprobs":[{}]`, 1), strings.Replace(visibleJSON, `"phase":`, `"unknown":`, 1)} {
|
|
if _, err := ConvertResponse(Responses, Chat, []byte(`{"status":"completed","output":[`+bad+`]}`)); err == nil {
|
|
t.Fatal("accepted invalid output")
|
|
}
|
|
}
|
|
}
|