dsh-web/internal/api/runtime_test.go

package api

import (
	"bufio"
	"context"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"git.misaka.ren/m1saka/dsh-web/internal/docker"
	"git.misaka.ren/m1saka/dsh-web/internal/store"
)

func runtimeTestAPI(t *testing.T, handler http.HandlerFunc) (*API, string) {
	t.Helper()
	s, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(s.Close)
	if err := s.CreateUser(store.User{ID: "owner", Username: "owner"}, "unused"); err != nil {
		t.Fatal(err)
	}
	sid, _, err := s.Session("owner")
	if err != nil {
		t.Fatal(err)
	}
	server := httptest.NewServer(handler)
	t.Cleanup(server.Close)
	a := &API{s: s, root: t.TempDir(), origin: "http://panel.test", docker: docker.New(server.URL)}
	if err := a.initAuth(); err != nil {
		t.Fatal(err)
	}
	registerTestAccess(t, a, sid)
	return a, sid
}
func runtimeRequest(method, path, body, sid string) *http.Request {
	r := httptest.NewRequest(method, path, strings.NewReader(body))
	r.Host = "panel.test"
	r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
	addTestAccess(r, sid)
	return r
}

func TestRuntimeProxyRejectsErrorState(t *testing.T) {
	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		t.Error("error-state proxy must not contact Docker")
	})
	if err := a.s.AddInstance(store.Instance{ID: "broken", UID: "owner", Status: "error", Created: time.Now()}); err != nil {
		t.Fatal(err)
	}
	w := httptest.NewRecorder()
	a.proxy(w, runtimeRequest("GET", "/dsh/broken/", "", sid))
	if w.Code != http.StatusConflict {
		t.Fatalf("status = %d", w.Code)
	}
}

func TestRuntimeCreateLimitAndPolicy(t *testing.T) {
	if os.Geteuid() != 0 {
		t.Skip("chown requires root")
	}
	t.Setenv("HOST_DATA_ROOT", "/host/data")
	t.Setenv("PANEL_CONTAINER_ID", "panel")
	imageID := "sha256:" + strings.Repeat("a", 64)
	var mu sync.Mutex
	var bodies []map[string]any
	var calls []string
	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		mu.Lock()
		defer mu.Unlock()
		calls = append(calls, r.Method+" "+r.URL.Path)
		switch {
		case strings.Contains(r.URL.Path, "/images/"):
			writeJSON(w, map[string]any{"Id": imageID, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3"}}})
		case strings.HasSuffix(r.URL.Path, "/containers/create"):
			var body map[string]any
			if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
				t.Error(err)
			}
			bodies = append(bodies, body)
			writeJSON(w, map[string]string{"Id": fmt.Sprintf("container%d", len(bodies))})
		default:
			w.WriteHeader(204)
		}
	})
	body := fmt.Sprintf("{\"name\":\"one\",\"image_id\":%q,\"version\":\"untrusted\"}", imageID)
	seedVerifiedImage(t, a, imageID, "1.2.3")
	var wg sync.WaitGroup
	statuses := make(chan int, 4)
	for n := 0; n < 4; n++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			w := httptest.NewRecorder()
			a.instances(w, runtimeRequest("POST", "/api/instances", body, sid))
			statuses <- w.Code
		}()
	}
	wg.Wait()
	close(statuses)
	successes, limits := 0, 0
	for status := range statuses {
		if status == 200 {
			successes++
		} else if status == 409 {
			limits++
		} else {
			t.Errorf("status %d", status)
		}
	}
	if successes != 2 || limits != 2 {
		t.Fatalf("success=%d limit=%d", successes, limits)
	}
	instances, err := a.s.Instances("owner")
	if err != nil || len(instances) != 2 {
		t.Fatalf("instances=%v err=%v", instances, err)
	}
	for _, i := range instances {
		if i.Version != "1.2.3" || i.ContainerID == "" || i.Status != "running" {
			t.Errorf("%+v", i)
		}
		if _, err := os.Stat(filepath.Join(a.root, "workspaces", "owner", "instances", i.ID, "home")); err != nil {
			t.Error(err)
		}
	}
	for _, b := range bodies {
		h := b["HostConfig"].(map[string]any)
		mounts := h["Mounts"].([]any)
		if len(mounts) != 1 {
			t.Fatalf("unexpected mounts: %v", mounts)
		}
		mount := mounts[0].(map[string]any)
		if mount["Target"] != "/workspace" || !strings.HasSuffix(mount["Source"].(string), "/home") {
			t.Fatal(mount)
		}
		if !strings.Contains(fmt.Sprint(b["Env"]), "DSH_HOME=/workspace") {
			t.Fatal("wrong data root", b["Env"])
		}
		if b["User"] != "1000:1000" || h["ReadonlyRootfs"] != true || h["Memory"] != float64(2<<30) || h["NanoCpus"] != float64(2e9) || h["PidsLimit"] != float64(256) {
			t.Errorf("policy %v", b)
		}
		if h["PortBindings"] != nil || h["PublishAllPorts"] != nil {
			t.Error("ports published")
		}
		if fmt.Sprint(h["CapDrop"]) != "[ALL]" || !strings.Contains(fmt.Sprint(h["Tmpfs"]), "exec") {
			t.Error(h)
		}
		if strings.Contains(fmt.Sprint(b["Cmd"]), "public-url") || !strings.Contains(fmt.Sprint(b["Cmd"]), "--trusted-host panel.test") {
			t.Error(b["Cmd"])
		}
		if !strings.Contains(fmt.Sprint(h["Mounts"]), "/host/data/workspaces/owner") {
			t.Error(h["Mounts"])
		}
	}
	if !strings.Contains(strings.Join(calls, "\n"), "/connect") {
		t.Fatal("panel not connected")
	}
}

func TestRuntimeLifecycleAndOwnership(t *testing.T) {
	t.Setenv("PANEL_CONTAINER_ID", "")
	var calls []string
	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		calls = append(calls, r.Method+" "+r.URL.Path)
		if strings.HasSuffix(r.URL.Path, "/json") {
			io.WriteString(w, "{\"State\":{\"Status\":\"running\"}}")
		} else if strings.HasSuffix(r.URL.Path, "/logs") {
			io.WriteString(w, "dsh web: http://localhost/?token=secret-value\nAuthorization: Bearer another-secret\n")
		} else {
			w.WriteHeader(204)
		}
	})
	i := store.Instance{ID: "instance", UID: "owner", ContainerID: "container"}
	if err := a.s.AddInstance(i); err != nil {
		t.Fatal(err)
	}
	dir := filepath.Join(a.root, "workspaces", "owner", "instances", i.ID, "home")
	if err := os.MkdirAll(dir, 0755); err != nil {
		t.Fatal(err)
	}
	for _, action := range []string{"start", "stop", "restart"} {
		w := httptest.NewRecorder()
		a.instance(w, runtimeRequest("POST", "/api/instances/instance/"+action, "", sid))
		if w.Code != 200 {
			t.Fatal(w.Code, w.Body.String())
		}
		w = httptest.NewRecorder()
		a.instance(w, runtimeRequest("GET", "/api/instances/instance/"+action, "", sid))
		if w.Code != 405 {
			t.Fatal(w.Code)
		}
	}
	w := httptest.NewRecorder()
	a.instance(w, runtimeRequest("GET", "/api/instances/instance/logs", "", sid))
	if strings.Contains(w.Body.String(), "secret-value") || strings.Contains(w.Body.String(), "another-secret") {
		t.Fatal("credentials in logs")
	}
	foreign := store.Instance{ID: "foreign", UID: "someone", ContainerID: "other"}
	a.s.AddInstance(foreign)
	before := len(calls)
	for _, path := range []string{"/api/instances/foreign/start", "/api/instances/foreign/logs"} {
		w = httptest.NewRecorder()
		a.instance(w, runtimeRequest("POST", path, "", sid))
		if w.Code != 404 {
			t.Fatal(w.Code)
		}
	}
	w = httptest.NewRecorder()
	a.proxy(w, runtimeRequest("GET", "/dsh/foreign/", "", sid))
	if w.Code != 404 || len(calls) != before {
		t.Fatal("owner check bypass")
	}
	w = httptest.NewRecorder()
	a.instance(w, runtimeRequest("DELETE", "/api/instances/instance", "", sid))
	if w.Code != 200 {
		t.Fatal(w.Code, w.Body.String())
	}
	if _, err := os.Stat(dir); err != nil {
		t.Fatal("data removed", err)
	}
	if _, err := a.s.GetInstance(i.ID); err == nil {
		t.Fatal("row retained")
	}
	if !strings.Contains(strings.Join(calls, "\n"), "DELETE /v1.47/containers/container") {
		t.Fatal(calls)
	}
}

func TestRuntimeTokenOriginAndExchange(t *testing.T) {
	logs := "dsh web: http://localhost/?token=old\ndsh web: http://localhost/?token=latest%2Btoken\x1b[0m\n"
	if got := latestRuntimeToken(logs); got != "latest+token" {
		t.Fatal(got)
	}
	if got := latestRuntimeToken(logs + `{"uri":"/?token=old"}`); got != "latest+token" {
		t.Fatalf("proxy log replaced startup token: %q", got)
	}
	if strings.Contains(redactRuntimeLogs(logs), "latest") || strings.Contains(redactRuntimeLogs(logs), "old") {
		t.Fatal("token not redacted")
	}
	a := &API{origin: "http://panel.test"}
	for _, tc := range []struct {
		method, origin, upgrade string
		want                    bool
	}{{"GET", "", "", true}, {"POST", "", "", false}, {"POST", "http://panel.test", "", true}, {"POST", "http://evil.test", "", false}, {"GET", "", "websocket", false}, {"GET", "http://panel.test", "websocket", true}} {
		r := runtimeRequest(tc.method, "/dsh/id/", "", "")
		r.Header.Set("Origin", tc.origin)
		r.Header.Set("Upgrade", tc.upgrade)
		if a.runtimeOriginAllowed(r) != tc.want {
			t.Errorf("%+v", tc)
		}
	}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "GET" || r.URL.Query().Get("token") != "latest+token" || r.Host != "panel.test" {
			t.Error("invalid exchange")
		}
		http.SetCookie(w, &http.Cookie{Name: "dsh_auth", Value: "server-secret", Path: "/"})
		w.Header().Set("Location", "/")
		w.WriteHeader(303)
	}))
	defer server.Close()
	target, _ := url.Parse(server.URL)
	cookie, err := exchangeRuntimeCookie(context.Background(), target, "panel.test", "latest+token")
	if err != nil || cookie != "dsh_auth=server-secret" {
		t.Fatalf("exchange failed: %v", err)
	}
}

func TestRuntimeProxyHeadersRedirectAndSSE(t *testing.T) {
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Cookie") != "dsh_auth=server-secret" || r.Header.Get("Authorization") != "" || r.Header.Get("X-CSRF-Token") != "" || r.URL.Query().Get("token") != "" {
			t.Error("credential boundary violated")
		}
		if r.Host != "panel.test" {
			t.Error(r.Host)
		}
		http.SetCookie(w, &http.Cookie{Name: "dsh_auth", Value: "not-for-browser"})
		if r.URL.Path == "/redirect" {
			w.Header().Set("Location", "/next?token=not-for-browser")
			w.WriteHeader(303)
			return
		}
		if r.URL.Path != "/events" {
			t.Error(r.URL.Path)
		}
		w.Header().Set("Content-Type", "text/event-stream")
		io.WriteString(w, "data: first\n\n")
		w.(http.Flusher).Flush()
		<-r.Context().Done()
	}))
	defer upstream.Close()
	target, _ := url.Parse(upstream.URL)
	proxy := httptest.NewServer(runtimeReverseProxy(target, "/dsh/id", "panel.test", "dsh_auth=server-secret"))
	defer proxy.Close()
	client := &http.Client{Timeout: 3 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
	for _, path := range []string{"redirect", "events"} {
		req, _ := http.NewRequest("GET", proxy.URL+"/dsh/id/"+path+"?token=browser-token", nil)
		req.Header.Set("Cookie", "dsh_session=panel-secret")
		req.Header.Set("Authorization", "Bearer panel-secret")
		req.Header.Set("X-CSRF-Token", "panel-secret")
		resp, err := client.Do(req)
		if err != nil {
			t.Fatal(err)
		}
		if resp.Header.Get("Set-Cookie") != "" {
			t.Fatal("cookie escaped")
		}
		if path == "redirect" {
			if resp.Header.Get("Location") != "/dsh/id/next" {
				t.Fatal(resp.Header.Get("Location"))
			}
		} else {
			line, err := bufio.NewReader(resp.Body).ReadString('\n')
			if err != nil || line != "data: first\n" {
				t.Fatal("SSE did not flush", err)
			}
		}
		resp.Body.Close()
	}
}

func TestRuntimeProxyPreservesComboQuery(t *testing.T) {
	const query = "?@deepseek-ai/dsh-client-modules/client.js,@deepseek-ai/dsh-client-ui/client.js&rev=abc"
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.RawQuery != query {
			t.Errorf("combo query changed: %q", r.URL.RawQuery)
		}
		w.Header().Set("Location", "/plugins/?"+query+"&%74oken=secret")
		w.WriteHeader(http.StatusTemporaryRedirect)
	}))
	defer upstream.Close()
	target, _ := url.Parse(upstream.URL)
	proxy := runtimeReverseProxy(target, "/dsh/id", "panel.test", "")
	for _, suffix := range []string{"", "&token=secret", "&%74oken=secret&token=second"} {
		w := httptest.NewRecorder()
		proxy.ServeHTTP(w, httptest.NewRequest("GET", "http://panel.test/dsh/id/plugins/?"+query+suffix, nil))
		if w.Code != http.StatusTemporaryRedirect || w.Header().Get("Location") != "/dsh/id/plugins/?"+query {
			t.Fatalf("unexpected combo redirect: %d %q", w.Code, w.Header().Get("Location"))
		}
	}
}

func TestRuntimeProxyWebSocketUpgrade(t *testing.T) {
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/socket" || r.Header.Get("Cookie") != "dsh_auth=secret" || r.Header.Get("Authorization") != "" {
			t.Error("invalid upgraded request")
		}
		conn, rw, err := w.(http.Hijacker).Hijack()
		if err != nil {
			t.Error(err)
			return
		}
		defer conn.Close()
		rw.WriteString("HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n")
		rw.Flush()
		data := make([]byte, 4)
		if _, err := io.ReadFull(rw, data); err != nil {
			t.Error(err)
			return
		}
		rw.Write(data)
		rw.Flush()
	}))
	defer upstream.Close()
	target, _ := url.Parse(upstream.URL)
	proxy := httptest.NewServer(runtimeReverseProxy(target, "/dsh/id", "panel.test", "dsh_auth=secret"))
	defer proxy.Close()
	conn, err := net.DialTimeout("tcp", strings.TrimPrefix(proxy.URL, "http://"), time.Second)
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()
	conn.SetDeadline(time.Now().Add(3 * time.Second))
	fmt.Fprint(conn, "GET /dsh/id/socket HTTP/1.1\r\nHost: panel.test\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nAuthorization: Bearer panel-secret\r\n\r\n")
	reader := bufio.NewReader(conn)
	resp, err := http.ReadResponse(reader, nil)
	if err != nil || resp.StatusCode != 101 {
		t.Fatalf("upgrade: %v %v", resp, err)
	}
	fmt.Fprint(conn, "ping")
	data := make([]byte, 4)
	if _, err := io.ReadFull(reader, data); err != nil || string(data) != "ping" {
		t.Fatal("upgrade tunnel failed", err)
	}
}

func TestRuntimeCreateRollbackAndImageRejection(t *testing.T) {
	if os.Geteuid() != 0 {
		t.Skip("chown requires root")
	}
	t.Setenv("PANEL_CONTAINER_ID", "")
	t.Setenv("HOST_DATA_ROOT", "")
	imageID := "sha256:" + strings.Repeat("b", 64)
	for _, role := range []string{"foreign", "runtime"} {
		t.Run(role, func(t *testing.T) {
			var calls []string
			a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				calls = append(calls, r.Method+" "+r.URL.Path)
				switch {
				case strings.Contains(r.URL.Path, "/images/"):
					writeJSON(w, map[string]any{"Id": imageID, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": role, "dsh-mgr.version": "1"}}})
				case strings.HasSuffix(r.URL.Path, "/containers/create"):
					writeJSON(w, map[string]string{"Id": "container"})
				case strings.HasSuffix(r.URL.Path, "/start"):
					w.WriteHeader(500)
					io.WriteString(w, "token=must-not-leak")
				default:
					w.WriteHeader(204)
				}
			})
			seedVerifiedImage(t, a, imageID, "1")
			w := httptest.NewRecorder()
			a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf("{\"name\":\"test\",\"image_id\":%q}", imageID), sid))
			want := 400
			if role == "runtime" {
				want = 502
			}
			if w.Code != want {
				t.Fatal(w.Code, w.Body.String())
			}
			if strings.Contains(w.Body.String(), "must-not-leak") {
				t.Fatal("docker error leaked")
			}
			instances, err := a.s.Instances("owner")
			if err != nil || len(instances) != 0 {
				t.Fatal("reservation not rolled back", err)
			}
			if role == "foreign" && len(calls) != 1 {
				t.Fatal("untrusted image used")
			}
			if role == "runtime" && !strings.Contains(strings.Join(calls, "\n"), "DELETE /v1.47/containers/container") {
				t.Fatal("container not cleaned")
			}
		})
	}
}

func TestRuntimeProxyMissingNetwork(t *testing.T) {
	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "{\"State\":{\"Running\":true},\"NetworkSettings\":{\"Networks\":null}}")
	})
	if err := a.s.AddInstance(store.Instance{ID: "instance", UID: "owner", ContainerID: "container"}); err != nil {
		t.Fatal(err)
	}
	w := httptest.NewRecorder()
	a.proxy(w, runtimeRequest("GET", "/dsh/instance/", "", sid))
	if w.Code != 502 {
		t.Fatal(w.Code)
	}
}

func TestRuntimeCookieCacheRestart(t *testing.T) {
	exchanges, logs := 0, 0
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		exchanges++
		token := r.URL.Query().Get("token")
		if token != "first" && token != "second" {
			t.Error("wrong token")
		}
		http.SetCookie(w, &http.Cookie{Name: "dsh_auth", Value: token})
		w.WriteHeader(303)
	}))
	defer upstream.Close()
	target, _ := url.Parse(upstream.URL)
	a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		logs++
		token := "first"
		if r.URL.Query().Get("since") == "2.000000000" {
			token = "second"
		} else if r.URL.Query().Get("since") != "1.000000000" {
			t.Error("missing startup filter")
		}
		io.WriteString(w, "dsh web: http://localhost/?token="+token+"\n")
	})
	i := store.Instance{ID: "cache-test", ContainerID: "container"}
	defer runtimeAuthCache.Delete(runtimeAuthKey{a, i.ID})
	for n, started := range []string{"1970-01-01T00:00:01Z", "1970-01-01T00:00:01Z", "1970-01-01T00:00:02Z"} {
		cookie, err := a.runtimeCookie(context.Background(), i, started, target, "panel.test")
		want := "dsh_auth=first"
		if n == 2 {
			want = "dsh_auth=second"
		}
		if err != nil || cookie != want {
			t.Fatal("cookie exchange", err)
		}
	}
	if exchanges != 2 || logs != 2 {
		t.Fatalf("exchanges=%d logs=%d", exchanges, logs)
	}
}

func TestRuntimeCookieCurrentStartupLargeMultiplexedLogs(t *testing.T) {
	upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Query().Get("token") != "current" {
			t.Error("stale token exchanged")
		}
		http.SetCookie(w, &http.Cookie{Name: "dsh_auth", Value: "current"})
		w.WriteHeader(303)
	}))
	defer upstream.Close()
	target, _ := url.Parse(upstream.URL)
	a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Query().Get("since") != "2.123456789" {
			t.Error("logs not filtered by StartedAt")
			io.WriteString(w, strings.Repeat("old logs\n", 200000))
			return
		}
		for _, payload := range []string{strings.Repeat("startup noise\n", 100000), "http://localhost/?token=current\n"} {
			header := [8]byte{1}
			binary.BigEndian.PutUint32(header[4:], uint32(len(payload)))
			w.Write(header[:])
			io.WriteString(w, payload)
		}
	})
	i := store.Instance{ID: "large-log", ContainerID: "container"}
	defer runtimeAuthCache.Delete(runtimeAuthKey{a, i.ID})
	cookie, err := a.runtimeCookie(context.Background(), i, "1970-01-01T00:00:02.123456789Z", target, "panel.test")
	if err != nil || cookie != "dsh_auth=current" {
		t.Fatalf("cookie=%s err=%v", cookie, err)
	}
}

func TestRuntimeCookieRejectsTruncatedLogs(t *testing.T) {
	a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "http://localhost/?token=partial\n"+strings.Repeat("x", 8<<20))
	})
	i := store.Instance{ID: "oversize", ContainerID: "container"}
	defer runtimeAuthCache.Delete(runtimeAuthKey{a, i.ID})
	if _, err := a.runtimeCookie(context.Background(), i, "1970-01-01T00:00:01Z", nil, "panel.test"); err == nil {
		t.Fatal("truncated log accepted")
	}
}

func TestRuntimeCreateRejectsTrailingJSON(t *testing.T) {
	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		t.Error("invalid request reached Docker")
		w.WriteHeader(500)
	})
	body := fmt.Sprintf(`{"name":"test","image_id":%q} {}`, "sha256:"+strings.Repeat("a", 64))
	w := httptest.NewRecorder()
	a.instances(w, runtimeRequest("POST", "/api/instances", body, sid))
	if w.Code != 400 {
		t.Fatal(w.Code, w.Body.String())
	}
}

func TestRuntimeBlocksPendingUpgrade(t *testing.T) {
	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		t.Error("pending upgrade reached Docker")
		w.WriteHeader(500)
	})
	if err := a.s.AddInstance(store.Instance{ID: "pending", UID: "owner", ContainerID: "old"}); err != nil {
		t.Fatal(err)
	}
	if _, err := a.s.DB.Exec("CREATE TABLE upgrade_journal(instance_id TEXT,phase TEXT); INSERT INTO upgrade_journal VALUES('pending','snapshotting')"); err != nil {
		t.Fatal(err)
	}
	for _, tc := range []struct{ method, path string }{{"POST", "/api/instances/pending/start"}, {"POST", "/api/instances/pending/restart"}, {"DELETE", "/api/instances/pending"}, {"GET", "/dsh/pending/"}} {
		w := httptest.NewRecorder()
		r := runtimeRequest(tc.method, tc.path, "", sid)
		if strings.HasPrefix(tc.path, "/dsh/") {
			a.proxy(w, r)
		} else {
			a.instance(w, r)
		}
		if w.Code != 409 {
			t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String())
		}
	}
}