package api

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"git.misaka.ren/m1saka/dsh-web/internal/docker"
	"git.misaka.ren/m1saka/dsh-web/internal/store"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"sync/atomic"
	"testing"
	"time"
)

func TestBuildCheckRenamedContainerRetainsIdentity(t *testing.T) {
	a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "GET" || r.URL.Path != "/v1.47/containers/known/json" {
			t.Errorf("unsafe lookup %s %s", r.Method, r.URL.Path)
			w.WriteHeader(404)
			return
		}
		fmt.Fprint(w, `{"Id":"known","Name":"/renamed","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
	})
	if err := a.ensureBuildChecks(); err != nil {
		t.Fatal(err)
	}
	if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
		t.Fatal(err)
	}
	if err := a.recoverBuildChecks(context.Background()); err == nil {
		t.Fatal("renamed identity accepted")
	}
	var n int
	if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
		t.Fatalf("record lost %d %v", n, err)
	}
}
func TestRuntimeNetworkDeletionLostResponseAndReplacement(t *testing.T) {
	for _, foreign := range []bool{false, true} {
		t.Run(fmt.Sprint(foreign), func(t *testing.T) {
			t.Setenv("PANEL_CONTAINER_ID", "")
			deleted := false
			a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Path == "/v1.47/images/image/json" {
					writeImageInspect(w, "image", "1.2.3")
					return
				}
				if r.Method == "GET" {
					if deleted {
						if r.URL.Path != "/v1.47/networks/known" {
							t.Error("did not use persisted ID")
						}
						w.WriteHeader(404)
						return
					}
					owner := "net"
					if foreign {
						owner = "foreign"
					}
					fmt.Fprintf(w, `{"Id":"known","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, owner)
					return
				}
				if foreign {
					t.Error("deleted foreign network")
				}
				deleted = true
				w.WriteHeader(500)
			})
			if err := a.setRuntimeCreatePending("net", "network_pending", false); err != nil {
				t.Fatal(err)
			}
			if err := a.removeRuntimeNetwork(context.Background(), "net"); err == nil {
				t.Fatal("first error missing")
			}
			if foreign {
				return
			}
			if err := a.removeRuntimeNetwork(context.Background(), "net"); err != nil {
				t.Fatalf("confirmed deletion did not converge: %v", err)
			}
		})
	}
}

func TestNetworkCreateLostResponse(t *testing.T) {
	for _, mode := range []string{"present", "late", "foreign"} {
		t.Run(mode, func(t *testing.T) {
			t.Setenv("PANEL_CONTAINER_ID", "")
			late := false
			deletes := 0
			a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Path == "/v1.47/images/image/json" {
					writeImageInspect(w, "image", "1.2.3")
					return
				}
				switch r.Method + " " + r.URL.Path {
				case "POST /v1.47/networks/create":
					w.WriteHeader(500)
				case "GET /v1.47/containers/dsh-inst-net/json":
					w.WriteHeader(404)
				case "GET /v1.47/networks/dsh-inst-net":
					if mode == "late" && !late {
						w.WriteHeader(404)
						return
					}
					id := "net"
					if mode == "foreign" {
						id = "other"
					}
					fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, id)
				case "DELETE /v1.47/networks/network-id":
					deletes++
					w.WriteHeader(204)
				default:
					t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
					w.WriteHeader(500)
				}
			})
			seedVerifiedImage(t, a, "image", "1.2.3")
			i := store.Instance{ID: "net", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
			if err := a.s.AddInstance(i); err != nil {
				t.Fatal(err)
			}
			if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
				t.Fatal("lost response accepted")
			}
			if mode == "present" {
				if deletes != 1 {
					t.Fatal("network leaked")
				}
				return
			}
			saved, err := a.s.GetInstance("net")
			if err != nil || saved.Status != "error" {
				t.Fatalf("lost reservation %+v %v", saved, err)
			}
			late = true
			w := httptest.NewRecorder()
			a.instance(w, runtimeRequest("DELETE", "/api/instances/net", "", sid))
			if mode == "foreign" {
				if deletes != 0 || w.Code != 502 {
					t.Fatal("foreign network touched")
				}
				return
			}
			if w.Code != 200 || deletes != 1 {
				t.Fatalf("late network retry %d %d", w.Code, deletes)
			}
		})
	}
}

func TestBuildCheckReopenWorkerWithoutQueuedBuilds(t *testing.T) {
	path := filepath.Join(t.TempDir(), "recovery.db")
	s, err := store.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	a := &API{s: s}
	if err = a.ensureBuildChecks(); err != nil {
		t.Fatal(err)
	}
	if _, err = s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id) VALUES('task','image')"); err != nil {
		t.Fatal(err)
	}
	s.Close()
	s, err = store.Open(path)
	if err != nil {
		t.Fatal(err)
	}
	defer s.Close()
	var attempts atomic.Int32
	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" {
			fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
			return
		}
		if r.Method == "DELETE" {
			if attempts.Add(1) == 1 {
				w.WriteHeader(500)
			} else {
				w.WriteHeader(204)
			}
			return
		}
		t.Errorf("unexpected %s", r.Method)
	}))
	defer srv.Close()
	a = &API{s: s, docker: docker.New(srv.URL)}
	if err = a.StartImageBuilds(); err != nil {
		t.Fatal(err)
	}
	deadline := time.Now().Add(4 * time.Second)
	for {
		var n int
		if err = s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil {
			t.Fatal(err)
		}
		if n == 0 {
			break
		}
		if time.Now().After(deadline) {
			t.Fatal("reopen worker did not retry")
		}
		time.Sleep(10 * time.Millisecond)
	}
	if attempts.Load() != 2 {
		t.Fatalf("attempts %d", attempts.Load())
	}
}

func TestBuildTerminalFailureDoesNotBlockNextTask(t *testing.T) {
	a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) })
	buildQueue.Lock()
	err := a.ensureBuilds()
	buildQueue.Unlock()
	if err != nil {
		t.Fatal(err)
	}
	for _, id := range []string{"a", "b"} {
		if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at,release_id) VALUES(?,'1.2.3','queued',?,'release')", id, id); err != nil {
			t.Fatal(err)
		}
	}
	if _, err = a.s.DB.Exec("CREATE TRIGGER fail_a BEFORE UPDATE ON image_builds WHEN NEW.id='a' AND NEW.status='failed' BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
		t.Fatal(err)
	}
	a.startBuildWorker()
	deadline := time.Now().Add(4 * time.Second)
	for {
		var status string
		if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='b'").Scan(&status); err != nil {
			t.Fatal(err)
		}
		if status == "failed" {
			break
		}
		if time.Now().After(deadline) {
			t.Fatal("A terminal retry blocked B")
		}
		time.Sleep(10 * time.Millisecond)
	}
	if _, err = a.s.DB.Exec("DROP TRIGGER fail_a"); err != nil {
		t.Fatal(err)
	}
	for {
		var status string
		if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='a'").Scan(&status); err != nil {
			t.Fatal(err)
		}
		if status == "failed" {
			break
		}
		if time.Now().After(deadline) {
			t.Fatal("A never converged")
		}
		time.Sleep(10 * time.Millisecond)
	}
}

func TestCreateLostResponseRecovery(t *testing.T) {
	for _, mode := range []string{"cleanup", "retry", "foreign", "inspect-failure", "late"} {
		t.Run(mode, func(t *testing.T) {
			t.Setenv("PANEL_CONTAINER_ID", "")
			deleted := 0
			retry := false
			a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Path == "/v1.47/images/image/json" {
					writeImageInspect(w, "image", "1.2.3")
					return
				}
				switch r.Method + " " + r.URL.Path {
				case "POST /v1.47/networks/create":
					fmt.Fprint(w, `{"Id":"dsh-inst-lost"}`)
				case "GET /v1.47/networks/dsh-inst-lost":
					fmt.Fprint(w, `{"Id":"dsh-inst-lost","Name":"dsh-inst-lost","Driver":"bridge","Labels":{"dsh-mgr.instance":"lost"}}`)
				case "POST /v1.47/containers/create":
					fmt.Fprint(w, "{")
				case "GET /v1.47/containers/dsh-inst-lost/json":
					if mode == "late" && !retry {
						w.WriteHeader(404)
						return
					}
					if mode == "inspect-failure" && !retry {
						w.WriteHeader(500)
						return
					}
					owner := "owner"
					if mode == "foreign" {
						owner = "another-panel"
					}
					fmt.Fprintf(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":%q,"dsh-mgr.role":"dsh"}}}`, owner)
				case "DELETE /v1.47/containers/created":
					if mode == "retry" && !retry {
						w.WriteHeader(500)
						return
					}
					deleted++
					w.WriteHeader(204)
				case "DELETE /v1.47/networks/dsh-inst-lost":
					w.WriteHeader(204)
				default:
					t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
					w.WriteHeader(500)
				}
			})
			seedVerifiedImage(t, a, "image", "1.2.3")
			i := store.Instance{ID: "lost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
			if err := a.s.AddInstance(i); err != nil {
				t.Fatal(err)
			}
			if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
				t.Fatal("accepted lost response")
			}
			if mode == "cleanup" {
				if _, err := a.s.GetInstance(i.ID); err == nil || deleted != 1 {
					t.Fatal("cleanup incomplete")
				}
				return
			}
			saved, err := a.s.GetInstance(i.ID)
			if err != nil || saved.Status != "error" || deleted != 0 {
				t.Fatalf("lost recovery record: %+v %v", saved, err)
			}
			retry = true
			w := httptest.NewRecorder()
			a.instance(w, runtimeRequest("DELETE", "/api/instances/lost", "", sid))
			if mode == "foreign" {
				if w.Code != 502 || deleted != 0 {
					t.Fatal("foreign resource touched")
				}
				return
			}
			if w.Code != 200 || deleted != 1 {
				t.Fatalf("retry failed %d %s", w.Code, w.Body.String())
			}
		})
	}
}

func TestBuildCheckRecoveryIdentityAndRetry(t *testing.T) {
	for _, mode := range []string{"lost", "delete-failure", "foreign", "absent"} {
		t.Run(mode, func(t *testing.T) {
			deleted := 0
			retry := false
			a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				switch {
				case r.Method == "POST" && r.URL.Path == "/v1.47/containers/create":
					if r.URL.Query().Get("name") != buildCheckName("task") {
						t.Error("missing task name")
					}
					if mode == "delete-failure" {
						fmt.Fprint(w, `{"Id":"check"}`)
					} else {
						fmt.Fprint(w, "{")
					}
				case r.Method == "GET":
					if mode == "absent" && !retry {
						w.WriteHeader(404)
						return
					}
					if !retry && mode == "lost" {
						w.WriteHeader(500)
						return
					}
					id := "task"
					if mode == "foreign" {
						id = "another-panel"
					}
					fmt.Fprintf(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":%q}}}`, id)
				case r.Method == "DELETE":
					if !retry {
						w.WriteHeader(500)
						return
					}
					deleted++
					w.WriteHeader(204)
				default:
					w.WriteHeader(500)
				}
			})
			if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
				t.Fatal("accepted failed validation")
			}
			var count int
			if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil {
				t.Fatal(err)
			}
			if count != 1 {
				t.Fatal("lost cleanup record")
			}
			retry = true
			err := a.recoverBuildChecks(context.Background())
			if mode == "foreign" {
				if err == nil || deleted != 0 {
					t.Fatal("foreign container deleted")
				}
				return
			}
			if err != nil {
				t.Fatal(err)
			}
			if deleted != 1 {
				t.Fatal("not recovered")
			}
			if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil || count != 0 {
				t.Fatalf("record retained %d %v", count, err)
			}
		})
	}
}

func TestReconcileRecoversLostRuntimeIdentity(t *testing.T) {
	t.Setenv("PANEL_CONTAINER_ID", "")
	a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/v1.47/images/image/json" {
			writeImageInspect(w, "image", "1.2.3")
			return
		}
		if r.Method != "GET" {
			t.Fatalf("destructive recovery: %s", r.Method)
		}
		if r.URL.Path == "/v1.47/containers/json" {
			fmt.Fprint(w, `[{"Id":"created","State":"running","Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner"}}]`)
			return
		}
		fmt.Fprint(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
	})
	if err := a.s.AddInstance(store.Instance{ID: "lost", UID: "owner", ImageID: "image", Status: "creating"}); err != nil {
		t.Fatal(err)
	}
	if err := a.Reconcile(context.Background()); err != nil {
		t.Fatal(err)
	}
	i, err := a.s.GetInstance("lost")
	if err != nil || i.ContainerID != "created" || i.Status != "error" {
		t.Fatalf("not recovered: %+v %v", i, err)
	}
}

func TestBuildResultTransactionAndRetry(t *testing.T) {
	for _, table := range []string{"images", "image_builds"} {
		t.Run(table, func(t *testing.T) {
			a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker request") })
			buildQueue.Lock()
			err := a.ensureBuilds()
			buildQueue.Unlock()
			if err != nil {
				t.Fatal(err)
			}
			b := buildRecord{ID: "task", Version: "1.2.3"}
			if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('task','1.2.3','validating','now')"); err != nil {
				t.Fatal(err)
			}
			event := "INSERT"
			if table == "image_builds" {
				event = "UPDATE"
			}
			if _, err = a.s.DB.Exec("CREATE TRIGGER fail_result BEFORE " + event + " ON " + table + " BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
				t.Fatal(err)
			}
			if err = a.persistBuildResult(b, "success", "logs", "", "image", "now"); err == nil {
				t.Fatal("injection ignored")
			}
			var status string
			var count int
			if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "validating" {
				t.Fatalf("partial terminal write %s %v", status, err)
			}
			if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 0 {
				t.Fatalf("partial index write %d %v", count, err)
			}
			done := make(chan struct{})
			go func() { a.finishBuild(b, "success", "logs", "", "image", "now"); close(done) }()
			select {
			case <-done:
				t.Fatal("retry abandoned")
			case <-time.After(30 * time.Millisecond):
			}
			if _, err = a.s.DB.Exec("DROP TRIGGER fail_result"); err != nil {
				t.Fatal(err)
			}
			select {
			case <-done:
			case <-time.After(4 * time.Second):
				t.Fatal("terminal write not retried")
			}
			if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "success" {
				t.Fatalf("terminal %s %v", status, err)
			}
			if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 1 {
				t.Fatalf("index %d %v", count, err)
			}
		})
	}
}

func TestBuildTerminalRetryStopsWhenDatabaseCloses(t *testing.T) {
	a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker call") })
	a.s.Close()
	done := make(chan struct{})
	go func() { a.finishBuild(buildRecord{ID: "closed"}, "failed", "", "", "", "now"); close(done) }()
	select {
	case <-done:
	case <-time.After(time.Second):
		t.Fatal("closed database retry did not stop")
	}
}

func TestBuildCheckDefiniteRejectionAndExternalDeletion(t *testing.T) {
	for _, rejected := range []bool{true, false} {
		t.Run(fmt.Sprint(rejected), func(t *testing.T) {
			a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				if r.Method == "POST" && r.URL.Path == "/v1.47/containers/create" {
					w.WriteHeader(400)
					return
				}
				w.WriteHeader(404)
			})
			if rejected {
				if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
					t.Fatal("rejection accepted")
				}
			} else {
				if err := a.ensureBuildChecks(); err != nil {
					t.Fatal(err)
				}
				if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
					t.Fatal(err)
				}
				if err := a.recoverBuildChecks(context.Background()); err != nil {
					t.Fatal(err)
				}
			}
			var n int
			if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 {
				t.Fatalf("definite outcome retained: %d %v", n, err)
			}
		})
	}
}

func TestUncertainPendingRetainsLateResources(t *testing.T) {
	for _, column := range []string{"container_pending", "network_pending"} {
		for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
			t.Run(column+"/"+stamp, func(t *testing.T) {
				t.Setenv("PANEL_CONTAINER_ID", "")
				late := false
				deletes := 0
				a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
					if r.URL.Path == "/v1.47/images/image/json" {
						writeImageInspect(w, "image", "1.2.3")
						return
					}
					switch r.Method + " " + r.URL.Path {
					case "GET /v1.47/containers/dsh-inst-ghost/json":
						if !late || column != "container_pending" {
							w.WriteHeader(404)
							return
						}
						fmt.Fprint(w, `{"Id":"container-id","Name":"/dsh-inst-ghost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"ghost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
					case "GET /v1.47/networks/dsh-inst-ghost":
						if !late || column != "network_pending" {
							w.WriteHeader(404)
							return
						}
						fmt.Fprint(w, `{"Id":"network-id","Name":"dsh-inst-ghost","Driver":"bridge","Labels":{"dsh-mgr.instance":"ghost"}}`)
					case "DELETE /v1.47/containers/container-id", "DELETE /v1.47/networks/network-id":
						deletes++
						w.WriteHeader(204)
					default:
						t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
						w.WriteHeader(500)
					}
				})
				i := store.Instance{ID: "ghost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "error"}
				if err := a.s.AddInstance(i); err != nil {
					t.Fatal(err)
				}
				if err := a.setRuntimeCreatePending(i.ID, column, true); err != nil {
					t.Fatal(err)
				}
				if _, err := a.s.DB.Exec("UPDATE runtime_creates SET "+pendingColumns[column].stamp+"=? WHERE instance_id=?", stamp, i.ID); err != nil {
					t.Fatal(err)
				}
				for attempt := 0; attempt < 2; attempt++ {
					w := httptest.NewRecorder()
					a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
					if w.Code != 502 {
						t.Fatalf("uncertainty discarded: %d %s", w.Code, w.Body.String())
					}
					if _, pending, err := a.pendingSince(i.ID, column); err != nil || !pending {
						t.Fatalf("pending lost: %v %v", pending, err)
					}
					if _, err := a.s.GetInstance(i.ID); err != nil {
						t.Fatal(err)
					}
				}
				late = true
				w := httptest.NewRecorder()
				a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
				if w.Code != 200 || deletes != 1 {
					t.Fatalf("late cleanup: %d %s deletes=%d", w.Code, w.Body.String(), deletes)
				}
				if _, err := a.s.GetInstance(i.ID); err == nil {
					t.Fatal("instance retained")
				}
			})
		}
	}
}

func TestBuildCheckUncertainRowRetainsLateContainer(t *testing.T) {
	for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
		t.Run(stamp, func(t *testing.T) {
			late := false
			deletes := 0
			a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				switch r.Method + " " + r.URL.Path {
				case "GET /v1.47/containers/dsh-build-check-task/json":
					if !late {
						w.WriteHeader(404)
						return
					}
					fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
				case "DELETE /v1.47/containers/check":
					deletes++
					w.WriteHeader(204)
				default:
					t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
					w.WriteHeader(500)
				}
			})
			if err := a.ensureBuildChecks(); err != nil {
				t.Fatal(err)
			}
			if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES('task','image',?)", stamp); err != nil {
				t.Fatal(err)
			}
			for attempt := 0; attempt < 2; attempt++ {
				if err := a.recoverBuildChecks(context.Background()); !errors.Is(err, errBuildCheckUncertain) {
					t.Fatalf("uncertainty discarded: %v", err)
				}
				var n int
				if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
					t.Fatalf("record lost: %d %v", n, err)
				}
			}
			late = true
			if err := a.recoverBuildChecks(context.Background()); err != nil {
				t.Fatal(err)
			}
			var n int
			if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 || deletes != 1 {
				t.Fatalf("late cleanup: rows=%d deletes=%d err=%v", n, deletes, err)
			}
		})
	}
}

func TestRuntimeNetworkPanelDisconnect(t *testing.T) {
	for _, mode := range []string{"not-connected", "connected", "panel-name", "other-connected", "lost-disconnect", "forbidden", "inspect-forbidden", "foreign"} {
		t.Run(mode, func(t *testing.T) {
			panel := "panel-id"
			if mode == "panel-name" || mode == "inspect-forbidden" {
				panel = "panel"
			}
			t.Setenv("PANEL_CONTAINER_ID", panel)
			connected := mode != "not-connected"
			disconnects, deletes := 0, 0
			a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
				if r.URL.Path == "/v1.47/images/image/json" {
					writeImageInspect(w, "image", "1.2.3")
					return
				}
				switch r.Method + " " + r.URL.Path {
				case "GET /v1.47/networks/dsh-inst-net", "GET /v1.47/networks/network-id":
					owner := "net"
					if mode == "foreign" {
						owner = "other"
					}
					containers := "{}"
					if connected {
						containers = `{"panel-id":{}}`
					}
					if mode == "other-connected" {
						containers = `{"other-id":{}}`
					}
					fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q},"Containers":%s}`, owner, containers)
				case "GET /v1.47/containers/panel/json", "GET /v1.47/containers/panel-id/json":
					if mode == "inspect-forbidden" {
						w.WriteHeader(403)
						return
					}
					fmt.Fprint(w, `{"Id":"panel-id"}`)
				case "POST /v1.47/networks/network-id/disconnect":
					disconnects++
					var body struct {
						Container string
						Force     bool
					}
					if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Container != "panel-id" || !body.Force {
						t.Errorf("disconnect identity: %+v %v", body, err)
					}
					if mode == "not-connected" || mode == "other-connected" || mode == "foreign" {
						t.Error("unsafe disconnect")
						w.WriteHeader(500)
						return
					}
					if mode == "forbidden" {
						w.WriteHeader(403)
						return
					}
					connected = false
					if mode == "lost-disconnect" {
						w.WriteHeader(500)
						return
					}
					w.WriteHeader(204)
				case "DELETE /v1.47/networks/network-id":
					deletes++
					w.WriteHeader(204)
				default:
					t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
					w.WriteHeader(500)
				}
			})
			if err := a.setRuntimeCreatePending("net", "network_pending", true); err != nil {
				t.Fatal(err)
			}
			err := a.recoverRuntimeNetwork(context.Background(), "net")
			wantError := mode == "lost-disconnect" || mode == "forbidden" || mode == "inspect-forbidden" || mode == "foreign"
			if (err != nil) != wantError {
				t.Fatalf("recovery error=%v wantError=%v", err, wantError)
			}
			if wantError {
				if deletes != 0 {
					t.Fatal("deleted after failure")
				}
				if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || !pending {
					t.Fatalf("pending lost: %v %v", pending, err)
				}
				if mode != "lost-disconnect" {
					return
				}
				if err := a.recoverRuntimeNetwork(context.Background(), "net"); err != nil {
					t.Fatalf("disconnect retry: %v", err)
				}
			}
			wantDisconnects := 1
			if mode == "not-connected" || mode == "other-connected" {
				wantDisconnects = 0
			}
			if deletes != 1 || disconnects != wantDisconnects {
				t.Fatalf("deletes=%d disconnects=%d", deletes, disconnects)
			}
			if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || pending {
				t.Fatalf("pending retained: %v %v", pending, err)
			}
		})
	}
}