m1saka
9a71772771
Add exact-image compatibility policy, experimental admission, paginated build history and shared reauthentication checks. Preserve uncertain Docker creation records, bound frontend resource waits and renew runtime sessions proactively. Lazy-load and split Monaco, add regression and opt-in live Docker fault tests, and record full race and real-browser acceptance results.
829 lines
26 KiB
Go
829 lines
26 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestBuildCheckRenamedContainerRetainsIdentity(t *testing.T) {
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "GET" || r.URL.Path != "/v1.47/containers/known/json" {
|
|
t.Errorf("unsafe lookup %s %s", r.Method, r.URL.Path)
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"Id":"known","Name":"/renamed","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
|
|
})
|
|
if err := a.ensureBuildChecks(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.recoverBuildChecks(context.Background()); err == nil {
|
|
t.Fatal("renamed identity accepted")
|
|
}
|
|
var n int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
|
|
t.Fatalf("record lost %d %v", n, err)
|
|
}
|
|
}
|
|
func TestRuntimeNetworkDeletionLostResponseAndReplacement(t *testing.T) {
|
|
for _, foreign := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(foreign), func(t *testing.T) {
|
|
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
deleted := false
|
|
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
if r.Method == "GET" {
|
|
if deleted {
|
|
if r.URL.Path != "/v1.47/networks/known" {
|
|
t.Error("did not use persisted ID")
|
|
}
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
owner := "net"
|
|
if foreign {
|
|
owner = "foreign"
|
|
}
|
|
fmt.Fprintf(w, `{"Id":"known","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, owner)
|
|
return
|
|
}
|
|
if foreign {
|
|
t.Error("deleted foreign network")
|
|
}
|
|
deleted = true
|
|
w.WriteHeader(500)
|
|
})
|
|
if err := a.setRuntimeCreatePending("net", "network_pending", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.removeRuntimeNetwork(context.Background(), "net"); err == nil {
|
|
t.Fatal("first error missing")
|
|
}
|
|
if foreign {
|
|
return
|
|
}
|
|
if err := a.removeRuntimeNetwork(context.Background(), "net"); err != nil {
|
|
t.Fatalf("confirmed deletion did not converge: %v", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
func TestStartIntentsUncertainCreationKeepsRecoveryAccessible(t *testing.T) {
|
|
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
if r.Method != "GET" {
|
|
t.Error("unexpected destructive call")
|
|
}
|
|
w.WriteHeader(404)
|
|
})
|
|
if err := a.s.EnsureIntents(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.s.ReserveIntent(store.Intent{ID: "intent", UID: "owner"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.s.ClaimIntent("intent", store.Instance{ID: "pending", UID: "owner", Status: "creating"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.setRuntimeCreatePending("pending", "container_pending", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if err := a.StartIntents(ctx); err != nil {
|
|
t.Fatalf("uncertain creation blocked startup: %v", err)
|
|
}
|
|
i, err := a.s.GetInstance("pending")
|
|
if err != nil || i.Status != "error" {
|
|
t.Fatalf("recovery reservation missing %+v %v", i, err)
|
|
}
|
|
list, err := a.s.Intents("owner")
|
|
if err != nil || len(list) != 1 || list[0].Status != "failed" {
|
|
t.Fatalf("intent not terminated %+v %v", list, err)
|
|
}
|
|
}
|
|
|
|
func TestNetworkCreateLostResponse(t *testing.T) {
|
|
for _, mode := range []string{"present", "late", "foreign"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
late := false
|
|
deletes := 0
|
|
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
switch r.Method + " " + r.URL.Path {
|
|
case "POST /v1.47/networks/create":
|
|
w.WriteHeader(500)
|
|
case "GET /v1.47/containers/dsh-inst-net/json":
|
|
w.WriteHeader(404)
|
|
case "GET /v1.47/networks/dsh-inst-net":
|
|
if mode == "late" && !late {
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
id := "net"
|
|
if mode == "foreign" {
|
|
id = "other"
|
|
}
|
|
fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, id)
|
|
case "DELETE /v1.47/networks/network-id":
|
|
deletes++
|
|
w.WriteHeader(204)
|
|
default:
|
|
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
w.WriteHeader(500)
|
|
}
|
|
})
|
|
seedVerifiedImage(t, a, "image", "1.2.3")
|
|
i := store.Instance{ID: "net", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
|
|
if err := a.s.AddInstance(i); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
|
|
t.Fatal("lost response accepted")
|
|
}
|
|
if mode == "present" {
|
|
if deletes != 1 {
|
|
t.Fatal("network leaked")
|
|
}
|
|
return
|
|
}
|
|
saved, err := a.s.GetInstance("net")
|
|
if err != nil || saved.Status != "error" {
|
|
t.Fatalf("lost reservation %+v %v", saved, err)
|
|
}
|
|
late = true
|
|
w := httptest.NewRecorder()
|
|
a.instance(w, runtimeRequest("DELETE", "/api/instances/net", "", sid))
|
|
if mode == "foreign" {
|
|
if deletes != 0 || w.Code != 502 {
|
|
t.Fatal("foreign network touched")
|
|
}
|
|
return
|
|
}
|
|
if w.Code != 200 || deletes != 1 {
|
|
t.Fatalf("late network retry %d %d", w.Code, deletes)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuildCheckReopenWorkerWithoutQueuedBuilds(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "recovery.db")
|
|
s, err := store.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := &API{s: s}
|
|
if err = a.ensureBuildChecks(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id) VALUES('task','image')"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
s.Close()
|
|
s, err = store.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
var attempts atomic.Int32
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == "GET" {
|
|
fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
|
|
return
|
|
}
|
|
if r.Method == "DELETE" {
|
|
if attempts.Add(1) == 1 {
|
|
w.WriteHeader(500)
|
|
} else {
|
|
w.WriteHeader(204)
|
|
}
|
|
return
|
|
}
|
|
t.Errorf("unexpected %s", r.Method)
|
|
}))
|
|
defer srv.Close()
|
|
a = &API{s: s, docker: docker.New(srv.URL)}
|
|
if err = a.StartImageBuilds(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deadline := time.Now().Add(4 * time.Second)
|
|
for {
|
|
var n int
|
|
if err = s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n == 0 {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("reopen worker did not retry")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if attempts.Load() != 2 {
|
|
t.Fatalf("attempts %d", attempts.Load())
|
|
}
|
|
}
|
|
|
|
func TestBuildTerminalFailureDoesNotBlockNextTask(t *testing.T) {
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) })
|
|
buildQueue.Lock()
|
|
err := a.ensureBuilds()
|
|
buildQueue.Unlock()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, id := range []string{"a", "b"} {
|
|
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES(?,'1.2.3','queued',?)", id, id); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if _, err = a.s.DB.Exec("CREATE TRIGGER fail_a BEFORE UPDATE ON image_builds WHEN NEW.id='a' AND NEW.status='failed' BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.startBuildWorker()
|
|
deadline := time.Now().Add(4 * time.Second)
|
|
for {
|
|
var status string
|
|
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='b'").Scan(&status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status == "failed" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("A terminal retry blocked B")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if _, err = a.s.DB.Exec("DROP TRIGGER fail_a"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for {
|
|
var status string
|
|
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='a'").Scan(&status); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if status == "failed" {
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("A never converged")
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestCreateLostResponseRecovery(t *testing.T) {
|
|
for _, mode := range []string{"cleanup", "retry", "foreign", "inspect-failure", "late"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
deleted := 0
|
|
retry := false
|
|
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
switch r.Method + " " + r.URL.Path {
|
|
case "POST /v1.47/networks/create":
|
|
fmt.Fprint(w, `{"Id":"dsh-inst-lost"}`)
|
|
case "GET /v1.47/networks/dsh-inst-lost":
|
|
fmt.Fprint(w, `{"Id":"dsh-inst-lost","Name":"dsh-inst-lost","Driver":"bridge","Labels":{"dsh-mgr.instance":"lost"}}`)
|
|
case "POST /v1.47/containers/create":
|
|
fmt.Fprint(w, "{")
|
|
case "GET /v1.47/containers/dsh-inst-lost/json":
|
|
if mode == "late" && !retry {
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
if mode == "inspect-failure" && !retry {
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
owner := "owner"
|
|
if mode == "foreign" {
|
|
owner = "another-panel"
|
|
}
|
|
fmt.Fprintf(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":%q,"dsh-mgr.role":"dsh"}}}`, owner)
|
|
case "DELETE /v1.47/containers/created":
|
|
if mode == "retry" && !retry {
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
deleted++
|
|
w.WriteHeader(204)
|
|
case "DELETE /v1.47/networks/dsh-inst-lost":
|
|
w.WriteHeader(204)
|
|
default:
|
|
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
w.WriteHeader(500)
|
|
}
|
|
})
|
|
seedVerifiedImage(t, a, "image", "1.2.3")
|
|
i := store.Instance{ID: "lost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
|
|
if err := a.s.AddInstance(i); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
|
|
t.Fatal("accepted lost response")
|
|
}
|
|
if mode == "cleanup" {
|
|
if _, err := a.s.GetInstance(i.ID); err == nil || deleted != 1 {
|
|
t.Fatal("cleanup incomplete")
|
|
}
|
|
return
|
|
}
|
|
saved, err := a.s.GetInstance(i.ID)
|
|
if err != nil || saved.Status != "error" || deleted != 0 {
|
|
t.Fatalf("lost recovery record: %+v %v", saved, err)
|
|
}
|
|
retry = true
|
|
w := httptest.NewRecorder()
|
|
a.instance(w, runtimeRequest("DELETE", "/api/instances/lost", "", sid))
|
|
if mode == "foreign" {
|
|
if w.Code != 502 || deleted != 0 {
|
|
t.Fatal("foreign resource touched")
|
|
}
|
|
return
|
|
}
|
|
if w.Code != 200 || deleted != 1 {
|
|
t.Fatalf("retry failed %d %s", w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuildCheckRecoveryIdentityAndRetry(t *testing.T) {
|
|
for _, mode := range []string{"lost", "delete-failure", "foreign", "absent"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
deleted := 0
|
|
retry := false
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case r.Method == "POST" && r.URL.Path == "/v1.47/containers/create":
|
|
if r.URL.Query().Get("name") != buildCheckName("task") {
|
|
t.Error("missing task name")
|
|
}
|
|
if mode == "delete-failure" {
|
|
fmt.Fprint(w, `{"Id":"check"}`)
|
|
} else {
|
|
fmt.Fprint(w, "{")
|
|
}
|
|
case r.Method == "GET":
|
|
if mode == "absent" && !retry {
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
if !retry && mode == "lost" {
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
id := "task"
|
|
if mode == "foreign" {
|
|
id = "another-panel"
|
|
}
|
|
fmt.Fprintf(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":%q}}}`, id)
|
|
case r.Method == "DELETE":
|
|
if !retry {
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
deleted++
|
|
w.WriteHeader(204)
|
|
default:
|
|
w.WriteHeader(500)
|
|
}
|
|
})
|
|
if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
|
|
t.Fatal("accepted failed validation")
|
|
}
|
|
var count int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if count != 1 {
|
|
t.Fatal("lost cleanup record")
|
|
}
|
|
retry = true
|
|
err := a.recoverBuildChecks(context.Background())
|
|
if mode == "foreign" {
|
|
if err == nil || deleted != 0 {
|
|
t.Fatal("foreign container deleted")
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if deleted != 1 {
|
|
t.Fatal("not recovered")
|
|
}
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("record retained %d %v", count, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestReconcileRecoversLostRuntimeIdentity(t *testing.T) {
|
|
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
if r.Method != "GET" {
|
|
t.Fatalf("destructive recovery: %s", r.Method)
|
|
}
|
|
if r.URL.Path == "/v1.47/containers/json" {
|
|
fmt.Fprint(w, `[{"Id":"created","State":"running","Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner"}}]`)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
|
|
})
|
|
if err := a.s.AddInstance(store.Instance{ID: "lost", UID: "owner", ImageID: "image", Status: "creating"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.Reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
i, err := a.s.GetInstance("lost")
|
|
if err != nil || i.ContainerID != "created" || i.Status != "error" {
|
|
t.Fatalf("not recovered: %+v %v", i, err)
|
|
}
|
|
}
|
|
|
|
func TestBuildResultTransactionAndRetry(t *testing.T) {
|
|
for _, table := range []string{"images", "image_builds"} {
|
|
t.Run(table, func(t *testing.T) {
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker request") })
|
|
buildQueue.Lock()
|
|
err := a.ensureBuilds()
|
|
buildQueue.Unlock()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
b := buildRecord{ID: "task", Version: "1.2.3"}
|
|
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('task','1.2.3','validating','now')"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
event := "INSERT"
|
|
if table == "image_builds" {
|
|
event = "UPDATE"
|
|
}
|
|
if _, err = a.s.DB.Exec("CREATE TRIGGER fail_result BEFORE " + event + " ON " + table + " BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = a.persistBuildResult(b, "success", "logs", "", "image", "now"); err == nil {
|
|
t.Fatal("injection ignored")
|
|
}
|
|
var status string
|
|
var count int
|
|
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "validating" {
|
|
t.Fatalf("partial terminal write %s %v", status, err)
|
|
}
|
|
if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 0 {
|
|
t.Fatalf("partial index write %d %v", count, err)
|
|
}
|
|
done := make(chan struct{})
|
|
go func() { a.finishBuild(b, "success", "logs", "", "image", "now"); close(done) }()
|
|
select {
|
|
case <-done:
|
|
t.Fatal("retry abandoned")
|
|
case <-time.After(30 * time.Millisecond):
|
|
}
|
|
if _, err = a.s.DB.Exec("DROP TRIGGER fail_result"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-done:
|
|
case <-time.After(4 * time.Second):
|
|
t.Fatal("terminal write not retried")
|
|
}
|
|
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "success" {
|
|
t.Fatalf("terminal %s %v", status, err)
|
|
}
|
|
if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 1 {
|
|
t.Fatalf("index %d %v", count, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIntentRecoveryMissingReservationDoesNotTouchDocker(t *testing.T) {
|
|
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
t.Error("missing reservation must not authorize Docker cleanup")
|
|
})
|
|
if err := a.s.EnsureIntents(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.s.ReserveIntent(store.Intent{ID: "missing", UID: "owner"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.s.DB.Exec("UPDATE creation_intents SET status='creating',instance_id='gone' WHERE id='missing'"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.recoverIntents(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
list, err := a.s.Intents("owner")
|
|
if err != nil || len(list) != 1 || list[0].Status != "failed" {
|
|
t.Fatalf("intent not terminated: %+v %v", list, err)
|
|
}
|
|
}
|
|
|
|
func TestBuildTerminalRetryStopsWhenDatabaseCloses(t *testing.T) {
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker call") })
|
|
a.s.Close()
|
|
done := make(chan struct{})
|
|
go func() { a.finishBuild(buildRecord{ID: "closed"}, "failed", "", "", "", "now"); close(done) }()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("closed database retry did not stop")
|
|
}
|
|
}
|
|
|
|
func TestBuildCheckDefiniteRejectionAndExternalDeletion(t *testing.T) {
|
|
for _, rejected := range []bool{true, false} {
|
|
t.Run(fmt.Sprint(rejected), func(t *testing.T) {
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == "POST" && r.URL.Path == "/v1.47/containers/create" {
|
|
w.WriteHeader(400)
|
|
return
|
|
}
|
|
w.WriteHeader(404)
|
|
})
|
|
if rejected {
|
|
if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
|
|
t.Fatal("rejection accepted")
|
|
}
|
|
} else {
|
|
if err := a.ensureBuildChecks(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.recoverBuildChecks(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var n int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 {
|
|
t.Fatalf("definite outcome retained: %d %v", n, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestUncertainPendingRetainsLateResources(t *testing.T) {
|
|
for _, column := range []string{"container_pending", "network_pending"} {
|
|
for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
|
|
t.Run(column+"/"+stamp, func(t *testing.T) {
|
|
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
late := false
|
|
deletes := 0
|
|
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
switch r.Method + " " + r.URL.Path {
|
|
case "GET /v1.47/containers/dsh-inst-ghost/json":
|
|
if !late || column != "container_pending" {
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"Id":"container-id","Name":"/dsh-inst-ghost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"ghost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
|
|
case "GET /v1.47/networks/dsh-inst-ghost":
|
|
if !late || column != "network_pending" {
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"Id":"network-id","Name":"dsh-inst-ghost","Driver":"bridge","Labels":{"dsh-mgr.instance":"ghost"}}`)
|
|
case "DELETE /v1.47/containers/container-id", "DELETE /v1.47/networks/network-id":
|
|
deletes++
|
|
w.WriteHeader(204)
|
|
default:
|
|
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
w.WriteHeader(500)
|
|
}
|
|
})
|
|
i := store.Instance{ID: "ghost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "error"}
|
|
if err := a.s.AddInstance(i); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.setRuntimeCreatePending(i.ID, column, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.s.DB.Exec("UPDATE runtime_creates SET "+pendingColumns[column].stamp+"=? WHERE instance_id=?", stamp, i.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for attempt := 0; attempt < 2; attempt++ {
|
|
w := httptest.NewRecorder()
|
|
a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
|
|
if w.Code != 502 {
|
|
t.Fatalf("uncertainty discarded: %d %s", w.Code, w.Body.String())
|
|
}
|
|
if _, pending, err := a.pendingSince(i.ID, column); err != nil || !pending {
|
|
t.Fatalf("pending lost: %v %v", pending, err)
|
|
}
|
|
if _, err := a.s.GetInstance(i.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
late = true
|
|
w := httptest.NewRecorder()
|
|
a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
|
|
if w.Code != 200 || deletes != 1 {
|
|
t.Fatalf("late cleanup: %d %s deletes=%d", w.Code, w.Body.String(), deletes)
|
|
}
|
|
if _, err := a.s.GetInstance(i.ID); err == nil {
|
|
t.Fatal("instance retained")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBuildCheckUncertainRowRetainsLateContainer(t *testing.T) {
|
|
for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
|
|
t.Run(stamp, func(t *testing.T) {
|
|
late := false
|
|
deletes := 0
|
|
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
switch r.Method + " " + r.URL.Path {
|
|
case "GET /v1.47/containers/dsh-build-check-task/json":
|
|
if !late {
|
|
w.WriteHeader(404)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
|
|
case "DELETE /v1.47/containers/check":
|
|
deletes++
|
|
w.WriteHeader(204)
|
|
default:
|
|
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
w.WriteHeader(500)
|
|
}
|
|
})
|
|
if err := a.ensureBuildChecks(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES('task','image',?)", stamp); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for attempt := 0; attempt < 2; attempt++ {
|
|
if err := a.recoverBuildChecks(context.Background()); !errors.Is(err, errBuildCheckUncertain) {
|
|
t.Fatalf("uncertainty discarded: %v", err)
|
|
}
|
|
var n int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
|
|
t.Fatalf("record lost: %d %v", n, err)
|
|
}
|
|
}
|
|
late = true
|
|
if err := a.recoverBuildChecks(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var n int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 || deletes != 1 {
|
|
t.Fatalf("late cleanup: rows=%d deletes=%d err=%v", n, deletes, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRuntimeNetworkPanelDisconnect(t *testing.T) {
|
|
for _, mode := range []string{"not-connected", "connected", "panel-name", "other-connected", "lost-disconnect", "forbidden", "inspect-forbidden", "foreign"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
panel := "panel-id"
|
|
if mode == "panel-name" || mode == "inspect-forbidden" {
|
|
panel = "panel"
|
|
}
|
|
t.Setenv("PANEL_CONTAINER_ID", panel)
|
|
connected := mode != "not-connected"
|
|
disconnects, deletes := 0, 0
|
|
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
if r.URL.Path == "/v1.47/images/image/json" {
|
|
writeImageInspect(w, "image", "1.2.3")
|
|
return
|
|
}
|
|
switch r.Method + " " + r.URL.Path {
|
|
case "GET /v1.47/networks/dsh-inst-net", "GET /v1.47/networks/network-id":
|
|
owner := "net"
|
|
if mode == "foreign" {
|
|
owner = "other"
|
|
}
|
|
containers := "{}"
|
|
if connected {
|
|
containers = `{"panel-id":{}}`
|
|
}
|
|
if mode == "other-connected" {
|
|
containers = `{"other-id":{}}`
|
|
}
|
|
fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q},"Containers":%s}`, owner, containers)
|
|
case "GET /v1.47/containers/panel/json", "GET /v1.47/containers/panel-id/json":
|
|
if mode == "inspect-forbidden" {
|
|
w.WriteHeader(403)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"Id":"panel-id"}`)
|
|
case "POST /v1.47/networks/network-id/disconnect":
|
|
disconnects++
|
|
var body struct {
|
|
Container string
|
|
Force bool
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Container != "panel-id" || !body.Force {
|
|
t.Errorf("disconnect identity: %+v %v", body, err)
|
|
}
|
|
if mode == "not-connected" || mode == "other-connected" || mode == "foreign" {
|
|
t.Error("unsafe disconnect")
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
if mode == "forbidden" {
|
|
w.WriteHeader(403)
|
|
return
|
|
}
|
|
connected = false
|
|
if mode == "lost-disconnect" {
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
w.WriteHeader(204)
|
|
case "DELETE /v1.47/networks/network-id":
|
|
deletes++
|
|
w.WriteHeader(204)
|
|
default:
|
|
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
w.WriteHeader(500)
|
|
}
|
|
})
|
|
if err := a.setRuntimeCreatePending("net", "network_pending", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := a.recoverRuntimeNetwork(context.Background(), "net")
|
|
wantError := mode == "lost-disconnect" || mode == "forbidden" || mode == "inspect-forbidden" || mode == "foreign"
|
|
if (err != nil) != wantError {
|
|
t.Fatalf("recovery error=%v wantError=%v", err, wantError)
|
|
}
|
|
if wantError {
|
|
if deletes != 0 {
|
|
t.Fatal("deleted after failure")
|
|
}
|
|
if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || !pending {
|
|
t.Fatalf("pending lost: %v %v", pending, err)
|
|
}
|
|
if mode != "lost-disconnect" {
|
|
return
|
|
}
|
|
if err := a.recoverRuntimeNetwork(context.Background(), "net"); err != nil {
|
|
t.Fatalf("disconnect retry: %v", err)
|
|
}
|
|
}
|
|
wantDisconnects := 1
|
|
if mode == "not-connected" || mode == "other-connected" {
|
|
wantDisconnects = 0
|
|
}
|
|
if deletes != 1 || disconnects != wantDisconnects {
|
|
t.Fatalf("deletes=%d disconnects=%d", deletes, disconnects)
|
|
}
|
|
if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || pending {
|
|
t.Fatalf("pending retained: %v %v", pending, err)
|
|
}
|
|
})
|
|
}
|
|
}
|