Fix audit findings: lock architecture, security, deployment hardening

Close all 14 findings from the codebase audit (1 high, 5 medium, 8 low):

- Locking: replace the global API and workspace mutexes with per-instance
  and per-user keyed locks (new internal/api/locks.go) so a slow restore,
  upgrade or transfer no longer blocks unrelated users or instances; move
  Docker I/O out of the build queue and invalidate concurrent image-index
  observations with a generation counter; serialize instance deletion
  behind in-flight creation.
- Security: equalize login timing for unknown usernames, periodically
  purge expired sessions/refresh tokens/lock rows, URL-encode orphan IDs
  in the frontend, upgrade vitest 3.x->5.x to clear 3 npm audit
  vulnerabilities (2 critical RCE).
- Deployment: add a -healthcheck mode probing /healthz (used by Dockerfile
  HEALTHCHECK and compose), make ADMIN_PASSWORD optional in compose with
  removal guidance after first start, exclude test scripts from the image,
  and cap panel resources (cpus: 2, pids_limit: 512).
- Reliability: check rows.Err() on admin listing loops, surface intent
  polling errors without sticking, disable close during in-flight create.

Verified: go build/vet, full go test -race -count=1 ./..., frontend
tsc build + 32 tests, npm audit 0, compose config, real-container
healthcheck.

internal/api/admin.go

@@ -51,6 +51,10 @@ func (a *API) users(w http.ResponseWriter, r *http.Request) {
 			v.Created = time.Unix(created, 0)
 			out = append(out, v)
 		}
+		if err := rows.Err(); err != nil {
+			fail(w, 500, "database error")
+			return
+		}
 		writeJSON(w, out)
 	case "POST":
 		var in struct {
@@ -208,5 +212,9 @@ func (a *API) audit(w http.ResponseWriter, r *http.Request) {
 		}
 		out = append(out, map[string]any{"id": id, "user_id": uid, "action": action, "target": target, "detail": detail, "created_at": time.Unix(at, 0)})
 	}
+	if err := rows.Err(); err != nil {
+		fail(w, 500, "database error")
+		return
+	}
 	writeJSON(w, out)
 }

internal/api/api.go

@@ -19,8 +19,10 @@ type API struct {
 	origin           string
 	root             string
 	docker           *docker.Client
-	mu               sync.Mutex
+	readersMu        sync.Mutex
 	workspaceReaders map[string]int
+	instanceLocks    keyedLocks
+	userLocks        keyedLocks
 	loginMu          sync.Mutex
 	loginSources     map[string]loginSource
 	trustedProxies   []*net.IPNet
@@ -29,6 +31,9 @@ type API struct {
 	authKey          []byte
 	authErr          error
 	authLoginMu      sync.Mutex
+
+	// Protected by buildQueue; invalidates image observations across deletions.
+	imageDeleteGeneration uint64
 }
 
 func New(s *store.Store, root string) *API {
@@ -134,6 +139,11 @@ func (a *API) user(r *http.Request) (store.User, string, error) {
 	}
 	return u, csrf, nil
 }
+
+// dummyPasswordHash keeps login latency uniform for unknown usernames so the
+// response timing cannot reveal whether an account exists.
+var dummyPasswordHash, _ = bcrypt.GenerateFromPassword([]byte("timing-equalization"), bcrypt.DefaultCost)
+
 func (a *API) login(w http.ResponseWriter, r *http.Request) {
 	if r.Method != "POST" {
 		fail(w, 405, "method not allowed")
@@ -180,6 +190,7 @@ func (a *API) login(w http.ResponseWriter, r *http.Request) {
 	}
 	h, e := a.s.Password(in.Username)
 	if e != nil {
+		_ = bcrypt.CompareHashAndPassword(dummyPasswordHash, []byte(in.Password))
 		fail(w, 401, "invalid credentials")
 		return
 	}

internal/api/auth_tokens.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
@@ -9,6 +10,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
+	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -191,6 +193,34 @@ func (a *API) accountLocked(username string) (bool, error) {
 	}
 	return until > time.Now().Unix(), err
 }
+
+// RunSessionCleanup removes expired sessions, their refresh tokens and elapsed
+// login locks every ten minutes; without it the sessions table grows without
+// bound because SessionUser only filters rows on read.
+func (a *API) RunSessionCleanup(ctx context.Context) {
+	ticker := time.NewTicker(10 * time.Minute)
+	defer ticker.Stop()
+	for {
+		a.cleanupSessions(time.Now().Unix())
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
+
+func (a *API) cleanupSessions(now int64) {
+	for _, q := range []struct{ name, sql string }{
+		{"refresh tokens", "DELETE FROM auth_refresh WHERE sid IN (SELECT id FROM sessions WHERE expires<=?)"},
+		{"sessions", "DELETE FROM sessions WHERE expires<=?"},
+		{"login locks", "DELETE FROM auth_failures WHERE locked_until>0 AND locked_until<=?"},
+	} {
+		if _, err := a.s.DB.Exec(q.sql, now); err != nil {
+			log.Printf("session cleanup (%s): %v", q.name, err)
+		}
+	}
+}
 func (a *API) failedLogin(username string) error {
 	_, err := a.s.DB.Exec(`INSERT INTO auth_failures(username,failures,locked_until) VALUES(?,1,0)
  ON CONFLICT(username) DO UPDATE SET

internal/api/auth_tokens_test.go

@@ -213,3 +213,52 @@ func TestUnsafeAuthKeyFailsClosed(t *testing.T) {
 		t.Fatal(w.Code)
 	}
 }
+
+func TestSessionCleanupRemovesExpiredRows(t *testing.T) {
+	a, _ := authFixture(t)
+	expired, _, e := a.s.Session("alice")
+	if e != nil {
+		t.Fatal(e)
+	}
+	current, _, e := a.s.Session("alice")
+	if e != nil {
+		t.Fatal(e)
+	}
+	past := time.Now().Add(-time.Minute).Unix()
+	if _, e = a.s.DB.Exec("UPDATE sessions SET expires=? WHERE id=?", past, expired); e != nil {
+		t.Fatal(e)
+	}
+	token, e := refreshToken()
+	if e != nil {
+		t.Fatal(e)
+	}
+	if _, e = a.s.DB.Exec("INSERT INTO auth_refresh(sid,token_hash) VALUES(?,?)", expired, refreshHash(token)); e != nil {
+		t.Fatal(e)
+	}
+	future := time.Now().Add(time.Hour).Unix()
+	if _, e = a.s.DB.Exec("INSERT INTO auth_failures(username,failures,locked_until) VALUES('expired-lock',5,?)", past); e != nil {
+		t.Fatal(e)
+	}
+	if _, e = a.s.DB.Exec("INSERT INTO auth_failures(username,failures,locked_until) VALUES('active-lock',5,?)", future); e != nil {
+		t.Fatal(e)
+	}
+	a.cleanupSessions(time.Now().Unix())
+	var n int
+	for _, q := range []string{
+		"SELECT count(*) FROM sessions WHERE id='" + expired + "'",
+		"SELECT count(*) FROM auth_refresh WHERE sid='" + expired + "'",
+		"SELECT count(*) FROM auth_failures WHERE username='expired-lock'",
+	} {
+		if err := a.s.DB.QueryRow(q).Scan(&n); err != nil || n != 0 {
+			t.Fatalf("expired row retained: %s (%d, %v)", q, n, err)
+		}
+	}
+	for _, q := range []string{
+		"SELECT count(*) FROM sessions WHERE id='" + current + "'",
+		"SELECT count(*) FROM auth_failures WHERE username='active-lock'",
+	} {
+		if err := a.s.DB.QueryRow(q).Scan(&n); err != nil || n != 1 {
+			t.Fatalf("live row removed: %s (%d, %v)", q, n, err)
+		}
+	}
+}

internal/api/config.go

@@ -63,20 +63,23 @@ func (a *API) config(w http.ResponseWriter, r *http.Request, i store.Instance, a
 			}
 		}
 	}
-	a.mu.Lock()
-	locked := true
+	inst := a.instanceLocks.Acquire(i.ID)
+	locked := inst
 	defer func() {
-		if locked {
-			a.mu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
-	if r.Method != http.MethodGet && a.workspaceReaders[i.ID] > 0 {
-		fail(w, 409, "instance files busy; retry later")
-		return
-	}
 	if r.Method != http.MethodGet {
-		workspaceMu.Lock()
-		defer workspaceMu.Unlock()
+		a.readersMu.Lock()
+		readers := a.workspaceReaders[i.ID]
+		a.readersMu.Unlock()
+		if readers > 0 {
+			fail(w, 409, "instance files busy; retry later")
+			return
+		}
+		user := a.userLocks.Acquire(i.UID)
+		defer user.Release()
 	}
 	current, e := a.s.GetInstance(i.ID)
 	if e != nil || current.UID != i.UID {
@@ -266,8 +269,8 @@ func (a *API) config(w http.ResponseWriter, r *http.Request, i store.Instance, a
 					fail(w, 400, "invalid backup")
 					return
 				}
-				a.mu.Unlock()
-				locked = false
+				locked.Release()
+				locked = nil
 				http.ServeContent(w, r, id+".dshbak", st.ModTime(), f)
 				return
 			}

internal/api/config_body_test.go

@@ -66,17 +66,14 @@ func TestConfigSlowBodyRevalidatesWithoutLocks(t *testing.T) {
 				case <-time.After(3 * time.Second):
 					t.Fatal("body read did not start")
 				}
-				if !a.mu.TryLock() {
-					t.Fatal("slow body holds API lock")
+				if a.instanceLocks.held() || a.userLocks.held() {
+					t.Fatal("slow body holds API locks")
 				}
 				if change == "lease" {
+					a.readersMu.Lock()
 					a.workspaceReaders[i.ID]++
+					a.readersMu.Unlock()
 				}
-				a.mu.Unlock()
-				if !workspaceMu.TryLock() {
-					t.Fatal("slow body holds workspace lock")
-				}
-				workspaceMu.Unlock()
 				want := 409
 				if change == "deleted" {
 					if _, err := a.s.DB.Exec("DELETE FROM instances WHERE id=?", i.ID); err != nil {
@@ -136,17 +133,17 @@ func TestConfigBodyErrorsReturnWithoutLocks(t *testing.T) {
 				r.Header.Set("Content-Type", "multipart/form-data; boundary=test")
 			}
 			w := &configDeadlineWriter{ResponseRecorder: httptest.NewRecorder()}
-			a.mu.Lock()
-			workspaceMu.Lock()
+			inst := a.instanceLocks.Acquire("")
+			user := a.userLocks.Acquire("")
 			done := make(chan struct{})
 			go func() { defer close(done); a.config(w, r, store.Instance{}, "backups") }()
 			select {
 			case <-done:
-				workspaceMu.Unlock()
-				a.mu.Unlock()
+				user.Release()
+				inst.Release()
 			case <-time.After(3 * time.Second):
-				workspaceMu.Unlock()
-				a.mu.Unlock()
+				user.Release()
+				inst.Release()
 				t.Fatal("body error waited for locks")
 			}
 			if w.Code != tc.want {
@@ -191,14 +188,12 @@ func TestConfigSlowResponseReleasesLocks(t *testing.T) {
 			case <-time.After(3 * time.Second):
 				t.Fatal("response write did not start")
 			}
-			if !a.mu.TryLock() {
-				t.Fatal("slow response holds API lock")
+			if a.instanceLocks.held() {
+				t.Fatal("slow response holds instance lock")
 			}
-			a.mu.Unlock()
-			if !workspaceMu.TryLock() {
-				t.Fatal("slow response holds workspace lock")
+			if a.userLocks.held() {
+				t.Fatal("slow response holds user lock")
 			}
-			workspaceMu.Unlock()
 
 		})
 	}

internal/api/files.go

@@ -13,7 +13,6 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -34,8 +33,6 @@ const (
 var zipSlots = make(chan struct{}, 4)
 var errZipLimit = errors.New("archive limit exceeded")
 
-var workspaceMu sync.Mutex
-
 func renameNoReplace(root *os.Root, old, next string) error {
 	anchor, err := root.Open(".")
 	if err != nil {
@@ -155,37 +152,41 @@ func (a *API) workspace(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if instanceID != "" {
-		// Pin this instance's home without holding the global lifecycle lock
+		// Pin this instance's home without holding the instance lifecycle lock
 		// across client reads/writes. Lifecycle mutations reject active leases.
-		a.mu.Lock()
+		inst := a.instanceLocks.Acquire(instanceID)
 		i, e := a.s.GetInstance(instanceID)
 		if e != nil || i.UID != u.ID {
-			a.mu.Unlock()
+			inst.Release()
 			fail(w, 404, "instance not found")
 			return
 		}
 		if r.Method != http.MethodGet && (i.Status == "error" || a.upgradeRecoveryRequired(i)) {
-			a.mu.Unlock()
+			inst.Release()
 			fail(w, 409, "instance requires recovery")
 			return
 		}
+		a.readersMu.Lock()
 		if a.workspaceReaders == nil {
 			a.workspaceReaders = make(map[string]int)
 		}
 		a.workspaceReaders[instanceID]++
-		a.mu.Unlock()
+		a.readersMu.Unlock()
+		inst.Release()
 		defer func() {
-			a.mu.Lock()
-			a.workspaceReaders[instanceID]--
-			if a.workspaceReaders[instanceID] == 0 {
-				delete(a.workspaceReaders, instanceID)
+			a.readersMu.Lock()
+			if a.workspaceReaders != nil {
+				a.workspaceReaders[instanceID]--
+				if a.workspaceReaders[instanceID] == 0 {
+					delete(a.workspaceReaders, instanceID)
+				}
 			}
-			a.mu.Unlock()
+			a.readersMu.Unlock()
 		}()
 	}
-	workspaceMu.Lock()
+	user := a.userLocks.Acquire(u.ID)
 	root, err := a.openWorkspace(u, instanceID)
-	workspaceMu.Unlock()
+	user.Release()
 	if err != nil {
 		fail(w, 500, "storage unavailable")
 		return
@@ -195,7 +196,7 @@ func (a *API) workspace(w http.ResponseWriter, r *http.Request) {
 	case "usage":
 		a.wsUsage(w, r, root, u)
 	case "tree":
-		a.wsTree(w, r, root)
+		a.wsTree(w, r, root, u)
 	case "mkdir":
 		a.wsMkdir(w, r, root, u)
 	case "rename":
@@ -223,9 +224,9 @@ func (a *API) wsUsage(w http.ResponseWriter, r *http.Request, root *os.Root, u s
 		fail(w, 405, "method not allowed")
 		return
 	}
-	workspaceMu.Lock()
+	user := a.userLocks.Acquire(u.ID)
 	n, e := a.userUsage(u)
-	workspaceMu.Unlock()
+	user.Release()
 	if e != nil {
 		fail(w, 500, "usage unavailable")
 		return
@@ -233,12 +234,12 @@ func (a *API) wsUsage(w http.ResponseWriter, r *http.Request, root *os.Root, u s
 	writeJSON(w, map[string]int64{"used_bytes": n, "quota_bytes": u.Quota})
 }
 
-func (a *API) wsTree(w http.ResponseWriter, r *http.Request, root *os.Root) {
-	workspaceMu.Lock()
-	locked := true
+func (a *API) wsTree(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
+	user := a.userLocks.Acquire(u.ID)
+	locked := user
 	defer func() {
-		if locked {
-			workspaceMu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
 	if r.Method != http.MethodGet {
@@ -281,8 +282,8 @@ func (a *API) wsTree(w http.ResponseWriter, r *http.Request, root *os.Root) {
 		out = append(out, map[string]any{"name": de.Name(), "path": p, "is_dir": st.IsDir(), "size": sz, "mod_time": st.ModTime()})
 	}
 	sort.Slice(out, func(i, j int) bool { return out[i]["name"].(string) < out[j]["name"].(string) })
-	workspaceMu.Unlock()
-	locked = false
+	locked.Release()
+	locked = nil
 	writeJSON(w, out)
 }
 
@@ -299,11 +300,11 @@ func (a *API) wsMkdir(w http.ResponseWriter, r *http.Request, root *os.Root, u s
 		return
 	}
 	p, e := safeRel(x.Path)
-	workspaceMu.Lock()
-	locked := true
+	user := a.userLocks.Acquire(u.ID)
+	locked := user
 	defer func() {
-		if locked {
-			workspaceMu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
 	if e != nil || p == "." || rootSafe(root, path.Dir(p), false) != nil {
@@ -318,8 +319,8 @@ func (a *API) wsMkdir(w http.ResponseWriter, r *http.Request, root *os.Root, u s
 		_ = f.Chown(1000, 1000)
 		f.Close()
 	}
-	workspaceMu.Unlock()
-	locked = false
+	locked.Release()
+	locked = nil
 	a.s.AddAudit(u.ID, "workspace.mkdir", p, "")
 	writeJSON(w, map[string]bool{"ok": true})
 }
@@ -339,11 +340,11 @@ func (a *API) wsRename(w http.ResponseWriter, r *http.Request, root *os.Root, u
 	}
 	p, e := safeRel(x.Path)
 	q, e2 := safeRel(x.NewPath)
-	workspaceMu.Lock()
-	locked := true
+	user := a.userLocks.Acquire(u.ID)
+	locked := user
 	defer func() {
-		if locked {
-			workspaceMu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
 	if e != nil || e2 != nil || p == "." || q == "." || rootSafe(root, p, false) != nil || rootSafe(root, path.Dir(q), false) != nil {
@@ -363,18 +364,18 @@ func (a *API) wsRename(w http.ResponseWriter, r *http.Request, root *os.Root, u
 		fail(w, 400, "rename failed")
 		return
 	}
-	workspaceMu.Unlock()
-	locked = false
+	locked.Release()
+	locked = nil
 	a.s.AddAudit(u.ID, "workspace.rename", p, q)
 	writeJSON(w, map[string]bool{"ok": true})
 }
 
 func (a *API) wsDelete(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
-	workspaceMu.Lock()
-	locked := true
+	user := a.userLocks.Acquire(u.ID)
+	locked := user
 	defer func() {
-		if locked {
-			workspaceMu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
 	if r.Method != http.MethodDelete {
@@ -402,8 +403,8 @@ func (a *API) wsDelete(w http.ResponseWriter, r *http.Request, root *os.Root, u
 		fail(w, 400, "delete failed")
 		return
 	}
-	workspaceMu.Unlock()
-	locked = false
+	locked.Release()
+	locked = nil
 	a.s.AddAudit(u.ID, "workspace.delete", p, "")
 	writeJSON(w, map[string]bool{"ok": true})
 }
@@ -437,11 +438,11 @@ func (a *API) wsUpload(w http.ResponseWriter, r *http.Request, root *os.Root, u
 		return
 	}
 	defer r.MultipartForm.RemoveAll()
-	workspaceMu.Lock()
-	locked := true
+	user := a.userLocks.Acquire(u.ID)
+	locked := user
 	defer func() {
-		if locked {
-			workspaceMu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
 	dir, e := safeRel(r.FormValue("path"))
@@ -503,8 +504,8 @@ func (a *API) wsUpload(w http.ResponseWriter, r *http.Request, root *os.Root, u
 		fail(w, 400, "upload failed")
 		return
 	}
-	workspaceMu.Unlock()
-	locked = false
+	locked.Release()
+	locked = nil
 	a.s.AddAudit(u.ID, "workspace.upload", target, "")
 	writeJSON(w, map[string]bool{"ok": true})
 }

internal/api/files_test.go

@@ -326,14 +326,12 @@ func TestZipTransferNoLocksAndConcurrencyLimit(t *testing.T) {
 	case <-time.After(2 * time.Second):
 		t.Fatal("download not started")
 	}
-	if !workspaceMu.TryLock() {
-		t.Fatal("workspace lock held across write")
+	if a.userLocks.held() {
+		t.Fatal("user write lock held across write")
 	}
-	workspaceMu.Unlock()
-	if !a.mu.TryLock() {
-		t.Fatal("lifecycle lock held across write")
+	if a.instanceLocks.held() {
+		t.Fatal("instance lock held across write")
 	}
-	a.mu.Unlock()
 	if len(w.deadlines) != 1 || w.deadlines[0].IsZero() {
 		t.Fatal("missing deadline")
 	}

internal/api/images.go

@@ -75,34 +75,38 @@ func (a *API) builds(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	buildQueue.Lock()
-	defer buildQueue.Unlock()
-	if err := a.ensureBuilds(); err != nil {
-		fail(w, 500, "database error")
-		return
-	}
 	if r.Method == http.MethodGet {
-		rows, err := a.s.DB.Query("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds ORDER BY created_at DESC,id DESC")
+		buildQueue.Lock()
+		err := a.ensureBuilds()
+		var rows *sql.Rows
+		if err == nil {
+			rows, err = a.s.DB.Query("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds ORDER BY created_at DESC,id DESC")
+		}
+		result := []buildRecord{}
+		if err == nil {
+			for rows.Next() {
+				var b buildRecord
+				var finished, recipe, platform sql.NullString
+				if err = rows.Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished); err != nil {
+					break
+				}
+				b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
+				result = append(result, b)
+			}
+			scanErr := rows.Err()
+			rows.Close()
+			if err == nil {
+				err = scanErr
+			}
+		}
+		buildQueue.Unlock()
 		if err != nil {
 			fail(w, 500, "database error")
 			return
 		}
-		result := []buildRecord{}
-		for rows.Next() {
-			var b buildRecord
-			var finished, recipe, platform sql.NullString
-			if err = rows.Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished); err != nil {
-				break
-			}
-			b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
-			result = append(result, b)
-		}
-		scanErr := rows.Err()
-		rows.Close()
-		if err != nil || scanErr != nil {
-			fail(w, 500, "database error")
-			return
-		}
+		// Docker image reconciliation is I/O-bound and updates rows with
+		// compare-and-set, so it must not hold the queue lock that instance
+		// creation and upgrades also need.
 		if err := a.reconcileBuilds(r.Context(), result); err != nil {
 			fail(w, 502, "docker image reconciliation failed")
 			return
@@ -111,7 +115,7 @@ func (a *API) builds(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, result)
 		return
 	}
-	b, err := a.enqueueBuildLocked(r.Context(), u.ID, in.Version)
+	b, err := a.enqueueBuild(r.Context(), u.ID, in.Version)
 	if err != nil {
 		status := 502
 		if errors.Is(err, errBuildRateLimited) {
@@ -128,54 +132,110 @@ func (a *API) builds(w http.ResponseWriter, r *http.Request) {
 
 var errBuildRateLimited = errors.New("build limit: three new tasks per user per hour")
 
-func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecord, error) {
-	buildQueue.Lock()
-	defer buildQueue.Unlock()
-	if !validBuildVersion(version) {
-		return buildRecord{}, errors.New("a fixed version is required")
-	}
-	if err := a.ensureBuilds(); err != nil {
-		return buildRecord{}, err
-	}
-	return a.enqueueBuildLocked(ctx, uid, version)
-}
-
-func (a *API) enqueueBuildLocked(ctx context.Context, uid, version string) (buildRecord, error) {
+// lookupActiveBuild returns the active or successful record for a version; it
+// performs no I/O and must be called with the build queue held.
+func (a *API) lookupActiveBuild(version string) (buildRecord, bool, error) {
 	var b buildRecord
 	var finished, recipe, platform sql.NullString
 	err := a.s.DB.QueryRow("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds WHERE version=? AND recipe_rev=? AND platform=? AND status IN ('queued','running','validating','success')", version, imageRecipeRev, imagePlatform).Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished)
 	b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
-	if err == nil && b.Status == "success" {
-		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
-		id, lookupErr := a.localBuildImage(ctx, b.Version, b.ImageID)
+	if errors.Is(err, sql.ErrNoRows) {
+		return b, false, nil
+	}
+	if err != nil {
+		return b, false, err
+	}
+	return b, true, nil
+}
+
+func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecord, error) {
+	if !validBuildVersion(version) {
+		return buildRecord{}, errors.New("a fixed version is required")
+	}
+	buildQueue.Lock()
+	err := a.ensureBuilds()
+	var b buildRecord
+	var found bool
+	if err == nil {
+		b, found, err = a.lookupActiveBuild(version)
+	}
+	buildQueue.Unlock()
+	if err != nil {
+		return b, err
+	}
+	if found && b.Status != "success" {
+		a.startBuildWorker()
+		return b, nil
+	}
+	if found {
+		// Verifying a previously built image talks to Docker; keep that I/O
+		// out of the queue lock shared with instance creation and upgrades.
+		verify, cancel := context.WithTimeout(ctx, 15*time.Second)
+		id, lookupErr := a.localBuildImage(verify, b.Version, b.ImageID)
 		cancel()
 		if lookupErr != nil {
 			return b, lookupErr
 		}
-		if id != b.ImageID || id == "" {
-			if _, err = a.s.DB.Exec("UPDATE image_builds SET status='missing',error='image no longer available' WHERE id=?", b.ID); err == nil {
-				err = sql.ErrNoRows
-			}
+		if id != "" && id == b.ImageID {
+			a.startBuildWorker()
+			return b, nil
 		}
-	}
-	if errors.Is(err, sql.ErrNoRows) {
-		var count int
-		if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests WHERE user_id=? AND created_at>?", uid, time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)).Scan(&count); err != nil {
+		buildQueue.Lock()
+		if _, err = a.s.DB.Exec("UPDATE image_builds SET status='missing',error='image no longer available' WHERE id=? AND status='success'", b.ID); err == nil {
+			// Another request may have queued a replacement while the image
+			// was being verified; honor it instead of inserting a duplicate.
+			b, found, err = a.lookupActiveBuild(version)
+		}
+		if err == nil && !found {
+			b, err = a.insertBuild(uid, version)
+		}
+		buildQueue.Unlock()
+		if err != nil {
 			return b, err
 		}
-		if count >= 3 {
-			return b, errBuildRateLimited
-		}
-		b = buildRecord{ID: store.ID(), Version: version, Status: "queued", Created: time.Now().UTC().Format(time.RFC3339Nano), RequestedBy: uid, RecipeRev: imageRecipeRev, Platform: imagePlatform}
-		err = a.insertBuildRequest(b, uid)
+		a.startBuildWorker()
+		return b, nil
 	}
+	buildQueue.Lock()
+	defer buildQueue.Unlock()
+	// Re-check after the unlocked phases: another request may have inserted.
+	b, found, err = a.lookupActiveBuild(version)
 	if err != nil {
 		return b, err
 	}
-	a.startBuildWorker()
+	if found {
+		return b, nil
+	}
+	b, err = a.insertBuild(uid, version)
+	if err != nil {
+		return b, err
+	}
+	a.startBuildWorkerLocked()
 	return b, nil
 }
+
+// insertBuild applies the per-user rate limit and inserts a queued record;
+// call with the build queue held.
+func (a *API) insertBuild(uid, version string) (buildRecord, error) {
+	var count int
+	if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests WHERE user_id=? AND created_at>?", uid, time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)).Scan(&count); err != nil {
+		return buildRecord{}, err
+	}
+	if count >= 3 {
+		return buildRecord{}, errBuildRateLimited
+	}
+	b := buildRecord{ID: store.ID(), Version: version, Status: "queued", Created: time.Now().UTC().Format(time.RFC3339Nano), RequestedBy: uid, RecipeRev: imageRecipeRev, Platform: imagePlatform}
+	return b, a.insertBuildRequest(b, uid)
+}
 func (a *API) startBuildWorker() {
+	buildQueue.Lock()
+	defer buildQueue.Unlock()
+	a.startBuildWorkerLocked()
+}
+
+// startBuildWorkerLocked requires the caller to hold buildQueue: the active map
+// is only safe under that lock.
+func (a *API) startBuildWorkerLocked() {
 	if buildQueue.active[a.s.DB] {
 		return
 	}

internal/api/images_delete.go

@@ -26,9 +26,11 @@ func (a *API) deleteImage(w http.ResponseWriter, r *http.Request) {
 		fail(w, 400, "invalid image_id")
 		return
 	}
-	// Upgrades also acquire a.mu before buildQueue. Keep both through deletion.
-	a.mu.Lock()
-	defer a.mu.Unlock()
+	// Hold the build queue through the checks and the Docker deletion so a
+	// record cannot flip to missing mid-flight. Upgrades write their journal
+	// entry before creating containers; the transaction below re-checks that
+	// journal, and an upgrade racing an administrator deletion fails safely
+	// with its own rollback path.
 	buildQueue.Lock()
 	defer buildQueue.Unlock()
 	if buildQueue.active[a.s.DB] {
@@ -43,7 +45,7 @@ func (a *API) deleteImage(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	defer tx.Rollback()
-	// Take SQLite's writer lock before checking intents: reservation does not use a.mu.
+	// Take SQLite's writer lock before checking intents: reservation does not use the build queue.
 	if _, err = tx.ExecContext(ctx, "UPDATE images SET status=status WHERE image_id=?", id); err != nil {
 		fail(w, 500, "image index unavailable")
 		return
@@ -132,6 +134,9 @@ func (a *API) deleteImage(w http.ResponseWriter, r *http.Request) {
 		fail(w, status, "docker image deletion failed")
 		return
 	}
+	// Docker deletion is already authoritative, even if the DB commit fails.
+	// Any inventory observation taken before it must be discarded.
+	a.imageDeleteGeneration++
 	if err = tx.Commit(); err != nil {
 		fail(w, 500, "image deleted but recording failed")
 		return

internal/api/images_enqueue_concurrency_test.go

@@ -0,0 +1,128 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestEnqueueBuildConcurrentMissingImageReturnsReplacement(t *testing.T) {
+	inspections := make(chan chan struct{}, 2)
+	stop := make(chan struct{})
+	a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			t.Errorf("unexpected Docker method: %s", r.Method)
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		switch r.URL.Path {
+		case "/v1.47/images/json":
+			writeJSON(w, []map[string]any{{"Id": "old-image", "Labels": map[string]string{
+				"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": imageRecipeRev,
+			}}})
+		case "/v1.47/images/old-image/json":
+			release := make(chan struct{})
+			select {
+			case inspections <- release:
+			case <-stop:
+				return
+			}
+			select {
+			case <-release:
+				w.WriteHeader(http.StatusNotFound)
+			case <-stop:
+			case <-r.Context().Done():
+			}
+		default:
+			t.Errorf("unexpected Docker path: %s", r.URL.Path)
+			w.WriteHeader(http.StatusInternalServerError)
+		}
+	})
+	seedVerifiedImage(t, a, "old-image", "1.2.3")
+	var oldID string
+	if err := a.s.DB.QueryRow("SELECT id FROM image_builds WHERE image_id='old-image'").Scan(&oldID); err != nil {
+		t.Fatal(err)
+	}
+	// Keep the replacement queued without starting a background worker.
+	buildQueue.Lock()
+	buildQueue.active[a.s.DB] = true
+	buildQueue.Unlock()
+	ctx, cancel := context.WithCancel(context.Background())
+	var submissions sync.WaitGroup
+	t.Cleanup(func() {
+		close(stop)
+		cancel()
+		submissions.Wait()
+		buildQueue.Lock()
+		delete(buildQueue.active, a.s.DB)
+		delete(buildQueue.initialized, a.s.DB)
+		buildQueue.Unlock()
+	})
+	type result struct {
+		build buildRecord
+		err   error
+	}
+	submit := func() <-chan result {
+		done := make(chan result, 1)
+		submissions.Add(1)
+		go func() {
+			defer submissions.Done()
+			b, err := a.enqueueBuild(ctx, "u", "1.2.3")
+			done <- result{b, err}
+		}()
+		return done
+	}
+	waitInspection := func() chan struct{} {
+		t.Helper()
+		select {
+		case release := <-inspections:
+			return release
+		case <-time.After(3 * time.Second):
+			t.Fatal("submission did not inspect the old image")
+			return nil
+		}
+	}
+	waitResult := func(done <-chan result) buildRecord {
+		t.Helper()
+		select {
+		case got := <-done:
+			if got.err != nil {
+				t.Fatal(got.err)
+			}
+			return got.build
+		case <-time.After(3 * time.Second):
+			t.Fatal("submission did not finish")
+			return buildRecord{}
+		}
+	}
+	firstDone := submit()
+	firstRelease := waitInspection()
+	secondDone := submit()
+	secondRelease := waitInspection()
+	close(firstRelease)
+	first := waitResult(firstDone)
+	if first.ID == "" || first.ID == oldID || first.Status != "queued" {
+		t.Fatalf("first submission did not queue a replacement: %+v", first)
+	}
+	close(secondRelease)
+	second := waitResult(secondDone)
+	if second.ID != first.ID || second.Status != "queued" {
+		t.Errorf("second submission returned id=%q status=%q; want replacement id=%q status=queued", second.ID, second.Status, first.ID)
+	}
+	var status string
+	if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id=?", oldID).Scan(&status); err != nil || status != "missing" {
+		t.Errorf("old build status=%q, err=%v; want missing", status, err)
+	}
+	for query, want := range map[string]int{
+		"SELECT count(*) FROM image_builds":                       2,
+		"SELECT count(*) FROM image_builds WHERE status='queued'": 1,
+		"SELECT count(*) FROM image_build_requests":               1,
+	} {
+		var count int
+		if err := a.s.DB.QueryRow(query).Scan(&count); err != nil || count != want {
+			t.Errorf("%s: count=%d, err=%v; want %d", query, count, err, want)
+		}
+	}
+}

internal/api/images_index_concurrency_test.go

@@ -0,0 +1,140 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+func TestImageIndexDoesNotUndoConcurrentDelete(t *testing.T) {
+	for _, initial := range []string{"available", "missing"} {
+		t.Run(initial, func(t *testing.T) {
+			id := "sha256:" + strings.Repeat("a", 64)
+			blocker := "sha256:" + strings.Repeat("b", 64)
+			reached, release := make(chan struct{}), make(chan struct{})
+			var once sync.Once
+			var reachedOnce sync.Once
+			var deleted atomic.Bool
+			unblock := func() { once.Do(func() { close(release) }) }
+			a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
+				switch {
+				case r.Method == http.MethodDelete:
+					deleted.Store(true)
+					writeJSON(w, []any{})
+				case r.URL.Path == "/v1.47/images/"+id+"/json":
+					if deleted.Load() {
+						w.WriteHeader(http.StatusNotFound)
+						return
+					}
+					writeImageInspect(w, id, "1.2.3")
+				case r.URL.Path == "/v1.47/images/"+blocker+"/json":
+					reachedOnce.Do(func() { close(reached) })
+					select {
+					case <-release:
+					case <-r.Context().Done():
+						return
+					}
+					w.WriteHeader(http.StatusNotFound)
+				case r.URL.Path == "/v1.47/images/json", r.URL.Path == "/v1.47/containers/json":
+					writeJSON(w, []any{})
+				default:
+					t.Errorf("unexpected Docker request: %s %s", r.Method, r.URL)
+					w.WriteHeader(500)
+				}
+			})
+			defer unblock()
+			seedVerifiedImage(t, a, id, "1.2.3")
+			for _, q := range []struct {
+				sql  string
+				args []any
+			}{
+				{"UPDATE users SET admin=1 WHERE id='u'", nil},
+				{"INSERT INTO images(image_id,version,status,built_at) VALUES(?,'1.2.3',?,'2'),(?,'2.0.0','missing','1')", []any{id, initial, blocker}},
+			} {
+				if _, err := a.s.DB.Exec(q.sql, q.args...); err != nil {
+					t.Fatal(err)
+				}
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			type outcome struct {
+				images []imageRecord
+				err    error
+			}
+			done := make(chan outcome, 1)
+			go func() { images, err := a.imageIndex(ctx); done <- outcome{images, err} }()
+			select {
+			case <-reached:
+			case <-ctx.Done():
+				t.Fatal("index did not reach second image")
+			}
+			r := httptest.NewRequest(http.MethodDelete, "/api/images?image_id="+id, nil).WithContext(ctx)
+			r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
+			addTestAccess(r, sid)
+			w := httptest.NewRecorder()
+			a.images(w, r)
+			if w.Code != 200 {
+				t.Fatalf("delete: %d %s", w.Code, w.Body.String())
+			}
+			unblock()
+			select {
+			case out := <-done:
+				if out.err != nil {
+					t.Fatal(out.err)
+				}
+				matches := 0
+				for _, im := range out.images {
+					if im.ImageID == id {
+						matches++
+					}
+					if im.ImageID == id && im.Status != "missing" {
+						t.Errorf("returned deleted image as %s", im.Status)
+					}
+				}
+				if matches != 1 {
+					t.Errorf("returned %d records for deleted image, want 1", matches)
+				}
+			case <-ctx.Done():
+				t.Fatal("index did not finish")
+			}
+			var status string
+			if err := a.s.DB.QueryRow("SELECT status FROM images WHERE image_id=?", id).Scan(&status); err != nil {
+				t.Fatal(err)
+			}
+			if status != "missing" {
+				t.Errorf("persisted deleted image as %s", status)
+			}
+		})
+	}
+}
+
+func TestImageIndexBoundsInvalidatedSnapshots(t *testing.T) {
+	var a *API
+	var calls atomic.Int32
+	a, _ = imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1.47/images/json" {
+			t.Errorf("unexpected request: %s", r.URL)
+			w.WriteHeader(500)
+			return
+		}
+		// Model a completed deletion during every unlocked observation phase.
+		buildQueue.Lock()
+		a.imageDeleteGeneration++
+		buildQueue.Unlock()
+		calls.Add(1)
+		writeJSON(w, []any{})
+	})
+	images, err := a.imageIndex(context.Background())
+	if !errors.Is(err, errImageIndexChanged) || images != nil {
+		t.Fatalf("images=%v err=%v; want no inventory and invalidation error", images, err)
+	}
+	if calls.Load() != 3 {
+		t.Fatalf("attempts=%d, want 3", calls.Load())
+	}
+}

internal/api/images_policy.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -72,49 +73,69 @@ func (a *API) imageMatches(ctx context.Context, id, version string) (bool, error
 	return im.ID == id && im.platform() == imagePlatform && im.Config.Labels["dsh-mgr.role"] == "runtime" && im.Config.Labels["dsh-mgr.version"] == version && im.Config.Labels["dsh-mgr.recipe"] == imageRecipeRev, nil
 }
 
+var errImageIndexChanged = errors.New("image inventory changed during reconciliation")
+
 func (a *API) imageIndex(ctx context.Context) ([]imageRecord, error) {
+	// Retry stale observations without holding the queue across Docker I/O.
+	// Bound retries so sustained deletions cannot keep a request alive forever.
+	for attempt := 0; attempt < 3; attempt++ {
+		result, err := a.imageIndexSnapshot(ctx)
+		if !errors.Is(err, errImageIndexChanged) {
+			return result, err
+		}
+	}
+	return nil, errImageIndexChanged
+}
+
+func (a *API) imageIndexSnapshot(ctx context.Context) ([]imageRecord, error) {
+	// Phase 1: read the managed index under the queue lock (DB only).
 	buildQueue.Lock()
-	defer buildQueue.Unlock()
-	if err := a.ensureBuilds(); err != nil {
-		return nil, err
+	generation := a.imageDeleteGeneration
+	err := a.ensureBuilds()
+	if err == nil {
+		_, err = a.s.DB.ExecContext(ctx, `INSERT OR IGNORE INTO images(image_id,version,recipe_rev,platform,tag,status,built_at)
+		SELECT image_id,version,recipe_rev,platform,'','missing',COALESCE(finished_at,created_at) FROM image_builds WHERE status IN ('success','missing') AND image_id<>'' ORDER BY created_at DESC,id DESC`)
 	}
-	if _, err := a.s.DB.ExecContext(ctx, `INSERT OR IGNORE INTO images(image_id,version,recipe_rev,platform,tag,status,built_at)
-		SELECT image_id,version,recipe_rev,platform,'','missing',COALESCE(finished_at,created_at) FROM image_builds WHERE status IN ('success','missing') AND image_id<>'' ORDER BY created_at DESC,id DESC`); err != nil {
-		return nil, err
-	}
-	rows, err := a.s.DB.QueryContext(ctx, `SELECT image_id,version,COALESCE(recipe_rev,''),COALESCE(platform,''),tag,status,built_at,
+	var rows *sql.Rows
+	if err == nil {
+		rows, err = a.s.DB.QueryContext(ctx, `SELECT image_id,version,COALESCE(recipe_rev,''),COALESCE(platform,''),tag,status,built_at,
 		(SELECT count(*) FROM instances WHERE instances.image_id=images.image_id) FROM images ORDER BY built_at DESC,image_id`)
-	if err != nil {
-		return nil, err
 	}
 	result := []imageRecord{}
-	for rows.Next() {
-		var im imageRecord
-		if err = rows.Scan(&im.ImageID, &im.Version, &im.RecipeRev, &im.Platform, &im.Tag, &im.Status, &im.Created, &im.UsageCount); err != nil {
-			break
+	if err == nil {
+		for rows.Next() {
+			var im imageRecord
+			if err = rows.Scan(&im.ImageID, &im.Version, &im.RecipeRev, &im.Platform, &im.Tag, &im.Status, &im.Created, &im.UsageCount); err != nil {
+				break
+			}
+			im.ID = im.ImageID
+			result = append(result, im)
+		}
+		scanErr := rows.Err()
+		rows.Close()
+		if err == nil {
+			err = scanErr
 		}
-		im.ID = im.ImageID
-		result = append(result, im)
 	}
-	scanErr := rows.Err()
-	rows.Close()
+	buildQueue.Unlock()
 	if err != nil {
 		return nil, err
 	}
-	if scanErr != nil {
-		return nil, scanErr
+	// Phase 2: inspect Docker without the queue lock; per-image state comes
+	// from real daemon observations, and instance creation must not wait here.
+	known := make(map[string]bool, len(result))
+	for _, im := range result {
+		known[im.ImageID] = true
 	}
-	filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime"}})
 	var local []struct {
 		ID     string
 		Labels map[string]string
 	}
-	if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &local); err != nil {
-		return nil, err
-	}
-	known := make(map[string]bool, len(result))
-	for _, im := range result {
-		known[im.ImageID] = true
+	{
+		filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime"}})
+		if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &local); err != nil {
+			return nil, err
+		}
 	}
 	for _, im := range local {
 		if im.ID != "" && !known[im.ID] && im.Labels["dsh-mgr.role"] == "runtime" {
@@ -154,6 +175,12 @@ func (a *API) imageIndex(ctx context.Context) ([]imageRecord, error) {
 			im.Status = "available"
 		}
 	}
+	// Phase 3: write observed state back under the queue lock (DB only).
+	buildQueue.Lock()
+	defer buildQueue.Unlock()
+	if generation != a.imageDeleteGeneration {
+		return nil, errImageIndexChanged
+	}
 	tx, err := a.s.DB.BeginTx(ctx, nil)
 	if err != nil {
 		return nil, err
@@ -179,7 +206,7 @@ func (a *API) StartImageBuilds() error {
 	if err := a.ensureBuilds(); err != nil {
 		return err
 	}
-	a.startBuildWorker()
+	a.startBuildWorkerLocked()
 	return nil
 }
 

internal/api/locks.go

@@ -0,0 +1,56 @@
+package api
+
+import "sync"
+
+// keyedLocks hands out one mutex per string key so slow work for one instance
+// or user never blocks unrelated work behind a shared global lock. Registry
+// entries are removed once the last holder releases, keeping the map bounded.
+type keyedLocks struct {
+	mu    sync.Mutex
+	locks map[string]*keyedLock
+}
+
+type keyedLock struct {
+	owner *keyedLocks
+	key   string
+	mu    sync.Mutex
+	refs  int
+}
+
+// Acquire registers interest in the key's lock (blocking callers keep the
+// registry entry alive) and then blocks until the mutex is held.
+func (k *keyedLocks) Acquire(key string) *keyedLock {
+	k.mu.Lock()
+	if k.locks == nil {
+		k.locks = make(map[string]*keyedLock)
+	}
+	l, ok := k.locks[key]
+	if !ok {
+		l = &keyedLock{owner: k, key: key}
+		k.locks[key] = l
+	}
+	l.refs++
+	k.mu.Unlock()
+	l.mu.Lock()
+	return l
+}
+
+// Release frees the mutex and drops the registry entry when no holder or
+// waiter remains.
+func (l *keyedLock) Release() {
+	l.mu.Unlock()
+	k := l.owner
+	k.mu.Lock()
+	l.refs--
+	if l.refs == 0 {
+		delete(k.locks, l.key)
+	}
+	k.mu.Unlock()
+}
+
+// held reports whether any keyed lock is currently held or being waited for.
+func (k *keyedLocks) held() bool {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	return len(k.locks) > 0
+}

internal/api/orphans.go

@@ -101,8 +101,9 @@ func (a *API) orphans(w http.ResponseWriter, r *http.Request) {
 	}
 	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
 	defer cancel()
-	a.mu.Lock()
-	defer a.mu.Unlock()
+	// checkOrphan re-verifies instance rows, upgrade journal entries and mount
+	// identities before removal, so no lifecycle lock is needed here: a
+	// container is only reclaimed when its instance row is already gone.
 	switch r.Method {
 	case "GET":
 		filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=dsh"}})

internal/api/quota_write.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-// Call with workspaceMu held through the write. Runtime writes remain soft-limited.
+// Call with the owner's user lock held through the write. Runtime writes remain soft-limited.
 func (a *API) quotaGrowth(uid string, growth int64) error {
 	var u store.User
 	u.ID = uid

internal/api/quota_write_test.go

@@ -76,8 +76,8 @@ func TestQuotaSerializedWriters(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			workspaceMu.Lock()
-			defer workspaceMu.Unlock()
+			user := a.userLocks.Acquire(i.UID)
+			defer user.Release()
 			if a.quotaGrowth(i.UID, 4) == nil {
 				if err := os.WriteFile(filepath.Join(home, name), []byte("1234"), 0600); err != nil {
 					t.Error(err)

internal/api/reconcile.go

@@ -11,9 +11,8 @@ import (
 
 // Reconcile checks persisted identities against Docker without deleting unknown
 // containers. Orphans require explicit administrator review, never blind adoption.
+// Runs before the HTTP server starts, so no request can race this pass.
 func (a *API) Reconcile(ctx context.Context) error {
-	a.mu.Lock()
-	defer a.mu.Unlock()
 	filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.instance"}})
 	var containers []struct {
 		ID     string

internal/api/runtime.go

@@ -130,8 +130,12 @@ func (a *API) instances(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance, trustedHost string) (store.Instance, error) {
-	a.mu.Lock()
-	defer a.mu.Unlock()
+	// The instance row is visible to its owner while creating, so hold the
+	// instance lock for the whole creation: a concurrent DELETE or file lease
+	// then waits instead of interleaving with container setup. Concurrent
+	// creations of different instances remain independent.
+	inst := a.instanceLocks.Acquire(i.ID)
+	defer inst.Release()
 	if _, err := a.s.GetInstance(i.ID); err != nil {
 		return i, errors.New("instance reservation unavailable")
 	}
@@ -317,8 +321,8 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if action == "" && r.Method == "DELETE" {
-		a.mu.Lock()
-		defer a.mu.Unlock()
+		inst := a.instanceLocks.Acquire(p[0])
+		defer inst.Release()
 		i, err = a.s.GetInstance(p[0])
 		if err != nil || i.UID != u.ID {
 			fail(w, 404, "not found")
@@ -328,7 +332,10 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
 			fail(w, 409, "instance requires recovery")
 			return
 		}
-		if a.workspaceReaders[i.ID] > 0 {
+		a.readersMu.Lock()
+		readers := a.workspaceReaders[i.ID]
+		a.readersMu.Unlock()
+		if readers > 0 {
 			fail(w, 409, "instance files busy; retry later")
 			return
 		}
@@ -359,8 +366,8 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
 		fail(w, 405, "method not allowed")
 		return
 	}
-	a.mu.Lock()
-	defer a.mu.Unlock()
+	inst := a.instanceLocks.Acquire(p[0])
+	defer inst.Release()
 	i, err = a.s.GetInstance(p[0])
 	if err != nil || i.UID != u.ID {
 		fail(w, 404, "not found")
@@ -370,7 +377,10 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
 		fail(w, 409, "instance requires recovery")
 		return
 	}
-	if a.workspaceReaders[i.ID] > 0 {
+	a.readersMu.Lock()
+	readers := a.workspaceReaders[i.ID]
+	a.readersMu.Unlock()
+	if readers > 0 {
 		fail(w, 409, "instance files busy; retry later")
 		return
 	}
@@ -740,14 +750,18 @@ func (a *API) proxy(w http.ResponseWriter, r *http.Request) {
 		fail(w, 401, "authentication required")
 		return
 	}
-	a.mu.Lock()
-	locked := true
+	p := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/dsh/"), "/", 2)
+	if p[0] == "" {
+		fail(w, 404, "not found")
+		return
+	}
+	inst := a.instanceLocks.Acquire(p[0])
+	locked := inst
 	defer func() {
-		if locked {
-			a.mu.Unlock()
+		if locked != nil {
+			locked.Release()
 		}
 	}()
-	p := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/dsh/"), "/", 2)
 	i, err := a.s.GetInstance(p[0])
 	if err != nil || i.UID != u.ID {
 		fail(w, 404, "not found")
@@ -807,8 +821,8 @@ func (a *API) proxy(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	cancel()
-	a.mu.Unlock()
-	locked = false
+	locked.Release()
+	locked = nil
 	proxy := runtimeReverseProxy(target, prefix, host, cookies)
 	a.invalidateRejectedRuntimeCookie(proxy, i, c.State.StartedAt, host, cookies)
 	if bridge {

internal/api/runtime_test.go

@@ -67,6 +67,45 @@ func TestRuntimeProxyRejectsErrorState(t *testing.T) {
 	}
 }
 
+// createRuntime holds the instance lock during creation; a concurrent DELETE
+// must wait instead of removing the row while the container is still starting.
+func TestInstanceDeleteWaitsForInFlightCreate(t *testing.T) {
+	a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/networks/") {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		t.Errorf("unexpected Docker call %s %s", r.Method, r.URL.Path)
+	})
+	if err := a.s.AddInstance(store.Instance{ID: "creating", UID: "owner", Status: "creating", Created: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	inst := a.instanceLocks.Acquire("creating")
+	done := make(chan *httptest.ResponseRecorder, 1)
+	go func() {
+		w := httptest.NewRecorder()
+		a.instance(w, runtimeRequest(http.MethodDelete, "/api/instances/creating", "", sid))
+		done <- w
+	}()
+	select {
+	case w := <-done:
+		t.Fatalf("delete interleaved with creation: %d %s", w.Code, w.Body.String())
+	case <-time.After(300 * time.Millisecond):
+	}
+	inst.Release()
+	select {
+	case w := <-done:
+		if w.Code != 200 {
+			t.Fatalf("status %d: %s", w.Code, w.Body.String())
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("delete stalled after creation finished")
+	}
+	if _, err := a.s.GetInstance("creating"); err == nil {
+		t.Fatal("instance row retained")
+	}
+}
+
 func TestRuntimeCreateLimitAndPolicy(t *testing.T) {
 	if os.Geteuid() != 0 {
 		t.Skip("chown requires root")

internal/api/upgrade.go

@@ -242,7 +242,7 @@ func upgradeCopy(ctx context.Context, r *os.Root, source, target string, limit i
 	return upgradeSync(r)
 }
 
-// Call while holding a.mu before any operation that can start or mutate a runtime.
+// Call while holding the instance lock before any operation that can start or mutate a runtime.
 func (a *API) upgradeRecoveryRequired(i store.Instance) bool {
 	var exists int
 	if err := a.s.DB.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='upgrade_journal'").Scan(&exists); err != nil {
@@ -275,14 +275,17 @@ func (a *API) upgrade(w http.ResponseWriter, r *http.Request, i store.Instance)
 		fail(w, 400, "invalid image_id")
 		return
 	}
-	a.mu.Lock()
-	defer a.mu.Unlock()
-	if a.workspaceReaders[i.ID] > 0 {
+	inst := a.instanceLocks.Acquire(i.ID)
+	defer inst.Release()
+	a.readersMu.Lock()
+	readers := a.workspaceReaders[i.ID]
+	a.readersMu.Unlock()
+	if readers > 0 {
 		fail(w, 409, "instance files busy; retry later")
 		return
 	}
-	workspaceMu.Lock()
-	defer workspaceMu.Unlock()
+	user := a.userLocks.Acquire(i.UID)
+	defer user.Release()
 	current, err := a.s.GetInstance(i.ID)
 	if err != nil || current.UID != i.UID {
 		fail(w, 404, "not found")

internal/api/workspace_concurrency_test.go

@@ -55,11 +55,12 @@ func TestSlowDownloadDoesNotHoldLifecycleLock(t *testing.T) {
 	case <-time.After(3 * time.Second):
 		t.Fatal("download not started")
 	}
-	if !a.mu.TryLock() {
-		t.Fatal("slow network holds global lifecycle lock")
+	if a.instanceLocks.held() {
+		t.Fatal("slow network holds instance lifecycle lock")
 	}
+	a.readersMu.Lock()
 	active := a.workspaceReaders["one"]
-	a.mu.Unlock()
+	a.readersMu.Unlock()
 	if active != 1 {
 		t.Fatalf("lease count %d", active)
 	}
@@ -174,14 +175,12 @@ func TestSlowUploadDoesNotHoldLifecycleLock(t *testing.T) {
 	case <-time.After(3 * time.Second):
 		t.Fatal("upload not reading body")
 	}
-	if !a.mu.TryLock() {
-		t.Fatal("slow upload holds global lock")
+	if a.instanceLocks.held() {
+		t.Fatal("slow upload holds instance lock")
 	}
-	a.mu.Unlock()
-	if !workspaceMu.TryLock() {
-		t.Fatal("slow upload holds workspace write lock")
+	if a.userLocks.held() {
+		t.Fatal("slow upload holds user write lock")
 	}
-	workspaceMu.Unlock()
 }
 
 func TestWorkspaceErrorResponseReleasesLocks(t *testing.T) {
@@ -204,16 +203,96 @@ func TestWorkspaceErrorResponseReleasesLocks(t *testing.T) {
 	case <-time.After(3 * time.Second):
 		t.Fatal("response not started")
 	}
-	if !workspaceMu.TryLock() {
-		t.Fatal("error response holds workspace lock")
+	if a.userLocks.held() {
+		t.Fatal("error response holds user write lock")
 	}
-	workspaceMu.Unlock()
-	if !a.mu.TryLock() {
-		t.Fatal("error response holds lifecycle lock")
+	if a.instanceLocks.held() {
+		t.Fatal("error response holds instance lock")
 	}
+	a.readersMu.Lock()
 	leases := len(a.workspaceReaders)
-	a.mu.Unlock()
+	a.readersMu.Unlock()
 	if leases != 0 {
 		t.Fatal("response retains lease")
 	}
 }
+
+// A restore or upgrade holds its instance and user locks for minutes; that
+// must never block file operations or lifecycle actions of other users.
+func TestLongMutationDoesNotBlockOtherUsers(t *testing.T) {
+	a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/stop"):
+			w.WriteHeader(http.StatusNoContent)
+		case r.Method == "GET" && strings.HasSuffix(r.URL.Path, "/json"):
+			writeJSON(w, map[string]any{"Id": "cid-two", "State": map[string]any{"Status": "exited", "Running": false}})
+		default:
+			t.Errorf("unexpected Docker call %s %s", r.Method, r.URL.Path)
+		}
+	})
+	if err := a.s.CreateUser(store.User{ID: "other", Username: "other"}, "unused"); err != nil {
+		t.Fatal(err)
+	}
+	otherSid, _, err := a.s.Session("other")
+	if err != nil {
+		t.Fatal(err)
+	}
+	registerTestAccess(t, a, otherSid)
+	if err := a.s.AddInstance(store.Instance{ID: "one", UID: "owner"}); err != nil {
+		t.Fatal(err)
+	}
+	if err := a.s.AddInstance(store.Instance{ID: "two", UID: "other", ContainerID: "cid-two", Status: "running", Created: time.Now()}); err != nil {
+		t.Fatal(err)
+	}
+	for _, dir := range []string{"workspaces/owner/instances/one/home", "workspaces/other/instances/two/home"} {
+		if err := os.MkdirAll(filepath.Join(a.root, dir), 0750); err != nil {
+			t.Fatal(err)
+		}
+	}
+	// Simulate a mutation paused mid-flight (restore/upgrade state).
+	inst := a.instanceLocks.Acquire("one")
+	defer inst.Release()
+	user := a.userLocks.Acquire("owner")
+	defer user.Release()
+
+	treeDone := make(chan *httptest.ResponseRecorder, 1)
+	go func() {
+		w := httptest.NewRecorder()
+		a.workspace(w, runtimeRequest("GET", "/api/ws/tree?instance=two", "", otherSid))
+		treeDone <- w
+	}()
+	select {
+	case w := <-treeDone:
+		if w.Code != 200 {
+			t.Fatalf("other user tree: %d %s", w.Code, w.Body.String())
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("other user file operation blocked by long mutation")
+	}
+	stopDone := make(chan *httptest.ResponseRecorder, 1)
+	go func() {
+		w := httptest.NewRecorder()
+		a.instance(w, runtimeRequest("POST", "/api/instances/two/stop", "", otherSid))
+		stopDone <- w
+	}()
+	select {
+	case w := <-stopDone:
+		if w.Code != 200 {
+			t.Fatalf("other instance stop: %d %s", w.Code, w.Body.String())
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("other instance lifecycle blocked by long mutation")
+	}
+	// The same user's mutation is still serialized by the user lock.
+	blocked := make(chan struct{})
+	go func() {
+		user := a.userLocks.Acquire("owner")
+		user.Release()
+		close(blocked)
+	}()
+	select {
+	case <-blocked:
+		t.Fatal("owner lock not held during mutation")
+	case <-time.After(300 * time.Millisecond):
+	}
+}