m1saka/dsh-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden image admission, resource recovery and session renewal

Browse Source 
Add exact-image compatibility policy, experimental admission, paginated build history and shared reauthentication checks. Preserve uncertain Docker creation records, bound frontend resource waits and renew runtime sessions proactively.

Lazy-load and split Monaco, add regression and opt-in live Docker fault tests, and record full race and real-browser acceptance results.
This commit is contained in:
m1saka
2026-10-14 13:41:54 +08:00
parent 66abb76318
commit 9a71772771
62 changed files with 4679 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/admin.go
+2
View File
@@ -91,6 +91,8 @@ func (a *API) users(w http.ResponseWriter, r *http.Request) {
fail(w, 400, "positive quota and user ID required")
return
}
user := a.userLocks.Acquire(in.ID)
defer user.Release()
result, err := a.s.DB.Exec("UPDATE users SET quota=? WHERE id=?", in.Quota, in.ID)
if err != nil {
fail(w, 500, "quota update failed")
internal/api/admission.go
+208
View File
@@ -0,0 +1,208 @@
package api
import (
"context"
"database/sql"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"sync"
"time"
)
// Admission writes serialize with new runtime operations, including their Docker side effects.
var admissionMu sync.RWMutex
var errBuildUnsupported = errors.New("version is not supported for normal builds; administrator experimental build required")
func defaultAdmission(version, recipe, platform string) string {
if recipe == "7" && platform == "linux/amd64" {
switch version {
case "0.2.0-rc.1":
return "compatible"
case "0.2.0-rc.2":
return "recommended"
}
}
return "existing_only"
}
func supportedLayout(recipe string) bool {
switch recipe {
case "3", "4", "5", "6", "7":
return true
}
return false
}
func normalBuildAllowed(version string) bool {
return positiveAdmission(defaultAdmission(version, imageRecipeRev, imagePlatform))
}
func positiveAdmission(state string) bool { return state == "recommended" || state == "compatible" }
func (a *API) compatibility(w http.ResponseWriter, r *http.Request) {
if _, _, err := a.user(r); err != nil {
fail(w, 401, "authentication required")
return
}
if r.Method != http.MethodGet {
fail(w, 405, "method not allowed")
return
}
entries := []map[string]any{}
for _, v := range []string{"0.2.0-rc.1", "0.2.0-rc.2"} {
entries = append(entries, map[string]any{"version": v, "recipe_rev": imageRecipeRev, "platform": imagePlatform, "state": defaultAdmission(v, imageRecipeRev, imagePlatform), "build_allowed": normalBuildAllowed(v), "reason": "only linux/amd64 recipe 7 is built-in; successful exact image verification still required"})
}
writeJSON(w, map[string]any{"recipe_rev": imageRecipeRev, "platform": imagePlatform, "entries": entries, "unknown": map[string]any{"state": "existing_only", "build_allowed": false, "reason": "not accepted; administrator experimental build and explicit admission required"}, "experimental_admin_only": true})
}
func (a *API) evaluateImage(ctx context.Context, id string) (imageRecord, error) {
im := imageRecord{ID: id, ImageID: id, Status: "missing", Admission: "existing_only", AdmissionReason: "local image missing"}
actual, err := a.inspectLocalImage(ctx, id)
if err != nil || actual == nil {
return im, err
}
im.Version = actual.Config.Labels["dsh-mgr.version"]
im.RecipeRev = actual.Config.Labels["dsh-mgr.recipe"]
im.Platform = actual.platform()
im.Created = actual.Created
im.Status = "unverified"
im.AdmissionReason = "successful exact image identity verification required"
if actual.ID != id || actual.Config.Labels["dsh-mgr.role"] != "runtime" {
im.Version = ""
return im, nil
}
var count, experimental int
err = a.s.DB.QueryRowContext(ctx, "SELECT count(*),COALESCE(min(experimental),0) FROM image_builds WHERE image_id=? AND version=? AND recipe_rev=? AND platform=? AND status='success'", id, im.Version, im.RecipeRev, im.Platform).Scan(&count, &experimental)
if err != nil {
return im, err
}
var state, evidence string
err = a.s.DB.QueryRowContext(ctx, "SELECT state,evidence FROM image_admissions WHERE image_id=? AND version=? AND recipe_rev=? AND platform=?", id, im.Version, im.RecipeRev, im.Platform).Scan(&state, &evidence)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return im, err
}
if state == "blocked" {
im.Admission = state
im.Status = state
im.AdmissionReason = evidence
return im, nil
}
if count == 0 {
return im, nil
}
im.Admission = defaultAdmission(im.Version, im.RecipeRev, im.Platform)
im.AdmissionReason = "no accepted compatibility evidence for this tuple"
if positiveAdmission(im.Admission) {
im.AdmissionReason = "built-in exact tuple compatibility"
}
if experimental != 0 {
im.Admission = "existing_only"
im.Status = "experimental"
im.AdmissionReason = "experimental validation succeeded; administrator acceptance required"
} else {
im.Status = "existing_only"
}
if state != "" {
im.Admission = state
im.AdmissionReason = evidence
im.Status = state
}
if !supportedLayout(im.RecipeRev) || im.Platform != imagePlatform {
im.Admission = "existing_only"
im.Status = "existing_only"
im.AdmissionReason = "historical validation retained; current mount layout or platform not supported"
}
im.Selectable = positiveAdmission(im.Admission)
if im.Selectable {
im.Status = "available"
}
return im, nil
}
func (a *API) imageAdmission(w http.ResponseWriter, r *http.Request) {
u, _, err := a.user(r)
if err != nil {
fail(w, 401, "authentication required")
return
}
if !u.Admin {
fail(w, 403, "administrator required")
return
}
if r.Method != http.MethodPost {
fail(w, 405, "method not allowed")
return
}
var in struct {
ImageID string `json:"image_id"`
State string `json:"state"`
Evidence string `json:"evidence"`
}
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 8192))
dec.DisallowUnknownFields()
if dec.Decode(&in) != nil || dec.Decode(new(any)) != io.EOF || !runtimeImageID.MatchString(in.ImageID) || strings.TrimSpace(in.Evidence) == "" || (!positiveAdmission(in.State) && in.State != "existing_only" && in.State != "blocked") {
fail(w, 400, "image_id, valid state and evidence required")
return
}
if len(in.Evidence) > 4096 {
fail(w, 400, "evidence must not exceed 4096 UTF-8 bytes")
return
}
admissionMu.Lock()
defer admissionMu.Unlock()
buildQueue.Lock()
err = a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
fail(w, 500, "database error")
return
}
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
im, err := a.evaluateImage(ctx, in.ImageID)
if err != nil {
fail(w, 502, "image inspection failed")
return
}
if im.Status == "missing" || im.Version == "" || im.RecipeRev == "" || im.Platform == "" {
fail(w, 400, "real runtime image identity required")
return
}
if positiveAdmission(in.State) {
var count int
err = a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM image_builds WHERE image_id=? AND version=? AND recipe_rev=? AND platform=? AND status='success'", im.ImageID, im.Version, im.RecipeRev, im.Platform).Scan(&count)
if err != nil || count == 0 || !supportedLayout(im.RecipeRev) || im.Platform != imagePlatform {
fail(w, 400, "successful exact validation and supported mount layout/platform required")
return
}
}
buildQueue.Lock()
defer buildQueue.Unlock()
tx, err := a.s.DB.BeginTx(ctx, nil)
if err != nil {
fail(w, 500, "database error")
return
}
defer tx.Rollback()
_, err = tx.ExecContext(ctx, "INSERT INTO image_admissions(image_id,version,recipe_rev,platform,state,evidence,admitted_by,admitted_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(image_id,version,recipe_rev,platform) DO UPDATE SET state=excluded.state,evidence=excluded.evidence,admitted_by=excluded.admitted_by,admitted_at=excluded.admitted_at", im.ImageID, im.Version, im.RecipeRev, im.Platform, in.State, strings.TrimSpace(in.Evidence), u.ID, time.Now().UTC().Format(time.RFC3339Nano))
if err != nil {
fail(w, 500, "database error")
return
}
if _, err = tx.ExecContext(ctx, "INSERT INTO audit(uid,action,target,detail,at) VALUES(?,?,?,?,?)", u.ID, "image.admission", im.ImageID, in.State+": "+in.Evidence, time.Now().Unix()); err != nil {
fail(w, 500, "audit persistence failed")
return
}
if err = tx.Commit(); err != nil {
fail(w, 500, "database error")
return
}
a.imageDeleteGeneration++
im.Admission = in.State
im.AdmissionReason = strings.TrimSpace(in.Evidence)
im.Selectable = positiveAdmission(in.State)
im.Status = in.State
if im.Selectable {
im.Status = "available"
}
writeJSON(w, im)
}
internal/api/admission_routes_test.go
+116
View File
@@ -0,0 +1,116 @@
package api
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestAdmissionEvidenceUTF8Limit(t *testing.T) {
id := "sha256:" + strings.Repeat("d", 64)
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { writeImageInspect(w, id, "0.2.0-rc.2") })
seedVerifiedImage(t, a, id, "0.2.0-rc.2")
if _, err := a.s.DB.Exec("UPDATE users SET admin=1"); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name, evidence string
want int
}{
{"ascii boundary", strings.Repeat("a", 4096), 200},
{"ascii over", strings.Repeat("a", 4097), 400},
{"Chinese regression", strings.Repeat("中", 1400), 400},
{"Chinese below", strings.Repeat("中", 1365), 200},
{"Chinese boundary", strings.Repeat("中", 1365) + "a", 200},
{"Chinese over", strings.Repeat("中", 1365) + "ab", 400},
{"emoji boundary", strings.Repeat("😀", 1024), 200},
{"emoji over", strings.Repeat("😀", 1024) + "a", 400},
{"two byte boundary", strings.Repeat("é", 2048), 200},
{"raw whitespace counts", " " + strings.Repeat("a", 4096), 400},
} {
t.Run(tc.name, func(t *testing.T) {
body, err := json.Marshal(map[string]string{"image_id": id, "state": "compatible", "evidence": tc.evidence})
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", string(body), sid))
if w.Code != tc.want {
t.Fatal(w.Code, w.Body.String())
}
if tc.want == 400 && !strings.Contains(w.Body.String(), "4096 UTF-8 bytes") {
t.Fatal(w.Body.String())
}
if tc.want == 200 {
var evidence string
if err := a.s.DB.QueryRow("SELECT evidence FROM image_admissions WHERE image_id=?", id).Scan(&evidence); err != nil || evidence != tc.evidence {
t.Fatal(evidence, err)
}
}
})
}
}
func TestAdmissionRoutesEnforceAuthorization(t *testing.T) {
id := "sha256:" + strings.Repeat("d", 64)
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { writeImageInspect(w, id, "0.2.0-rc.2") })
seedVerifiedImage(t, a, id, "0.2.0-rc.2")
_, csrf, err := a.s.SessionUser(sid)
if err != nil {
t.Fatal(err)
}
body := fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":"isolated validation reference"}`, id)
request := func(method, path, payload string, auth, token bool) *httptest.ResponseRecorder {
session := ""
if auth {
session = sid
}
r := runtimeRequest(method, path, payload, session)
r.Header.Set("Origin", "http://panel.test")
if token {
r.Header.Set("X-CSRF-Token", csrf)
}
w := httptest.NewRecorder()
a.Handler().ServeHTTP(w, r)
return w
}
if w := request("GET", "/api/compatibility", "", false, false); w.Code != 401 {
t.Fatal(w.Code, w.Body.String())
}
if w := request("GET", "/api/compatibility", "", true, false); w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
if w := request("POST", "/api/images/admission", body, true, true); w.Code != 403 {
t.Fatal("nonadmin admission", w.Code, w.Body.String())
}
if w := request("POST", "/api/builds", `{"version":"9.8.7","experimental":true}`, true, true); w.Code != 403 {
t.Fatal("nonadmin experiment", w.Code, w.Body.String())
}
if _, err = a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='owner'"); err != nil {
t.Fatal(err)
}
if w := request("POST", "/api/images/admission", body, true, false); w.Code != 403 {
t.Fatal("missing CSRF", w.Code, w.Body.String())
}
if w := request("POST", "/api/images/admission", body, true, true); w.Code != 200 {
t.Fatal("admin admission", w.Code, w.Body.String())
}
for _, payload := range []string{
body + "{}",
body + strings.Repeat(" ", 8192),
fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":" "}`, id),
fmt.Sprintf(`{"image_id":%q,"state":"available","evidence":"invalid state"}`, id),
fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":"test","dockerfile":"not allowed"}`, id),
} {
if w := request("POST", "/api/images/admission", payload, true, true); w.Code != 400 {
t.Fatal(w.Code, w.Body.String())
}
}
var count int
if err = a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='image.admission'").Scan(&count); err != nil || count != 1 {
t.Fatal(count, err)
}
}
internal/api/admission_test.go
+274
View File
@@ -0,0 +1,274 @@
package api
import (
"context"
"encoding/json"
"fmt"
"git.misaka.ren/m1saka/dsh-web/internal/store"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestExperimentalBuildSuccessRequiresAcceptance(t *testing.T) {
id := "sha256:" + strings.Repeat("d", 64)
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/json":
writeJSON(w, []any{map[string]any{"Id": id, "Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "9.8.7", "dsh-mgr.recipe": imageRecipeRev}}})
case "/v1.47/images/" + id + "/json":
writeImageInspect(w, id, "9.8.7")
case "/v1.47/containers/create":
writeJSON(w, map[string]string{"Id": "check"})
case "/v1.47/containers/check/exec":
writeJSON(w, map[string]string{"Id": "probe"})
case "/v1.47/exec/probe/json":
writeJSON(w, map[string]any{"Running": false, "ExitCode": 0})
case "/v1.47/containers/check/json":
writeJSON(w, map[string]any{"State": map[string]bool{"Running": true}})
default:
w.WriteHeader(204)
}
})
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='u'"); err != nil {
t.Fatal(err)
}
w := imageRequest(a.builds, sid, "POST", `{"version":"9.8.7","experimental":true}`)
if w.Code != 202 {
t.Fatal(w.Code, w.Body.String())
}
var b buildRecord
if err := json.Unmarshal(w.Body.Bytes(), &b); err != nil || !b.Experimental {
t.Fatal(b, err)
}
deadline := time.Now().Add(5 * time.Second)
for {
var state string
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id=?", b.ID).Scan(&state); err != nil {
t.Fatal(err)
}
if state == "success" {
break
}
if state == "failed" || time.Now().After(deadline) {
t.Fatal(state)
}
time.Sleep(10 * time.Millisecond)
}
im, err := a.evaluateImage(context.Background(), id)
if err != nil || im.Selectable || im.Status != "experimental" {
t.Fatal(im, err)
}
}
func TestQueuedUnsupportedPlatformFailsBeforeDocker(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unsupported build contacted Docker") })
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
t.Fatal(err)
}
previous := imagePlatform
imagePlatform = "linux/arm64"
defer func() { imagePlatform = previous }()
b := buildRecord{ID: "unsupported", Version: "0.2.0-rc.2", Status: "queued", RecipeRev: imageRecipeRev, Platform: imagePlatform}
if err := a.insertBuildRequest(b, "u"); err != nil {
t.Fatal(err)
}
a.runImageBuild(b)
var state, detail string
if err := a.s.DB.QueryRow("SELECT status,error FROM image_builds WHERE id=?", b.ID).Scan(&state, &detail); err != nil || state != "failed" || !strings.Contains(detail, "experimental") {
t.Fatal(state, detail, err)
}
}
func TestAdmissionDowngradeWaitsForCreate(t *testing.T) {
id := "sha256:" + strings.Repeat("e", 64)
entered, release := make(chan struct{}), make(chan struct{})
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "/images/"):
writeImageInspect(w, id, "0.2.0-rc.1")
case strings.HasSuffix(r.URL.Path, "/networks/create"):
close(entered)
<-release
w.WriteHeader(204)
case strings.HasSuffix(r.URL.Path, "/containers/create"):
writeJSON(w, map[string]string{"Id": "created"})
default:
w.WriteHeader(204)
}
})
seedVerifiedImage(t, a, id, "0.2.0-rc.1")
if _, err := a.s.DB.Exec("UPDATE users SET admin=1"); err != nil {
t.Fatal(err)
}
created := make(chan int, 1)
go func() {
w := httptest.NewRecorder()
a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf(`{"name":"race","image_id":%q}`, id), sid))
created <- w.Code
}()
select {
case <-entered:
case <-time.After(3 * time.Second):
close(release)
t.Fatal("create did not reach network")
}
downgraded := make(chan int, 1)
go func() {
w := httptest.NewRecorder()
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", fmt.Sprintf(`{"image_id":%q,"state":"blocked","evidence":"downgrade test"}`, id), sid))
downgraded <- w.Code
}()
select {
case status := <-downgraded:
close(release)
t.Fatalf("admission raced active create: %d", status)
case <-time.After(30 * time.Millisecond):
}
close(release)
if status := <-created; status != 200 {
t.Fatal(status)
}
if status := <-downgraded; status != 200 {
t.Fatal(status)
}
w := httptest.NewRecorder()
a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf(`{"name":"denied","image_id":%q}`, id), sid))
if w.Code != 400 {
t.Fatal(w.Code, w.Body.String())
}
}
func TestAdmissionHistoricalCreateAndBlock(t *testing.T) {
id := "sha256:" + strings.Repeat("a", 64)
var body map[string]any
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "/images/"):
writeJSON(w, map[string]any{"Id": id, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": "4"}}})
case strings.HasSuffix(r.URL.Path, "/containers/create"):
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Error(err)
}
writeJSON(w, map[string]string{"Id": "created"})
default:
w.WriteHeader(204)
}
})
seedVerifiedImage(t, a, id, "1.2.3")
if _, err := a.s.DB.Exec("UPDATE image_builds SET recipe_rev='4'; DELETE FROM image_admissions; UPDATE users SET admin=1"); err != nil {
t.Fatal(err)
}
im, err := a.evaluateImage(context.Background(), id)
if err != nil || im.Status != "existing_only" || im.Selectable {
t.Fatal(im, err)
}
records := []buildRecord{{ID: "unused", ImageID: id, Version: "1.2.3", RecipeRev: "4", Platform: imagePlatform, Status: "success"}}
if err := a.reconcileBuilds(context.Background(), records); err != nil || records[0].Status != "success" {
t.Fatal(records, err)
}
admit := func(state string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", fmt.Sprintf(`{"image_id":%q,"state":%q,"evidence":"legacy layout explicitly accepted"}`, id, state), sid))
return w
}
if w := admit("compatible"); w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
w := httptest.NewRecorder()
a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf(`{"name":"legacy","image_id":%q}`, id), sid))
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
host := body["HostConfig"].(map[string]any)
mounts := host["Mounts"].([]any)
if len(mounts) != 2 || mounts[0].(map[string]any)["Target"] != "/data/dsh-home" || mounts[1].(map[string]any)["Target"] != "/workspace" || !strings.Contains(fmt.Sprint(body["Env"]), "DSH_HOME=/data/dsh-home") {
t.Fatal(body)
}
var instance store.Instance
if err := json.Unmarshal(w.Body.Bytes(), &instance); err != nil {
t.Fatal(err)
}
if w := admit("blocked"); w.Code != 200 {
t.Fatal(w.Body.String())
}
for _, action := range []string{"restore", "backups", "profile"} {
w := httptest.NewRecorder()
a.config(w, runtimeRequest("POST", "/api/instances/"+instance.ID+"/"+action, `{}`, sid), instance, action)
if w.Code != 409 {
t.Fatal(action, w.Code, w.Body.String())
}
}
for _, action := range []string{"start", "restart"} {
w := httptest.NewRecorder()
a.instance(w, runtimeRequest("POST", "/api/instances/"+instance.ID+"/"+action, "", sid))
if w.Code != 409 {
t.Fatal(action, w.Code, w.Body.String())
}
}
if _, err := a.s.DB.Exec("CREATE TRIGGER audit_fail BEFORE INSERT ON audit BEGIN SELECT RAISE(FAIL,'audit unavailable'); END"); err != nil {
t.Fatal(err)
}
if w := admit("compatible"); w.Code != 500 {
t.Fatal(w.Code, w.Body.String())
}
im, err = a.evaluateImage(context.Background(), id)
if err != nil || im.Admission != "blocked" {
t.Fatal(im, err)
}
}
func TestAdmissionExperimentalAndPreflight(t *testing.T) {
id := "sha256:" + strings.Repeat("b", 64)
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { writeImageInspect(w, id, "9.8.7") })
seedVerifiedImage(t, a, id, "9.8.7")
if _, err := a.s.DB.Exec("DELETE FROM image_admissions; UPDATE image_builds SET experimental=1"); err != nil {
t.Fatal(err)
}
im, err := a.evaluateImage(context.Background(), id)
if err != nil || im.Status != "experimental" || im.Selectable {
t.Fatal(im, err)
}
for _, tc := range []struct {
handler http.HandlerFunc
body string
want int
}{{a.builds, `{"version":"9.8.7"}`, 400}, {a.builds, `{"version":"9.8.7","experimental":true}`, 403}, {a.intents, `{"name":"unknown","version":"9.8.7"}`, 400}, {a.instances, fmt.Sprintf(`{"name":"unknown","image_id":%q}`, id), 400}} {
w := httptest.NewRecorder()
tc.handler(w, runtimeRequest("POST", "/api/test", tc.body, sid))
if w.Code != tc.want {
t.Fatal(w.Code, w.Body.String())
}
}
var n int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests").Scan(&n); err != nil || n != 0 {
t.Fatal(n, err)
}
if err := a.s.DB.QueryRow("SELECT count(*) FROM creation_intents").Scan(&n); err != nil || n != 0 {
t.Fatal(n, err)
}
if _, err := a.s.DB.Exec("UPDATE users SET admin=1"); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":"accepted after startup validation"}`, id), sid))
if w.Code != 200 {
t.Fatal(w.Body.String())
}
im, err = a.evaluateImage(context.Background(), id)
if err != nil || !im.Selectable {
t.Fatal(im, err)
}
if _, err := a.s.DB.Exec("UPDATE image_builds SET status='failed'"); err != nil {
t.Fatal(err)
}
im, err = a.evaluateImage(context.Background(), id)
if err != nil || im.Selectable || im.Status != "unverified" {
t.Fatal(im, err)
}
}
internal/api/api.go
+10
View File
@@ -34,6 +34,13 @@ type API struct {
// Protected by buildQueue; invalidates image observations across deletions.
imageDeleteGeneration uint64
// Per-image admission evaluations cached for the intent poller; guarded
// by evalCacheMu and invalidated by imageDeleteGeneration (buildQueue)
// plus a short TTL so Docker observations stay fresh.
evalCacheMu sync.Mutex
evalCache map[string]evalCacheEntry
evalCacheGen uint64
}
func New(s *store.Store, root string) *API {
@@ -62,8 +69,11 @@ func (a *API) Handler() http.Handler {
m.HandleFunc("/api/instances", a.instances)
m.HandleFunc("/api/instances/", a.instance)
m.HandleFunc("/api/images", a.images)
m.HandleFunc("/api/images/admission", a.imageAdmission)
m.HandleFunc("/api/compatibility", a.compatibility)
m.HandleFunc("/api/latest", a.latest)
m.HandleFunc("/api/builds", a.builds)
m.HandleFunc("/api/builds/", a.buildDetail)
m.HandleFunc("/api/intents", a.intents)
m.HandleFunc("/api/intents/", a.intent)
m.HandleFunc("/api/ws/", a.workspace)
internal/api/auth_tokens.go
+30 -4
View File
@@ -157,9 +157,23 @@ func (a *API) refresh(w http.ResponseWriter, r *http.Request) {
fail(w, 401, "authentication required")
return
}
u, csrf, err := a.s.SessionUser(sid.Value)
var uid, csrf string
err = a.s.DB.QueryRow("SELECT uid,csrf FROM sessions WHERE id=? AND expires>?", sid.Value, time.Now().Unix()).Scan(&uid, &csrf)
if err != nil {
fail(w, 401, "authentication required")
if errors.Is(err, sql.ErrNoRows) {
fail(w, 401, "authentication required")
} else {
fail(w, 500, "session error")
}
return
}
u, err := a.s.UserByID(uid)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
fail(w, 401, "authentication required")
} else {
fail(w, 500, "session error")
}
return
}
old, err := r.Cookie("dsh_refresh")
@@ -178,8 +192,20 @@ func (a *API) refresh(w http.ResponseWriter, r *http.Request) {
return
}
n, err := result.RowsAffected()
if err != nil || n != 1 {
fail(w, 401, "authentication required")
if err != nil {
fail(w, 500, "session error")
return
}
if n != 1 {
var active bool
err = a.s.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sessions JOIN auth_refresh ON auth_refresh.sid=sessions.id WHERE sessions.id=? AND sessions.expires>?)", sid.Value, time.Now().Unix()).Scan(&active)
if err != nil {
fail(w, 500, "session error")
} else if active {
fail(w, 409, "refresh_conflict")
} else {
fail(w, 401, "authentication required")
}
return
}
a.authCookies(w, r, sid.Value, u.ID, csrf, token)
internal/api/auth_tokens_test.go
+21 -3
View File
@@ -49,6 +49,21 @@ func cookieValue(cs []*http.Cookie, name string) string {
}
return ""
}
func TestRefreshSessionLookupFailureIsServerError(t *testing.T) {
a, req := authFixture(t)
w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil)
cookies := w.Result().Cookies()
a.s.Close()
r := httptest.NewRequest("POST", "/api/refresh", nil)
for _, c := range cookies {
r.AddCookie(c)
}
w = httptest.NewRecorder()
a.refresh(w, r)
if w.Code != 500 || len(w.Result().Cookies()) != 0 {
t.Fatal(w.Code, w.Header(), w.Body.String())
}
}
func TestAccessRefreshLifecycle(t *testing.T) {
a, req := authFixture(t)
w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil)
@@ -92,9 +107,12 @@ func TestAccessRefreshLifecycle(t *testing.T) {
if cookieValue(fresh, "dsh_refresh") == cookieValue(cs, "dsh_refresh") {
t.Fatal("not rotated")
}
if w = req("POST", "/api/refresh", "", csrf, cs); w.Code != 401 {
if w = req("POST", "/api/refresh", "", csrf, cs); w.Code != 409 {
t.Fatal("replay accepted", w.Code)
}
if len(w.Result().Cookies()) != 0 || !strings.Contains(w.Body.String(), "refresh_conflict") {
t.Fatal("conflict must not issue credentials", w.Header(), w.Body.String())
}
if w = req("GET", "/api/me", "", "", fresh); w.Code != 200 {
t.Fatal(w.Code)
}
@@ -104,7 +122,7 @@ func TestAccessRefreshLifecycle(t *testing.T) {
if w = req("GET", "/api/me", "", "", fresh); w.Code != 401 {
t.Fatal("logout access valid")
}
if w = req("POST", "/api/refresh", "", csrf, fresh); w.Code == 200 {
if w = req("POST", "/api/refresh", "", csrf, fresh); w.Code != 403 {
t.Fatal("logout refresh valid")
}
}
@@ -180,7 +198,7 @@ func TestRefreshOriginConcurrentRotationAndJWTBinding(t *testing.T) {
for code := range codes {
counts[code]++
}
if counts[200] != 1 || counts[401] != 1 {
if counts[200] != 1 || counts[409] != 1 {
t.Fatal(counts)
}
b := New(a.s, a.root)
internal/api/builds_read.go
+112
View File
@@ -0,0 +1,112 @@
package api
import (
"context"
"database/sql"
"errors"
"net/http"
"strconv"
"strings"
"time"
)
// Summary queries never load log bodies; history cannot inflate a page response.
const buildSummaryColumns = "id,version,status,error,image_id,created_at,requested_by,COALESCE(recipe_rev,''),COALESCE(platform,''),COALESCE(finished_at,''),experimental"
func scanBuildSummary(row interface{ Scan(...any) error }, b *buildRecord) error {
return row.Scan(&b.ID, &b.Version, &b.Status, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &b.RecipeRev, &b.Platform, &b.Finished, &b.Experimental)
}
func (a *API) listBuilds(w http.ResponseWriter, r *http.Request) {
limit, offset := 20, 0
for key, dest := range map[string]*int{"limit": &limit, "offset": &offset} {
if values, ok := r.URL.Query()[key]; ok {
if len(values) != 1 {
fail(w, 400, "invalid pagination")
return
}
value, err := strconv.Atoi(values[0])
if err != nil || value < 0 || (key == "limit" && (value < 1 || value > 100)) {
fail(w, 400, "invalid pagination")
return
}
*dest = value
}
}
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
fail(w, 500, "database error")
return
}
rows, err := a.s.DB.QueryContext(ctx, "SELECT "+buildSummaryColumns+" FROM image_builds ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?", limit+1, offset)
if err != nil {
fail(w, 500, "database error")
return
}
result := []buildRecord{}
for rows.Next() {
var b buildRecord
if err = scanBuildSummary(rows, &b); err != nil {
break
}
result = append(result, b)
}
if err == nil {
err = rows.Err()
}
rows.Close()
if err != nil {
fail(w, 500, "database error")
return
}
hasMore := len(result) > limit
if hasMore {
result = result[:limit]
}
if err := a.reconcileBuilds(ctx, result); err != nil {
fail(w, 502, "docker image reconciliation failed")
return
}
a.startBuildWorker()
w.Header().Set("X-Has-More", strconv.FormatBool(hasMore))
writeJSON(w, result)
}
func (a *API) buildDetail(w http.ResponseWriter, r *http.Request) {
if _, _, err := a.user(r); err != nil {
fail(w, 401, "authentication required")
return
}
if r.Method != http.MethodGet {
fail(w, 405, "method not allowed")
return
}
id := strings.TrimPrefix(r.URL.Path, "/api/builds/")
if id == "" || len(id) > 128 || strings.Contains(id, "/") {
fail(w, 404, "build not found")
return
}
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
defer cancel()
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
var b buildRecord
if err == nil {
err = a.s.DB.QueryRowContext(ctx, "SELECT "+buildSummaryColumns+",logs FROM image_builds WHERE id=?", id).Scan(&b.ID, &b.Version, &b.Status, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &b.RecipeRev, &b.Platform, &b.Finished, &b.Experimental, &b.Logs)
}
if errors.Is(err, sql.ErrNoRows) {
fail(w, 404, "build not found")
return
}
if err != nil {
fail(w, 500, "database error")
return
}
b.Logs = sanitizeBuildLog(b.Logs)
writeJSON(w, b)
}
internal/api/builds_read_test.go
+167
View File
@@ -0,0 +1,167 @@
package api
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestBuildHistoryPaginationAndOnDemandLogs(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Errorf("unexpected Docker request: %s", r.URL) })
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
t.Fatal(err)
}
for n := 0; n < 125; n++ {
_, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,logs,created_at) VALUES(?,?,?,?,?)", fmt.Sprintf("task-%03d", n), "1.2.3", "failed", "token=secret-value"+string(rune(10))+strings.Repeat("x", buildLogLimit), "2026-01-01")
if err != nil {
t.Fatal(err)
}
}
request := func(target, session, method string) *httptest.ResponseRecorder {
r := httptest.NewRequest(method, target, nil)
if session != "" {
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: session})
addTestAccess(r, session)
}
w := httptest.NewRecorder()
if strings.HasPrefix(r.URL.Path, "/api/builds/") {
a.buildDetail(w, r)
} else {
a.builds(w, r)
}
return w
}
for _, tc := range []struct {
query string
count int
first, more string
}{
{"", 20, "task-124", "true"}, {"?limit=100", 100, "task-124", "true"},
{"?limit=20&offset=120", 5, "task-004", "false"}, {"?offset=125", 0, "", "false"},
{"?limit=20&offset=105", 20, "task-019", "false"}, {"?limit=20&offset=104", 20, "task-020", "true"},
} {
w := request("/api/builds"+tc.query, sid, "GET")
var records []buildRecord
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &records) != nil || len(records) != tc.count {
t.Fatalf("%s: %d %s", tc.query, w.Code, w.Body.String())
}
if tc.count > 0 && records[0].ID != tc.first {
t.Fatal(records[0].ID)
}
if w.Header().Get("X-Has-More") != tc.more || strings.Contains(w.Body.String(), "logs") || strings.Contains(w.Body.String(), "secret-value") {
t.Fatal("unbounded or invalid summary response", w.Body.String())
}
}
for _, query := range []string{"?limit=0", "?limit=101", "?limit=-1", "?offset=-1", "?offset=x", "?limit=1&limit=2", "?offset=9999999999999999999999999"} {
if w := request("/api/builds"+query, sid, "GET"); w.Code != 400 {
t.Fatal(query, w.Code)
}
}
w := request("/api/builds/task-000", sid, "GET")
var b buildRecord
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &b) != nil || b.ID != "task-000" || b.Logs == "" || len(b.Logs) > buildLogLimit || strings.Contains(b.Logs, "secret-value") {
t.Fatal(w.Code, w.Body.String())
}
for _, path := range []string{"/api/builds/missing", "/api/builds/", "/api/builds/task-000/extra"} {
if w := request(path, sid, "GET"); w.Code != 404 {
t.Fatal(path, w.Code)
}
}
if w := request("/api/builds/task-000", "", "GET"); w.Code != 401 {
t.Fatal(w.Code)
}
if w := request("/api/builds", "", "GET"); w.Code != 401 {
t.Fatal(w.Code)
}
if w := request("/api/builds/task-000", sid, "POST"); w.Code != 405 {
t.Fatal(w.Code)
}
}
func TestBuildDetailRouteAndAccess(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker call") })
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
t.Fatal(err)
}
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('task','1.2.3','failed','now')"); err != nil {
t.Fatal(err)
}
_, csrf, err := a.s.SessionUser(sid)
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
path, method string
access bool
status int
}{
{"/api/builds/task", "GET", true, 200}, {"/api/builds/task", "GET", false, 401},
{"/api/builds/task", "POST", true, 405}, {"/api/builds/", "GET", true, 404},
{"/api/builds/task%2Fextra", "GET", true, 404}, {"/api/builds/task/extra", "GET", true, 404},
} {
r := httptest.NewRequest(tc.method, tc.path, nil)
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
if tc.access {
addTestAccess(r, sid)
}
if tc.method != "GET" {
r.Header.Set("Origin", "http://example.com")
r.Header.Set("X-CSRF-Token", csrf)
}
w := httptest.NewRecorder()
a.Handler().ServeHTTP(w, r)
if w.Code != tc.status {
t.Fatalf("%s %s: %d %s", tc.method, tc.path, w.Code, w.Body.String())
}
}
}
func TestBuildPageReconcilesOnlySelectedRecords(t *testing.T) {
unavailable := false
calls := 0
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
calls++
if unavailable {
w.WriteHeader(500)
return
}
if !strings.HasPrefix(r.URL.Path, "/v1.47/images/") {
t.Errorf("unexpected request %s", r.URL)
}
w.WriteHeader(404)
})
seedVerifiedImage(t, a, "first-image", "1.2.3")
seedVerifiedImage(t, a, "second-image", "1.2.4")
if _, err := a.s.DB.Exec("UPDATE image_builds SET created_at=CASE version WHEN '1.2.3' THEN 'z' ELSE 'a' END"); err != nil {
t.Fatal(err)
}
request := func(offset int) *httptest.ResponseRecorder {
r := httptest.NewRequest("GET", fmt.Sprintf("/api/builds?limit=1&offset=%d", offset), nil)
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
addTestAccess(r, sid)
w := httptest.NewRecorder()
a.builds(w, r)
return w
}
w := request(0)
if w.Code != 200 || calls != 1 || !strings.Contains(w.Body.String(), "missing") {
t.Fatal(w.Code, calls, w.Body.String())
}
var state string
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE version='1.2.4'").Scan(&state); err != nil || state != "success" {
t.Fatal(state, err)
}
unavailable = true
if w = request(1); w.Code != 502 {
t.Fatal(w.Code, w.Body.String())
}
}
internal/api/config.go
+21 -4
View File
@@ -7,7 +7,6 @@ import (
"errors"
"git.misaka.ren/m1saka/dsh-web/internal/backup"
"git.misaka.ren/m1saka/dsh-web/internal/store"
"golang.org/x/crypto/bcrypt"
"gopkg.in/yaml.v3"
"io"
"net/http"
@@ -87,6 +86,26 @@ func (a *API) config(w http.ResponseWriter, r *http.Request, i store.Instance, a
return
}
i = current
if r.Method != http.MethodGet {
admissionMu.RLock()
defer admissionMu.RUnlock()
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
fail(w, 500, "database error")
return
}
var blocked int
if err = a.s.DB.QueryRowContext(r.Context(), "SELECT count(*) FROM image_admissions WHERE image_id=? AND state='blocked'", i.ImageID).Scan(&blocked); err != nil {
fail(w, 500, "database error")
return
}
if blocked > 0 {
fail(w, 409, "image is blocked; configuration mutations and automatic restarts denied")
return
}
}
if i.Status == "error" || a.upgradeRecoveryRequired(i) {
fail(w, 409, "instance recovery required")
return
@@ -345,9 +364,7 @@ func (a *API) config(w http.ResponseWriter, r *http.Request, i store.Instance, a
fail(w, 400, "login password required")
return
}
var hash string
if a.s.DB.QueryRow("SELECT pass_hash FROM users WHERE id=?", i.UID).Scan(&hash) != nil || bcrypt.CompareHashAndPassword([]byte(hash), []byte(in.Password)) != nil {
fail(w, 403, "incorrect login password")
if !a.reauthenticate(w, i.UID, in.Password) {
return
}
if wasRunning {
internal/api/files.go
+4
View File
@@ -470,6 +470,10 @@ func (a *API) wsUpload(w http.ResponseWriter, r *http.Request, root *os.Root, u
fail(w, 409, "target exists")
return
}
if err := a.s.DB.QueryRow("SELECT quota FROM users WHERE id=?", u.ID).Scan(&u.Quota); err != nil {
fail(w, 413, "quota unavailable")
return
}
used, e := a.userUsage(u)
if e != nil || h.Size > maxUpload || used+h.Size > u.Quota {
fail(w, 413, "quota or upload limit exceeded")
internal/api/files_test.go
+3
View File
@@ -446,6 +446,9 @@ func TestWorkspaceSafetyAndUpload(t *testing.T) {
defer s.Close()
a := &API{s: s, root: t.TempDir()}
u := store.User{ID: "user", Quota: 8}
if err := s.CreateUser(u, "unused"); err != nil {
t.Fatal(err)
}
root, e := a.openWorkspace(u)
if e != nil {
t.Fatal(e)
internal/api/home_policy_test.go
+86
View File
@@ -0,0 +1,86 @@
package api
import (
"archive/zip"
"bytes"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"git.misaka.ren/m1saka/dsh-web/internal/store"
)
// Full-home access is an explicit owner capability, not a credential write-only
// boundary. Keep ownership, authentication and traversal checks independent.
func TestFullHomeOwnerPolicy(t *testing.T) {
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker operation") })
for _, i := range []store.Instance{{ID: "owned", UID: "owner", Status: "running"}, {ID: "foreign", UID: "other", Status: "running"}} {
if err := a.s.AddInstance(i); err != nil {
t.Fatal(err)
}
home := filepath.Join(a.root, "workspaces", i.UID, "instances", i.ID, "home")
if err := os.MkdirAll(filepath.Join(home, "profiles", "web"), 0700); err != nil {
t.Fatal(err)
}
for path, contents := range map[string]string{".credentials.yaml": "synthetic-test-credential", "dsh-ssh.json": "synthetic-test-ssh", "profiles/web/cordis.patch.yml": "old: value"} {
if err := os.WriteFile(filepath.Join(home, path), []byte(contents), 0600); err != nil {
t.Fatal(err)
}
}
}
request := func(method, path, body, session string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
a.workspace(w, runtimeRequest(method, path, body, session))
return w
}
for _, name := range []string{".credentials.yaml", "dsh-ssh.json"} {
w := request("GET", "/api/ws/download?instance=owned&path="+name, "", sid)
if w.Code != 200 || w.Body.Len() == 0 {
t.Fatal(name, w.Code, w.Body.String())
}
if w = request("GET", "/api/ws/download?instance=foreign&path="+name, "", sid); w.Code != 404 {
t.Fatal("cross-owner read", w.Code)
}
if w = request("GET", "/api/ws/download?instance=owned&path="+name, "", ""); w.Code != 401 {
t.Fatal("anonymous read", w.Code)
}
}
w := request("GET", "/api/ws/download?instance=owned&path=.", "", sid)
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
archive, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
if err != nil {
t.Fatal(err)
}
found := false
for _, f := range archive.File {
if f.Name == ".credentials.yaml" {
r, e := f.Open()
if e != nil {
t.Fatal(e)
}
data, e := io.ReadAll(r)
r.Close()
if e != nil || string(data) != "synthetic-test-credential" {
t.Fatal("wrong credential archive entry", e)
}
found = true
}
}
if !found {
t.Fatal("owner archive unexpectedly filters credentials")
}
// Running state intentionally does not force the controlled profile workflow.
w = request("POST", "/api/ws/rename?instance=owned", `{"path":"profiles/web/cordis.patch.yml","new_path":"profiles/web/saved.yml"}`, sid)
if w.Code != 200 {
t.Fatal("owner profile rename blocked", w.Code, w.Body.String())
}
w = request("GET", "/api/ws/download?instance=owned&path=../../other/instances/foreign/home/.credentials.yaml", "", sid)
if w.Code != 400 && w.Code != 404 {
t.Fatal("traversal allowed", w.Code)
}
}
internal/api/images.go
+256 -93
View File
@@ -29,10 +29,13 @@ func (a *API) ensureBuilds() error {
return err
}
for _, q := range []string{
"CREATE INDEX IF NOT EXISTS image_builds_history ON image_builds(created_at DESC,id DESC)",
"ALTER TABLE image_builds ADD COLUMN requested_by TEXT NOT NULL DEFAULT ''",
"ALTER TABLE image_builds ADD COLUMN recipe_rev TEXT",
"ALTER TABLE image_builds ADD COLUMN platform TEXT",
"ALTER TABLE image_builds ADD COLUMN finished_at TEXT",
"ALTER TABLE image_builds ADD COLUMN experimental INTEGER NOT NULL DEFAULT 0",
"CREATE TABLE IF NOT EXISTS image_admissions(image_id TEXT NOT NULL,version TEXT NOT NULL,recipe_rev TEXT NOT NULL,platform TEXT NOT NULL,state TEXT NOT NULL,evidence TEXT NOT NULL,admitted_by TEXT NOT NULL,admitted_at TEXT NOT NULL,PRIMARY KEY(image_id,version,recipe_rev,platform))",
"ALTER TABLE images ADD COLUMN recipe_rev TEXT",
"ALTER TABLE images ADD COLUMN platform TEXT",
"ALTER TABLE images ADD COLUMN usage_count INTEGER NOT NULL DEFAULT 0",
@@ -45,6 +48,9 @@ func (a *API) ensureBuilds() error {
if err != nil {
return err
}
if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='version requires administrator experimental build' WHERE status='queued' AND experimental=0 AND version NOT IN ('0.2.0-rc.1','0.2.0-rc.2')"); err != nil {
return err
}
if !buildQueue.initialized[a.s.DB] {
if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='image template identity is unknown' WHERE status='queued' AND (recipe_rev IS NULL OR recipe_rev<>? OR platform IS NULL OR platform<>?)", imageRecipeRev, imagePlatform); err != nil {
return err
@@ -66,7 +72,10 @@ func (a *API) builds(w http.ResponseWriter, r *http.Request) {
fail(w, 405, "method not allowed")
return
}
var in struct{ Version string }
var in struct {
Version string `json:"version"`
Experimental bool `json:"experimental"`
}
if r.Method == http.MethodPost {
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
dec.DisallowUnknownFields()
@@ -76,48 +85,19 @@ func (a *API) builds(w http.ResponseWriter, r *http.Request) {
}
}
if r.Method == http.MethodGet {
buildQueue.Lock()
err := a.ensureBuilds()
var rows *sql.Rows
if err == nil {
rows, err = a.s.DB.Query("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds ORDER BY created_at DESC,id DESC")
}
result := []buildRecord{}
if err == nil {
for rows.Next() {
var b buildRecord
var finished, recipe, platform sql.NullString
if err = rows.Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished); err != nil {
break
}
b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
result = append(result, b)
}
scanErr := rows.Err()
rows.Close()
if err == nil {
err = scanErr
}
}
buildQueue.Unlock()
if err != nil {
fail(w, 500, "database error")
return
}
// Docker image reconciliation is I/O-bound and updates rows with
// compare-and-set, so it must not hold the queue lock that instance
// creation and upgrades also need.
if err := a.reconcileBuilds(r.Context(), result); err != nil {
fail(w, 502, "docker image reconciliation failed")
return
}
a.startBuildWorker()
writeJSON(w, result)
a.listBuilds(w, r)
return
}
b, err := a.enqueueBuild(r.Context(), u.ID, in.Version)
if in.Experimental && !u.Admin {
fail(w, 403, "administrator required")
return
}
b, err := a.enqueueBuild(r.Context(), u.ID, in.Version, in.Experimental)
if err != nil {
status := 502
if errors.Is(err, errBuildUnsupported) {
status = 400
}
if errors.Is(err, errBuildRateLimited) {
status = 429
w.Header().Set("Retry-After", "3600")
@@ -137,7 +117,7 @@ var errBuildRateLimited = errors.New("build limit: three new tasks per user per
func (a *API) lookupActiveBuild(version string) (buildRecord, bool, error) {
var b buildRecord
var finished, recipe, platform sql.NullString
err := a.s.DB.QueryRow("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds WHERE version=? AND recipe_rev=? AND platform=? AND status IN ('queued','running','validating','success')", version, imageRecipeRev, imagePlatform).Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished)
err := a.s.DB.QueryRow("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at,experimental FROM image_builds WHERE version=? AND recipe_rev=? AND platform=? AND status IN ('queued','running','validating','success')", version, imageRecipeRev, imagePlatform).Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished, &b.Experimental)
b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
if errors.Is(err, sql.ErrNoRows) {
return b, false, nil
@@ -148,7 +128,17 @@ func (a *API) lookupActiveBuild(version string) (buildRecord, bool, error) {
return b, true, nil
}
func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecord, error) {
func (a *API) enqueueBuild(ctx context.Context, uid, version string, experiment ...bool) (buildRecord, error) {
experimental := len(experiment) > 0 && experiment[0]
if experimental {
u, err := a.s.UserByID(uid)
if err != nil || !u.Admin {
return buildRecord{}, errors.New("administrator required")
}
}
if !experimental && !normalBuildAllowed(version) {
return buildRecord{}, errBuildUnsupported
}
if !validBuildVersion(version) {
return buildRecord{}, errors.New("a fixed version is required")
}
@@ -163,6 +153,9 @@ func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecor
if err != nil {
return b, err
}
if found && !experimental && b.Experimental && b.Status != "success" {
return b, errBuildUnsupported
}
if found && b.Status != "success" {
a.startBuildWorker()
return b, nil
@@ -177,6 +170,15 @@ func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecor
return b, lookupErr
}
if id != "" && id == b.ImageID {
if !experimental {
im, e := a.evaluateImage(ctx, id)
if e != nil {
return b, e
}
if !im.Selectable {
return b, errBuildUnsupported
}
}
a.startBuildWorker()
return b, nil
}
@@ -187,12 +189,15 @@ func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecor
b, found, err = a.lookupActiveBuild(version)
}
if err == nil && !found {
b, err = a.insertBuild(uid, version)
b, err = a.insertBuild(uid, version, experimental)
}
buildQueue.Unlock()
if err != nil {
return b, err
}
if !experimental && found && b.Experimental {
return b, errBuildUnsupported
}
a.startBuildWorker()
return b, nil
}
@@ -204,9 +209,12 @@ func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecor
return b, err
}
if found {
if !experimental && b.Experimental {
return b, errBuildUnsupported
}
return b, nil
}
b, err = a.insertBuild(uid, version)
b, err = a.insertBuild(uid, version, experimental)
if err != nil {
return b, err
}
@@ -216,7 +224,7 @@ func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecor
// insertBuild applies the per-user rate limit and inserts a queued record;
// call with the build queue held.
func (a *API) insertBuild(uid, version string) (buildRecord, error) {
func (a *API) insertBuild(uid, version string, experiment ...bool) (buildRecord, error) {
var count int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests WHERE user_id=? AND created_at>?", uid, time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)).Scan(&count); err != nil {
return buildRecord{}, err
@@ -225,6 +233,7 @@ func (a *API) insertBuild(uid, version string) (buildRecord, error) {
return buildRecord{}, errBuildRateLimited
}
b := buildRecord{ID: store.ID(), Version: version, Status: "queued", Created: time.Now().UTC().Format(time.RFC3339Nano), RequestedBy: uid, RecipeRev: imageRecipeRev, Platform: imagePlatform}
b.Experimental = len(experiment) > 0 && experiment[0]
return b, a.insertBuildRequest(b, uid)
}
func (a *API) startBuildWorker() {
@@ -241,22 +250,56 @@ func (a *API) startBuildWorkerLocked() {
}
buildQueue.active[a.s.DB] = true
go func() {
buildWorker.Lock()
defer buildWorker.Unlock()
var lastCleanupAudit time.Time
var nextCleanup time.Time
var recoveryErr error
for {
select {
case <-a.s.Done():
buildQueue.Lock()
delete(buildQueue.active, a.s.DB)
buildQueue.Unlock()
return
default:
}
if time.Now().After(nextCleanup) {
buildWorker.Lock()
recovery, cancel := context.WithTimeout(context.Background(), 20*time.Second)
recoveryErr = a.recoverBuildChecks(recovery)
cancel()
buildWorker.Unlock()
if errors.Is(recoveryErr, errBuildCheckUncertain) {
nextCleanup = time.Now().Add(time.Minute)
}
}
if recoveryErr != nil {
if time.Since(lastCleanupAudit) >= time.Minute {
a.s.AddAudit("", "build.cleanup_error", "", sanitizeBuildLog(recoveryErr.Error()))
lastCleanupAudit = time.Now()
}
}
buildQueue.Lock()
var b buildRecord
err := a.s.DB.QueryRow("SELECT id,version FROM image_builds WHERE status='queued' ORDER BY created_at,id LIMIT 1").Scan(&b.ID, &b.Version)
err := a.s.DB.QueryRow("SELECT id,version,experimental,requested_by FROM image_builds WHERE status='queued' ORDER BY created_at,id LIMIT 1").Scan(&b.ID, &b.Version, &b.Experimental, &b.RequestedBy)
if err == nil {
_, err = a.s.DB.Exec("UPDATE image_builds SET status='running' WHERE id=?", b.ID)
}
if err != nil {
if !errors.Is(err, sql.ErrNoRows) || recoveryErr != nil {
buildQueue.Unlock()
select {
case <-a.s.Done():
case <-time.After(time.Second):
}
continue
}
delete(buildQueue.active, a.s.DB)
buildQueue.Unlock()
return
}
buildQueue.Unlock()
a.runImageBuild(b)
nextCleanup = time.Time{}
}
}()
}
@@ -282,6 +325,7 @@ func imageBuildContext() (io.Reader, error) {
return &buf, nil
}
func (a *API) runImageBuild(b buildRecord) {
buildWorker.Lock()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
logs := ""
@@ -301,6 +345,15 @@ func (a *API) runImageBuild(b buildRecord) {
}
}
tarball, err := imageBuildContext()
if !b.Experimental && !normalBuildAllowed(b.Version) {
err = errBuildUnsupported
}
if b.Experimental {
u, e := a.s.UserByID(b.RequestedBy)
if e != nil || !u.Admin {
err = errors.New("experimental build requester is not an administrator")
}
}
var id string
if err == nil {
id, err = a.localBuildImage(ctx, b.Version, "")
@@ -325,7 +378,7 @@ func (a *API) runImageBuild(b buildRecord) {
_, err = a.s.DB.Exec("UPDATE image_builds SET status='validating',image_id=? WHERE id=?", id, b.ID)
}
if err == nil {
err = a.verifyBuiltImage(ctx, id, b.Version)
err = a.verifyBuiltImage(ctx, id, b.Version, b.ID)
}
if err == nil {
err = logErr
@@ -337,31 +390,53 @@ func (a *API) runImageBuild(b buildRecord) {
appendLog(string(rune(10)) + detail)
}
finished := time.Now().UTC().Format(time.RFC3339Nano)
_, _ = a.s.DB.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, id, finished, b.ID)
if status == "success" && id != "" {
_, _ = a.s.DB.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", id, b.Version, imageRecipeRev, imagePlatform, "dsh-web:"+b.Version+"-"+imageRecipeRev, "available", finished)
buildWorker.Unlock()
if err := a.persistBuildResult(b, status, logs, detail, id, finished); err != nil {
go a.finishBuild(b, status, logs, detail, id, finished)
}
}
func (a *API) verifyBuiltImage(parent context.Context, image, version string) (result error) {
func (a *API) verifyBuiltImage(parent context.Context, image, version string, buildIDs ...string) (result error) {
ctx, cancel := context.WithTimeout(parent, 90*time.Second)
defer cancel()
id := store.ID()
if len(buildIDs) != 0 {
id = buildIDs[0]
}
if err := a.ensureBuildChecks(); err != nil {
return err
}
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES(?,?,?)", id, image, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
return err
}
var created struct {
ID string `json:"Id"`
}
config := map[string]any{"Image": image, "User": "1000:1000", "Labels": map[string]string{"dsh-mgr.role": "build-check"}, "HostConfig": map[string]any{"NetworkMode": "none", "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges:true"}, "Memory": int64(1 << 30), "NanoCpus": int64(1e9), "PidsLimit": 256, "Tmpfs": map[string]string{"/tmp": "rw,exec,nosuid,nodev,size=256m,mode=1777", "/workspace": "rw,nosuid,nodev,size=256m,uid=1000,gid=1000"}}}
if err := a.docker.Do(ctx, http.MethodPost, "/containers/create", config, &created); err != nil {
rejected := false
defer func() {
cleanup, c := context.WithTimeout(context.Background(), 10*time.Second)
defer c()
var err error
if rejected {
_, err = a.s.DB.Exec("DELETE FROM image_build_checks WHERE build_id=?", id)
} else if created.ID != "" {
err = a.deleteBuildCheck(cleanup, id, created.ID)
} else {
err = a.cleanupBuildCheck(cleanup, id, image)
}
result = errors.Join(result, err)
}()
config := map[string]any{"Image": image, "User": "1000:1000", "Labels": map[string]string{"dsh-mgr.role": "build-check", "dsh-mgr.build": id}, "HostConfig": map[string]any{"NetworkMode": "none", "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges:true"}, "Memory": int64(1 << 30), "NanoCpus": int64(1e9), "PidsLimit": 256, "Tmpfs": map[string]string{"/tmp": "rw,exec,nosuid,nodev,size=256m,mode=1777", "/workspace": "rw,nosuid,nodev,size=256m,uid=1000,gid=1000"}}}
if err := a.docker.Do(ctx, http.MethodPost, "/containers/create?name="+url.QueryEscape(buildCheckName(id)), config, &created); err != nil {
rejected = dockerCreateRejected(err)
return err
}
if created.ID == "" {
return errors.New("verification container ID missing")
}
if _, err := a.s.DB.Exec("UPDATE image_build_checks SET container_id=? WHERE build_id=?", created.ID, id); err != nil {
return err
}
path := "/containers/" + url.PathEscape(created.ID)
defer func() {
cleanup, c := context.WithTimeout(context.Background(), 10*time.Second)
defer c()
err := a.docker.Do(cleanup, http.MethodDelete, path+"?force=1&v=1", nil, nil)
result = errors.Join(result, err)
}()
if err := a.docker.Do(ctx, http.MethodPost, path+"/start", nil, nil); err != nil {
return err
}
@@ -428,28 +503,32 @@ var buildQueue = struct {
var buildWorker sync.Mutex
type imageRecord struct {
ID string `json:"id"`
ImageID string `json:"image_id"`
Version string `json:"version"`
Tag string `json:"tag,omitempty"`
Status string `json:"status"`
Created string `json:"created_at"`
RecipeRev string `json:"recipe_rev,omitempty"`
Platform string `json:"platform,omitempty"`
UsageCount int `json:"usage_count"`
Admission string `json:"admission"`
AdmissionReason string `json:"admission_reason"`
Selectable bool `json:"selectable"`
ID string `json:"id"`
ImageID string `json:"image_id"`
Version string `json:"version"`
Tag string `json:"tag,omitempty"`
Status string `json:"status"`
Created string `json:"created_at"`
RecipeRev string `json:"recipe_rev,omitempty"`
Platform string `json:"platform,omitempty"`
UsageCount int `json:"usage_count"`
}
type buildRecord struct {
ImageID string `json:"image_id,omitempty"`
ID string `json:"id"`
Version string `json:"version"`
Status string `json:"status"`
Logs string `json:"logs"`
Error string `json:"error,omitempty"`
Created string `json:"created_at"`
RequestedBy string `json:"requested_by,omitempty"`
RecipeRev string `json:"recipe_rev,omitempty"`
Platform string `json:"platform,omitempty"`
Finished string `json:"finished_at,omitempty"`
Experimental bool `json:"experimental"`
ImageID string `json:"image_id,omitempty"`
ID string `json:"id"`
Version string `json:"version"`
Status string `json:"status"`
Logs string `json:"logs,omitempty"`
Error string `json:"error,omitempty"`
Created string `json:"created_at"`
RequestedBy string `json:"requested_by,omitempty"`
RecipeRev string `json:"recipe_rev,omitempty"`
Platform string `json:"platform,omitempty"`
Finished string `json:"finished_at,omitempty"`
}
func (a *API) images(w http.ResponseWriter, r *http.Request) {
@@ -475,26 +554,110 @@ func (a *API) images(w http.ResponseWriter, r *http.Request) {
writeJSON(w, result)
}
func (a *API) verifiedImages(ctx context.Context) (map[string]string, error) {
buildQueue.Lock()
defer buildQueue.Unlock()
if err := a.ensureBuilds(); err != nil {
return nil, err
// verifiedImagesTTL bounds how long a cached admission evaluation is reused
// by the one-second intent poller.
var verifiedImagesTTL = 5 * time.Second
type evalCacheEntry struct {
im imageRecord
at time.Time
}
// evaluateImageCached reuses recent per-image evaluations for the intent
// poller so repeated ticks do not hammer Docker. Deletions and admission
// writes bump imageDeleteGeneration and invalidate every entry.
func (a *API) evaluateImageCached(ctx context.Context, gen uint64, id string) (imageRecord, error) {
a.evalCacheMu.Lock()
if entry, ok := a.evalCache[id]; ok && a.evalCacheGen == gen && time.Since(entry.at) < verifiedImagesTTL {
a.evalCacheMu.Unlock()
return entry.im, nil
}
rows, err := a.s.DB.QueryContext(ctx, "SELECT image_id,version FROM image_builds WHERE status='success' AND image_id<>'' AND recipe_rev=? AND platform=?", imageRecipeRev, imagePlatform)
a.evalCacheMu.Unlock()
im, err := a.evaluateImage(ctx, id)
if err != nil {
return im, err
}
a.evalCacheMu.Lock()
if a.evalCache == nil || a.evalCacheGen != gen {
a.evalCache, a.evalCacheGen = make(map[string]evalCacheEntry), gen
}
a.evalCache[id] = evalCacheEntry{im: im, at: time.Now()}
a.evalCacheMu.Unlock()
return im, nil
}
func (a *API) successfulImageIDs(ctx context.Context) ([]string, error) {
rows, err := a.s.DB.QueryContext(ctx, "SELECT DISTINCT image_id FROM image_builds WHERE status='success' AND image_id<>''")
if err != nil {
return nil, err
}
defer rows.Close()
images := make(map[string]string)
ids := []string{}
for rows.Next() {
var id, version string
if err := rows.Scan(&id, &version); err != nil {
var id string
if err = rows.Scan(&id); err != nil {
rows.Close()
return nil, err
}
images[id] = version
ids = append(ids, id)
}
return images, rows.Err()
err = rows.Err()
rows.Close()
if err != nil {
return nil, err
}
return ids, nil
}
func (a *API) verifiedImages(ctx context.Context) (map[string]string, error) {
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
return nil, err
}
ids, err := a.successfulImageIDs(ctx)
if err != nil {
return nil, err
}
result := map[string]string{}
for _, id := range ids {
im, err := a.evaluateImage(ctx, id)
if err != nil {
return nil, err
}
if im.Selectable {
result[id] = im.Version
}
}
return result, nil
}
// verifiedImagesPoll is the intent poller's view: build rows are reread every
// tick, but per-image Docker evaluation is cached briefly so a stuck waiting
// intent cannot turn the one-second loop into Docker hammering.
func (a *API) verifiedImagesPoll(ctx context.Context) (map[string]string, error) {
buildQueue.Lock()
err := a.ensureBuilds()
gen := a.imageDeleteGeneration
buildQueue.Unlock()
if err != nil {
return nil, err
}
ids, err := a.successfulImageIDs(ctx)
if err != nil {
return nil, err
}
result := map[string]string{}
for _, id := range ids {
im, err := a.evaluateImageCached(ctx, gen, id)
if err != nil {
return nil, err
}
if im.Selectable {
result[id] = im.Version
}
}
return result, nil
}
func (a *API) latest(w http.ResponseWriter, r *http.Request) {
if _, _, err := a.user(r); err != nil {
@@ -510,7 +673,7 @@ func (a *API) latest(w http.ResponseWriter, r *http.Request) {
fail(w, 502, "npm version lookup failed")
return
}
writeJSON(w, map[string]string{"version": v, "checked_at": time.Now().UTC().Format(time.RFC3339)})
writeJSON(w, map[string]any{"version": v, "checked_at": time.Now().UTC().Format(time.RFC3339), "supported": normalBuildAllowed(v), "build_allowed": normalBuildAllowed(v), "admission": defaultAdmission(v, imageRecipeRev, imagePlatform), "admission_reason": "normal builds require a built-in tested tuple; unknown versions require administrator experimental validation and admission"})
}
func fetchLatest(ctx context.Context, client *http.Client) (string, error) {
return fetchNPMVersion(ctx, client, "latest")
internal/api/images_delete.go
+2
View File
@@ -31,6 +31,8 @@ func (a *API) deleteImage(w http.ResponseWriter, r *http.Request) {
// entry before creating containers; the transaction below re-checks that
// journal, and an upgrade racing an administrator deletion fails safely
// with its own rollback path.
admissionMu.Lock()
defer admissionMu.Unlock()
buildQueue.Lock()
defer buildQueue.Unlock()
if buildQueue.active[a.s.DB] {
internal/api/images_enqueue_concurrency_test.go
+3 -3
View File
@@ -20,7 +20,7 @@ func TestEnqueueBuildConcurrentMissingImageReturnsReplacement(t *testing.T) {
switch r.URL.Path {
case "/v1.47/images/json":
writeJSON(w, []map[string]any{{"Id": "old-image", "Labels": map[string]string{
"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": imageRecipeRev,
"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev,
}}})
case "/v1.47/images/old-image/json":
release := make(chan struct{})
@@ -40,7 +40,7 @@ func TestEnqueueBuildConcurrentMissingImageReturnsReplacement(t *testing.T) {
w.WriteHeader(http.StatusInternalServerError)
}
})
seedVerifiedImage(t, a, "old-image", "1.2.3")
seedVerifiedImage(t, a, "old-image", "0.2.0-rc.1")
var oldID string
if err := a.s.DB.QueryRow("SELECT id FROM image_builds WHERE image_id='old-image'").Scan(&oldID); err != nil {
t.Fatal(err)
@@ -69,7 +69,7 @@ func TestEnqueueBuildConcurrentMissingImageReturnsReplacement(t *testing.T) {
submissions.Add(1)
go func() {
defer submissions.Done()
b, err := a.enqueueBuild(ctx, "u", "1.2.3")
b, err := a.enqueueBuild(ctx, "u", "0.2.0-rc.1")
done <- result{b, err}
}()
return done
internal/api/images_policy.go
+8 -15
View File
@@ -146,6 +146,8 @@ func (a *API) imageIndexSnapshot(ctx context.Context) ([]imageRecord, error) {
for i := range result {
im := &result[i]
im.Status = "missing"
im.Admission = "existing_only"
im.AdmissionReason = "local image missing"
actual, err := a.inspectLocalImage(ctx, im.ImageID)
if err != nil {
return nil, err
@@ -167,13 +169,11 @@ func (a *API) imageIndexSnapshot(ctx context.Context) ([]imageRecord, error) {
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM instances WHERE image_id=?", im.ImageID).Scan(&im.UsageCount); err != nil {
return nil, err
}
var verified int
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM image_builds WHERE image_id=? AND version=? AND recipe_rev=? AND platform=? AND status='success'", im.ImageID, im.Version, imageRecipeRev, imagePlatform).Scan(&verified); err != nil {
policy, err := a.evaluateImage(ctx, im.ImageID)
if err != nil {
return nil, err
}
if verified > 0 && actual.ID == im.ImageID && actual.Config.Labels["dsh-mgr.role"] == "runtime" && im.RecipeRev == imageRecipeRev && im.Platform == imagePlatform {
im.Status = "available"
}
im.Status, im.Admission, im.AdmissionReason, im.Selectable = policy.Status, policy.Admission, policy.AdmissionReason, policy.Selectable
}
// Phase 3: write observed state back under the queue lock (DB only).
buildQueue.Lock()
@@ -240,7 +240,7 @@ func (a *API) insertBuildRequest(b buildRecord, user string) error {
return err
}
defer tx.Rollback()
if _, err = tx.Exec("INSERT INTO image_builds(id,version,status,created_at,requested_by,recipe_rev,platform) VALUES(?,?,?,?,?,?,?)", b.ID, b.Version, b.Status, b.Created, b.RequestedBy, b.RecipeRev, b.Platform); err != nil {
if _, err = tx.Exec("INSERT INTO image_builds(id,version,status,created_at,requested_by,recipe_rev,platform,experimental) VALUES(?,?,?,?,?,?,?,?)", b.ID, b.Version, b.Status, b.Created, b.RequestedBy, b.RecipeRev, b.Platform, b.Experimental); err != nil {
return err
}
if _, err = tx.Exec("INSERT INTO image_build_requests(user_id,build_id,created_at) VALUES(?,?,?)", user, b.ID, b.Created); err != nil {
@@ -257,18 +257,11 @@ func (a *API) reconcileBuilds(parent context.Context, records []buildRecord) err
if b.Status != "success" {
continue
}
if b.RecipeRev != imageRecipeRev || b.Platform != imagePlatform {
b.Status, b.Error = "missing", "image template identity is unknown"
if _, err := a.s.DB.Exec("UPDATE image_builds SET status=?,error=? WHERE id=? AND status='success'", b.Status, b.Error, b.ID); err != nil {
return err
}
continue
}
id, err := a.localBuildImage(ctx, b.Version, b.ImageID)
actual, err := a.inspectLocalImage(ctx, b.ImageID)
if err != nil {
return err
}
if id == "" || id != b.ImageID {
if actual == nil || actual.ID != b.ImageID || actual.Config.Labels["dsh-mgr.role"] != "runtime" || actual.Config.Labels["dsh-mgr.version"] != b.Version || actual.Config.Labels["dsh-mgr.recipe"] != b.RecipeRev || actual.platform() != b.Platform {
b.Status, b.Error = "missing", "image no longer available"
if _, err := a.s.DB.Exec("UPDATE image_builds SET status=?,error=? WHERE id=? AND status='success'", b.Status, b.Error, b.ID); err != nil {
return err
internal/api/images_policy_test.go
+68 -27
View File
@@ -20,13 +20,13 @@ func TestImageLegacyMigrationNeverCertifies(t *testing.T) {
io.WriteString(w, "[]")
return
}
writeImageInspect(w, "legacy-image", "1.2.3")
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
})
_, err := a.s.DB.Exec(`CREATE TABLE image_builds(id TEXT PRIMARY KEY,version TEXT NOT NULL,status TEXT NOT NULL,logs TEXT NOT NULL DEFAULT '',error TEXT NOT NULL DEFAULT '',image_id TEXT NOT NULL DEFAULT '',created_at TEXT NOT NULL);
CREATE UNIQUE INDEX image_builds_version ON image_builds(version) WHERE status IN ('queued','running','validating','success');
INSERT INTO image_builds(id,version,status,image_id,created_at) VALUES('legacy','1.2.3','success','legacy-image','now'),('queued','1.2.4','queued','','now');
INSERT INTO image_builds(id,version,status,image_id,created_at) VALUES('legacy','0.2.0-rc.1','success','legacy-image','now'),('queued','0.2.0-rc.2','queued','','now');
CREATE TABLE images(image_id TEXT PRIMARY KEY,version TEXT NOT NULL,tag TEXT NOT NULL DEFAULT '',status TEXT NOT NULL,built_at TEXT NOT NULL);
INSERT INTO images VALUES('legacy-image','1.2.3','','available','now');`)
INSERT INTO images VALUES('legacy-image','0.2.0-rc.1','','available','now');`)
if err != nil {
t.Fatal(err)
}
@@ -50,9 +50,9 @@ func TestImageLegacyMigrationNeverCertifies(t *testing.T) {
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='queued'").Scan(&status); err != nil || status != "failed" {
t.Fatal(status, err)
}
seedVerifiedImage(t, a, "new-image", "1.2.3")
seedVerifiedImage(t, a, "new-image", "0.2.0-rc.1")
verified, err = a.verifiedImages(context.Background())
if err != nil || len(verified) != 1 || verified["new-image"] != "1.2.3" {
if err != nil || len(verified) != 1 || verified["new-image"] != "0.2.0-rc.1" {
t.Fatal(verified, err)
}
}
@@ -73,7 +73,7 @@ func TestImageInspectIdentity(t *testing.T) {
return
}
parts := strings.Split(imagePlatform, "/")
labels := map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": imageRecipeRev}
labels := map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}
im := map[string]any{"Id": "image", "Os": parts[0], "Architecture": parts[1], "Config": map[string]any{"Labels": labels}}
switch mode {
case "recipe":
@@ -96,7 +96,7 @@ func TestImageInspectIdentity(t *testing.T) {
}
writeJSON(w, im)
})
ok, err := a.imageMatches(context.Background(), "image", "1.2.3")
ok, err := a.imageMatches(context.Background(), "image", "0.2.0-rc.1")
if ok != (mode == "valid") || (err != nil) != (mode == "unavailable") {
t.Fatal(ok, err)
}
@@ -129,8 +129,8 @@ func TestImageIndexLocalInventory(t *testing.T) {
return
}
parts := strings.Split(imagePlatform, "/")
im := inspectedImage{ID: "image", Os: parts[0], Architecture: parts[1], Created: "2026-01-01T00:00:00Z", RepoTags: []string{"dsh-web:1.2.3"}}
im.Config.Labels = map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": imageRecipeRev}
im := inspectedImage{ID: "image", Os: parts[0], Architecture: parts[1], Created: "2026-01-01T00:00:00Z", RepoTags: []string{"dsh-web:0.2.0-rc.1"}}
im.Config.Labels = map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}
if mode == "old-recipe" {
im.Config.Labels["dsh-mgr.recipe"] = "1"
}
@@ -140,7 +140,7 @@ func TestImageIndexLocalInventory(t *testing.T) {
writeJSON(w, im)
})
if mode != "external" {
seedVerifiedImage(t, a, "image", "1.2.3")
seedVerifiedImage(t, a, "image", "0.2.0-rc.1")
}
if mode == "old-recipe" {
if _, err := a.s.DB.Exec("UPDATE image_builds SET recipe_rev='1'"); err != nil {
@@ -155,8 +155,12 @@ func TestImageIndexLocalInventory(t *testing.T) {
}
return
}
wantStatus := "unverified"
if mode == "old-recipe" {
wantStatus = "existing_only"
}
var images []imageRecord
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != "unverified" || images[0].Version != "1.2.3" || images[0].Created != "2026-01-01T00:00:00Z" || images[0].Tag != "dsh-web:1.2.3" {
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != wantStatus || images[0].Version != "0.2.0-rc.1" || images[0].Created != "2026-01-01T00:00:00Z" || images[0].Tag != "dsh-web:0.2.0-rc.1" {
t.Fatal(w.Body.String(), err)
}
if mode == "other-platform" && images[0].Platform != strings.Split(imagePlatform, "/")[0]+"/arm/v7" {
@@ -193,10 +197,10 @@ func TestImageIndexUsageMissingAndDockerFailure(t *testing.T) {
case 2:
w.WriteHeader(404)
default:
writeImageInspect(w, "image", "1.2.3")
writeImageInspect(w, "image", "0.2.0-rc.1")
}
})
seedVerifiedImage(t, a, "image", "1.2.3")
seedVerifiedImage(t, a, "image", "0.2.0-rc.1")
if _, err := a.s.DB.Exec(`INSERT INTO instances(id,uid,name,image_id,status,created) VALUES('one','u','one','image','running',0),('two','other-user','two','image','stopped',0),('three','u','three','other','running',0)`); err != nil {
t.Fatal(err)
}
@@ -237,21 +241,21 @@ func TestImageIndexUsageMissingAndDockerFailure(t *testing.T) {
func TestBuildRateLimitDoesNotPenalizeSharedTask(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/verified/json" {
writeImageInspect(w, "verified", "1.2.3")
writeImageInspect(w, "verified", "0.2.0-rc.1")
return
}
io.WriteString(w, `[{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
io.WriteString(w, `[{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
})
seedVerifiedImage(t, a, "verified", "1.2.3")
seedVerifiedImage(t, a, "verified", "0.2.0-rc.1")
for _, id := range []string{"a", "b", "c"} {
if _, err := a.s.DB.Exec("INSERT INTO image_build_requests VALUES('u',?,?)", id, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
t.Fatal(err)
}
}
if w := imageRequest(a.builds, sid, "POST", `{"version":"1.2.3"}`); w.Code != 202 {
if w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`); w.Code != 202 {
t.Fatal(w.Code, w.Body.String())
}
if w := imageRequest(a.builds, sid, "POST", `{"version":"1.2.4"}`); w.Code != 429 || w.Header().Get("Retry-After") == "" {
if w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.2"}`); w.Code != 429 || w.Header().Get("Retry-After") == "" {
t.Fatal(w.Code, w.Body.String())
}
var n int
@@ -314,7 +318,7 @@ func TestUnknownDockerDiskBlocksBuild(t *testing.T) {
io.WriteString(w, "[]")
})
t.Setenv("DSH_DOCKER_ROOT_VISIBLE", "")
w := imageRequest(a.builds, sid, "POST", `{"version":"1.2.3"}`)
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
if w.Code != 202 {
t.Fatal(w.Body.String())
}
@@ -343,19 +347,19 @@ func TestExactNPMMetadata(t *testing.T) {
status int
ok bool
}{
{`{"version":"1.2.3"}`, 200, true},
{`{"version":"1.2.4"}`, 200, false},
{`{"version":"0.2.0-rc.1"}`, 200, true},
{`{"version":"0.2.0-rc.2"}`, 200, false},
{`{"version":"latest"}`, 200, false},
{`{}`, 404, false},
{`invalid`, 200, false},
} {
client := &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
if r.URL.Host != "registry.npmjs.org" || !strings.HasSuffix(r.URL.Path, "/1.2.3") {
if r.URL.Host != "registry.npmjs.org" || !strings.HasSuffix(r.URL.Path, "/0.2.0-rc.1") {
t.Error(r.URL)
}
return &http.Response{StatusCode: test.status, Body: io.NopCloser(strings.NewReader(test.body)), Header: make(http.Header)}, nil
})}
_, err := fetchNPMVersion(context.Background(), client, "1.2.3")
_, err := fetchNPMVersion(context.Background(), client, "0.2.0-rc.1")
if (err == nil) != test.ok {
t.Fatalf("%s: %v", test.body, err)
}
@@ -363,19 +367,56 @@ func TestExactNPMMetadata(t *testing.T) {
}
func TestBuildListReconcilesMissingAndLatestTimestamp(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "[]") })
seedVerifiedImage(t, a, "deleted", "1.2.3")
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) })
seedVerifiedImage(t, a, "deleted", "0.2.0-rc.1")
w := imageRequest(a.builds, sid, "GET", "")
var records []buildRecord
if err := json.Unmarshal(w.Body.Bytes(), &records); err != nil || len(records) != 1 || records[0].Status != "missing" {
t.Fatal(w.Body.String())
}
w = imageRequest(a.latest, sid, "GET", "")
var latest map[string]string
var latest struct {
Version string `json:"version"`
CheckedAt string `json:"checked_at"`
}
if err := json.Unmarshal(w.Body.Bytes(), &latest); err != nil {
t.Fatal(err)
}
if _, err := time.Parse(time.RFC3339, latest["checked_at"]); err != nil || latest["version"] != "1.2.3" {
if _, err := time.Parse(time.RFC3339, latest.CheckedAt); err != nil || latest.Version != "0.2.0-rc.1" {
t.Fatal(latest, err)
}
}
// The one-second intent poller must reuse per-image evaluations instead of
// re-inspecting Docker every tick, and a generation bump (deletion or
// admission change) must invalidate the cache.
func TestIntentPollerCachesImageEvaluations(t *testing.T) {
inspects := 0
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
inspects++
writeImageInspect(w, "image", "0.2.0-rc.1")
return
}
w.WriteHeader(404)
})
seedVerifiedImage(t, a, "image", "0.2.0-rc.1")
for n := 0; n < 3; n++ {
verified, err := a.verifiedImagesPoll(context.Background())
if err != nil || len(verified) != 1 || verified["image"] != "0.2.0-rc.1" {
t.Fatal(verified, err)
}
}
if inspects != 1 {
t.Fatalf("poller re-inspected Docker %d times for the same image", inspects)
}
buildQueue.Lock()
a.imageDeleteGeneration++
buildQueue.Unlock()
if _, err := a.verifiedImagesPoll(context.Background()); err != nil {
t.Fatal(err)
}
if inspects != 2 {
t.Fatalf("generation bump did not invalidate the cache: %d inspects", inspects)
}
}
internal/api/images_test.go
+31 -28
View File
@@ -26,7 +26,7 @@ func imageTestAPI(t *testing.T, h http.HandlerFunc) (*API, string) {
t.Cleanup(func() { buildDiskAvailable = oldDiskAvailable })
oldClient := npmBuildClient
npmBuildClient = &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"version":"1.2.3"}`))}, nil
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"version":"0.2.0-rc.1"}`))}, nil
})}
t.Cleanup(func() { npmBuildClient = oldClient })
root := t.TempDir()
@@ -86,19 +86,19 @@ func imageRequest(h http.HandlerFunc, sid, method, body string) *httptest.Respon
func TestImageListContract(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/unverified/json" {
writeImageInspect(w, "unverified", "1.2.3")
writeImageInspect(w, "unverified", "0.2.0-rc.1")
return
}
if r.URL.Path == "/v1.47/images/sha256:abc/json" {
writeImageInspect(w, "sha256:abc", "1.2.3")
writeImageInspect(w, "sha256:abc", "0.2.0-rc.1")
return
}
if !strings.Contains(r.URL.Query().Get("filters"), "dsh-mgr.role=runtime") {
t.Error("missing label filter")
}
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"},"RepoTags":["runtime:1.2.3"]},{"Id":"other","Labels":{}},{"Id":"unverified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"9.9.9"}}]`)
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"},"RepoTags":["runtime:0.2.0-rc.1"]},{"Id":"other","Labels":{}},{"Id":"unverified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"9.9.9"}}]`)
})
seedVerifiedImage(t, a, "sha256:abc", "1.2.3")
seedVerifiedImage(t, a, "sha256:abc", "0.2.0-rc.1")
if w := imageRequest(a.images, "", "GET", ""); w.Code != 401 {
t.Fatal(w.Code)
}
@@ -125,6 +125,9 @@ func seedVerifiedImage(t *testing.T, a *API, id, version string) {
if _, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,image_id,created_at,recipe_rev,platform) VALUES(?,?,'success',?,'now',?,?)", store.ID(), version, id, imageRecipeRev, imagePlatform); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("INSERT OR REPLACE INTO image_admissions(image_id,version,recipe_rev,platform,state,evidence,admitted_by,admitted_at) VALUES(?,?,?,?,'compatible','test fixture explicit acceptance','test','now')", id, version, imageRecipeRev, imagePlatform); err != nil {
t.Fatal(err)
}
}
func TestImageContextAndSanitizing(t *testing.T) {
r, err := imageBuildContext()
@@ -163,12 +166,12 @@ func TestImageContextAndSanitizing(t *testing.T) {
if len(sanitizeBuildLog(strings.Repeat("x", 2*buildLogLimit))) > buildLogLimit {
t.Fatal("unbounded")
}
for _, v := range []string{"latest", "*", "^1.2.3", "1.2", "1.2.3;id", "https://example.com", ""} {
for _, v := range []string{"latest", "*", "^0.2.0-rc.1", "1.2", "0.2.0-rc.1;id", "https://example.com", ""} {
if validBuildVersion(v) {
t.Fatal(v)
}
}
if !validBuildVersion("1.2.3-beta.1") {
if !validBuildVersion("0.2.0-rc.1-beta.1") {
t.Fatal("fixed prerelease rejected")
}
}
@@ -181,10 +184,10 @@ func TestLatestOfficialRegistry(t *testing.T) {
if r.URL.Host != "registry.npmjs.org" || r.URL.Scheme != "https" {
t.Fatal(r.URL)
}
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"version":"1.2.3"}`)), Header: make(http.Header)}, nil
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"version":"0.2.0-rc.1"}`)), Header: make(http.Header)}, nil
})}
v, err := fetchLatest(context.Background(), client)
if err != nil || v != "1.2.3" {
if err != nil || v != "0.2.0-rc.1" {
t.Fatalf("%s %v", v, err)
}
}
@@ -193,21 +196,21 @@ func TestBuildDedupAndVerification(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "1.2.3")
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
case "/v1.47/images/json":
if calls.Load() == 0 {
io.WriteString(w, "[]")
} else {
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
}
case "/v1.47/build":
calls.Add(1)
if r.URL.Query().Get("t") != "dsh-web:1.2.3-"+imageRecipeRev {
if r.URL.Query().Get("t") != "dsh-web:0.2.0-rc.1-"+imageRecipeRev {
t.Error("unexpected tag", r.URL.Query())
}
var args map[string]string
json.Unmarshal([]byte(r.URL.Query().Get("buildargs")), &args)
if args["DSH_VERSION"] != "1.2.3" {
if args["DSH_VERSION"] != "0.2.0-rc.1" {
t.Error(args)
}
io.Copy(io.Discard, r.Body)
@@ -230,7 +233,7 @@ func TestBuildDedupAndVerification(t *testing.T) {
w.WriteHeader(204)
case "/v1.47/containers/check/exec":
var cfg struct{ Cmd []string }
if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil || len(cfg.Cmd) != 3 || !strings.Contains(cfg.Cmd[2], "execFileSync('dsh',['--version']") || !strings.Contains(cfg.Cmd[2], `actual!=="1.2.3"`) {
if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil || len(cfg.Cmd) != 3 || !strings.Contains(cfg.Cmd[2], "execFileSync('dsh',['--version']") || !strings.Contains(cfg.Cmd[2], `actual!=="0.2.0-rc.1"`) {
t.Errorf("missing version verification: %+v %v", cfg, err)
}
io.WriteString(w, `{"Id":"probe"}`)
@@ -252,7 +255,7 @@ func TestBuildDedupAndVerification(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
responses <- imageRequest(a.builds, sid, "POST", `{"version":"1.2.3"}`)
responses <- imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
}()
}
wg.Wait()
@@ -290,7 +293,7 @@ func TestBuildDedupAndVerification(t *testing.T) {
if calls.Load() != 1 || deleted.Load() != 1 {
t.Fatalf("builds %d cleanup %d", calls.Load(), deleted.Load())
}
for _, body := range []string{`{"version":"latest"}`, `{"version":"1.2.3","url":"bad"}`, `{"version":"1.2.3"} {}`} {
for _, body := range []string{`{"version":"latest"}`, `{"version":"0.2.0-rc.1","url":"bad"}`, `{"version":"0.2.0-rc.1"} {}`} {
if w := imageRequest(a.builds, sid, "POST", body); w.Code != 400 {
t.Fatal(w.Code)
}
@@ -310,7 +313,7 @@ func TestVerificationFailureCleansUp(t *testing.T) {
io.WriteString(w, `{"message":"cannot start"}`)
}
})
if err := a.verifyBuiltImage(context.Background(), "bad", "1.2.3"); err == nil {
if err := a.verifyBuiltImage(context.Background(), "bad", "0.2.0-rc.1"); err == nil {
t.Fatal("accepted failed start")
}
if !deleted.Load() {
@@ -325,10 +328,10 @@ func TestBuildReuseAndMissingImage(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "1.2.3")
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
case "/v1.47/images/json":
if exists.Load() {
io.WriteString(w, `[{"Id":"sha256:reuse","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
io.WriteString(w, `[{"Id":"sha256:reuse","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
} else {
io.WriteString(w, "[]")
}
@@ -373,7 +376,7 @@ func TestBuildReuseAndMissingImage(t *testing.T) {
}
post := func() buildRecord {
t.Helper()
w := imageRequest(a.builds, sid, "POST", `{"version":"1.2.3"}`)
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
if w.Code != 202 {
t.Fatal(w.Body.String())
}
@@ -405,7 +408,7 @@ func TestBuildRestartRecovery(t *testing.T) {
if err := a.ensureBuilds(); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('old','1.2.3','running','now')"); err != nil {
if _, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('old','0.2.0-rc.1','running','now')"); err != nil {
t.Fatal(err)
}
delete(buildQueue.initialized, a.s.DB)
@@ -427,9 +430,9 @@ func TestFailedVerificationNeverAvailable(t *testing.T) {
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "1.2.3")
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
case "/v1.47/images/json":
writeJSON(w, []any{map[string]any{"Id": imageID, "Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": imageRecipeRev}}})
writeJSON(w, []any{map[string]any{"Id": imageID, "Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}}})
case "/v1.47/containers/create":
io.WriteString(w, `{"Id":"check"}`)
case "/v1.47/containers/check/start", "/v1.47/exec/probe/start":
@@ -442,13 +445,13 @@ func TestFailedVerificationNeverAvailable(t *testing.T) {
deleted.Store(true)
w.WriteHeader(204)
case "/v1.47/images/" + imageID + "/json":
writeImageInspect(w, imageID, "1.2.3")
writeImageInspect(w, imageID, "0.2.0-rc.1")
default:
t.Errorf("unexpected Docker call %s", r.URL)
w.WriteHeader(500)
}
})
w := imageRequest(a.builds, sid, "POST", `{"version":"1.2.3"}`)
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
if w.Code != 202 {
t.Fatal(w.Body.String())
}
@@ -481,12 +484,12 @@ func TestFailedVerificationNeverAvailable(t *testing.T) {
func TestLocalBuildImagePrefersPersistedID(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/verified/json" {
writeImageInspect(w, "verified", "1.2.3")
writeImageInspect(w, "verified", "0.2.0-rc.1")
return
}
io.WriteString(w, `[{"Id":"newer","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"1.2.3","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
io.WriteString(w, `[{"Id":"newer","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
})
id, err := a.localBuildImage(context.Background(), "1.2.3", "verified")
id, err := a.localBuildImage(context.Background(), "0.2.0-rc.1", "verified")
if err != nil || id != "verified" {
t.Fatalf("id=%s err=%v", id, err)
}
internal/api/intents.go
+25 -4
View File
@@ -2,6 +2,7 @@ package api
import (
"context"
"database/sql"
"encoding/json"
"errors"
"io"
@@ -41,6 +42,10 @@ func (a *API) intents(w http.ResponseWriter, r *http.Request) {
fail(w, 400, "name and fixed version required")
return
}
if !normalBuildAllowed(in.Version) {
fail(w, 400, errBuildUnsupported.Error())
return
}
i := store.Intent{ID: store.ID(), UID: u.ID, Name: strings.TrimSpace(in.Name), Version: in.Version, Host: a.runtimeHost(r), Status: "waiting"}
if err = a.s.ReserveIntent(i); err != nil {
if errors.Is(err, store.ErrInstanceLimit) {
@@ -124,21 +129,37 @@ func (a *API) recoverIntents(ctx context.Context) error {
continue
}
i, err := a.s.GetInstance(in.InstanceID)
if errors.Is(err, sql.ErrNoRows) {
a.finishIntent(in.ID, "failed", "instance reservation missing after process restart")
continue
}
if err != nil {
return err
}
if err == nil && i.Status == "running" {
a.finishIntent(in.ID, "success", "")
continue
}
cleanup, cancel := context.WithTimeout(ctx, 30*time.Second)
// Docker may have created the container before its ID reached SQLite.
err = a.removeRuntimeContainer(cleanup, runtimeNetwork(in.InstanceID))
if err == nil && i.ContainerID == "" {
err = a.recoverRuntimeContainer(cleanup, &i)
}
if err == nil && i.ContainerID != "" {
err = a.removeRuntimeContainer(cleanup, i.ContainerID)
}
if err == nil {
err = a.removeRuntimeNetwork(cleanup, in.InstanceID)
}
cancel()
if err != nil {
return err
a.s.SetInstance(in.InstanceID, "error")
a.finishIntent(in.ID, "failed", "instance creation requires recovery: "+err.Error())
a.s.AddAudit(in.UID, "instance.recovery_error", in.InstanceID, err.Error())
continue
}
a.s.DeleteInstance(in.InstanceID)
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=?", in.InstanceID)
_, _ = a.s.DB.Exec("DELETE FROM runtime_networks WHERE instance_id=?", in.InstanceID)
a.finishIntent(in.ID, "failed", "instance creation interrupted by process restart")
}
return nil
@@ -180,7 +201,7 @@ func (a *API) processIntents(ctx context.Context) {
continue
}
work, cancel := context.WithTimeout(ctx, 90*time.Second)
verified, err := a.verifiedImages(work)
verified, err := a.verifiedImagesPoll(work)
if err != nil {
cancel()
continue
internal/api/intents_test.go
+18 -10
View File
@@ -17,7 +17,7 @@ func TestIntentsReserveCancel(t *testing.T) {
}
for n := 0; n < 3; n++ {
w := httptest.NewRecorder()
a.intents(w, runtimeRequest("POST", "/api/intents", `{"name":"one","version":"1.2.3"}`, sid))
a.intents(w, runtimeRequest("POST", "/api/intents", `{"name":"one","version":"0.2.0-rc.1"}`, sid))
want := 202
if n == 2 {
want = 409
@@ -49,7 +49,7 @@ func TestIntentBuildSuccessUsesRealRuntime(t *testing.T) {
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "/images/"):
writeJSON(w, map[string]any{"Id": image, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3"}}})
writeJSON(w, map[string]any{"Id": image, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}}})
case strings.HasSuffix(r.URL.Path, "/containers/create"):
creates++
writeJSON(w, map[string]string{"Id": "runtime-created"})
@@ -57,15 +57,15 @@ func TestIntentBuildSuccessUsesRealRuntime(t *testing.T) {
w.WriteHeader(204)
}
})
seedVerifiedImage(t, a, image, "1.2.3")
seedVerifiedImage(t, a, image, "0.2.0-rc.1")
if err := a.s.EnsureIntents(); err != nil {
t.Fatal(err)
}
var build string
if err := a.s.DB.QueryRow("SELECT id FROM image_builds WHERE version='1.2.3'").Scan(&build); err != nil {
if err := a.s.DB.QueryRow("SELECT id FROM image_builds WHERE version='0.2.0-rc.1'").Scan(&build); err != nil {
t.Fatal(err)
}
in := store.Intent{ID: store.ID(), UID: "owner", Name: "one", Version: "1.2.3", Host: "panel.test"}
in := store.Intent{ID: store.ID(), UID: "owner", Name: "one", Version: "0.2.0-rc.1", Host: "panel.test"}
if err := a.s.ReserveIntent(in); err != nil {
t.Fatal(err)
}
@@ -79,21 +79,21 @@ func TestIntentBuildSuccessUsesRealRuntime(t *testing.T) {
t.Fatalf("%+v err=%v creates=%d", list, err, creates)
}
instances, err := a.s.Instances("owner")
if err != nil || len(instances) != 1 || instances[0].ImageID != image || instances[0].Version != "1.2.3" {
if err != nil || len(instances) != 1 || instances[0].ImageID != image || instances[0].Version != "0.2.0-rc.1" {
t.Fatalf("%+v %v", instances, err)
}
}
func TestIntentFailedBuildReleasesAndCancelPreservesBuild(t *testing.T) {
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("failed intent contacted Docker") })
seedVerifiedImage(t, a, "sha256:"+strings.Repeat("b", 64), "1.2.3")
seedVerifiedImage(t, a, "sha256:"+strings.Repeat("b", 64), "0.2.0-rc.1")
if err := a.s.EnsureIntents(); err != nil {
t.Fatal(err)
}
var build string
a.s.DB.QueryRow("SELECT id FROM image_builds LIMIT 1").Scan(&build)
for _, id := range []string{"cancel", "fail"} {
if err := a.s.ReserveIntent(store.Intent{ID: id, UID: "owner", Version: "1.2.3"}); err != nil {
if err := a.s.ReserveIntent(store.Intent{ID: id, UID: "owner", Version: "0.2.0-rc.1"}); err != nil {
t.Fatal(err)
}
a.s.DB.Exec("UPDATE creation_intents SET build_id=? WHERE id=?", build, id)
@@ -118,6 +118,14 @@ func TestIntentFailedBuildReleasesAndCancelPreservesBuild(t *testing.T) {
func TestIntentRestartCleansInterruptedCreationAndRetainsWaiting(t *testing.T) {
deletes := 0
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" && r.URL.Path == "/v1.47/networks/dsh-inst-reserved" {
w.Write([]byte(`{"Id":"network","Name":"dsh-inst-reserved","Driver":"bridge","Labels":{"dsh-mgr.instance":"reserved"}}`))
return
}
if r.Method == "GET" && r.URL.Path == "/v1.47/containers/dsh-inst-reserved/json" {
w.Write([]byte(`{"Id":"recovered","Name":"/dsh-inst-reserved","Config":{"Labels":{"dsh-mgr.instance":"reserved","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`))
return
}
if r.Method == "DELETE" {
deletes++
}
@@ -127,11 +135,11 @@ func TestIntentRestartCleansInterruptedCreationAndRetainsWaiting(t *testing.T) {
t.Fatal(err)
}
for _, id := range []string{"waiting", "creating"} {
if err := a.s.ReserveIntent(store.Intent{ID: id, UID: "owner", Version: "1.2.3"}); err != nil {
if err := a.s.ReserveIntent(store.Intent{ID: id, UID: "owner", Version: "0.2.0-rc.1"}); err != nil {
t.Fatal(err)
}
}
if err := a.s.ClaimIntent("creating", store.Instance{ID: "reserved", UID: "owner", Version: "1.2.3", Status: "creating"}); err != nil {
if err := a.s.ClaimIntent("creating", store.Instance{ID: "reserved", UID: "owner", Version: "0.2.0-rc.1", Status: "creating"}); err != nil {
t.Fatal(err)
}
if err := a.recoverIntents(context.Background()); err != nil {
internal/api/password.go
+1 -20
View File
@@ -37,26 +37,7 @@ func (a *API) changePassword(w http.ResponseWriter, r *http.Request) {
fail(w, 401, "authentication required")
return
}
locked, err := a.accountLocked(u.Username)
if err != nil {
fail(w, 503, "authentication unavailable")
return
}
if locked {
fail(w, 429, "account temporarily locked; retry later")
return
}
currentHash, err := a.s.Password(u.Username)
if err != nil {
fail(w, 503, "authentication unavailable")
return
}
if len(in.CurrentPassword) > 72 || bcrypt.CompareHashAndPassword([]byte(currentHash), []byte(in.CurrentPassword)) != nil {
if err = a.failedLogin(u.Username); err != nil {
fail(w, 503, "authentication unavailable")
return
}
fail(w, 400, "invalid current password")
if !a.reauthenticateLocked(w, u.ID, in.CurrentPassword, http.StatusBadRequest) {
return
}
hash, err := bcrypt.GenerateFromPassword([]byte(in.Password), bcrypt.DefaultCost)
internal/api/reauth.go
+45
View File
@@ -0,0 +1,45 @@
package api
import (
"net/http"
"golang.org/x/crypto/bcrypt"
)
func (a *API) reauthenticate(w http.ResponseWriter, uid, password string) bool {
a.authLoginMu.Lock()
defer a.authLoginMu.Unlock()
return a.reauthenticateLocked(w, uid, password, http.StatusForbidden)
}
// The caller holds authLoginMu; never acquire instance or user locks while holding it.
func (a *API) reauthenticateLocked(w http.ResponseWriter, uid, password string, invalidStatus int) bool {
u, err := a.s.UserByID(uid)
if err != nil {
fail(w, 503, "authentication unavailable")
return false
}
locked, err := a.accountLocked(u.Username)
if err != nil {
fail(w, 503, "authentication unavailable")
return false
}
if locked {
fail(w, 429, "account temporarily locked; retry later")
return false
}
hash, err := a.s.Password(u.Username)
if err != nil {
fail(w, 503, "authentication unavailable")
return false
}
if len(password) > 72 || bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) != nil {
if err = a.failedLogin(u.Username); err != nil {
fail(w, 503, "authentication unavailable")
return false
}
fail(w, invalidStatus, "invalid current password")
return false
}
return true
}
internal/api/reauth_test.go
+335
View File
@@ -0,0 +1,335 @@
package api
import (
"bytes"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"git.misaka.ren/m1saka/dsh-web/internal/store"
"golang.org/x/crypto/bcrypt"
)
func TestReauthSlowUpgradeDoesNotBlockOtherLogin(t *testing.T) {
inspecting, release := make(chan struct{}), make(chan struct{})
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || !strings.HasSuffix(r.URL.Path, "/containers/old/json") {
t.Errorf("unexpected Docker request: %s %s", r.Method, r.URL.Path)
w.WriteHeader(500)
return
}
close(inspecting)
<-release
w.WriteHeader(503)
})
hash, err := bcrypt.GenerateFromPassword([]byte("original-password"), bcrypt.MinCost)
if err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("UPDATE users SET pass_hash=? WHERE id='owner'", string(hash)); err != nil {
t.Fatal(err)
}
if err := a.s.CreateUser(store.User{ID: "other", Username: "other"}, string(hash)); err != nil {
t.Fatal(err)
}
i := store.Instance{ID: "one", UID: "owner", ContainerID: "old", Status: "exited", Created: time.Now()}
if err := a.s.AddInstance(i); err != nil {
t.Fatal(err)
}
upgradeDone := make(chan struct{})
go func() {
defer close(upgradeDone)
a.upgrade(httptest.NewRecorder(), runtimeRequest("POST", "/api/instances/one/upgrade", `{"image_id":"sha256:`+strings.Repeat("a", 64)+`","password":"original-password"}`, sid), i)
}()
defer func() {
close(release)
select {
case <-upgradeDone:
case <-time.After(5 * time.Second):
t.Error("upgrade did not finish after releasing Docker")
}
}()
select {
case <-inspecting:
case <-upgradeDone:
t.Fatal("upgrade did not reach Docker after reauthentication")
case <-time.After(5 * time.Second):
t.Fatal("upgrade did not reach inspection")
}
loginDone := make(chan *httptest.ResponseRecorder, 1)
go func() {
w := httptest.NewRecorder()
r := runtimeRequest("POST", "/api/login", resetLogin("other", "original-password"), "")
r.Header.Set("Origin", a.origin)
a.login(w, r)
loginDone <- w
}()
select {
case w := <-loginDone:
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
case <-time.After(5 * time.Second):
t.Fatal("slow upgrade held authentication lock and blocked another user's login")
}
select {
case <-upgradeDone:
t.Fatal("upgrade finished before Docker was released")
default:
}
}
func TestReauthConcurrentFailures(t *testing.T) {
a, i, _ := configTestAPI(t, false)
start := make(chan struct{})
done := make(chan int, 10)
for n := 0; n < 10; n++ {
go func() {
<-start
w := httptest.NewRecorder()
a.reauthenticate(w, i.UID, "wrong")
done <- w.Code
}()
}
close(start)
counts := map[int]int{}
for n := 0; n < 10; n++ {
select {
case code := <-done:
counts[code]++
case <-time.After(5 * time.Second):
t.Fatal("reauthentication deadlocked")
}
}
if counts[403] != 5 || counts[429] != 5 {
t.Fatal(counts)
}
}
func TestReauthOldBackupPasswordIndependent(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("requires chown")
}
a, i, home := configTestAPI(t, false)
configWrite(t, filepath.Join(home, ".credentials.yaml"), "original")
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
var result struct {
ID string `json:"id"`
}
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.ID == "" {
t.Fatal(w.Code, w.Body.String())
}
if _, err := a.s.DB.Exec("UPDATE users SET pass_hash='changed' WHERE id=?", i.UID); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("INSERT INTO auth_failures VALUES('owner',5,?)", time.Now().Add(time.Hour).Unix()); err != nil {
t.Fatal(err)
}
configWrite(t, filepath.Join(home, ".credentials.yaml"), "changed")
w = configCall(a, i, "POST", "restore", `{"backup_id":"`+result.ID+`","password":"original-password"}`)
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
content, err := os.ReadFile(filepath.Join(home, ".credentials.yaml"))
if err != nil || string(content) != "original" {
t.Fatal(string(content), err)
}
var failures int
if err := a.s.DB.QueryRow("SELECT failures FROM auth_failures WHERE username='owner'").Scan(&failures); err != nil || failures != 5 {
t.Fatal(failures, err)
}
}
func TestReauthSharedFailureLock(t *testing.T) {
a, i, _ := configTestAPI(t, false)
sid, _, err := a.s.Session(i.UID)
if err != nil {
t.Fatal(err)
}
registerTestAccess(t, a, sid)
backup := func(password string) *httptest.ResponseRecorder {
return configCall(a, i, "POST", "backups", `{"password":"`+password+`"}`)
}
upgrade := func(password string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
a.upgrade(w, runtimeRequest("POST", "/api/instances/one/upgrade", `{"image_id":"sha256:`+strings.Repeat("a", 64)+`","password":"`+password+`"}`, sid), i)
return w
}
for _, call := range []func(string) *httptest.ResponseRecorder{backup, upgrade, backup, upgrade} {
if w := call("wrong"); w.Code != 403 {
t.Fatal(w.Code, w.Body.String())
}
}
w := httptest.NewRecorder()
a.changePassword(w, runtimeRequest("POST", changePath, changeBody("wrong", resetSecret), sid))
if w.Code != 400 {
t.Fatal(w.Code, w.Body.String())
}
for _, call := range []func(string) *httptest.ResponseRecorder{backup, upgrade} {
if w := call("original-password"); w.Code != 429 {
t.Fatal(w.Code, w.Body.String())
}
}
w = httptest.NewRecorder()
r := runtimeRequest("POST", "/api/login", resetLogin("owner", "original-password"), "")
r.Header.Set("Origin", a.origin)
a.login(w, r)
if w.Code != 429 {
t.Fatal(w.Code, w.Body.String())
}
var failures int
if err := a.s.DB.QueryRow("SELECT failures FROM auth_failures WHERE username='owner'").Scan(&failures); err != nil || failures != 5 {
t.Fatal(failures, err)
}
if _, err := a.s.DB.Exec("UPDATE auth_failures SET locked_until=?", time.Now().Add(-time.Minute).Unix()); err != nil {
t.Fatal(err)
}
if w := upgrade("wrong"); w.Code != 403 {
t.Fatal(w.Code, w.Body.String())
}
if err := a.s.DB.QueryRow("SELECT failures FROM auth_failures WHERE username='owner'").Scan(&failures); err != nil || failures != 1 {
t.Fatal(failures, err)
}
}
func TestReauthFailureStorageUnavailable(t *testing.T) {
a, i, _ := configTestAPI(t, false)
if _, err := a.s.DB.Exec("DROP TABLE auth_failures"); err != nil {
t.Fatal(err)
}
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
if w.Code != 503 {
t.Fatal(w.Code, w.Body.String())
}
}
type quotaPausedBody struct {
io.Reader
started chan struct{}
resume chan struct{}
}
func (b *quotaPausedBody) Read(p []byte) (int, error) {
if b.started != nil {
close(b.started)
b.started = nil
<-b.resume
}
return b.Reader.Read(p)
}
func TestReauthQuotaInFlightUpload(t *testing.T) {
a, sid := runtimeTestAPI(t, func(http.ResponseWriter, *http.Request) {})
if _, err := a.s.DB.Exec("UPDATE users SET admin=1,quota=1048576 WHERE id='owner'"); err != nil {
t.Fatal(err)
}
u, err := a.s.UserByID("owner")
if err != nil {
t.Fatal(err)
}
dir := filepath.Join(a.root, "workspaces", u.ID)
if err := os.MkdirAll(dir, 0700); err != nil {
t.Fatal(err)
}
root, err := os.OpenRoot(dir)
if err != nil {
t.Fatal(err)
}
defer root.Close()
var body bytes.Buffer
form := multipart.NewWriter(&body)
part, err := form.CreateFormFile("file", "large.txt")
if err != nil {
t.Fatal(err)
}
part.Write(bytes.Repeat([]byte("x"), 4096))
form.Close()
started, resume := make(chan struct{}), make(chan struct{})
r := httptest.NewRequest("POST", "/api/ws/upload", &quotaPausedBody{Reader: &body, started: started, resume: resume})
r.Header.Set("Content-Type", form.FormDataContentType())
done := make(chan *httptest.ResponseRecorder, 1)
go func() {
w := httptest.NewRecorder()
a.wsUpload(w, r, root, u)
done <- w
}()
select {
case <-started:
case <-time.After(5 * time.Second):
close(resume)
t.Fatal("upload did not start reading")
}
w := httptest.NewRecorder()
a.users(w, runtimeRequest("PATCH", "/api/users", `{"id":"owner","quota_bytes":1024}`, sid))
close(resume)
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
select {
case w := <-done:
if w.Code != 413 {
t.Fatal(w.Code, w.Body.String())
}
case <-time.After(5 * time.Second):
t.Fatal("upload deadlocked")
}
entries, err := os.ReadDir(dir)
if err != nil || len(entries) != 0 {
t.Fatal("rejected upload left files", entries, err)
}
}
func TestReauthQuotaUpdateUsesUserLock(t *testing.T) {
a, sid := runtimeTestAPI(t, func(http.ResponseWriter, *http.Request) {})
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='owner'"); err != nil {
t.Fatal(err)
}
lock := a.userLocks.Acquire("owner")
defer func() {
if lock != nil {
lock.Release()
}
}()
done := make(chan *httptest.ResponseRecorder, 1)
go func() {
w := httptest.NewRecorder()
a.users(w, runtimeRequest("PATCH", "/api/users", `{"id":"owner","quota_bytes":1024}`, sid))
done <- w
}()
deadline := time.Now().Add(5 * time.Second)
for {
a.userLocks.mu.Lock()
refs := lock.refs
a.userLocks.mu.Unlock()
if refs == 2 {
break
}
select {
case w := <-done:
t.Fatal("quota update bypassed user lock", w.Code)
default:
}
if time.Now().After(deadline) {
t.Fatal("quota update did not wait on user lock")
}
runtime.Gosched()
}
lock.Release()
lock = nil
select {
case w := <-done:
if w.Code != 200 {
t.Fatal(w.Code, w.Body.String())
}
case <-time.After(5 * time.Second):
t.Fatal("quota update deadlocked")
}
}
internal/api/reconcile.go
+14
View File
@@ -44,6 +44,20 @@ func (a *API) Reconcile(ctx context.Context) error {
}
known := make(map[string]record)
for _, r := range records {
if r.container == "" && (r.status == "creating" || r.status == "error") {
i, loadErr := a.s.GetInstance(r.id)
if loadErr != nil {
return loadErr
}
if recoveryErr := a.recoverRuntimeContainer(ctx, &i); recoveryErr != nil {
a.s.AddAudit(r.uid, "reconcile.recovery_error", r.id, recoveryErr.Error())
}
r.container = i.ContainerID
r.status = "error"
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET status='error' WHERE id=?", r.id); err != nil {
return err
}
}
known[r.id] = r
}
matched := make(map[string]bool)
internal/api/resource_recovery.go
+305
View File
@@ -0,0 +1,305 @@
package api
import (
"context"
"database/sql"
"encoding/json"
"errors"
"git.misaka.ren/m1saka/dsh-web/internal/store"
"log"
"net/http"
"net/url"
"os"
"strings"
"time"
)
// pendingColumns whitelists the runtime_creates flag columns so no caller can
// concatenate an unvalidated column name into SQL.
type pendingColumn struct{ flag, stamp string }
var pendingColumns = map[string]pendingColumn{
"container_pending": {flag: "container_pending", stamp: "container_pending_at"},
"network_pending": {flag: "network_pending", stamp: "network_pending_at"},
}
// ensureColumn adds a TEXT column, tolerating databases that already have it.
func (a *API) ensureColumn(table, column string) error {
_, err := a.s.DB.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " TEXT NOT NULL DEFAULT ''")
if err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate column") {
return nil
}
return err
}
func (a *API) recoverRuntimeContainer(ctx context.Context, i *store.Instance) error {
c, err := a.docker.Inspect(ctx, runtimeNetwork(i.ID))
if dockerNotFound(err) {
_, pending, checkErr := a.pendingSince(i.ID, "container_pending")
if checkErr != nil {
return checkErr
}
if pending {
return errors.New("container creation outcome remains uncertain")
}
return nil
}
if err != nil {
return err
}
if c.ID == "" || strings.TrimPrefix(c.Name, "/") != runtimeNetwork(i.ID) || c.Config.Labels["dsh-mgr.instance"] != i.ID || c.Config.Labels["dsh-mgr.owner"] != i.UID || c.Config.Labels["dsh-mgr.role"] != "dsh" || c.Image != i.ImageID {
return errors.New("runtime container identity mismatch")
}
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET container_id=? WHERE id=? AND container_id=''", c.ID, i.ID); err != nil {
return err
}
i.ContainerID = c.ID
if err = a.setRuntimeCreatePending(i.ID, "container_pending", false); err != nil {
return err
}
return nil
}
func (a *API) ensureRuntimeCreates() error {
_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_creates(instance_id TEXT PRIMARY KEY,container_pending INTEGER NOT NULL DEFAULT 0,network_pending INTEGER NOT NULL DEFAULT 0,container_pending_at TEXT NOT NULL DEFAULT '',network_pending_at TEXT NOT NULL DEFAULT '')")
if err != nil {
return err
}
if err = a.ensureColumn("runtime_creates", "container_pending_at"); err != nil {
return err
}
if err = a.ensureColumn("runtime_creates", "network_pending_at"); err != nil {
return err
}
_, err = a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_networks(instance_id TEXT PRIMARY KEY,network_id TEXT NOT NULL)")
return err
}
func (a *API) setRuntimeCreatePending(id, column string, pending bool) error {
if err := a.ensureRuntimeCreates(); err != nil {
return err
}
c, ok := pendingColumns[column]
if !ok {
return errors.New("unknown pending column")
}
stamp := ""
if pending {
stamp = time.Now().UTC().Format(time.RFC3339Nano)
}
_, err := a.s.DB.Exec("INSERT INTO runtime_creates(instance_id,"+c.flag+","+c.stamp+") VALUES(?,?,?) ON CONFLICT(instance_id) DO UPDATE SET "+c.flag+"=excluded."+c.flag+","+c.stamp+"=excluded."+c.stamp, id, pending, stamp)
return err
}
// pendingSince reports when a pending flag was raised. A legacy row without a
// timestamp reports a zero time without resolving the uncertainty.
func (a *API) pendingSince(id, column string) (time.Time, bool, error) {
if err := a.ensureRuntimeCreates(); err != nil {
return time.Time{}, false, err
}
c, ok := pendingColumns[column]
if !ok {
return time.Time{}, false, errors.New("unknown pending column")
}
var pending bool
var stamp string
err := a.s.DB.QueryRow("SELECT "+c.flag+",COALESCE("+c.stamp+",'') FROM runtime_creates WHERE instance_id=?", id).Scan(&pending, &stamp)
if errors.Is(err, sql.ErrNoRows) {
return time.Time{}, false, nil
}
if err != nil {
return time.Time{}, false, err
}
if !pending {
return time.Time{}, false, nil
}
if stamp == "" {
return time.Time{}, true, nil
}
since, perr := time.Parse(time.RFC3339Nano, stamp)
if perr != nil {
return time.Time{}, true, nil
}
return since, true, nil
}
func (a *API) recoverRuntimeNetwork(ctx context.Context, id string) error {
_, pending, err := a.pendingSince(id, "network_pending")
if err != nil {
return err
}
var recorded string
err = a.s.DB.QueryRowContext(ctx, "SELECT network_id FROM runtime_networks WHERE instance_id=?", id).Scan(&recorded)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
lookup := runtimeNetwork(id)
if recorded != "" {
lookup = recorded
}
var network struct {
ID, Name, Driver string
Labels map[string]string
Containers map[string]json.RawMessage
}
err = a.docker.Do(ctx, http.MethodGet, "/networks/"+url.PathEscape(lookup), nil, &network)
if dockerNotFound(err) {
if pending && recorded == "" {
return errors.New("network creation outcome remains uncertain")
}
return a.setRuntimeCreatePending(id, "network_pending", false)
}
if err != nil {
return err
}
if network.ID == "" || (recorded != "" && recorded != network.ID) || network.Name != runtimeNetwork(id) || network.Driver != "bridge" || network.Labels["dsh-mgr.instance"] != id {
return errors.New("runtime network identity mismatch")
}
if _, err = a.s.DB.ExecContext(ctx, "INSERT INTO runtime_networks(instance_id,network_id) VALUES(?,?) ON CONFLICT(instance_id) DO UPDATE SET network_id=excluded.network_id", id, network.ID); err != nil {
return err
}
if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" && len(network.Containers) != 0 {
if _, connected := network.Containers[panel]; !connected {
c, err := a.docker.Inspect(ctx, panel)
if err != nil {
return err
}
panel = c.ID
}
if _, connected := network.Containers[panel]; connected {
err = a.docker.Do(ctx, http.MethodPost, "/networks/"+url.PathEscape(network.ID)+"/disconnect", map[string]any{"Container": panel, "Force": true}, nil)
if err != nil && !dockerNotFound(err) {
return err
}
}
}
err = a.docker.Do(ctx, http.MethodDelete, "/networks/"+url.PathEscape(network.ID), nil, nil)
if err != nil && !dockerNotFound(err) {
return err
}
return a.setRuntimeCreatePending(id, "network_pending", false)
}
func (a *API) ensureBuildChecks() error {
_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS image_build_checks(build_id TEXT PRIMARY KEY,image_id TEXT NOT NULL)")
if err != nil {
return err
}
if err = a.ensureColumn("image_build_checks", "container_id"); err != nil {
return err
}
return a.ensureColumn("image_build_checks", "created_at")
}
func buildCheckName(id string) string { return "dsh-build-check-" + id }
var errBuildCheckUncertain = errors.New("verification container creation outcome remains uncertain")
func (a *API) cleanupBuildCheck(ctx context.Context, id, image string) error {
var recorded string
if err := a.s.DB.QueryRowContext(ctx, "SELECT container_id FROM image_build_checks WHERE build_id=?", id).Scan(&recorded); err != nil {
return err
}
lookup := buildCheckName(id)
if recorded != "" {
lookup = recorded
}
c, err := a.docker.Inspect(ctx, lookup)
if dockerNotFound(err) {
if recorded != "" {
_, err := a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id)
return err
}
// A name lookup cannot rule out a create still running in Docker.
return errBuildCheckUncertain
}
if err != nil {
return err
}
if c.ID == "" || (recorded != "" && c.ID != recorded) || strings.TrimPrefix(c.Name, "/") != buildCheckName(id) || c.Image != image || c.Config.Labels["dsh-mgr.role"] != "build-check" || c.Config.Labels["dsh-mgr.build"] != id {
return errors.New("build verification container identity mismatch")
}
return a.deleteBuildCheck(ctx, id, c.ID)
}
func (a *API) deleteBuildCheck(ctx context.Context, id, container string) error {
if _, err := a.s.DB.ExecContext(ctx, "UPDATE image_build_checks SET container_id=? WHERE build_id=?", container, id); err != nil {
return err
}
err := a.docker.Do(ctx, http.MethodDelete, "/containers/"+url.PathEscape(container)+"?force=1&v=1", nil, nil)
if err != nil && !dockerNotFound(err) {
return err
}
_, err = a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id)
return err
}
func (a *API) recoverBuildChecks(ctx context.Context) error {
if err := a.ensureBuildChecks(); err != nil {
return err
}
rows, err := a.s.DB.QueryContext(ctx, "SELECT build_id,image_id,COALESCE(created_at,'') FROM image_build_checks")
if err != nil {
return err
}
type check struct{ id, image, createdAt string }
var checks []check
for rows.Next() {
var c check
if err = rows.Scan(&c.id, &c.image, &c.createdAt); err != nil {
break
}
checks = append(checks, c)
}
if err == nil {
err = rows.Err()
}
rows.Close()
if err != nil {
return err
}
var failures []error
missing := false
for _, c := range checks {
err := a.cleanupBuildCheck(ctx, c.id, c.image)
if errors.Is(err, errBuildCheckUncertain) {
missing = true
} else {
failures = append(failures, err)
}
}
if err := errors.Join(failures...); err != nil {
return err
}
if missing {
return errBuildCheckUncertain
}
return nil
}
func (a *API) persistBuildResult(b buildRecord, status, logs, detail, image, finished string) error {
tx, err := a.s.DB.Begin()
if err != nil {
return err
}
defer tx.Rollback()
if status == "success" && image != "" {
_, err = tx.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", image, b.Version, imageRecipeRev, imagePlatform, "dsh-web:"+b.Version+"-"+imageRecipeRev, "unverified", finished)
if err != nil {
return err
}
}
if _, err = tx.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, image, finished, b.ID); err != nil {
return err
}
return tx.Commit()
}
func (a *API) finishBuild(b buildRecord, status, logs, detail, image, finished string) {
for attempt := 0; ; attempt++ {
err := a.persistBuildResult(b, status, logs, detail, image, finished)
if err == nil {
return
}
if attempt%60 == 0 {
log.Printf("build %s: terminal transaction failed; retrying: %s", b.ID, sanitizeBuildLog(err.Error()))
}
select {
case <-a.s.Done():
return
case <-time.After(time.Second):
}
}
}
internal/api/resource_recovery_live_test.go
+407
View File
@@ -0,0 +1,407 @@
package api
import (
"bytes"
"context"
"crypto/rand"
"encoding/hex"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.misaka.ren/m1saka/dsh-web/internal/docker"
"git.misaka.ren/m1saka/dsh-web/internal/store"
)
const liveRecoveryLabel = "dsh-test.recovery-run"
type liveRecoveryResource struct{ kind, id string }
type liveRecoveryFault struct {
method, path string
release chan struct{}
done chan struct{}
once sync.Once
}
func (f *liveRecoveryFault) unblock() { f.once.Do(func() { close(f.release) }) }
type liveRecoveryProxy struct {
t *testing.T
run string
transport *http.Transport
mu sync.Mutex
handlers sync.WaitGroup
resources []liveRecoveryResource
fault *liveRecoveryFault
}
func liveRequire(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}
func (p *liveRecoveryProxy) arm(method, path string, delayed bool) *liveRecoveryFault {
f := &liveRecoveryFault{method: method, path: path, release: make(chan struct{}), done: make(chan struct{})}
if !delayed {
f.unblock()
}
p.mu.Lock()
p.fault = f
p.mu.Unlock()
p.t.Cleanup(f.unblock)
return f
}
func (p *liveRecoveryProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
p.handlers.Add(1)
defer p.handlers.Done()
body, err := io.ReadAll(r.Body)
if err != nil {
p.t.Error(err)
panic(http.ErrAbortHandler)
}
path := strings.TrimPrefix(r.URL.Path, "/v1.47")
create := r.Method == "POST" && (path == "/containers/create" || path == "/networks/create")
p.mu.Lock()
allowed := r.Method == "GET"
if create {
var spec struct {
Name string
Labels map[string]string
}
if json.Unmarshal(body, &spec) == nil {
name := spec.Name
if path == "/containers/create" {
name = r.URL.Query().Get("name")
}
allowed = spec.Labels[liveRecoveryLabel] == p.run && strings.Contains(name, p.run)
}
} else if !allowed {
for _, resource := range p.resources {
base := "/" + resource.kind + "/" + resource.id
if path == base || strings.HasPrefix(path, base+"/") {
allowed = true
}
}
if strings.HasSuffix(path, "/connect") || strings.HasSuffix(path, "/disconnect") {
var spec struct{ Container string }
_ = json.Unmarshal(body, &spec)
owned := false
for _, resource := range p.resources {
if resource.kind == "containers" && spec.Container == resource.id {
owned = true
}
}
allowed = allowed && owned
}
}
var fault *liveRecoveryFault
if p.fault != nil && p.fault.method == r.Method && p.fault.path == path {
fault = p.fault
p.fault = nil
}
p.mu.Unlock()
if !allowed {
p.t.Errorf("blocked out-of-scope request: %s %s", r.Method, path)
panic(http.ErrAbortHandler)
}
if fault != nil {
defer close(fault.done)
select {
case <-fault.release:
default:
conn, _, hijackErr := w.(http.Hijacker).Hijack()
if hijackErr != nil {
p.t.Error(hijackErr)
return
}
_ = conn.Close()
<-fault.release
}
}
// A disconnected test client must not cancel the deliberately late daemon operation.
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
req := r.Clone(ctx)
req.URL.Scheme, req.URL.Host, req.RequestURI = "http", "docker", ""
req.Body = io.NopCloser(bytes.NewReader(body))
if len(body) == 0 {
req.Body = http.NoBody
}
resp, err := p.transport.RoundTrip(req)
if err != nil {
p.t.Error(err)
panic(http.ErrAbortHandler)
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
if err != nil {
p.t.Error(err)
panic(http.ErrAbortHandler)
}
if create && resp.StatusCode >= 200 && resp.StatusCode < 300 {
var result struct{ ID string }
if err := json.Unmarshal(data, &result); err != nil || result.ID == "" {
p.t.Errorf("missing real create ID: %s", data)
} else {
kind := "containers"
if path == "/networks/create" {
kind = "networks"
}
p.mu.Lock()
p.resources = append(p.resources, liveRecoveryResource{kind, result.ID})
p.mu.Unlock()
p.t.Logf("created %s ID=%s run=%s", kind, result.ID, p.run)
}
}
if fault != nil {
p.t.Logf("dropped real response: %s %s status=%d", r.Method, path, resp.StatusCode)
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
p.t.Errorf("fault did not follow successful daemon operation: %s", data)
}
panic(http.ErrAbortHandler)
}
for key, values := range resp.Header {
for _, value := range values {
w.Header().Add(key, value)
}
}
w.WriteHeader(resp.StatusCode)
_, _ = w.Write(data)
}
func newLiveRecovery(t *testing.T) (*API, *liveRecoveryProxy, string) {
t.Helper()
if os.Getenv("DSH_DOCKER_LIVE_TEST") != "1" {
t.Skip("opt in with DSH_DOCKER_LIVE_TEST=1; uses /var/run/docker.sock and local node:24-alpine only")
}
t.Setenv("PANEL_CONTAINER_ID", "")
var nonce [12]byte
_, err := rand.Read(nonce[:])
liveRequire(t, err)
p := &liveRecoveryProxy{t: t, run: "live-" + hex.EncodeToString(nonce[:]), transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
}}}
real := docker.New("unix:///var/run/docker.sock")
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
var image struct{ ID string }
liveRequire(t, real.Do(ctx, "GET", "/images/node:24-alpine/json", nil, &image))
s, err := store.Open(filepath.Join(t.TempDir(), "live.db"))
liveRequire(t, err)
t.Cleanup(s.Close)
server := httptest.NewServer(p)
t.Cleanup(func() {
server.Close()
p.handlers.Wait()
p.transport.CloseIdleConnections()
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
p.mu.Lock()
resources := append([]liveRecoveryResource(nil), p.resources...)
p.mu.Unlock()
for _, kind := range []string{"containers", "networks"} {
for _, resource := range resources {
if resource.kind != kind {
continue
}
path := "/" + kind + "/" + resource.id
err := real.Do(ctx, "DELETE", path+"?force=1", nil, nil)
if err != nil && !dockerNotFound(err) {
t.Errorf("cleanup %s: %v", path, err)
}
if kind == "containers" {
path += "/json"
}
if err := real.Do(ctx, "GET", path, nil, nil); !dockerNotFound(err) {
t.Errorf("residual %s: %v", path, err)
} else {
t.Logf("cleanup confirmed absent %s %s", kind, resource.id)
}
}
}
filters, _ := json.Marshal(map[string][]string{"label": {liveRecoveryLabel + "=" + p.run}})
for _, path := range []string{"/containers/json?all=1&", "/networks?"} {
var residual []json.RawMessage
if err := real.Do(ctx, "GET", path+"filters="+url.QueryEscape(string(filters)), nil, &residual); err != nil || len(residual) != 0 {
t.Errorf("label residual check: %s count=%d err=%v", p.run, len(residual), err)
} else {
t.Logf("label residual check empty: %s %s", p.run, path)
}
}
})
return &API{s: s, root: t.TempDir(), docker: docker.New(server.URL)}, p, image.ID
}
func liveContainerSpec(run, image string, labels map[string]string) map[string]any {
labels[liveRecoveryLabel] = run
return map[string]any{"Image": image, "Cmd": []string{"sleep", "600"}, "Labels": labels, "HostConfig": map[string]any{"NetworkMode": "none", "Memory": 64 << 20, "PidsLimit": 32}}
}
func TestDockerLiveRecoveryNetwork(t *testing.T) {
for _, panelMode := range []string{"unset", "unconnected", "connected-id", "connected-name"} {
t.Run(panelMode, func(t *testing.T) {
a, p, image := newLiveRecovery(t)
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
id := p.run
liveRequire(t, a.setRuntimeCreatePending(id, "network_pending", true))
p.arm("POST", "/networks/create", false)
err := a.docker.Do(ctx, "POST", "/networks/create", map[string]any{"Name": runtimeNetwork(id), "Driver": "bridge", "Labels": map[string]string{"dsh-mgr.instance": id, liveRecoveryLabel: p.run}}, nil)
if err == nil {
t.Fatal("create response was not lost")
}
var network struct{ ID string }
liveRequire(t, a.docker.Do(ctx, "GET", "/networks/"+runtimeNetwork(id), nil, &network))
if panelMode != "unset" {
name := p.run + "-panel"
var panel struct{ ID string }
spec := liveContainerSpec(p.run, image, map[string]string{})
if strings.HasPrefix(panelMode, "connected") {
spec["HostConfig"].(map[string]any)["NetworkMode"] = network.ID
}
liveRequire(t, a.docker.Do(ctx, "POST", "/containers/create?name="+name, spec, &panel))
liveRequire(t, a.docker.Do(ctx, "POST", "/containers/"+panel.ID+"/start", nil, nil))
t.Setenv("PANEL_CONTAINER_ID", panel.ID)
if panelMode == "connected-name" {
t.Setenv("PANEL_CONTAINER_ID", name)
}
var endpoints struct{ Containers map[string]json.RawMessage }
liveRequire(t, a.docker.Do(ctx, "GET", "/networks/"+network.ID, nil, &endpoints))
_, connected := endpoints.Containers[panel.ID]
if connected != strings.HasPrefix(panelMode, "connected") {
t.Fatalf("unexpected real panel connection: %v", connected)
}
}
p.arm("DELETE", "/networks/"+network.ID, false)
if err := a.removeRuntimeNetwork(ctx, id); err == nil {
t.Fatal("delete response was not lost")
}
var recorded string
liveRequire(t, a.s.DB.QueryRow("SELECT network_id FROM runtime_networks WHERE instance_id=?", id).Scan(&recorded))
if recorded != network.ID {
t.Fatalf("lost recorded identity: %s", recorded)
}
liveRequire(t, a.removeRuntimeNetwork(ctx, id))
_, pending, err := a.pendingSince(id, "network_pending")
liveRequire(t, err)
if pending {
t.Fatal("pending did not converge")
}
if err := a.docker.Do(ctx, "GET", "/networks/"+network.ID, nil, nil); !dockerNotFound(err) {
t.Fatalf("network still exists: %v", err)
}
})
}
}
func TestDockerLiveRecoveryLateContainer(t *testing.T) {
for _, build := range []bool{false, true} {
name := "runtime"
if build {
name = "build-check"
}
t.Run(name, func(t *testing.T) {
a, p, image := newLiveRecovery(t)
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
defer cancel()
i := store.Instance{ID: p.run, UID: p.run, ImageID: image, Status: "error", Created: time.Now()}
name := runtimeNetwork(i.ID)
labels := map[string]string{"dsh-mgr.instance": i.ID, "dsh-mgr.owner": i.UID, "dsh-mgr.role": "dsh"}
if build {
name = buildCheckName(i.ID)
labels = map[string]string{"dsh-mgr.build": i.ID, "dsh-mgr.role": "build-check"}
liveRequire(t, a.ensureBuildChecks())
_, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES(?,?,?)", i.ID, image, time.Now().Add(-time.Hour).Format(time.RFC3339Nano))
liveRequire(t, err)
} else {
liveRequire(t, a.s.AddInstance(i))
liveRequire(t, a.setRuntimeCreatePending(i.ID, "container_pending", true))
}
fault := p.arm("POST", "/containers/create", true)
if err := a.docker.Do(ctx, "POST", "/containers/create?name="+name, liveContainerSpec(p.run, image, labels), nil); err == nil {
t.Fatal("expected disconnected create")
}
if _, err := a.docker.Inspect(ctx, name); !dockerNotFound(err) {
t.Fatalf("expected real initial 404: %v", err)
}
for attempt := 0; attempt < 2; attempt++ {
if build {
if err := a.recoverBuildChecks(ctx); !errors.Is(err, errBuildCheckUncertain) {
t.Fatalf("uncertainty lost: %v", err)
}
var count int
liveRequire(t, a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks WHERE build_id=?", i.ID).Scan(&count))
if count != 1 {
t.Fatal("build record lost")
}
} else {
if err := a.recoverRuntimeContainer(ctx, &i); err == nil {
t.Fatal("uncertainty lost")
}
_, pending, err := a.pendingSince(i.ID, "container_pending")
liveRequire(t, err)
if !pending {
t.Fatal("pending lost")
}
}
}
fault.unblock()
select {
case <-fault.done:
case <-ctx.Done():
t.Fatal(ctx.Err())
}
container, err := a.docker.Inspect(ctx, name)
liveRequire(t, err)
p.arm("DELETE", "/containers/"+container.ID, false)
if build {
if err := a.recoverBuildChecks(ctx); err == nil {
t.Fatal("delete response was not lost")
}
var recorded string
liveRequire(t, a.s.DB.QueryRow("SELECT container_id FROM image_build_checks WHERE build_id=?", i.ID).Scan(&recorded))
if recorded != container.ID {
t.Fatal("build identity lost")
}
liveRequire(t, a.recoverBuildChecks(ctx))
var count int
liveRequire(t, a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count))
if count != 0 {
t.Fatal("build record retained")
}
} else {
liveRequire(t, a.recoverRuntimeContainer(ctx, &i))
_, pending, err := a.pendingSince(i.ID, "container_pending")
liveRequire(t, err)
if pending {
t.Fatal("runtime pending did not converge")
}
stored, err := a.s.GetInstance(i.ID)
liveRequire(t, err)
if stored.ContainerID != container.ID {
t.Fatal("runtime identity lost")
}
if err := a.removeRuntimeContainer(ctx, i.ContainerID); err == nil {
t.Fatal("delete response was not lost")
}
liveRequire(t, a.removeRuntimeContainer(ctx, i.ContainerID))
}
if _, err := a.docker.Inspect(ctx, container.ID); !dockerNotFound(err) {
t.Fatalf("late resource not reclaimed: %v", err)
}
})
}
}
internal/api/resource_recovery_test.go
+828
View File
@@ -0,0 +1,828 @@
package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"git.misaka.ren/m1saka/dsh-web/internal/docker"
"git.misaka.ren/m1saka/dsh-web/internal/store"
"net/http"
"net/http/httptest"
"path/filepath"
"sync/atomic"
"testing"
"time"
)
func TestBuildCheckRenamedContainerRetainsIdentity(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" || r.URL.Path != "/v1.47/containers/known/json" {
t.Errorf("unsafe lookup %s %s", r.Method, r.URL.Path)
w.WriteHeader(404)
return
}
fmt.Fprint(w, `{"Id":"known","Name":"/renamed","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
})
if err := a.ensureBuildChecks(); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
t.Fatal(err)
}
if err := a.recoverBuildChecks(context.Background()); err == nil {
t.Fatal("renamed identity accepted")
}
var n int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
t.Fatalf("record lost %d %v", n, err)
}
}
func TestRuntimeNetworkDeletionLostResponseAndReplacement(t *testing.T) {
for _, foreign := range []bool{false, true} {
t.Run(fmt.Sprint(foreign), func(t *testing.T) {
t.Setenv("PANEL_CONTAINER_ID", "")
deleted := false
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
if r.Method == "GET" {
if deleted {
if r.URL.Path != "/v1.47/networks/known" {
t.Error("did not use persisted ID")
}
w.WriteHeader(404)
return
}
owner := "net"
if foreign {
owner = "foreign"
}
fmt.Fprintf(w, `{"Id":"known","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, owner)
return
}
if foreign {
t.Error("deleted foreign network")
}
deleted = true
w.WriteHeader(500)
})
if err := a.setRuntimeCreatePending("net", "network_pending", false); err != nil {
t.Fatal(err)
}
if err := a.removeRuntimeNetwork(context.Background(), "net"); err == nil {
t.Fatal("first error missing")
}
if foreign {
return
}
if err := a.removeRuntimeNetwork(context.Background(), "net"); err != nil {
t.Fatalf("confirmed deletion did not converge: %v", err)
}
})
}
}
func TestStartIntentsUncertainCreationKeepsRecoveryAccessible(t *testing.T) {
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
if r.Method != "GET" {
t.Error("unexpected destructive call")
}
w.WriteHeader(404)
})
if err := a.s.EnsureIntents(); err != nil {
t.Fatal(err)
}
if err := a.s.ReserveIntent(store.Intent{ID: "intent", UID: "owner"}); err != nil {
t.Fatal(err)
}
if err := a.s.ClaimIntent("intent", store.Instance{ID: "pending", UID: "owner", Status: "creating"}); err != nil {
t.Fatal(err)
}
if err := a.setRuntimeCreatePending("pending", "container_pending", true); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if err := a.StartIntents(ctx); err != nil {
t.Fatalf("uncertain creation blocked startup: %v", err)
}
i, err := a.s.GetInstance("pending")
if err != nil || i.Status != "error" {
t.Fatalf("recovery reservation missing %+v %v", i, err)
}
list, err := a.s.Intents("owner")
if err != nil || len(list) != 1 || list[0].Status != "failed" {
t.Fatalf("intent not terminated %+v %v", list, err)
}
}
func TestNetworkCreateLostResponse(t *testing.T) {
for _, mode := range []string{"present", "late", "foreign"} {
t.Run(mode, func(t *testing.T) {
t.Setenv("PANEL_CONTAINER_ID", "")
late := false
deletes := 0
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
switch r.Method + " " + r.URL.Path {
case "POST /v1.47/networks/create":
w.WriteHeader(500)
case "GET /v1.47/containers/dsh-inst-net/json":
w.WriteHeader(404)
case "GET /v1.47/networks/dsh-inst-net":
if mode == "late" && !late {
w.WriteHeader(404)
return
}
id := "net"
if mode == "foreign" {
id = "other"
}
fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, id)
case "DELETE /v1.47/networks/network-id":
deletes++
w.WriteHeader(204)
default:
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
w.WriteHeader(500)
}
})
seedVerifiedImage(t, a, "image", "1.2.3")
i := store.Instance{ID: "net", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
if err := a.s.AddInstance(i); err != nil {
t.Fatal(err)
}
if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
t.Fatal("lost response accepted")
}
if mode == "present" {
if deletes != 1 {
t.Fatal("network leaked")
}
return
}
saved, err := a.s.GetInstance("net")
if err != nil || saved.Status != "error" {
t.Fatalf("lost reservation %+v %v", saved, err)
}
late = true
w := httptest.NewRecorder()
a.instance(w, runtimeRequest("DELETE", "/api/instances/net", "", sid))
if mode == "foreign" {
if deletes != 0 || w.Code != 502 {
t.Fatal("foreign network touched")
}
return
}
if w.Code != 200 || deletes != 1 {
t.Fatalf("late network retry %d %d", w.Code, deletes)
}
})
}
}
func TestBuildCheckReopenWorkerWithoutQueuedBuilds(t *testing.T) {
path := filepath.Join(t.TempDir(), "recovery.db")
s, err := store.Open(path)
if err != nil {
t.Fatal(err)
}
a := &API{s: s}
if err = a.ensureBuildChecks(); err != nil {
t.Fatal(err)
}
if _, err = s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id) VALUES('task','image')"); err != nil {
t.Fatal(err)
}
s.Close()
s, err = store.Open(path)
if err != nil {
t.Fatal(err)
}
defer s.Close()
var attempts atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" {
fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
return
}
if r.Method == "DELETE" {
if attempts.Add(1) == 1 {
w.WriteHeader(500)
} else {
w.WriteHeader(204)
}
return
}
t.Errorf("unexpected %s", r.Method)
}))
defer srv.Close()
a = &API{s: s, docker: docker.New(srv.URL)}
if err = a.StartImageBuilds(); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(4 * time.Second)
for {
var n int
if err = s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil {
t.Fatal(err)
}
if n == 0 {
break
}
if time.Now().After(deadline) {
t.Fatal("reopen worker did not retry")
}
time.Sleep(10 * time.Millisecond)
}
if attempts.Load() != 2 {
t.Fatalf("attempts %d", attempts.Load())
}
}
func TestBuildTerminalFailureDoesNotBlockNextTask(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) })
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
t.Fatal(err)
}
for _, id := range []string{"a", "b"} {
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES(?,'1.2.3','queued',?)", id, id); err != nil {
t.Fatal(err)
}
}
if _, err = a.s.DB.Exec("CREATE TRIGGER fail_a BEFORE UPDATE ON image_builds WHEN NEW.id='a' AND NEW.status='failed' BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
t.Fatal(err)
}
a.startBuildWorker()
deadline := time.Now().Add(4 * time.Second)
for {
var status string
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='b'").Scan(&status); err != nil {
t.Fatal(err)
}
if status == "failed" {
break
}
if time.Now().After(deadline) {
t.Fatal("A terminal retry blocked B")
}
time.Sleep(10 * time.Millisecond)
}
if _, err = a.s.DB.Exec("DROP TRIGGER fail_a"); err != nil {
t.Fatal(err)
}
for {
var status string
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='a'").Scan(&status); err != nil {
t.Fatal(err)
}
if status == "failed" {
break
}
if time.Now().After(deadline) {
t.Fatal("A never converged")
}
time.Sleep(10 * time.Millisecond)
}
}
func TestCreateLostResponseRecovery(t *testing.T) {
for _, mode := range []string{"cleanup", "retry", "foreign", "inspect-failure", "late"} {
t.Run(mode, func(t *testing.T) {
t.Setenv("PANEL_CONTAINER_ID", "")
deleted := 0
retry := false
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
switch r.Method + " " + r.URL.Path {
case "POST /v1.47/networks/create":
fmt.Fprint(w, `{"Id":"dsh-inst-lost"}`)
case "GET /v1.47/networks/dsh-inst-lost":
fmt.Fprint(w, `{"Id":"dsh-inst-lost","Name":"dsh-inst-lost","Driver":"bridge","Labels":{"dsh-mgr.instance":"lost"}}`)
case "POST /v1.47/containers/create":
fmt.Fprint(w, "{")
case "GET /v1.47/containers/dsh-inst-lost/json":
if mode == "late" && !retry {
w.WriteHeader(404)
return
}
if mode == "inspect-failure" && !retry {
w.WriteHeader(500)
return
}
owner := "owner"
if mode == "foreign" {
owner = "another-panel"
}
fmt.Fprintf(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":%q,"dsh-mgr.role":"dsh"}}}`, owner)
case "DELETE /v1.47/containers/created":
if mode == "retry" && !retry {
w.WriteHeader(500)
return
}
deleted++
w.WriteHeader(204)
case "DELETE /v1.47/networks/dsh-inst-lost":
w.WriteHeader(204)
default:
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
w.WriteHeader(500)
}
})
seedVerifiedImage(t, a, "image", "1.2.3")
i := store.Instance{ID: "lost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
if err := a.s.AddInstance(i); err != nil {
t.Fatal(err)
}
if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
t.Fatal("accepted lost response")
}
if mode == "cleanup" {
if _, err := a.s.GetInstance(i.ID); err == nil || deleted != 1 {
t.Fatal("cleanup incomplete")
}
return
}
saved, err := a.s.GetInstance(i.ID)
if err != nil || saved.Status != "error" || deleted != 0 {
t.Fatalf("lost recovery record: %+v %v", saved, err)
}
retry = true
w := httptest.NewRecorder()
a.instance(w, runtimeRequest("DELETE", "/api/instances/lost", "", sid))
if mode == "foreign" {
if w.Code != 502 || deleted != 0 {
t.Fatal("foreign resource touched")
}
return
}
if w.Code != 200 || deleted != 1 {
t.Fatalf("retry failed %d %s", w.Code, w.Body.String())
}
})
}
}
func TestBuildCheckRecoveryIdentityAndRetry(t *testing.T) {
for _, mode := range []string{"lost", "delete-failure", "foreign", "absent"} {
t.Run(mode, func(t *testing.T) {
deleted := 0
retry := false
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == "POST" && r.URL.Path == "/v1.47/containers/create":
if r.URL.Query().Get("name") != buildCheckName("task") {
t.Error("missing task name")
}
if mode == "delete-failure" {
fmt.Fprint(w, `{"Id":"check"}`)
} else {
fmt.Fprint(w, "{")
}
case r.Method == "GET":
if mode == "absent" && !retry {
w.WriteHeader(404)
return
}
if !retry && mode == "lost" {
w.WriteHeader(500)
return
}
id := "task"
if mode == "foreign" {
id = "another-panel"
}
fmt.Fprintf(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":%q}}}`, id)
case r.Method == "DELETE":
if !retry {
w.WriteHeader(500)
return
}
deleted++
w.WriteHeader(204)
default:
w.WriteHeader(500)
}
})
if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
t.Fatal("accepted failed validation")
}
var count int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil {
t.Fatal(err)
}
if count != 1 {
t.Fatal("lost cleanup record")
}
retry = true
err := a.recoverBuildChecks(context.Background())
if mode == "foreign" {
if err == nil || deleted != 0 {
t.Fatal("foreign container deleted")
}
return
}
if err != nil {
t.Fatal(err)
}
if deleted != 1 {
t.Fatal("not recovered")
}
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil || count != 0 {
t.Fatalf("record retained %d %v", count, err)
}
})
}
}
func TestReconcileRecoversLostRuntimeIdentity(t *testing.T) {
t.Setenv("PANEL_CONTAINER_ID", "")
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
if r.Method != "GET" {
t.Fatalf("destructive recovery: %s", r.Method)
}
if r.URL.Path == "/v1.47/containers/json" {
fmt.Fprint(w, `[{"Id":"created","State":"running","Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner"}}]`)
return
}
fmt.Fprint(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
})
if err := a.s.AddInstance(store.Instance{ID: "lost", UID: "owner", ImageID: "image", Status: "creating"}); err != nil {
t.Fatal(err)
}
if err := a.Reconcile(context.Background()); err != nil {
t.Fatal(err)
}
i, err := a.s.GetInstance("lost")
if err != nil || i.ContainerID != "created" || i.Status != "error" {
t.Fatalf("not recovered: %+v %v", i, err)
}
}
func TestBuildResultTransactionAndRetry(t *testing.T) {
for _, table := range []string{"images", "image_builds"} {
t.Run(table, func(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker request") })
buildQueue.Lock()
err := a.ensureBuilds()
buildQueue.Unlock()
if err != nil {
t.Fatal(err)
}
b := buildRecord{ID: "task", Version: "1.2.3"}
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('task','1.2.3','validating','now')"); err != nil {
t.Fatal(err)
}
event := "INSERT"
if table == "image_builds" {
event = "UPDATE"
}
if _, err = a.s.DB.Exec("CREATE TRIGGER fail_result BEFORE " + event + " ON " + table + " BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
t.Fatal(err)
}
if err = a.persistBuildResult(b, "success", "logs", "", "image", "now"); err == nil {
t.Fatal("injection ignored")
}
var status string
var count int
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "validating" {
t.Fatalf("partial terminal write %s %v", status, err)
}
if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 0 {
t.Fatalf("partial index write %d %v", count, err)
}
done := make(chan struct{})
go func() { a.finishBuild(b, "success", "logs", "", "image", "now"); close(done) }()
select {
case <-done:
t.Fatal("retry abandoned")
case <-time.After(30 * time.Millisecond):
}
if _, err = a.s.DB.Exec("DROP TRIGGER fail_result"); err != nil {
t.Fatal(err)
}
select {
case <-done:
case <-time.After(4 * time.Second):
t.Fatal("terminal write not retried")
}
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "success" {
t.Fatalf("terminal %s %v", status, err)
}
if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 1 {
t.Fatalf("index %d %v", count, err)
}
})
}
}
func TestIntentRecoveryMissingReservationDoesNotTouchDocker(t *testing.T) {
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
t.Error("missing reservation must not authorize Docker cleanup")
})
if err := a.s.EnsureIntents(); err != nil {
t.Fatal(err)
}
if err := a.s.ReserveIntent(store.Intent{ID: "missing", UID: "owner"}); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("UPDATE creation_intents SET status='creating',instance_id='gone' WHERE id='missing'"); err != nil {
t.Fatal(err)
}
if err := a.recoverIntents(context.Background()); err != nil {
t.Fatal(err)
}
list, err := a.s.Intents("owner")
if err != nil || len(list) != 1 || list[0].Status != "failed" {
t.Fatalf("intent not terminated: %+v %v", list, err)
}
}
func TestBuildTerminalRetryStopsWhenDatabaseCloses(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker call") })
a.s.Close()
done := make(chan struct{})
go func() { a.finishBuild(buildRecord{ID: "closed"}, "failed", "", "", "", "now"); close(done) }()
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("closed database retry did not stop")
}
}
func TestBuildCheckDefiniteRejectionAndExternalDeletion(t *testing.T) {
for _, rejected := range []bool{true, false} {
t.Run(fmt.Sprint(rejected), func(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method == "POST" && r.URL.Path == "/v1.47/containers/create" {
w.WriteHeader(400)
return
}
w.WriteHeader(404)
})
if rejected {
if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
t.Fatal("rejection accepted")
}
} else {
if err := a.ensureBuildChecks(); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
t.Fatal(err)
}
if err := a.recoverBuildChecks(context.Background()); err != nil {
t.Fatal(err)
}
}
var n int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 {
t.Fatalf("definite outcome retained: %d %v", n, err)
}
})
}
}
func TestUncertainPendingRetainsLateResources(t *testing.T) {
for _, column := range []string{"container_pending", "network_pending"} {
for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
t.Run(column+"/"+stamp, func(t *testing.T) {
t.Setenv("PANEL_CONTAINER_ID", "")
late := false
deletes := 0
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
switch r.Method + " " + r.URL.Path {
case "GET /v1.47/containers/dsh-inst-ghost/json":
if !late || column != "container_pending" {
w.WriteHeader(404)
return
}
fmt.Fprint(w, `{"Id":"container-id","Name":"/dsh-inst-ghost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"ghost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
case "GET /v1.47/networks/dsh-inst-ghost":
if !late || column != "network_pending" {
w.WriteHeader(404)
return
}
fmt.Fprint(w, `{"Id":"network-id","Name":"dsh-inst-ghost","Driver":"bridge","Labels":{"dsh-mgr.instance":"ghost"}}`)
case "DELETE /v1.47/containers/container-id", "DELETE /v1.47/networks/network-id":
deletes++
w.WriteHeader(204)
default:
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
w.WriteHeader(500)
}
})
i := store.Instance{ID: "ghost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "error"}
if err := a.s.AddInstance(i); err != nil {
t.Fatal(err)
}
if err := a.setRuntimeCreatePending(i.ID, column, true); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("UPDATE runtime_creates SET "+pendingColumns[column].stamp+"=? WHERE instance_id=?", stamp, i.ID); err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < 2; attempt++ {
w := httptest.NewRecorder()
a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
if w.Code != 502 {
t.Fatalf("uncertainty discarded: %d %s", w.Code, w.Body.String())
}
if _, pending, err := a.pendingSince(i.ID, column); err != nil || !pending {
t.Fatalf("pending lost: %v %v", pending, err)
}
if _, err := a.s.GetInstance(i.ID); err != nil {
t.Fatal(err)
}
}
late = true
w := httptest.NewRecorder()
a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
if w.Code != 200 || deletes != 1 {
t.Fatalf("late cleanup: %d %s deletes=%d", w.Code, w.Body.String(), deletes)
}
if _, err := a.s.GetInstance(i.ID); err == nil {
t.Fatal("instance retained")
}
})
}
}
}
func TestBuildCheckUncertainRowRetainsLateContainer(t *testing.T) {
for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
t.Run(stamp, func(t *testing.T) {
late := false
deletes := 0
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.Method + " " + r.URL.Path {
case "GET /v1.47/containers/dsh-build-check-task/json":
if !late {
w.WriteHeader(404)
return
}
fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
case "DELETE /v1.47/containers/check":
deletes++
w.WriteHeader(204)
default:
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
w.WriteHeader(500)
}
})
if err := a.ensureBuildChecks(); err != nil {
t.Fatal(err)
}
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES('task','image',?)", stamp); err != nil {
t.Fatal(err)
}
for attempt := 0; attempt < 2; attempt++ {
if err := a.recoverBuildChecks(context.Background()); !errors.Is(err, errBuildCheckUncertain) {
t.Fatalf("uncertainty discarded: %v", err)
}
var n int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
t.Fatalf("record lost: %d %v", n, err)
}
}
late = true
if err := a.recoverBuildChecks(context.Background()); err != nil {
t.Fatal(err)
}
var n int
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 || deletes != 1 {
t.Fatalf("late cleanup: rows=%d deletes=%d err=%v", n, deletes, err)
}
})
}
}
func TestRuntimeNetworkPanelDisconnect(t *testing.T) {
for _, mode := range []string{"not-connected", "connected", "panel-name", "other-connected", "lost-disconnect", "forbidden", "inspect-forbidden", "foreign"} {
t.Run(mode, func(t *testing.T) {
panel := "panel-id"
if mode == "panel-name" || mode == "inspect-forbidden" {
panel = "panel"
}
t.Setenv("PANEL_CONTAINER_ID", panel)
connected := mode != "not-connected"
disconnects, deletes := 0, 0
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/image/json" {
writeImageInspect(w, "image", "1.2.3")
return
}
switch r.Method + " " + r.URL.Path {
case "GET /v1.47/networks/dsh-inst-net", "GET /v1.47/networks/network-id":
owner := "net"
if mode == "foreign" {
owner = "other"
}
containers := "{}"
if connected {
containers = `{"panel-id":{}}`
}
if mode == "other-connected" {
containers = `{"other-id":{}}`
}
fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q},"Containers":%s}`, owner, containers)
case "GET /v1.47/containers/panel/json", "GET /v1.47/containers/panel-id/json":
if mode == "inspect-forbidden" {
w.WriteHeader(403)
return
}
fmt.Fprint(w, `{"Id":"panel-id"}`)
case "POST /v1.47/networks/network-id/disconnect":
disconnects++
var body struct {
Container string
Force bool
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Container != "panel-id" || !body.Force {
t.Errorf("disconnect identity: %+v %v", body, err)
}
if mode == "not-connected" || mode == "other-connected" || mode == "foreign" {
t.Error("unsafe disconnect")
w.WriteHeader(500)
return
}
if mode == "forbidden" {
w.WriteHeader(403)
return
}
connected = false
if mode == "lost-disconnect" {
w.WriteHeader(500)
return
}
w.WriteHeader(204)
case "DELETE /v1.47/networks/network-id":
deletes++
w.WriteHeader(204)
default:
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
w.WriteHeader(500)
}
})
if err := a.setRuntimeCreatePending("net", "network_pending", true); err != nil {
t.Fatal(err)
}
err := a.recoverRuntimeNetwork(context.Background(), "net")
wantError := mode == "lost-disconnect" || mode == "forbidden" || mode == "inspect-forbidden" || mode == "foreign"
if (err != nil) != wantError {
t.Fatalf("recovery error=%v wantError=%v", err, wantError)
}
if wantError {
if deletes != 0 {
t.Fatal("deleted after failure")
}
if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || !pending {
t.Fatalf("pending lost: %v %v", pending, err)
}
if mode != "lost-disconnect" {
return
}
if err := a.recoverRuntimeNetwork(context.Background(), "net"); err != nil {
t.Fatalf("disconnect retry: %v", err)
}
}
wantDisconnects := 1
if mode == "not-connected" || mode == "other-connected" {
wantDisconnects = 0
}
if deletes != 1 || disconnects != wantDisconnects {
t.Fatalf("deletes=%d disconnects=%d", deletes, disconnects)
}
if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || pending {
t.Fatalf("pending retained: %v %v", pending, err)
}
})
}
}
internal/api/runtime.go
+109 -14
View File
@@ -81,6 +81,7 @@ func (a *API) instances(w http.ResponseWriter, r *http.Request) {
ImageID string `json:"image_id"`
}
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
decoder.DisallowUnknownFields()
if decoder.Decode(&in) != nil || decoder.Decode(new(any)) != io.EOF || strings.TrimSpace(in.Name) == "" || len(in.Name) > 200 || !runtimeImageID.MatchString(in.ImageID) {
fail(w, 400, "invalid instance")
return
@@ -139,13 +140,25 @@ func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance,
if _, err := a.s.GetInstance(i.ID); err != nil {
return i, errors.New("instance reservation unavailable")
}
networkCreated, complete := false, false
networkCreated, networkAttempted, containerAttempted, complete := false, false, false, false
defer func() {
if complete {
return
}
cleanup, done := context.WithTimeout(context.Background(), 20*time.Second)
defer done()
if networkAttempted && !networkCreated {
if err := a.recoverRuntimeNetwork(cleanup, i.ID); err != nil {
a.s.SetInstance(i.ID, "error")
return
}
}
if containerAttempted && i.ContainerID == "" {
if err := a.recoverRuntimeContainer(cleanup, &i); err != nil {
a.s.SetInstance(i.ID, "error")
return
}
}
if i.ContainerID != "" {
if err := a.removeRuntimeContainer(cleanup, i.ContainerID); err != nil {
a.s.SetInstance(i.ID, "error")
@@ -159,7 +172,24 @@ func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance,
}
}
a.s.DeleteInstance(i.ID)
_, _ = a.s.DB.Exec("DELETE FROM runtime_networks WHERE instance_id=?", i.ID)
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=?", i.ID)
}()
admissionMu.RLock()
defer admissionMu.RUnlock()
buildQueue.Lock()
policyErr := a.ensureBuilds()
buildQueue.Unlock()
if policyErr != nil {
return i, policyErr
}
policy, policyErr := a.evaluateImage(ctx, i.ImageID)
if policyErr != nil {
return i, policyErr
}
if !policy.Selectable || policy.Version != i.Version {
return i, errors.New("image is not admitted for creation")
}
root, err := filepath.Abs(a.root)
if err != nil {
return i, errors.New("storage unavailable")
@@ -186,10 +216,30 @@ func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance,
return i, err
}
network := runtimeNetwork(i.ID)
if err := a.docker.Do(ctx, "POST", "/networks/create", map[string]any{"Name": network, "Driver": "bridge", "CheckDuplicate": true, "Labels": map[string]string{"dsh-mgr.instance": i.ID}}, nil); err != nil {
if err := a.setRuntimeCreatePending(i.ID, "network_pending", true); err != nil {
return i, err
}
networkAttempted = true
var createdNetwork struct {
ID string `json:"Id"`
}
if err := a.docker.Do(ctx, "POST", "/networks/create", map[string]any{"Name": network, "Driver": "bridge", "CheckDuplicate": true, "Labels": map[string]string{"dsh-mgr.instance": i.ID}}, &createdNetwork); err != nil {
if dockerCreateRejected(err) {
if clearErr := a.setRuntimeCreatePending(i.ID, "network_pending", false); clearErr == nil {
networkAttempted = false
}
}
return i, errors.New("network creation failed")
}
networkCreated = true
if createdNetwork.ID != "" {
if _, err := a.s.DB.ExecContext(ctx, "INSERT INTO runtime_networks(instance_id,network_id) VALUES(?,?)", i.ID, createdNetwork.ID); err != nil {
return i, err
}
}
if err := a.setRuntimeCreatePending(i.ID, "network_pending", false); err != nil {
return i, err
}
if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" {
if err := a.docker.Do(ctx, "POST", "/networks/"+network+"/connect", map[string]string{"Container": panel}, nil); err != nil {
return i, errors.New("panel network connection failed")
@@ -212,13 +262,33 @@ func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance,
},
},
}
if policy.RecipeRev != "7" {
sharedSource, err := toHostPath(root, hostRoot, filepath.Join(root, suffix, "files"))
if err != nil {
return i, err
}
body["Env"] = []string{"DSH_HOME=/data/dsh-home", "HOME=/data/dsh-home", "XDG_DATA_HOME=/data/dsh-home/.local/share", "XDG_CONFIG_HOME=/data/dsh-home/.config"}
body["HostConfig"].(map[string]any)["Mounts"] = []map[string]any{{"Type": "bind", "Source": homeSource, "Target": "/data/dsh-home"}, {"Type": "bind", "Source": sharedSource, "Target": "/workspace"}}
}
var created struct {
ID string `json:"Id"`
}
if err := a.setRuntimeCreatePending(i.ID, "container_pending", true); err != nil {
return i, err
}
containerAttempted = true
if err := a.docker.Do(ctx, "POST", "/containers/create?name="+network, body, &created); err != nil || created.ID == "" {
if dockerCreateRejected(err) {
if clearErr := a.setRuntimeCreatePending(i.ID, "container_pending", false); clearErr == nil {
containerAttempted = false
}
}
return i, errors.New("container creation failed")
}
i.ContainerID = created.ID
if err := a.setRuntimeCreatePending(i.ID, "container_pending", false); err != nil {
return i, err
}
if _, err := a.s.DB.ExecContext(ctx, "UPDATE instances SET container_id=? WHERE id=?", i.ContainerID, i.ID); err != nil {
return i, errors.New("database error")
}
@@ -230,6 +300,7 @@ func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance,
return i, errors.New("database error")
}
complete = true
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=? AND container_pending=0 AND network_pending=0", i.ID)
a.s.AddAudit(u.ID, "instance.create", i.ID, i.ImageID)
return i, nil
}
@@ -238,6 +309,10 @@ func dockerNotFound(err error) bool {
var e *docker.APIError
return errors.As(err, &e) && e.StatusCode == 404
}
func dockerCreateRejected(err error) bool {
var e *docker.APIError
return errors.As(err, &e) && e.StatusCode >= 400 && e.StatusCode < 500 && e.StatusCode != 408 && e.StatusCode != 409
}
func (a *API) removeRuntimeContainer(ctx context.Context, id string) error {
err := a.docker.Do(ctx, "DELETE", "/containers/"+url.PathEscape(id)+"?force=1", nil, nil)
if dockerNotFound(err) {
@@ -246,18 +321,7 @@ func (a *API) removeRuntimeContainer(ctx context.Context, id string) error {
return err
}
func (a *API) removeRuntimeNetwork(ctx context.Context, id string) error {
path := "/networks/" + runtimeNetwork(id)
if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" {
err := a.docker.Do(ctx, "POST", path+"/disconnect", map[string]any{"Container": panel, "Force": true}, nil)
if err != nil && !dockerNotFound(err) {
return err
}
}
err := a.docker.Do(ctx, "DELETE", path, nil, nil)
if dockerNotFound(err) {
return nil
}
return err
return a.recoverRuntimeNetwork(ctx, id)
}
func (a *API) instance(w http.ResponseWriter, r *http.Request) {
@@ -339,6 +403,12 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
fail(w, 409, "instance files busy; retry later")
return
}
if i.ContainerID == "" {
if err := a.recoverRuntimeContainer(ctx, &i); err != nil {
fail(w, 502, "container recovery failed")
return
}
}
if i.ContainerID != "" {
if err := a.removeRuntimeContainer(ctx, i.ContainerID); err != nil {
fail(w, 502, "container deletion failed")
@@ -353,7 +423,11 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
fail(w, 500, "database error")
return
}
_, _ = a.s.DB.Exec("DELETE FROM runtime_networks WHERE instance_id=?", i.ID)
a.s.AddAudit(u.ID, "instance.delete", i.ID, "data retained")
// The instance row is gone and its resources were just cleaned, so
// any pending flags cannot resolve later: drop the row unconditionally.
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=?", i.ID)
runtimeAuthCache.Delete(runtimeAuthKey{a, i.ID})
writeJSON(w, map[string]bool{"ok": true})
return
@@ -384,6 +458,27 @@ func (a *API) instance(w http.ResponseWriter, r *http.Request) {
fail(w, 409, "instance files busy; retry later")
return
}
admissionMu.RLock()
defer admissionMu.RUnlock()
if action != "stop" {
buildQueue.Lock()
e := a.ensureBuilds()
buildQueue.Unlock()
if e != nil {
fail(w, 500, "database error")
return
}
var blocked int
e = a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM image_admissions WHERE image_id=? AND state='blocked'", i.ImageID).Scan(&blocked)
if e != nil {
fail(w, 500, "database error")
return
}
if blocked > 0 {
fail(w, 409, "image is blocked; start/restart denied")
return
}
}
if err := a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(i.ContainerID)+"/"+action+"?t=10", nil, nil); err != nil {
fail(w, 502, "container action failed")
return
internal/api/runtime_session.go
+13 -5
View File
@@ -34,15 +34,23 @@ const runtimeSessionJS = `(function () {
let pending;
async function renew() {
if (pending) return pending;
pending = (async function () {
const run = async function () {
// Renew proactively: a successful /me does not extend the five-minute
// access lifetime. Web Locks serialize cookie rotation across pages.
const csrf = document.cookie.split("; ").find(c => c.startsWith("dsh_csrf="));
if (!csrf) return;
const response = await fetch("/api/refresh", {method:"POST", credentials:"same-origin", headers:{"X-CSRF-Token":csrf.slice(9)}});
if (response.status === 401 || response.status === 403) {
const check = await fetch("/api/me", {credentials:"same-origin"});
if (check.status === 401) location.reload();
if (response.status === 401 || response.status === 403 || response.status === 409) {
for (const delay of [0, 100, 250, 500]) {
if (delay) await new Promise(resolve => setTimeout(resolve, delay));
const check = await fetch("/api/me", {credentials:"same-origin"});
if (check.status !== 401) return;
}
if (response.status === 401 || response.status === 403) location.reload();
}
})().catch(function () {}).finally(function () { pending = undefined; });
};
pending = (navigator.locks ? navigator.locks.request("dsh-session-refresh", run) : run())
.catch(function () {}).finally(function () { pending = undefined; });
return pending;
}
setInterval(renew, 240000);
internal/api/runtime_test.go
+18 -3
View File
@@ -71,6 +71,14 @@ func TestRuntimeProxyRejectsErrorState(t *testing.T) {
// must wait instead of removing the row while the container is still starting.
func TestInstanceDeleteWaitsForInFlightCreate(t *testing.T) {
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.Method == "GET" && strings.Contains(r.URL.Path, "/networks/") {
w.WriteHeader(404)
return
}
if r.Method == http.MethodGet && r.URL.Path == "/v1.47/containers/dsh-inst-creating/json" {
w.WriteHeader(http.StatusNotFound)
return
}
if r.Method == http.MethodDelete && strings.Contains(r.URL.Path, "/networks/") {
w.WriteHeader(http.StatusNoContent)
return
@@ -122,7 +130,7 @@ func TestRuntimeCreateLimitAndPolicy(t *testing.T) {
calls = append(calls, r.Method+" "+r.URL.Path)
switch {
case strings.Contains(r.URL.Path, "/images/"):
writeJSON(w, map[string]any{"Id": imageID, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3"}}})
writeJSON(w, map[string]any{"Id": imageID, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": imageRecipeRev}}})
case strings.HasSuffix(r.URL.Path, "/containers/create"):
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
@@ -134,7 +142,7 @@ func TestRuntimeCreateLimitAndPolicy(t *testing.T) {
w.WriteHeader(204)
}
})
body := fmt.Sprintf("{\"name\":\"one\",\"image_id\":%q,\"version\":\"untrusted\"}", imageID)
body := fmt.Sprintf("{\"name\":\"one\",\"image_id\":%q}", imageID)
seedVerifiedImage(t, a, imageID, "1.2.3")
var wg sync.WaitGroup
statuses := make(chan int, 4)
@@ -213,6 +221,10 @@ func TestRuntimeLifecycleAndOwnership(t *testing.T) {
var calls []string
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
calls = append(calls, r.Method+" "+r.URL.Path)
if r.Method == "GET" && strings.Contains(r.URL.Path, "/networks/") {
w.WriteHeader(404)
return
}
if strings.HasSuffix(r.URL.Path, "/json") {
io.WriteString(w, "{\"State\":{\"Status\":\"running\"}}")
} else if strings.HasSuffix(r.URL.Path, "/logs") {
@@ -447,8 +459,11 @@ func TestRuntimeCreateRollbackAndImageRejection(t *testing.T) {
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
calls = append(calls, r.Method+" "+r.URL.Path)
switch {
case r.Method == "GET" && strings.Contains(r.URL.Path, "/networks/"):
name := strings.TrimPrefix(r.URL.Path, "/v1.47/networks/")
writeJSON(w, map[string]any{"Id": name, "Name": name, "Driver": "bridge", "Labels": map[string]string{"dsh-mgr.instance": strings.TrimPrefix(name, "dsh-inst-")}})
case strings.Contains(r.URL.Path, "/images/"):
writeJSON(w, map[string]any{"Id": imageID, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": role, "dsh-mgr.version": "1"}}})
writeJSON(w, map[string]any{"Id": imageID, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": role, "dsh-mgr.version": "1", "dsh-mgr.recipe": imageRecipeRev}}})
case strings.HasSuffix(r.URL.Path, "/containers/create"):
writeJSON(w, map[string]string{"Id": "container"})
case strings.HasSuffix(r.URL.Path, "/start"):
internal/api/upgrade.go
+27 -6
View File
@@ -16,7 +16,6 @@ import (
"git.misaka.ren/m1saka/dsh-web/internal/backup"
"git.misaka.ren/m1saka/dsh-web/internal/store"
"golang.org/x/crypto/bcrypt"
)
const upgradeSnapshotLimit int64 = 1 << 30
@@ -52,10 +51,31 @@ func upgradeWorkspace(config, host map[string]json.RawMessage, recipe, home, sha
if homeIndex == -1 {
return errors.New("instance home mount required")
}
if !supportedLayout(recipe) {
return errors.New("unsupported template layout")
}
if recipe != "7" {
if get(mounts[homeIndex], "Target") == "/workspace" {
return errors.New("workspace home requires template 7")
return errors.New("workspace home cannot be downgraded to legacy shared layout")
}
if sharedIndex == -1 || get(mounts[sharedIndex], "Target") != "/workspace" {
return errors.New("legacy shared workspace mount required")
}
var env []string
if raw := config["Env"]; len(raw) != 0 {
if err := json.Unmarshal(raw, &env); err != nil {
return err
}
}
kept := []string{}
for _, entry := range env {
key, _, _ := strings.Cut(entry, "=")
if key != "DSH_HOME" && key != "HOME" && key != "XDG_DATA_HOME" && key != "XDG_CONFIG_HOME" {
kept = append(kept, entry)
}
}
kept = append(kept, "DSH_HOME=/data/dsh-home", "HOME=/data/dsh-home", "XDG_DATA_HOME=/data/dsh-home/.local/share", "XDG_CONFIG_HOME=/data/dsh-home/.config")
config["Env"], _ = json.Marshal(kept)
return nil
}
for _, key := range []string{"Binds", "VolumesFrom"} {
@@ -271,6 +291,7 @@ func (a *API) upgrade(w http.ResponseWriter, r *http.Request, i store.Instance)
Password string `json:"password"`
}
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
dec.DisallowUnknownFields()
if dec.Decode(&in) != nil || !runtimeImageID.MatchString(in.ImageID) || dec.Decode(new(any)) != io.EOF {
fail(w, 400, "invalid image_id")
return
@@ -286,6 +307,8 @@ func (a *API) upgrade(w http.ResponseWriter, r *http.Request, i store.Instance)
}
user := a.userLocks.Acquire(i.UID)
defer user.Release()
admissionMu.RLock()
defer admissionMu.RUnlock()
current, err := a.s.GetInstance(i.ID)
if err != nil || current.UID != i.UID {
fail(w, 404, "not found")
@@ -296,9 +319,7 @@ func (a *API) upgrade(w http.ResponseWriter, r *http.Request, i store.Instance)
fail(w, 400, "login password required for pre-upgrade backup")
return
}
var passwordHash string
if a.s.DB.QueryRow("SELECT pass_hash FROM users WHERE id=?", i.UID).Scan(&passwordHash) != nil || bcrypt.CompareHashAndPassword([]byte(passwordHash), []byte(in.Password)) != nil {
fail(w, 403, "incorrect login password")
if !a.reauthenticate(w, i.UID, in.Password) {
return
}
if a.upgradeRecoveryRequired(i) || i.Status == "error" {
@@ -337,7 +358,7 @@ func (a *API) upgrade(w http.ResponseWriter, r *http.Request, i store.Instance)
ID string `json:"Id"`
Config struct{ Labels map[string]string }
}
if a.docker.Do(ctx, "GET", "/images/"+url.PathEscape(in.ImageID)+"/json", nil, &image) != nil || image.ID != in.ImageID || image.Config.Labels["dsh-mgr.role"] != "runtime" || image.Config.Labels["dsh-mgr.recipe"] != imageRecipeRev || verified[image.ID] == "" || verified[image.ID] != image.Config.Labels["dsh-mgr.version"] {
if a.docker.Do(ctx, "GET", "/images/"+url.PathEscape(in.ImageID)+"/json", nil, &image) != nil || image.ID != in.ImageID || image.Config.Labels["dsh-mgr.role"] != "runtime" || !supportedLayout(image.Config.Labels["dsh-mgr.recipe"]) || verified[image.ID] == "" || verified[image.ID] != image.Config.Labels["dsh-mgr.version"] {
fail(w, 400, "local runtime image required")
return
}
internal/api/upgrade_test.go
+12 -3
View File
@@ -27,7 +27,7 @@ func TestUpgradeWorkspaceCompatibility(t *testing.T) {
{"single-to-seven", "7", "/workspace", false, false},
{"migrated-to-seven", "7", "/workspace", true, false},
{"legacy-to-six", "6", "/data/dsh-home", true, false},
{"legacy-to-unknown", "", "/data/dsh-home", true, false},
{"legacy-to-unknown", "", "/data/dsh-home", true, true},
{"single-to-six", "6", "/workspace", false, true},
{"single-to-unknown", "", "/workspace", false, true},
} {
@@ -48,7 +48,13 @@ func TestUpgradeWorkspaceCompatibility(t *testing.T) {
if (err != nil) != tc.reject {
t.Fatalf("error=%v", err)
}
if tc.reject || tc.recipe != "7" {
if !tc.reject && tc.recipe != "7" {
if !strings.Contains(string(config["Env"]), "XDG_DATA_HOME=/data/dsh-home/.local/share") || string(host["Mounts"]) != string(raw) {
t.Fatal(config, host)
}
return
}
if tc.reject {
if string(host["Mounts"]) != string(raw) || string(config["Env"]) != before {
t.Fatal("legacy or rejected config changed")
}
@@ -149,7 +155,7 @@ func TestUpgradeLifecycle(t *testing.T) {
if mode == "invalid-image" {
role = "builder"
}
writeJSON(w, map[string]any{"Id": image, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": role, "dsh-mgr.version": "v2", "dsh-mgr.recipe": imageRecipeRev}}})
writeJSON(w, map[string]any{"Id": image, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": role, "dsh-mgr.version": "v2", "dsh-mgr.recipe": imageRecipeRev}}})
case p == "/containers/create":
if !strings.HasPrefix(r.URL.Query().Get("name"), "dsh-inst-one-upgrade-") {
t.Error("missing unique name")
@@ -238,6 +244,9 @@ func TestUpgradeLifecycle(t *testing.T) {
t.Fatal(err)
}
}
if _, err := a.s.DB.Exec("INSERT INTO image_admissions(image_id,version,recipe_rev,platform,state,evidence,admitted_by,admitted_at) VALUES(?,'v2',?,?,'compatible','test acceptance','test','now')", image, imageRecipeRev, imagePlatform); err != nil {
t.Fatal(err)
}
base := filepath.Join(a.root, "workspaces", "owner", "instances", "one")
home = filepath.Join(base, "home")
shared = filepath.Join(a.root, "workspaces", "owner", "files")
internal/store/store.go
+10 -4
View File
@@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/hex"
_ "modernc.org/sqlite"
"sync"
"time"
)
@@ -25,7 +26,11 @@ type Instance struct {
ContainerID string `json:"container_id"`
Created time.Time `json:"created_at"`
}
type Store struct{ DB *sql.DB }
type Store struct {
DB *sql.DB
closed chan struct{}
closeOnce sync.Once
}
func Open(path string) (*Store, error) {
db, e := sql.Open("sqlite", path)
@@ -42,10 +47,11 @@ func Open(path string) (*Store, error) {
db.Close()
return nil, e
}
return &Store{db}, nil
return &Store{DB: db, closed: make(chan struct{})}, nil
}
func (s *Store) Close() { s.DB.Close() }
func ID() string { b := make([]byte, 16); _, _ = rand.Read(b); return hex.EncodeToString(b) }
func (s *Store) Close() { s.closeOnce.Do(func() { close(s.closed); s.DB.Close() }) }
func (s *Store) Done() <-chan struct{} { return s.closed }
func ID() string { b := make([]byte, 16); _, _ = rand.Read(b); return hex.EncodeToString(b) }
func (s *Store) UserByName(n string) (User, error) {
var u User
var a, t int64