package api import ( "context" "database/sql" "encoding/json" "errors" "git.misaka.ren/m1saka/dsh-web/internal/store" "log" "net/http" "net/url" "os" "strings" "time" ) // pendingColumns whitelists the runtime_creates flag columns so no caller can // concatenate an unvalidated column name into SQL. type pendingColumn struct{ flag, stamp string } var pendingColumns = map[string]pendingColumn{ "container_pending": {flag: "container_pending", stamp: "container_pending_at"}, "network_pending": {flag: "network_pending", stamp: "network_pending_at"}, } // ensureColumn adds a TEXT column, tolerating databases that already have it. func (a *API) ensureColumn(table, column string) error { _, err := a.s.DB.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " TEXT NOT NULL DEFAULT ''") if err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate column") { return nil } return err } func (a *API) recoverRuntimeContainer(ctx context.Context, i *store.Instance) error { c, err := a.docker.Inspect(ctx, runtimeNetwork(i.ID)) if dockerNotFound(err) { _, pending, checkErr := a.pendingSince(i.ID, "container_pending") if checkErr != nil { return checkErr } if pending { return errors.New("container creation outcome remains uncertain") } return nil } if err != nil { return err } if c.ID == "" || strings.TrimPrefix(c.Name, "/") != runtimeNetwork(i.ID) || c.Config.Labels["dsh-mgr.instance"] != i.ID || c.Config.Labels["dsh-mgr.owner"] != i.UID || c.Config.Labels["dsh-mgr.role"] != "dsh" || c.Image != i.ImageID { return errors.New("runtime container identity mismatch") } if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET container_id=? WHERE id=? AND container_id=''", c.ID, i.ID); err != nil { return err } i.ContainerID = c.ID if err = a.setRuntimeCreatePending(i.ID, "container_pending", false); err != nil { return err } return nil } func (a *API) ensureRuntimeCreates() error { _, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_creates(instance_id TEXT PRIMARY KEY,container_pending INTEGER NOT NULL DEFAULT 0,network_pending INTEGER NOT NULL DEFAULT 0,container_pending_at TEXT NOT NULL DEFAULT '',network_pending_at TEXT NOT NULL DEFAULT '')") if err != nil { return err } if err = a.ensureColumn("runtime_creates", "container_pending_at"); err != nil { return err } if err = a.ensureColumn("runtime_creates", "network_pending_at"); err != nil { return err } _, err = a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_networks(instance_id TEXT PRIMARY KEY,network_id TEXT NOT NULL)") return err } func (a *API) setRuntimeCreatePending(id, column string, pending bool) error { if err := a.ensureRuntimeCreates(); err != nil { return err } c, ok := pendingColumns[column] if !ok { return errors.New("unknown pending column") } stamp := "" if pending { stamp = time.Now().UTC().Format(time.RFC3339Nano) } _, err := a.s.DB.Exec("INSERT INTO runtime_creates(instance_id,"+c.flag+","+c.stamp+") VALUES(?,?,?) ON CONFLICT(instance_id) DO UPDATE SET "+c.flag+"=excluded."+c.flag+","+c.stamp+"=excluded."+c.stamp, id, pending, stamp) return err } // pendingSince reports when a pending flag was raised. A legacy row without a // timestamp reports a zero time without resolving the uncertainty. func (a *API) pendingSince(id, column string) (time.Time, bool, error) { if err := a.ensureRuntimeCreates(); err != nil { return time.Time{}, false, err } c, ok := pendingColumns[column] if !ok { return time.Time{}, false, errors.New("unknown pending column") } var pending bool var stamp string err := a.s.DB.QueryRow("SELECT "+c.flag+",COALESCE("+c.stamp+",'') FROM runtime_creates WHERE instance_id=?", id).Scan(&pending, &stamp) if errors.Is(err, sql.ErrNoRows) { return time.Time{}, false, nil } if err != nil { return time.Time{}, false, err } if !pending { return time.Time{}, false, nil } if stamp == "" { return time.Time{}, true, nil } since, perr := time.Parse(time.RFC3339Nano, stamp) if perr != nil { return time.Time{}, true, nil } return since, true, nil } func (a *API) recoverRuntimeNetwork(ctx context.Context, id string) error { _, pending, err := a.pendingSince(id, "network_pending") if err != nil { return err } var recorded string err = a.s.DB.QueryRowContext(ctx, "SELECT network_id FROM runtime_networks WHERE instance_id=?", id).Scan(&recorded) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } lookup := runtimeNetwork(id) if recorded != "" { lookup = recorded } var network struct { ID, Name, Driver string Labels map[string]string Containers map[string]json.RawMessage } err = a.docker.Do(ctx, http.MethodGet, "/networks/"+url.PathEscape(lookup), nil, &network) if dockerNotFound(err) { if pending && recorded == "" { return errors.New("network creation outcome remains uncertain") } return a.setRuntimeCreatePending(id, "network_pending", false) } if err != nil { return err } if network.ID == "" || (recorded != "" && recorded != network.ID) || network.Name != runtimeNetwork(id) || network.Driver != "bridge" || network.Labels["dsh-mgr.instance"] != id { return errors.New("runtime network identity mismatch") } if _, err = a.s.DB.ExecContext(ctx, "INSERT INTO runtime_networks(instance_id,network_id) VALUES(?,?) ON CONFLICT(instance_id) DO UPDATE SET network_id=excluded.network_id", id, network.ID); err != nil { return err } if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" && len(network.Containers) != 0 { if _, connected := network.Containers[panel]; !connected { c, err := a.docker.Inspect(ctx, panel) if err != nil { return err } panel = c.ID } if _, connected := network.Containers[panel]; connected { err = a.docker.Do(ctx, http.MethodPost, "/networks/"+url.PathEscape(network.ID)+"/disconnect", map[string]any{"Container": panel, "Force": true}, nil) if err != nil && !dockerNotFound(err) { return err } } } err = a.docker.Do(ctx, http.MethodDelete, "/networks/"+url.PathEscape(network.ID), nil, nil) if err != nil && !dockerNotFound(err) { return err } return a.setRuntimeCreatePending(id, "network_pending", false) } func (a *API) ensureBuildChecks() error { _, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS image_build_checks(build_id TEXT PRIMARY KEY,image_id TEXT NOT NULL)") if err != nil { return err } if err = a.ensureColumn("image_build_checks", "container_id"); err != nil { return err } return a.ensureColumn("image_build_checks", "created_at") } func buildCheckName(id string) string { return "dsh-build-check-" + id } var errBuildCheckUncertain = errors.New("verification container creation outcome remains uncertain") func (a *API) cleanupBuildCheck(ctx context.Context, id, image string) error { var recorded string if err := a.s.DB.QueryRowContext(ctx, "SELECT container_id FROM image_build_checks WHERE build_id=?", id).Scan(&recorded); err != nil { return err } lookup := buildCheckName(id) if recorded != "" { lookup = recorded } c, err := a.docker.Inspect(ctx, lookup) if dockerNotFound(err) { if recorded != "" { _, err := a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id) return err } // A name lookup cannot rule out a create still running in Docker. return errBuildCheckUncertain } if err != nil { return err } if c.ID == "" || (recorded != "" && c.ID != recorded) || strings.TrimPrefix(c.Name, "/") != buildCheckName(id) || c.Image != image || c.Config.Labels["dsh-mgr.role"] != "build-check" || c.Config.Labels["dsh-mgr.build"] != id { return errors.New("build verification container identity mismatch") } return a.deleteBuildCheck(ctx, id, c.ID) } func (a *API) deleteBuildCheck(ctx context.Context, id, container string) error { if _, err := a.s.DB.ExecContext(ctx, "UPDATE image_build_checks SET container_id=? WHERE build_id=?", container, id); err != nil { return err } err := a.docker.Do(ctx, http.MethodDelete, "/containers/"+url.PathEscape(container)+"?force=1&v=1", nil, nil) if err != nil && !dockerNotFound(err) { return err } _, err = a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id) return err } func (a *API) recoverBuildChecks(ctx context.Context) error { if err := a.ensureBuildChecks(); err != nil { return err } rows, err := a.s.DB.QueryContext(ctx, "SELECT build_id,image_id,COALESCE(created_at,'') FROM image_build_checks") if err != nil { return err } type check struct{ id, image, createdAt string } var checks []check for rows.Next() { var c check if err = rows.Scan(&c.id, &c.image, &c.createdAt); err != nil { break } checks = append(checks, c) } if err == nil { err = rows.Err() } rows.Close() if err != nil { return err } var failures []error missing := false for _, c := range checks { err := a.cleanupBuildCheck(ctx, c.id, c.image) if errors.Is(err, errBuildCheckUncertain) { missing = true } else { failures = append(failures, err) } } if err := errors.Join(failures...); err != nil { return err } if missing { return errBuildCheckUncertain } return nil } func (a *API) persistBuildResult(b buildRecord, status, logs, detail, image, finished string) error { tx, err := a.s.DB.Begin() if err != nil { return err } defer tx.Rollback() if status == "success" && image != "" { tag := "dsh-web:" + b.Version + "-" + imageRecipeRev if b.ReleaseID != "" { tag = "dsh-web:release-" + b.SourceCommit } _, err = tx.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", image, b.Version, imageRecipeRev, imagePlatform, tag, "unverified", finished) if err != nil { return err } } if _, err = tx.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, image, finished, b.ID); err != nil { return err } return tx.Commit() } func (a *API) finishBuild(b buildRecord, status, logs, detail, image, finished string) { for attempt := 0; ; attempt++ { err := a.persistBuildResult(b, status, logs, detail, image, finished) if err == nil { return } if attempt%60 == 0 { log.Printf("build %s: terminal transaction failed; retrying: %s", b.ID, sanitizeBuildLog(err.Error())) } select { case <-a.s.Done(): return case <-time.After(time.Second): } } }