dsh-web/internal/api/resource_recovery.go

package api

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"git.misaka.ren/m1saka/dsh-web/internal/store"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

// pendingColumns whitelists the runtime_creates flag columns so no caller can
// concatenate an unvalidated column name into SQL.
type pendingColumn struct{ flag, stamp string }

var pendingColumns = map[string]pendingColumn{
	"container_pending": {flag: "container_pending", stamp: "container_pending_at"},
	"network_pending":   {flag: "network_pending", stamp: "network_pending_at"},
}

// ensureColumn adds a TEXT column, tolerating databases that already have it.
func (a *API) ensureColumn(table, column string) error {
	_, err := a.s.DB.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " TEXT NOT NULL DEFAULT ''")
	if err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate column") {
		return nil
	}
	return err
}

func (a *API) recoverRuntimeContainer(ctx context.Context, i *store.Instance) error {
	c, err := a.docker.Inspect(ctx, runtimeNetwork(i.ID))
	if dockerNotFound(err) {
		_, pending, checkErr := a.pendingSince(i.ID, "container_pending")
		if checkErr != nil {
			return checkErr
		}
		if pending {
			return errors.New("container creation outcome remains uncertain")
		}
		return nil
	}
	if err != nil {
		return err
	}
	if c.ID == "" || strings.TrimPrefix(c.Name, "/") != runtimeNetwork(i.ID) || c.Config.Labels["dsh-mgr.instance"] != i.ID || c.Config.Labels["dsh-mgr.owner"] != i.UID || c.Config.Labels["dsh-mgr.role"] != "dsh" || c.Image != i.ImageID {
		return errors.New("runtime container identity mismatch")
	}
	if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET container_id=? WHERE id=? AND container_id=''", c.ID, i.ID); err != nil {
		return err
	}
	i.ContainerID = c.ID
	if err = a.setRuntimeCreatePending(i.ID, "container_pending", false); err != nil {
		return err
	}
	return nil
}
func (a *API) ensureRuntimeCreates() error {
	_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_creates(instance_id TEXT PRIMARY KEY,container_pending INTEGER NOT NULL DEFAULT 0,network_pending INTEGER NOT NULL DEFAULT 0,container_pending_at TEXT NOT NULL DEFAULT '',network_pending_at TEXT NOT NULL DEFAULT '')")
	if err != nil {
		return err
	}
	if err = a.ensureColumn("runtime_creates", "container_pending_at"); err != nil {
		return err
	}
	if err = a.ensureColumn("runtime_creates", "network_pending_at"); err != nil {
		return err
	}
	_, err = a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_networks(instance_id TEXT PRIMARY KEY,network_id TEXT NOT NULL)")
	return err
}
func (a *API) setRuntimeCreatePending(id, column string, pending bool) error {
	if err := a.ensureRuntimeCreates(); err != nil {
		return err
	}
	c, ok := pendingColumns[column]
	if !ok {
		return errors.New("unknown pending column")
	}
	stamp := ""
	if pending {
		stamp = time.Now().UTC().Format(time.RFC3339Nano)
	}
	_, err := a.s.DB.Exec("INSERT INTO runtime_creates(instance_id,"+c.flag+","+c.stamp+") VALUES(?,?,?) ON CONFLICT(instance_id) DO UPDATE SET "+c.flag+"=excluded."+c.flag+","+c.stamp+"=excluded."+c.stamp, id, pending, stamp)
	return err
}

// pendingSince reports when a pending flag was raised. A legacy row without a
// timestamp reports a zero time without resolving the uncertainty.
func (a *API) pendingSince(id, column string) (time.Time, bool, error) {
	if err := a.ensureRuntimeCreates(); err != nil {
		return time.Time{}, false, err
	}
	c, ok := pendingColumns[column]
	if !ok {
		return time.Time{}, false, errors.New("unknown pending column")
	}
	var pending bool
	var stamp string
	err := a.s.DB.QueryRow("SELECT "+c.flag+",COALESCE("+c.stamp+",'') FROM runtime_creates WHERE instance_id=?", id).Scan(&pending, &stamp)
	if errors.Is(err, sql.ErrNoRows) {
		return time.Time{}, false, nil
	}
	if err != nil {
		return time.Time{}, false, err
	}
	if !pending {
		return time.Time{}, false, nil
	}
	if stamp == "" {
		return time.Time{}, true, nil
	}
	since, perr := time.Parse(time.RFC3339Nano, stamp)
	if perr != nil {
		return time.Time{}, true, nil
	}
	return since, true, nil
}
func (a *API) recoverRuntimeNetwork(ctx context.Context, id string) error {
	_, pending, err := a.pendingSince(id, "network_pending")
	if err != nil {
		return err
	}
	var recorded string
	err = a.s.DB.QueryRowContext(ctx, "SELECT network_id FROM runtime_networks WHERE instance_id=?", id).Scan(&recorded)
	if err != nil && !errors.Is(err, sql.ErrNoRows) {
		return err
	}
	lookup := runtimeNetwork(id)
	if recorded != "" {
		lookup = recorded
	}
	var network struct {
		ID, Name, Driver string
		Labels           map[string]string
		Containers       map[string]json.RawMessage
	}
	err = a.docker.Do(ctx, http.MethodGet, "/networks/"+url.PathEscape(lookup), nil, &network)
	if dockerNotFound(err) {
		if pending && recorded == "" {
			return errors.New("network creation outcome remains uncertain")
		}
		return a.setRuntimeCreatePending(id, "network_pending", false)
	}
	if err != nil {
		return err
	}
	if network.ID == "" || (recorded != "" && recorded != network.ID) || network.Name != runtimeNetwork(id) || network.Driver != "bridge" || network.Labels["dsh-mgr.instance"] != id {
		return errors.New("runtime network identity mismatch")
	}
	if _, err = a.s.DB.ExecContext(ctx, "INSERT INTO runtime_networks(instance_id,network_id) VALUES(?,?) ON CONFLICT(instance_id) DO UPDATE SET network_id=excluded.network_id", id, network.ID); err != nil {
		return err
	}
	if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" && len(network.Containers) != 0 {
		if _, connected := network.Containers[panel]; !connected {
			c, err := a.docker.Inspect(ctx, panel)
			if err != nil {
				return err
			}
			panel = c.ID
		}
		if _, connected := network.Containers[panel]; connected {
			err = a.docker.Do(ctx, http.MethodPost, "/networks/"+url.PathEscape(network.ID)+"/disconnect", map[string]any{"Container": panel, "Force": true}, nil)
			if err != nil && !dockerNotFound(err) {
				return err
			}
		}
	}
	err = a.docker.Do(ctx, http.MethodDelete, "/networks/"+url.PathEscape(network.ID), nil, nil)
	if err != nil && !dockerNotFound(err) {
		return err
	}
	return a.setRuntimeCreatePending(id, "network_pending", false)
}

func (a *API) ensureBuildChecks() error {
	_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS image_build_checks(build_id TEXT PRIMARY KEY,image_id TEXT NOT NULL)")
	if err != nil {
		return err
	}
	if err = a.ensureColumn("image_build_checks", "container_id"); err != nil {
		return err
	}
	return a.ensureColumn("image_build_checks", "created_at")
}
func buildCheckName(id string) string { return "dsh-build-check-" + id }

var errBuildCheckUncertain = errors.New("verification container creation outcome remains uncertain")

func (a *API) cleanupBuildCheck(ctx context.Context, id, image string) error {
	var recorded string
	if err := a.s.DB.QueryRowContext(ctx, "SELECT container_id FROM image_build_checks WHERE build_id=?", id).Scan(&recorded); err != nil {
		return err
	}
	lookup := buildCheckName(id)
	if recorded != "" {
		lookup = recorded
	}
	c, err := a.docker.Inspect(ctx, lookup)
	if dockerNotFound(err) {
		if recorded != "" {
			_, err := a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id)
			return err
		}
		// A name lookup cannot rule out a create still running in Docker.
		return errBuildCheckUncertain
	}
	if err != nil {
		return err
	}
	if c.ID == "" || (recorded != "" && c.ID != recorded) || strings.TrimPrefix(c.Name, "/") != buildCheckName(id) || c.Image != image || c.Config.Labels["dsh-mgr.role"] != "build-check" || c.Config.Labels["dsh-mgr.build"] != id {
		return errors.New("build verification container identity mismatch")
	}
	return a.deleteBuildCheck(ctx, id, c.ID)
}
func (a *API) deleteBuildCheck(ctx context.Context, id, container string) error {
	if _, err := a.s.DB.ExecContext(ctx, "UPDATE image_build_checks SET container_id=? WHERE build_id=?", container, id); err != nil {
		return err
	}
	err := a.docker.Do(ctx, http.MethodDelete, "/containers/"+url.PathEscape(container)+"?force=1&v=1", nil, nil)
	if err != nil && !dockerNotFound(err) {
		return err
	}
	_, err = a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id)
	return err
}
func (a *API) recoverBuildChecks(ctx context.Context) error {
	if err := a.ensureBuildChecks(); err != nil {
		return err
	}
	rows, err := a.s.DB.QueryContext(ctx, "SELECT build_id,image_id,COALESCE(created_at,'') FROM image_build_checks")
	if err != nil {
		return err
	}
	type check struct{ id, image, createdAt string }
	var checks []check
	for rows.Next() {
		var c check
		if err = rows.Scan(&c.id, &c.image, &c.createdAt); err != nil {
			break
		}
		checks = append(checks, c)
	}
	if err == nil {
		err = rows.Err()
	}
	rows.Close()
	if err != nil {
		return err
	}
	var failures []error
	missing := false
	for _, c := range checks {
		err := a.cleanupBuildCheck(ctx, c.id, c.image)
		if errors.Is(err, errBuildCheckUncertain) {
			missing = true
		} else {
			failures = append(failures, err)
		}
	}
	if err := errors.Join(failures...); err != nil {
		return err
	}
	if missing {
		return errBuildCheckUncertain
	}
	return nil
}
func (a *API) persistBuildResult(b buildRecord, status, logs, detail, image, finished string) error {
	tx, err := a.s.DB.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()
	if status == "success" && image != "" {
		tag := "dsh-web:" + b.Version + "-" + imageRecipeRev
		if b.ReleaseID != "" {
			tag = "dsh-web:release-" + b.SourceCommit
		}
		_, err = tx.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", image, b.Version, imageRecipeRev, imagePlatform, tag, "unverified", finished)
		if err != nil {
			return err
		}
	}
	if _, err = tx.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, image, finished, b.ID); err != nil {
		return err
	}
	return tx.Commit()
}
func (a *API) finishBuild(b buildRecord, status, logs, detail, image, finished string) {
	for attempt := 0; ; attempt++ {
		err := a.persistBuildResult(b, status, logs, detail, image, finished)
		if err == nil {
			return
		}
		if attempt%60 == 0 {
			log.Printf("build %s: terminal transaction failed; retrying: %s", b.ID, sanitizeBuildLog(err.Error()))
		}
		select {
		case <-a.s.Done():
			return
		case <-time.After(time.Second):
		}
	}
}