dsh-web/internal/api/images.go

package api

import (
	"archive/tar"
	"bytes"
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	imageassets "git.misaka.ren/m1saka/dsh-web/image"
	"git.misaka.ren/m1saka/dsh-web/internal/store"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"
)

// Caller holds buildQueue; interrupted jobs cannot be reported as successful.
func (a *API) ensureBuilds() error {
	if buildQueue.initialized[a.s.DB] {
		return nil
	}
	_, err := a.s.DB.Exec(`CREATE TABLE IF NOT EXISTS image_builds(id TEXT PRIMARY KEY,version TEXT NOT NULL,status TEXT NOT NULL,logs TEXT NOT NULL DEFAULT '',error TEXT NOT NULL DEFAULT '',image_id TEXT NOT NULL DEFAULT '',created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS image_build_requests(user_id TEXT NOT NULL,build_id TEXT PRIMARY KEY,created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS images(image_id TEXT PRIMARY KEY,version TEXT NOT NULL,recipe_rev TEXT,platform TEXT,tag TEXT NOT NULL DEFAULT '',status TEXT NOT NULL,built_at TEXT NOT NULL,usage_count INTEGER NOT NULL DEFAULT 0);`)
	if err != nil {
		return err
	}
	for _, q := range []string{
		"ALTER TABLE image_builds ADD COLUMN requested_by TEXT NOT NULL DEFAULT ''",
		"ALTER TABLE image_builds ADD COLUMN recipe_rev TEXT",
		"ALTER TABLE image_builds ADD COLUMN platform TEXT",
		"ALTER TABLE image_builds ADD COLUMN finished_at TEXT",
		"ALTER TABLE images ADD COLUMN recipe_rev TEXT",
		"ALTER TABLE images ADD COLUMN platform TEXT",
		"ALTER TABLE images ADD COLUMN usage_count INTEGER NOT NULL DEFAULT 0",
	} {
		if _, e := a.s.DB.Exec(q); e != nil && !strings.Contains(strings.ToLower(e.Error()), "duplicate column") {
			return e
		}
	}
	_, err = a.s.DB.Exec("DROP INDEX IF EXISTS image_builds_version; CREATE UNIQUE INDEX IF NOT EXISTS image_builds_identity ON image_builds(version,recipe_rev,platform) WHERE recipe_rev IS NOT NULL AND platform IS NOT NULL AND status IN ('queued','running','validating','success')")
	if err != nil {
		return err
	}
	if !buildQueue.initialized[a.s.DB] {
		if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='image template identity is unknown' WHERE status='queued' AND (recipe_rev IS NULL OR recipe_rev<>? OR platform IS NULL OR platform<>?)", imageRecipeRev, imagePlatform); err != nil {
			return err
		}
		if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='build interrupted by process restart' WHERE status IN ('running','validating')"); err != nil {
			return err
		}
		buildQueue.initialized[a.s.DB] = true
	}
	return nil
}
func (a *API) builds(w http.ResponseWriter, r *http.Request) {
	u, _, authErr := a.user(r)
	if authErr != nil {
		fail(w, 401, "authentication required")
		return
	}
	if r.Method != http.MethodGet && r.Method != http.MethodPost {
		fail(w, 405, "method not allowed")
		return
	}
	var in struct{ Version string }
	if r.Method == http.MethodPost {
		dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
		dec.DisallowUnknownFields()
		if dec.Decode(&in) != nil || dec.Decode(new(any)) != io.EOF || !validBuildVersion(in.Version) {
			fail(w, 400, "a fixed version is required")
			return
		}
	}
	buildQueue.Lock()
	defer buildQueue.Unlock()
	if err := a.ensureBuilds(); err != nil {
		fail(w, 500, "database error")
		return
	}
	if r.Method == http.MethodGet {
		rows, err := a.s.DB.Query("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds ORDER BY created_at DESC,id DESC")
		if err != nil {
			fail(w, 500, "database error")
			return
		}
		result := []buildRecord{}
		for rows.Next() {
			var b buildRecord
			var finished, recipe, platform sql.NullString
			if err = rows.Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished); err != nil {
				break
			}
			b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
			result = append(result, b)
		}
		scanErr := rows.Err()
		rows.Close()
		if err != nil || scanErr != nil {
			fail(w, 500, "database error")
			return
		}
		if err := a.reconcileBuilds(r.Context(), result); err != nil {
			fail(w, 502, "docker image reconciliation failed")
			return
		}
		a.startBuildWorker()
		writeJSON(w, result)
		return
	}
	b, err := a.enqueueBuildLocked(r.Context(), u.ID, in.Version)
	if err != nil {
		status := 502
		if errors.Is(err, errBuildRateLimited) {
			status = 429
			w.Header().Set("Retry-After", "3600")
		}
		fail(w, status, sanitizeBuildLog(err.Error()))
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusAccepted)
	writeJSON(w, b)
}

var errBuildRateLimited = errors.New("build limit: three new tasks per user per hour")

func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecord, error) {
	buildQueue.Lock()
	defer buildQueue.Unlock()
	if !validBuildVersion(version) {
		return buildRecord{}, errors.New("a fixed version is required")
	}
	if err := a.ensureBuilds(); err != nil {
		return buildRecord{}, err
	}
	return a.enqueueBuildLocked(ctx, uid, version)
}

func (a *API) enqueueBuildLocked(ctx context.Context, uid, version string) (buildRecord, error) {
	var b buildRecord
	var finished, recipe, platform sql.NullString
	err := a.s.DB.QueryRow("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds WHERE version=? AND recipe_rev=? AND platform=? AND status IN ('queued','running','validating','success')", version, imageRecipeRev, imagePlatform).Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished)
	b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
	if err == nil && b.Status == "success" {
		ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
		id, lookupErr := a.localBuildImage(ctx, b.Version, b.ImageID)
		cancel()
		if lookupErr != nil {
			return b, lookupErr
		}
		if id != b.ImageID || id == "" {
			if _, err = a.s.DB.Exec("UPDATE image_builds SET status='missing',error='image no longer available' WHERE id=?", b.ID); err == nil {
				err = sql.ErrNoRows
			}
		}
	}
	if errors.Is(err, sql.ErrNoRows) {
		var count int
		if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests WHERE user_id=? AND created_at>?", uid, time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)).Scan(&count); err != nil {
			return b, err
		}
		if count >= 3 {
			return b, errBuildRateLimited
		}
		b = buildRecord{ID: store.ID(), Version: version, Status: "queued", Created: time.Now().UTC().Format(time.RFC3339Nano), RequestedBy: uid, RecipeRev: imageRecipeRev, Platform: imagePlatform}
		err = a.insertBuildRequest(b, uid)
	}
	if err != nil {
		return b, err
	}
	a.startBuildWorker()
	return b, nil
}
func (a *API) startBuildWorker() {
	if buildQueue.active[a.s.DB] {
		return
	}
	buildQueue.active[a.s.DB] = true
	go func() {
		buildWorker.Lock()
		defer buildWorker.Unlock()
		for {
			buildQueue.Lock()
			var b buildRecord
			err := a.s.DB.QueryRow("SELECT id,version FROM image_builds WHERE status='queued' ORDER BY created_at,id LIMIT 1").Scan(&b.ID, &b.Version)
			if err == nil {
				_, err = a.s.DB.Exec("UPDATE image_builds SET status='running' WHERE id=?", b.ID)
			}
			if err != nil {
				delete(buildQueue.active, a.s.DB)
				buildQueue.Unlock()
				return
			}
			buildQueue.Unlock()
			a.runImageBuild(b)
		}
	}()
}
func imageBuildContext() (io.Reader, error) {
	var buf bytes.Buffer
	tw := tar.NewWriter(&buf)
	for _, name := range []string{"Dockerfile", "Caddyfile", "entrypoint.sh", "install-mobile.mjs", "install-free-search.mjs", "install-workspace.mjs", "init-profile.mjs"} {
		dataBytes, err := imageassets.Files.ReadFile(name)
		if err != nil {
			return nil, err
		}
		data := string(dataBytes)
		if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0755, Size: int64(len(data))}); err != nil {
			return nil, err
		}
		if _, err := tw.Write([]byte(data)); err != nil {
			return nil, err
		}
	}
	if err := tw.Close(); err != nil {
		return nil, err
	}
	return &buf, nil
}
func (a *API) runImageBuild(b buildRecord) {
	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
	defer cancel()
	logs := ""
	rawLogs := ""
	var logErr error
	appendLog := func(s string) {
		if len(rawLogs) < buildLogLimit {
			remaining := buildLogLimit - len(rawLogs)
			if len(s) > remaining {
				s = s[:remaining]
			}
			rawLogs += s
		}
		logs = sanitizeBuildLog(rawLogs)
		if logErr == nil {
			_, logErr = a.s.DB.Exec("UPDATE image_builds SET logs=? WHERE id=?", logs, b.ID)
		}
	}
	tarball, err := imageBuildContext()
	var id string
	if err == nil {
		id, err = a.localBuildImage(ctx, b.Version, "")
	}
	if err == nil && id == "" {
		_, err = fetchNPMVersion(ctx, npmBuildClient, b.Version)
		if err == nil {
			err = a.checkBuildDisk(ctx)
		}
	}
	if err == nil && id == "" {
		id, err = a.docker.Build(ctx, tarball, b.Version, "dsh-web:"+b.Version+"-"+imageRecipeRev, appendLog)
	}
	if err == nil {
		var matches bool
		matches, err = a.imageMatches(ctx, id, b.Version)
		if err == nil && !matches {
			err = errors.New("image template or platform mismatch")
		}
	}
	if err == nil {
		_, err = a.s.DB.Exec("UPDATE image_builds SET status='validating',image_id=? WHERE id=?", id, b.ID)
	}
	if err == nil {
		err = a.verifyBuiltImage(ctx, id, b.Version)
	}
	if err == nil {
		err = logErr
	}
	status, detail := "success", ""
	if err != nil {
		status = "failed"
		detail = sanitizeBuildLog(err.Error())
		appendLog(string(rune(10)) + detail)
	}
	finished := time.Now().UTC().Format(time.RFC3339Nano)
	_, _ = a.s.DB.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, id, finished, b.ID)
	if status == "success" && id != "" {
		_, _ = a.s.DB.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", id, b.Version, imageRecipeRev, imagePlatform, "dsh-web:"+b.Version+"-"+imageRecipeRev, "available", finished)
	}
}
func (a *API) verifyBuiltImage(parent context.Context, image, version string) (result error) {
	ctx, cancel := context.WithTimeout(parent, 90*time.Second)
	defer cancel()
	var created struct {
		ID string `json:"Id"`
	}
	config := map[string]any{"Image": image, "User": "1000:1000", "Labels": map[string]string{"dsh-mgr.role": "build-check"}, "HostConfig": map[string]any{"NetworkMode": "none", "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges:true"}, "Memory": int64(1 << 30), "NanoCpus": int64(1e9), "PidsLimit": 256, "Tmpfs": map[string]string{"/tmp": "rw,exec,nosuid,nodev,size=256m,mode=1777", "/workspace": "rw,nosuid,nodev,size=256m,uid=1000,gid=1000"}}}
	if err := a.docker.Do(ctx, http.MethodPost, "/containers/create", config, &created); err != nil {
		return err
	}
	if created.ID == "" {
		return errors.New("verification container ID missing")
	}
	path := "/containers/" + url.PathEscape(created.ID)
	defer func() {
		cleanup, c := context.WithTimeout(context.Background(), 10*time.Second)
		defer c()
		err := a.docker.Do(cleanup, http.MethodDelete, path+"?force=1&v=1", nil, nil)
		result = errors.Join(result, err)
	}()
	if err := a.docker.Do(ctx, http.MethodPost, path+"/start", nil, nil); err != nil {
		return err
	}
	script := `const end=Date.now()+60000;async function probe(){try{const r=await fetch('http://127.0.0.1:8080/',{redirect:'manual',signal:AbortSignal.timeout(2000)});if(r.status>=200&&r.status<500)process.exit(0);}catch{}if(Date.now()>end)process.exit(1);setTimeout(probe,500);}probe();`
	{
		expected, _ := json.Marshal(version)
		script = `const cp=require('node:child_process');const actual=cp.execFileSync('dsh',['--version'],{encoding:'utf8',timeout:20000}).trim();if(actual!==` + string(expected) + `)process.exit(2);` + script
	}
	script = `const fs=require('node:fs');for(const [k,v] of Object.entries({DSH_HOME:'/workspace',HOME:'/workspace',XDG_DATA_HOME:'/workspace/.local/share',XDG_CONFIG_HOME:'/workspace/.config'})){if(process.env[k]!==v)process.exit(3);}if(!fs.lstatSync('/data/dsh-home').isSymbolicLink()||fs.realpathSync('/data/dsh-home')!=='/workspace')process.exit(4);fs.writeFileSync('/workspace/.home-check','isolated');if(fs.readFileSync('/data/dsh-home/.home-check','utf8')!=='isolated')process.exit(5);fs.unlinkSync('/workspace/.home-check');` + script
	var exec struct {
		ID string `json:"Id"`
	}
	if err := a.docker.Do(ctx, http.MethodPost, path+"/exec", map[string]any{"Cmd": []string{"node", "-e", script}, "User": "1000:1000"}, &exec); err != nil {
		return err
	}
	if exec.ID == "" {
		return errors.New("verification exec ID missing")
	}
	ep := "/exec/" + url.PathEscape(exec.ID)
	if err := a.docker.Do(ctx, http.MethodPost, ep+"/start", map[string]bool{"Detach": true, "Tty": false}, nil); err != nil {
		return err
	}
	ticker := time.NewTicker(250 * time.Millisecond)
	defer ticker.Stop()
	for {
		var state struct {
			Running  bool
			ExitCode int
		}
		if err := a.docker.Do(ctx, http.MethodGet, ep+"/json", nil, &state); err != nil {
			return err
		}
		if !state.Running {
			if state.ExitCode != 0 {
				return fmt.Errorf("image startup verification failed (exit %d)", state.ExitCode)
			}
			container, err := a.docker.Inspect(ctx, created.ID)
			if err != nil {
				return err
			}
			if !container.State.Running {
				return errors.New("verification container exited")
			}
			return nil
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
		}
	}
}

const buildLogLimit = 64 << 10

var fixedVersion = regexp.MustCompile(`^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)(-[0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?$`)
var buildSecrets = regexp.MustCompile(`(?i)(authorization["']?[[:space:]]*[:=][[:space:]]*["']?(?:bearer[[:space:]]+|basic[[:space:]]+)?|(?:token|password|passwd|secret|api[_-]?key|cookie)["']?[[:space:]]*[:=][[:space:]]*["']?)[^[:space:]"'&,;]+`)
var buildURLCredentials = regexp.MustCompile(`(https?://)[^/@[:space:]]+:[^/@[:space:]]+@`)
var buildQueue = struct {
	sync.Mutex
	initialized map[*sql.DB]bool
	active      map[*sql.DB]bool
}{initialized: map[*sql.DB]bool{}, active: map[*sql.DB]bool{}}
var buildWorker sync.Mutex

type imageRecord struct {
	ID         string `json:"id"`
	ImageID    string `json:"image_id"`
	Version    string `json:"version"`
	Tag        string `json:"tag,omitempty"`
	Status     string `json:"status"`
	Created    string `json:"created_at"`
	RecipeRev  string `json:"recipe_rev,omitempty"`
	Platform   string `json:"platform,omitempty"`
	UsageCount int    `json:"usage_count"`
}
type buildRecord struct {
	ImageID     string `json:"image_id,omitempty"`
	ID          string `json:"id"`
	Version     string `json:"version"`
	Status      string `json:"status"`
	Logs        string `json:"logs"`
	Error       string `json:"error,omitempty"`
	Created     string `json:"created_at"`
	RequestedBy string `json:"requested_by,omitempty"`
	RecipeRev   string `json:"recipe_rev,omitempty"`
	Platform    string `json:"platform,omitempty"`
	Finished    string `json:"finished_at,omitempty"`
}

func (a *API) images(w http.ResponseWriter, r *http.Request) {
	if r.Method == http.MethodDelete {
		a.deleteImage(w, r)
		return
	}
	if _, _, err := a.user(r); err != nil {
		fail(w, 401, "authentication required")
		return
	}
	if r.Method != http.MethodGet {
		fail(w, 405, "method not allowed")
		return
	}
	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
	defer cancel()
	result, err := a.imageIndex(ctx)
	if err != nil {
		fail(w, 502, "image index reconciliation failed")
		return
	}
	writeJSON(w, result)
}

func (a *API) verifiedImages(ctx context.Context) (map[string]string, error) {
	buildQueue.Lock()
	defer buildQueue.Unlock()
	if err := a.ensureBuilds(); err != nil {
		return nil, err
	}
	rows, err := a.s.DB.QueryContext(ctx, "SELECT image_id,version FROM image_builds WHERE status='success' AND image_id<>'' AND recipe_rev=? AND platform=?", imageRecipeRev, imagePlatform)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	images := make(map[string]string)
	for rows.Next() {
		var id, version string
		if err := rows.Scan(&id, &version); err != nil {
			return nil, err
		}
		images[id] = version
	}
	return images, rows.Err()
}
func (a *API) latest(w http.ResponseWriter, r *http.Request) {
	if _, _, err := a.user(r); err != nil {
		fail(w, 401, "authentication required")
		return
	}
	if r.Method != http.MethodGet {
		fail(w, 405, "method not allowed")
		return
	}
	v, err := fetchLatest(r.Context(), npmBuildClient)
	if err != nil {
		fail(w, 502, "npm version lookup failed")
		return
	}
	writeJSON(w, map[string]string{"version": v, "checked_at": time.Now().UTC().Format(time.RFC3339)})
}
func fetchLatest(ctx context.Context, client *http.Client) (string, error) {
	return fetchNPMVersion(ctx, client, "latest")
}
func fetchNPMVersion(ctx context.Context, client *http.Client, version string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://registry.npmjs.org/@deepseek-ai%2Fdsh/"+url.PathEscape(version), nil)
	if err != nil {
		return "", err
	}
	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		return "", fmt.Errorf("npm status %d", resp.StatusCode)
	}
	var v struct{ Version string }
	if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&v); err != nil {
		return "", err
	}
	if !validBuildVersion(v.Version) || (version != "latest" && v.Version != version) {
		return "", errors.New("invalid npm version")
	}
	return v.Version, nil
}
func validBuildVersion(v string) bool { return len(v) <= 100 && fixedVersion.MatchString(v) }
func sanitizeBuildLog(s string) string {
	s = buildURLCredentials.ReplaceAllString(s, "$1[REDACTED]@")
	s = buildSecrets.ReplaceAllString(s, "$1[REDACTED]")
	s = strings.ToValidUTF8(s, "")
	if len(s) > buildLogLimit {
		s = strings.ToValidUTF8(s[:buildLogLimit], "")
	}
	return s
}

func (a *API) localBuildImage(ctx context.Context, version, preferred string) (string, error) {
	filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime", "dsh-mgr.version=" + version}})
	var images []struct {
		ID     string
		Labels map[string]string
	}
	if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &images); err != nil {
		return "", err
	}
	for _, im := range images {
		if im.ID != "" && im.Labels["dsh-mgr.role"] == "runtime" && im.Labels["dsh-mgr.version"] == version && im.Labels["dsh-mgr.recipe"] == imageRecipeRev && (preferred == "" || im.ID == preferred) {
			ok, err := a.imageMatches(ctx, im.ID, version)
			if err != nil {
				return "", err
			}
			if ok {
				return im.ID, nil
			}
		}
	}
	return "", nil
}