package api import ( "archive/tar" "bytes" "context" "database/sql" "encoding/json" "errors" "fmt" imageassets "git.misaka.ren/m1saka/dsh-web/image" "git.misaka.ren/m1saka/dsh-web/internal/store" "io" "net/http" "net/url" "regexp" "strings" "sync" "time" ) // Caller holds buildQueue; interrupted jobs cannot be reported as successful. func (a *API) ensureBuilds() error { if buildQueue.initialized[a.s.DB] { return nil } _, err := a.s.DB.Exec(`CREATE TABLE IF NOT EXISTS image_builds(id TEXT PRIMARY KEY,version TEXT NOT NULL,status TEXT NOT NULL,logs TEXT NOT NULL DEFAULT '',error TEXT NOT NULL DEFAULT '',image_id TEXT NOT NULL DEFAULT '',created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS image_build_requests(user_id TEXT NOT NULL,build_id TEXT PRIMARY KEY,created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS images(image_id TEXT PRIMARY KEY,version TEXT NOT NULL,recipe_rev TEXT,platform TEXT,tag TEXT NOT NULL DEFAULT '',status TEXT NOT NULL,built_at TEXT NOT NULL,usage_count INTEGER NOT NULL DEFAULT 0);`) if err != nil { return err } for _, q := range []string{ "ALTER TABLE image_builds ADD COLUMN requested_by TEXT NOT NULL DEFAULT ''", "ALTER TABLE image_builds ADD COLUMN recipe_rev TEXT", "ALTER TABLE image_builds ADD COLUMN platform TEXT", "ALTER TABLE image_builds ADD COLUMN finished_at TEXT", "ALTER TABLE images ADD COLUMN recipe_rev TEXT", "ALTER TABLE images ADD COLUMN platform TEXT", "ALTER TABLE images ADD COLUMN usage_count INTEGER NOT NULL DEFAULT 0", } { if _, e := a.s.DB.Exec(q); e != nil && !strings.Contains(strings.ToLower(e.Error()), "duplicate column") { return e } } _, err = a.s.DB.Exec("DROP INDEX IF EXISTS image_builds_version; CREATE UNIQUE INDEX IF NOT EXISTS image_builds_identity ON image_builds(version,recipe_rev,platform) WHERE recipe_rev IS NOT NULL AND platform IS NOT NULL AND status IN ('queued','running','validating','success')") if err != nil { return err } if !buildQueue.initialized[a.s.DB] { if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='image template identity is unknown' WHERE status='queued' AND (recipe_rev IS NULL OR recipe_rev<>? OR platform IS NULL OR platform<>?)", imageRecipeRev, imagePlatform); err != nil { return err } if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='build interrupted by process restart' WHERE status IN ('running','validating')"); err != nil { return err } buildQueue.initialized[a.s.DB] = true } return nil } func (a *API) builds(w http.ResponseWriter, r *http.Request) { u, _, authErr := a.user(r) if authErr != nil { fail(w, 401, "authentication required") return } if r.Method != http.MethodGet && r.Method != http.MethodPost { fail(w, 405, "method not allowed") return } var in struct{ Version string } if r.Method == http.MethodPost { dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)) dec.DisallowUnknownFields() if dec.Decode(&in) != nil || dec.Decode(new(any)) != io.EOF || !validBuildVersion(in.Version) { fail(w, 400, "a fixed version is required") return } } buildQueue.Lock() defer buildQueue.Unlock() if err := a.ensureBuilds(); err != nil { fail(w, 500, "database error") return } if r.Method == http.MethodGet { rows, err := a.s.DB.Query("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds ORDER BY created_at DESC,id DESC") if err != nil { fail(w, 500, "database error") return } result := []buildRecord{} for rows.Next() { var b buildRecord var finished, recipe, platform sql.NullString if err = rows.Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished); err != nil { break } b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String result = append(result, b) } scanErr := rows.Err() rows.Close() if err != nil || scanErr != nil { fail(w, 500, "database error") return } if err := a.reconcileBuilds(r.Context(), result); err != nil { fail(w, 502, "docker image reconciliation failed") return } a.startBuildWorker() writeJSON(w, result) return } b, err := a.enqueueBuildLocked(r.Context(), u.ID, in.Version) if err != nil { status := 502 if errors.Is(err, errBuildRateLimited) { status = 429 w.Header().Set("Retry-After", "3600") } fail(w, status, sanitizeBuildLog(err.Error())) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) writeJSON(w, b) } var errBuildRateLimited = errors.New("build limit: three new tasks per user per hour") func (a *API) enqueueBuild(ctx context.Context, uid, version string) (buildRecord, error) { buildQueue.Lock() defer buildQueue.Unlock() if !validBuildVersion(version) { return buildRecord{}, errors.New("a fixed version is required") } if err := a.ensureBuilds(); err != nil { return buildRecord{}, err } return a.enqueueBuildLocked(ctx, uid, version) } func (a *API) enqueueBuildLocked(ctx context.Context, uid, version string) (buildRecord, error) { var b buildRecord var finished, recipe, platform sql.NullString err := a.s.DB.QueryRow("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at FROM image_builds WHERE version=? AND recipe_rev=? AND platform=? AND status IN ('queued','running','validating','success')", version, imageRecipeRev, imagePlatform).Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished) b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String if err == nil && b.Status == "success" { ctx, cancel := context.WithTimeout(ctx, 15*time.Second) id, lookupErr := a.localBuildImage(ctx, b.Version, b.ImageID) cancel() if lookupErr != nil { return b, lookupErr } if id != b.ImageID || id == "" { if _, err = a.s.DB.Exec("UPDATE image_builds SET status='missing',error='image no longer available' WHERE id=?", b.ID); err == nil { err = sql.ErrNoRows } } } if errors.Is(err, sql.ErrNoRows) { var count int if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests WHERE user_id=? AND created_at>?", uid, time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)).Scan(&count); err != nil { return b, err } if count >= 3 { return b, errBuildRateLimited } b = buildRecord{ID: store.ID(), Version: version, Status: "queued", Created: time.Now().UTC().Format(time.RFC3339Nano), RequestedBy: uid, RecipeRev: imageRecipeRev, Platform: imagePlatform} err = a.insertBuildRequest(b, uid) } if err != nil { return b, err } a.startBuildWorker() return b, nil } func (a *API) startBuildWorker() { if buildQueue.active[a.s.DB] { return } buildQueue.active[a.s.DB] = true go func() { buildWorker.Lock() defer buildWorker.Unlock() for { buildQueue.Lock() var b buildRecord err := a.s.DB.QueryRow("SELECT id,version FROM image_builds WHERE status='queued' ORDER BY created_at,id LIMIT 1").Scan(&b.ID, &b.Version) if err == nil { _, err = a.s.DB.Exec("UPDATE image_builds SET status='running' WHERE id=?", b.ID) } if err != nil { delete(buildQueue.active, a.s.DB) buildQueue.Unlock() return } buildQueue.Unlock() a.runImageBuild(b) } }() } func imageBuildContext() (io.Reader, error) { var buf bytes.Buffer tw := tar.NewWriter(&buf) for _, name := range []string{"Dockerfile", "Caddyfile", "entrypoint.sh", "install-mobile.mjs", "install-free-search.mjs", "install-workspace.mjs", "init-profile.mjs"} { dataBytes, err := imageassets.Files.ReadFile(name) if err != nil { return nil, err } data := string(dataBytes) if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0755, Size: int64(len(data))}); err != nil { return nil, err } if _, err := tw.Write([]byte(data)); err != nil { return nil, err } } if err := tw.Close(); err != nil { return nil, err } return &buf, nil } func (a *API) runImageBuild(b buildRecord) { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() logs := "" rawLogs := "" var logErr error appendLog := func(s string) { if len(rawLogs) < buildLogLimit { remaining := buildLogLimit - len(rawLogs) if len(s) > remaining { s = s[:remaining] } rawLogs += s } logs = sanitizeBuildLog(rawLogs) if logErr == nil { _, logErr = a.s.DB.Exec("UPDATE image_builds SET logs=? WHERE id=?", logs, b.ID) } } tarball, err := imageBuildContext() var id string if err == nil { id, err = a.localBuildImage(ctx, b.Version, "") } if err == nil && id == "" { _, err = fetchNPMVersion(ctx, npmBuildClient, b.Version) if err == nil { err = a.checkBuildDisk(ctx) } } if err == nil && id == "" { id, err = a.docker.Build(ctx, tarball, b.Version, "dsh-web:"+b.Version+"-"+imageRecipeRev, appendLog) } if err == nil { var matches bool matches, err = a.imageMatches(ctx, id, b.Version) if err == nil && !matches { err = errors.New("image template or platform mismatch") } } if err == nil { _, err = a.s.DB.Exec("UPDATE image_builds SET status='validating',image_id=? WHERE id=?", id, b.ID) } if err == nil { err = a.verifyBuiltImage(ctx, id, b.Version) } if err == nil { err = logErr } status, detail := "success", "" if err != nil { status = "failed" detail = sanitizeBuildLog(err.Error()) appendLog(string(rune(10)) + detail) } finished := time.Now().UTC().Format(time.RFC3339Nano) _, _ = a.s.DB.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, id, finished, b.ID) if status == "success" && id != "" { _, _ = a.s.DB.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", id, b.Version, imageRecipeRev, imagePlatform, "dsh-web:"+b.Version+"-"+imageRecipeRev, "available", finished) } } func (a *API) verifyBuiltImage(parent context.Context, image, version string) (result error) { ctx, cancel := context.WithTimeout(parent, 90*time.Second) defer cancel() var created struct { ID string `json:"Id"` } config := map[string]any{"Image": image, "User": "1000:1000", "Labels": map[string]string{"dsh-mgr.role": "build-check"}, "HostConfig": map[string]any{"NetworkMode": "none", "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges:true"}, "Memory": int64(1 << 30), "NanoCpus": int64(1e9), "PidsLimit": 256, "Tmpfs": map[string]string{"/tmp": "rw,exec,nosuid,nodev,size=256m,mode=1777", "/workspace": "rw,nosuid,nodev,size=256m,uid=1000,gid=1000"}}} if err := a.docker.Do(ctx, http.MethodPost, "/containers/create", config, &created); err != nil { return err } if created.ID == "" { return errors.New("verification container ID missing") } path := "/containers/" + url.PathEscape(created.ID) defer func() { cleanup, c := context.WithTimeout(context.Background(), 10*time.Second) defer c() err := a.docker.Do(cleanup, http.MethodDelete, path+"?force=1&v=1", nil, nil) result = errors.Join(result, err) }() if err := a.docker.Do(ctx, http.MethodPost, path+"/start", nil, nil); err != nil { return err } script := `const end=Date.now()+60000;async function probe(){try{const r=await fetch('http://127.0.0.1:8080/',{redirect:'manual',signal:AbortSignal.timeout(2000)});if(r.status>=200&&r.status<500)process.exit(0);}catch{}if(Date.now()>end)process.exit(1);setTimeout(probe,500);}probe();` { expected, _ := json.Marshal(version) script = `const cp=require('node:child_process');const actual=cp.execFileSync('dsh',['--version'],{encoding:'utf8',timeout:20000}).trim();if(actual!==` + string(expected) + `)process.exit(2);` + script } script = `const fs=require('node:fs');for(const [k,v] of Object.entries({DSH_HOME:'/workspace',HOME:'/workspace',XDG_DATA_HOME:'/workspace/.local/share',XDG_CONFIG_HOME:'/workspace/.config'})){if(process.env[k]!==v)process.exit(3);}if(!fs.lstatSync('/data/dsh-home').isSymbolicLink()||fs.realpathSync('/data/dsh-home')!=='/workspace')process.exit(4);fs.writeFileSync('/workspace/.home-check','isolated');if(fs.readFileSync('/data/dsh-home/.home-check','utf8')!=='isolated')process.exit(5);fs.unlinkSync('/workspace/.home-check');` + script var exec struct { ID string `json:"Id"` } if err := a.docker.Do(ctx, http.MethodPost, path+"/exec", map[string]any{"Cmd": []string{"node", "-e", script}, "User": "1000:1000"}, &exec); err != nil { return err } if exec.ID == "" { return errors.New("verification exec ID missing") } ep := "/exec/" + url.PathEscape(exec.ID) if err := a.docker.Do(ctx, http.MethodPost, ep+"/start", map[string]bool{"Detach": true, "Tty": false}, nil); err != nil { return err } ticker := time.NewTicker(250 * time.Millisecond) defer ticker.Stop() for { var state struct { Running bool ExitCode int } if err := a.docker.Do(ctx, http.MethodGet, ep+"/json", nil, &state); err != nil { return err } if !state.Running { if state.ExitCode != 0 { return fmt.Errorf("image startup verification failed (exit %d)", state.ExitCode) } container, err := a.docker.Inspect(ctx, created.ID) if err != nil { return err } if !container.State.Running { return errors.New("verification container exited") } return nil } select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: } } } const buildLogLimit = 64 << 10 var fixedVersion = regexp.MustCompile(`^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)(-[0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?$`) var buildSecrets = regexp.MustCompile(`(?i)(authorization["']?[[:space:]]*[:=][[:space:]]*["']?(?:bearer[[:space:]]+|basic[[:space:]]+)?|(?:token|password|passwd|secret|api[_-]?key|cookie)["']?[[:space:]]*[:=][[:space:]]*["']?)[^[:space:]"'&,;]+`) var buildURLCredentials = regexp.MustCompile(`(https?://)[^/@[:space:]]+:[^/@[:space:]]+@`) var buildQueue = struct { sync.Mutex initialized map[*sql.DB]bool active map[*sql.DB]bool }{initialized: map[*sql.DB]bool{}, active: map[*sql.DB]bool{}} var buildWorker sync.Mutex type imageRecord struct { ID string `json:"id"` ImageID string `json:"image_id"` Version string `json:"version"` Tag string `json:"tag,omitempty"` Status string `json:"status"` Created string `json:"created_at"` RecipeRev string `json:"recipe_rev,omitempty"` Platform string `json:"platform,omitempty"` UsageCount int `json:"usage_count"` } type buildRecord struct { ImageID string `json:"image_id,omitempty"` ID string `json:"id"` Version string `json:"version"` Status string `json:"status"` Logs string `json:"logs"` Error string `json:"error,omitempty"` Created string `json:"created_at"` RequestedBy string `json:"requested_by,omitempty"` RecipeRev string `json:"recipe_rev,omitempty"` Platform string `json:"platform,omitempty"` Finished string `json:"finished_at,omitempty"` } func (a *API) images(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodDelete { a.deleteImage(w, r) return } if _, _, err := a.user(r); err != nil { fail(w, 401, "authentication required") return } if r.Method != http.MethodGet { fail(w, 405, "method not allowed") return } ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) defer cancel() result, err := a.imageIndex(ctx) if err != nil { fail(w, 502, "image index reconciliation failed") return } writeJSON(w, result) } func (a *API) verifiedImages(ctx context.Context) (map[string]string, error) { buildQueue.Lock() defer buildQueue.Unlock() if err := a.ensureBuilds(); err != nil { return nil, err } rows, err := a.s.DB.QueryContext(ctx, "SELECT image_id,version FROM image_builds WHERE status='success' AND image_id<>'' AND recipe_rev=? AND platform=?", imageRecipeRev, imagePlatform) if err != nil { return nil, err } defer rows.Close() images := make(map[string]string) for rows.Next() { var id, version string if err := rows.Scan(&id, &version); err != nil { return nil, err } images[id] = version } return images, rows.Err() } func (a *API) latest(w http.ResponseWriter, r *http.Request) { if _, _, err := a.user(r); err != nil { fail(w, 401, "authentication required") return } if r.Method != http.MethodGet { fail(w, 405, "method not allowed") return } v, err := fetchLatest(r.Context(), npmBuildClient) if err != nil { fail(w, 502, "npm version lookup failed") return } writeJSON(w, map[string]string{"version": v, "checked_at": time.Now().UTC().Format(time.RFC3339)}) } func fetchLatest(ctx context.Context, client *http.Client) (string, error) { return fetchNPMVersion(ctx, client, "latest") } func fetchNPMVersion(ctx context.Context, client *http.Client, version string) (string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://registry.npmjs.org/@deepseek-ai%2Fdsh/"+url.PathEscape(version), nil) if err != nil { return "", err } resp, err := client.Do(req) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != 200 { return "", fmt.Errorf("npm status %d", resp.StatusCode) } var v struct{ Version string } if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&v); err != nil { return "", err } if !validBuildVersion(v.Version) || (version != "latest" && v.Version != version) { return "", errors.New("invalid npm version") } return v.Version, nil } func validBuildVersion(v string) bool { return len(v) <= 100 && fixedVersion.MatchString(v) } func sanitizeBuildLog(s string) string { s = buildURLCredentials.ReplaceAllString(s, "$1[REDACTED]@") s = buildSecrets.ReplaceAllString(s, "$1[REDACTED]") s = strings.ToValidUTF8(s, "") if len(s) > buildLogLimit { s = strings.ToValidUTF8(s[:buildLogLimit], "") } return s } func (a *API) localBuildImage(ctx context.Context, version, preferred string) (string, error) { filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime", "dsh-mgr.version=" + version}}) var images []struct { ID string Labels map[string]string } if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &images); err != nil { return "", err } for _, im := range images { if im.ID != "" && im.Labels["dsh-mgr.role"] == "runtime" && im.Labels["dsh-mgr.version"] == version && im.Labels["dsh-mgr.recipe"] == imageRecipeRev && (preferred == "" || im.ID == preferred) { ok, err := a.imageMatches(ctx, im.ID, version) if err != nil { return "", err } if ok { return im.ID, nil } } } return "", nil }