m1saka
602917698a
Add snapshot usage and protected cleanup, preserve unsaved profile edits, avoid holding SQL connections during snapshot deletion, and fix download filenames. Include regression tests and API documentation.
1072 lines
32 KiB
Go
1072 lines
32 KiB
Go
package api
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"crypto/aes"
|
|
"crypto/cipher"
|
|
"crypto/rand"
|
|
"crypto/sha256"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httputil"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
)
|
|
|
|
var runtimeImageID = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)
|
|
var runtimeToken = regexp.MustCompile(`(?i)[?&]token=([^\s&#\x1b"'<>]+)`)
|
|
var runtimeStartupToken = regexp.MustCompile(`(?im)^(?:dsh web:\s*)?https?://[^\s]*[?&]token=([^\s&#\x1b"'<>]+)`)
|
|
var runtimeSecrets = regexp.MustCompile(`(?i)((?:token|api[_-]?key|authorization|password|secret|cookie)\s*[=:]\s*)[^\r\n]+`)
|
|
|
|
func runtimeNetwork(id string) string { return "dsh-inst-" + id }
|
|
|
|
func (a *API) instances(w http.ResponseWriter, r *http.Request) {
|
|
u, _, err := a.user(r)
|
|
if err != nil {
|
|
fail(w, 401, "authentication required")
|
|
return
|
|
}
|
|
if r.Method == http.MethodGet {
|
|
instances, err := a.s.Instances(u.ID)
|
|
if err != nil {
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
if instances == nil {
|
|
instances = []store.Instance{}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
|
defer cancel()
|
|
for n := range instances {
|
|
if instances[n].ContainerID == "" {
|
|
continue
|
|
}
|
|
c, err := a.docker.Inspect(ctx, instances[n].ContainerID)
|
|
if err == nil {
|
|
if instances[n].Status != "error" {
|
|
instances[n].Status = c.State.Status
|
|
}
|
|
} else {
|
|
instances[n].Status = "unavailable"
|
|
}
|
|
}
|
|
writeJSON(w, instances)
|
|
return
|
|
}
|
|
if r.Method != http.MethodPost {
|
|
fail(w, 405, "method not allowed")
|
|
return
|
|
}
|
|
var in struct {
|
|
Name string `json:"name"`
|
|
ImageID string `json:"image_id"`
|
|
}
|
|
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10))
|
|
decoder.DisallowUnknownFields()
|
|
if decoder.Decode(&in) != nil || decoder.Decode(new(any)) != io.EOF || strings.TrimSpace(in.Name) == "" || len(in.Name) > 200 || !runtimeImageID.MatchString(in.ImageID) {
|
|
fail(w, 400, "invalid instance")
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
|
defer cancel()
|
|
var image struct {
|
|
ID string `json:"Id"`
|
|
Config struct{ Labels map[string]string }
|
|
}
|
|
if err := a.docker.Do(ctx, "GET", "/images/"+url.PathEscape(in.ImageID)+"/json", nil, &image); err != nil {
|
|
fail(w, 400, "local runtime image required")
|
|
return
|
|
}
|
|
if image.ID != in.ImageID || image.Config.Labels["dsh-mgr.role"] != "runtime" || image.Config.Labels["dsh-mgr.version"] == "" {
|
|
fail(w, 400, "local runtime image required")
|
|
return
|
|
}
|
|
verified, err := a.verifiedImages(ctx)
|
|
if err != nil {
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
if verified[image.ID] != image.Config.Labels["dsh-mgr.version"] {
|
|
fail(w, 400, "verified runtime image required")
|
|
return
|
|
}
|
|
i := store.Instance{ID: store.ID(), UID: u.ID, Name: strings.TrimSpace(in.Name), ImageID: image.ID, Version: image.Config.Labels["dsh-mgr.version"], Status: "creating", Created: time.Now()}
|
|
if err := a.s.ReserveInstance(i); err != nil {
|
|
if errors.Is(err, store.ErrInstanceLimit) {
|
|
fail(w, 409, err.Error())
|
|
return
|
|
}
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
created, err := a.createRuntime(ctx, u, i, a.runtimeHost(r))
|
|
if err != nil {
|
|
fail(w, 502, err.Error())
|
|
return
|
|
}
|
|
writeJSON(w, created)
|
|
}
|
|
|
|
func (a *API) createRuntime(ctx context.Context, u store.User, i store.Instance, trustedHost string) (store.Instance, error) {
|
|
// The instance row is visible to its owner while creating, so hold the
|
|
// instance lock for the whole creation: a concurrent DELETE or file lease
|
|
// then waits instead of interleaving with container setup. Concurrent
|
|
// creations of different instances remain independent.
|
|
inst := a.instanceLocks.Acquire(i.ID)
|
|
defer inst.Release()
|
|
if _, err := a.s.GetInstance(i.ID); err != nil {
|
|
return i, errors.New("instance reservation unavailable")
|
|
}
|
|
networkCreated, networkAttempted, containerAttempted, complete := false, false, false, false
|
|
defer func() {
|
|
if complete {
|
|
return
|
|
}
|
|
cleanup, done := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer done()
|
|
if networkAttempted && !networkCreated {
|
|
if err := a.recoverRuntimeNetwork(cleanup, i.ID); err != nil {
|
|
a.s.SetInstance(i.ID, "error")
|
|
return
|
|
}
|
|
}
|
|
if containerAttempted && i.ContainerID == "" {
|
|
if err := a.recoverRuntimeContainer(cleanup, &i); err != nil {
|
|
a.s.SetInstance(i.ID, "error")
|
|
return
|
|
}
|
|
}
|
|
if i.ContainerID != "" {
|
|
if err := a.removeRuntimeContainer(cleanup, i.ContainerID); err != nil {
|
|
a.s.SetInstance(i.ID, "error")
|
|
return
|
|
}
|
|
}
|
|
if networkCreated {
|
|
if err := a.removeRuntimeNetwork(cleanup, i.ID); err != nil {
|
|
a.s.SetInstance(i.ID, "error")
|
|
return
|
|
}
|
|
}
|
|
if err := a.removeInstanceData(i); err != nil {
|
|
a.s.SetInstance(i.ID, "error")
|
|
return
|
|
}
|
|
a.s.DeleteInstance(i.ID)
|
|
_, _ = a.s.DB.Exec("DELETE FROM runtime_networks WHERE instance_id=?", i.ID)
|
|
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=?", i.ID)
|
|
}()
|
|
admissionMu.RLock()
|
|
defer admissionMu.RUnlock()
|
|
buildQueue.Lock()
|
|
policyErr := a.ensureBuilds()
|
|
buildQueue.Unlock()
|
|
if policyErr != nil {
|
|
return i, policyErr
|
|
}
|
|
policy, policyErr := a.evaluateImage(ctx, i.ImageID)
|
|
if policyErr != nil {
|
|
return i, policyErr
|
|
}
|
|
if !policy.Selectable || policy.Version != i.Version {
|
|
return i, errors.New("image is not admitted for creation")
|
|
}
|
|
root, err := filepath.Abs(a.root)
|
|
if err != nil {
|
|
return i, errors.New("storage unavailable")
|
|
}
|
|
suffix := filepath.Join("workspaces", u.ID)
|
|
for _, relative := range []string{filepath.Join(suffix, "instances", i.ID, "home")} {
|
|
dir := filepath.Join(root, relative)
|
|
if err := os.MkdirAll(dir, 0750); err != nil {
|
|
return i, errors.New("storage unavailable")
|
|
}
|
|
if err := os.Chown(dir, 1000, 1000); err != nil {
|
|
return i, errors.New("storage ownership unavailable")
|
|
}
|
|
}
|
|
hostRoot := os.Getenv("HOST_DATA_ROOT")
|
|
if hostRoot == "" {
|
|
hostRoot = root
|
|
}
|
|
if !filepath.IsAbs(hostRoot) {
|
|
return i, errors.New("HOST_DATA_ROOT must be absolute")
|
|
}
|
|
homeSource, err := toHostPath(root, hostRoot, filepath.Join(root, suffix, "instances", i.ID, "home"))
|
|
if err != nil {
|
|
return i, err
|
|
}
|
|
network := runtimeNetwork(i.ID)
|
|
if err := a.setRuntimeCreatePending(i.ID, "network_pending", true); err != nil {
|
|
return i, err
|
|
}
|
|
networkAttempted = true
|
|
var createdNetwork struct {
|
|
ID string `json:"Id"`
|
|
}
|
|
if err := a.docker.Do(ctx, "POST", "/networks/create", map[string]any{"Name": network, "Driver": "bridge", "CheckDuplicate": true, "Labels": map[string]string{"dsh-mgr.instance": i.ID}}, &createdNetwork); err != nil {
|
|
if dockerCreateRejected(err) {
|
|
if clearErr := a.setRuntimeCreatePending(i.ID, "network_pending", false); clearErr == nil {
|
|
networkAttempted = false
|
|
}
|
|
}
|
|
return i, errors.New("network creation failed")
|
|
}
|
|
networkCreated = true
|
|
if createdNetwork.ID != "" {
|
|
if _, err := a.s.DB.ExecContext(ctx, "INSERT INTO runtime_networks(instance_id,network_id) VALUES(?,?)", i.ID, createdNetwork.ID); err != nil {
|
|
return i, err
|
|
}
|
|
}
|
|
if err := a.setRuntimeCreatePending(i.ID, "network_pending", false); err != nil {
|
|
return i, err
|
|
}
|
|
if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" {
|
|
if err := a.docker.Do(ctx, "POST", "/networks/"+network+"/connect", map[string]string{"Container": panel}, nil); err != nil {
|
|
return i, errors.New("panel network connection failed")
|
|
}
|
|
}
|
|
host := trustedHost
|
|
body := map[string]any{
|
|
"Image": i.ImageID, "User": "1000:1000", "WorkingDir": "/workspace",
|
|
"Env": []string{"DSH_HOME=/workspace", "HOME=/workspace", "XDG_DATA_HOME=/workspace/.local/share", "XDG_CONFIG_HOME=/workspace/.config"},
|
|
"Cmd": []string{"web", "--no-open", "--port", "3080", "--trusted-host", host},
|
|
"Labels": map[string]string{"dsh-mgr.instance": i.ID, "dsh-mgr.owner": u.ID, "dsh-mgr.role": "dsh", "dsh-mgr.version": i.Version},
|
|
"HostConfig": map[string]any{
|
|
"NetworkMode": network, "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges"},
|
|
"Memory": int64(2 << 30), "NanoCpus": int64(2e9), "PidsLimit": 256,
|
|
"Tmpfs": map[string]string{"/tmp": "rw,exec,nosuid,nodev,size=256m"},
|
|
"LogConfig": map[string]any{"Type": "json-file", "Config": map[string]string{"max-size": "10m", "max-file": "3"}},
|
|
"Mounts": []map[string]any{
|
|
{"Type": "bind", "Source": homeSource, "Target": "/workspace"},
|
|
// DSH_HOME is the same bind mount as the visible workspace.
|
|
},
|
|
},
|
|
}
|
|
var created struct {
|
|
ID string `json:"Id"`
|
|
}
|
|
if err := a.setRuntimeCreatePending(i.ID, "container_pending", true); err != nil {
|
|
return i, err
|
|
}
|
|
containerAttempted = true
|
|
if err := a.docker.Do(ctx, "POST", "/containers/create?name="+network, body, &created); err != nil || created.ID == "" {
|
|
if dockerCreateRejected(err) {
|
|
if clearErr := a.setRuntimeCreatePending(i.ID, "container_pending", false); clearErr == nil {
|
|
containerAttempted = false
|
|
}
|
|
}
|
|
return i, errors.New("container creation failed")
|
|
}
|
|
i.ContainerID = created.ID
|
|
if err := a.setRuntimeCreatePending(i.ID, "container_pending", false); err != nil {
|
|
return i, err
|
|
}
|
|
if _, err := a.s.DB.ExecContext(ctx, "UPDATE instances SET container_id=? WHERE id=?", i.ContainerID, i.ID); err != nil {
|
|
return i, errors.New("database error")
|
|
}
|
|
if err := a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(i.ContainerID)+"/start", nil, nil); err != nil {
|
|
return i, errors.New("container start failed")
|
|
}
|
|
i.Status = "running"
|
|
if _, err := a.s.DB.ExecContext(ctx, "UPDATE instances SET status=? WHERE id=?", i.Status, i.ID); err != nil {
|
|
return i, errors.New("database error")
|
|
}
|
|
complete = true
|
|
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=? AND container_pending=0 AND network_pending=0", i.ID)
|
|
a.s.AddAudit(u.ID, "instance.create", i.ID, i.ImageID)
|
|
return i, nil
|
|
}
|
|
|
|
func dockerNotFound(err error) bool {
|
|
var e *docker.APIError
|
|
return errors.As(err, &e) && e.StatusCode == 404
|
|
}
|
|
func dockerCreateRejected(err error) bool {
|
|
var e *docker.APIError
|
|
return errors.As(err, &e) && e.StatusCode >= 400 && e.StatusCode < 500 && e.StatusCode != 408 && e.StatusCode != 409
|
|
}
|
|
func (a *API) removeRuntimeContainer(ctx context.Context, id string) error {
|
|
err := a.docker.Do(ctx, "DELETE", "/containers/"+url.PathEscape(id)+"?force=1", nil, nil)
|
|
if dockerNotFound(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
func (a *API) removeRuntimeNetwork(ctx context.Context, id string) error {
|
|
return a.recoverRuntimeNetwork(ctx, id)
|
|
}
|
|
|
|
func (a *API) instance(w http.ResponseWriter, r *http.Request) {
|
|
u, _, err := a.user(r)
|
|
if err != nil {
|
|
fail(w, 401, "authentication required")
|
|
return
|
|
}
|
|
p := strings.Split(strings.TrimPrefix(r.URL.Path, "/api/instances/"), "/")
|
|
if len(p) > 2 || p[0] == "" {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
i, err := a.s.GetInstance(p[0])
|
|
if err != nil || i.UID != u.ID {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
action := ""
|
|
if len(p) == 2 {
|
|
action = p[1]
|
|
}
|
|
switch action {
|
|
case "profile-preview":
|
|
a.profilePreview(w, r, i)
|
|
return
|
|
case "profile", "backups", "restore", "snapshots":
|
|
a.config(w, r, i, action)
|
|
return
|
|
case "upgrade":
|
|
a.upgrade(w, r, i)
|
|
return
|
|
}
|
|
w, flush := deferJSONResponse(w)
|
|
defer flush()
|
|
ctx, cancel := context.WithTimeout(r.Context(), 40*time.Second)
|
|
defer cancel()
|
|
if action == "logs" {
|
|
if r.Method != "GET" {
|
|
fail(w, 405, "method not allowed")
|
|
return
|
|
}
|
|
logs, err := a.docker.Logs(ctx, i.ContainerID)
|
|
if err != nil {
|
|
fail(w, 502, "logs unavailable")
|
|
return
|
|
}
|
|
writeJSON(w, map[string]string{"logs": redactRuntimeLogs(logs)})
|
|
return
|
|
}
|
|
if action == "" && r.Method == "GET" {
|
|
c, err := a.docker.Inspect(ctx, i.ContainerID)
|
|
if err != nil {
|
|
fail(w, 502, "container unavailable")
|
|
return
|
|
}
|
|
if i.Status != "error" {
|
|
i.Status = c.State.Status
|
|
}
|
|
writeJSON(w, i)
|
|
return
|
|
}
|
|
if action == "" && r.Method == "DELETE" {
|
|
inst := a.instanceLocks.Acquire(p[0])
|
|
defer inst.Release()
|
|
i, err = a.s.GetInstance(p[0])
|
|
if err != nil || i.UID != u.ID {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
if a.upgradeRecoveryRequired(i) {
|
|
fail(w, 409, "instance requires recovery")
|
|
return
|
|
}
|
|
a.readersMu.Lock()
|
|
readers := a.workspaceReaders[i.ID]
|
|
a.readersMu.Unlock()
|
|
if readers > 0 {
|
|
fail(w, 409, "instance files busy; retry later")
|
|
return
|
|
}
|
|
if i.ContainerID == "" {
|
|
if err := a.recoverRuntimeContainer(ctx, &i); err != nil {
|
|
fail(w, 502, "container recovery failed")
|
|
return
|
|
}
|
|
}
|
|
if i.ContainerID != "" {
|
|
if err := a.removeRuntimeContainer(ctx, i.ContainerID); err != nil {
|
|
fail(w, 502, "container deletion failed")
|
|
return
|
|
}
|
|
}
|
|
if err := a.removeRuntimeNetwork(ctx, i.ID); err != nil {
|
|
fail(w, 502, "network deletion failed")
|
|
return
|
|
}
|
|
if err := a.removeInstanceData(i); err != nil {
|
|
a.s.SetInstance(i.ID, "error")
|
|
fail(w, 500, "instance data deletion failed; retry deletion")
|
|
return
|
|
}
|
|
if _, err := a.s.DB.ExecContext(ctx, "DELETE FROM instances WHERE id=?", i.ID); err != nil {
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
_, _ = a.s.DB.Exec("DELETE FROM runtime_networks WHERE instance_id=?", i.ID)
|
|
a.s.AddAudit(u.ID, "instance.delete", i.ID, "instance data deleted")
|
|
// The instance row is gone and its resources were just cleaned, so
|
|
// any pending flags cannot resolve later: drop the row unconditionally.
|
|
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=?", i.ID)
|
|
runtimeAuthCache.Delete(runtimeAuthKey{a, i.ID})
|
|
writeJSON(w, map[string]bool{"ok": true})
|
|
return
|
|
}
|
|
if action != "start" && action != "stop" && action != "restart" {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
if r.Method != "POST" {
|
|
fail(w, 405, "method not allowed")
|
|
return
|
|
}
|
|
inst := a.instanceLocks.Acquire(p[0])
|
|
defer inst.Release()
|
|
i, err = a.s.GetInstance(p[0])
|
|
if err != nil || i.UID != u.ID {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
if action != "stop" && (i.Status == "error" || a.upgradeRecoveryRequired(i)) {
|
|
fail(w, 409, "instance requires recovery")
|
|
return
|
|
}
|
|
a.readersMu.Lock()
|
|
readers := a.workspaceReaders[i.ID]
|
|
a.readersMu.Unlock()
|
|
if readers > 0 {
|
|
fail(w, 409, "instance files busy; retry later")
|
|
return
|
|
}
|
|
admissionMu.RLock()
|
|
defer admissionMu.RUnlock()
|
|
if action != "stop" {
|
|
buildQueue.Lock()
|
|
e := a.ensureBuilds()
|
|
buildQueue.Unlock()
|
|
if e != nil {
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
var blocked int
|
|
e = a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM image_admissions WHERE image_id=? AND state='blocked'", i.ImageID).Scan(&blocked)
|
|
if e != nil {
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
if blocked > 0 {
|
|
fail(w, 409, "image is blocked; start/restart denied")
|
|
return
|
|
}
|
|
}
|
|
if err := a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(i.ContainerID)+"/"+action+"?t=10", nil, nil); err != nil {
|
|
fail(w, 502, "container action failed")
|
|
return
|
|
}
|
|
c, err := a.docker.Inspect(ctx, i.ContainerID)
|
|
if err != nil {
|
|
fail(w, 502, "container inspection failed")
|
|
return
|
|
}
|
|
if i.Status != "error" {
|
|
i.Status = c.State.Status
|
|
}
|
|
if _, err := a.s.DB.ExecContext(ctx, "UPDATE instances SET status=? WHERE id=?", i.Status, i.ID); err != nil {
|
|
fail(w, 500, "database error")
|
|
return
|
|
}
|
|
a.s.AddAudit(u.ID, "instance."+action, i.ID, i.Status)
|
|
writeJSON(w, i)
|
|
}
|
|
|
|
func redactRuntimeLogs(s string) string {
|
|
s = runtimeToken.ReplaceAllString(s, "?token=[REDACTED]")
|
|
return runtimeSecrets.ReplaceAllString(s, "${1}[REDACTED]")
|
|
}
|
|
func latestRuntimeToken(logs string) string {
|
|
matches := runtimeStartupToken.FindAllStringSubmatch(logs, -1)
|
|
if len(matches) == 0 {
|
|
return ""
|
|
}
|
|
token, err := url.QueryUnescape(matches[len(matches)-1][1])
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return token
|
|
}
|
|
func (a *API) runtimeHost(r *http.Request) string {
|
|
if u, err := url.Parse(a.origin); err == nil && u.Host != "" {
|
|
return u.Host
|
|
}
|
|
return r.Host
|
|
}
|
|
func (a *API) runtimeOriginAllowed(r *http.Request) bool {
|
|
origin := r.Header.Get("Origin")
|
|
if origin == "" {
|
|
return (r.Method == "GET" || r.Method == "HEAD") && !strings.EqualFold(r.Header.Get("Upgrade"), "websocket")
|
|
}
|
|
expected := strings.TrimRight(a.origin, "/")
|
|
if expected == "" {
|
|
scheme := "http"
|
|
if r.TLS != nil {
|
|
scheme = "https"
|
|
}
|
|
expected = scheme + "://" + r.Host
|
|
}
|
|
return origin == expected
|
|
}
|
|
|
|
var runtimeTransport = &http.Transport{Proxy: nil, DialContext: (&net.Dialer{Timeout: 10 * time.Second, KeepAlive: 30 * time.Second}).DialContext, ResponseHeaderTimeout: 20 * time.Second, IdleConnTimeout: 90 * time.Second}
|
|
|
|
type runtimeAuthKey struct {
|
|
api *API
|
|
id string
|
|
}
|
|
type runtimeAuth struct {
|
|
mu sync.Mutex
|
|
entries map[string]runtimeAuthEntry
|
|
}
|
|
|
|
type runtimeAuthEntry struct {
|
|
Container string
|
|
Started string
|
|
Token string
|
|
Host string
|
|
Cookies string
|
|
}
|
|
|
|
type runtimeAuthDisk struct {
|
|
Nonce string `json:"nonce"`
|
|
Data string `json:"data"`
|
|
}
|
|
|
|
var runtimeAuthCache sync.Map
|
|
|
|
func (a *API) runtimeCookie(ctx context.Context, i store.Instance, started string, target *url.URL, host string) (string, error) {
|
|
if a.authErr != nil || len(a.authKey) != 32 {
|
|
return "", errors.New("runtime authentication unavailable")
|
|
}
|
|
value, _ := runtimeAuthCache.LoadOrStore(runtimeAuthKey{a, i.ID}, &runtimeAuth{})
|
|
auth := value.(*runtimeAuth)
|
|
auth.mu.Lock()
|
|
defer auth.mu.Unlock()
|
|
if auth.entries == nil {
|
|
auth.entries = make(map[string]runtimeAuthEntry)
|
|
}
|
|
baseKey := i.ContainerID + "/" + started
|
|
key := baseKey + "/" + host
|
|
if entry, ok := auth.entries[key]; ok && entry.Cookies != "" {
|
|
return entry.Cookies, nil
|
|
}
|
|
if entry, ok := a.loadRuntimeAuth(i.ID, baseKey); ok {
|
|
if entry.Host == host && entry.Cookies != "" {
|
|
auth.entries[key] = entry
|
|
return entry.Cookies, nil
|
|
}
|
|
cookies, err := exchangeRuntimeCookie(ctx, target, host, entry.Token)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
entry.Host, entry.Cookies = host, cookies
|
|
if err := a.saveRuntimeAuth(i.ID, baseKey, entry); err != nil {
|
|
return "", errors.New("runtime authentication unavailable")
|
|
}
|
|
auth.entries[key] = entry
|
|
return cookies, nil
|
|
}
|
|
startedTime, err := time.Parse(time.RFC3339Nano, started)
|
|
if err != nil {
|
|
return "", errors.New("invalid start time")
|
|
}
|
|
since := fmt.Sprintf("%d.%09d", startedTime.Unix(), startedTime.Nanosecond())
|
|
query := url.Values{"stdout": {"1"}, "stderr": {"1"}, "follow": {"0"}, "since": {since}}
|
|
resp, err := a.docker.Stream(ctx, "GET", "/containers/"+url.PathEscape(i.ContainerID)+"/logs?"+query.Encode(), nil, "")
|
|
if err != nil {
|
|
return "", errors.New("logs unavailable")
|
|
}
|
|
defer resp.Body.Close()
|
|
token, err := runtimeTokenFromStream(resp.Body)
|
|
if err != nil {
|
|
return "", errors.New("logs unavailable")
|
|
}
|
|
if token == "" {
|
|
return "", errors.New("token unavailable")
|
|
}
|
|
cookies, err := exchangeRuntimeCookie(ctx, target, host, token)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
entry := runtimeAuthEntry{Container: i.ContainerID, Started: started, Token: token, Host: host, Cookies: cookies}
|
|
if err := a.saveRuntimeAuth(i.ID, baseKey, entry); err != nil {
|
|
return "", errors.New("runtime authentication unavailable")
|
|
}
|
|
auth.entries[key] = entry
|
|
return cookies, nil
|
|
}
|
|
|
|
func (a *API) runtimeAuthKey(id string) []byte {
|
|
h := sha256.New()
|
|
h.Write(a.authKey)
|
|
h.Write([]byte("runtime-auth/" + id))
|
|
return h.Sum(nil)
|
|
}
|
|
|
|
func (a *API) runtimeAuthPath(id, key string) string {
|
|
h := sha256.Sum256([]byte(key))
|
|
return filepath.Join(a.root, "runtime-auth", id, hex.EncodeToString(h[:])+".json")
|
|
}
|
|
|
|
func (a *API) loadRuntimeAuth(id, key string) (runtimeAuthEntry, bool) {
|
|
var zero runtimeAuthEntry
|
|
if len(a.authKey) != 32 {
|
|
return zero, false
|
|
}
|
|
f, err := os.OpenFile(a.runtimeAuthPath(id, key), os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
if err != nil {
|
|
return zero, false
|
|
}
|
|
defer f.Close()
|
|
st, err := f.Stat()
|
|
if err != nil || !st.Mode().IsRegular() || st.Mode().Perm() != 0600 || st.Size() > 64<<10 {
|
|
return zero, false
|
|
}
|
|
b, err := io.ReadAll(io.LimitReader(f, (64<<10)+1))
|
|
if err != nil || len(b) > 64<<10 {
|
|
return zero, false
|
|
}
|
|
var disk runtimeAuthDisk
|
|
if json.Unmarshal(b, &disk) != nil {
|
|
return zero, false
|
|
}
|
|
nonce, err := base64.RawStdEncoding.DecodeString(disk.Nonce)
|
|
if err != nil {
|
|
return zero, false
|
|
}
|
|
ciphertext, err := base64.RawStdEncoding.DecodeString(disk.Data)
|
|
if err != nil {
|
|
return zero, false
|
|
}
|
|
bl, err := aes.NewCipher(a.runtimeAuthKey(id))
|
|
if err != nil {
|
|
return zero, false
|
|
}
|
|
gcm, err := cipher.NewGCM(bl)
|
|
if err != nil || len(nonce) != gcm.NonceSize() {
|
|
return zero, false
|
|
}
|
|
plain, err := gcm.Open(nil, nonce, ciphertext, []byte(key))
|
|
if err != nil {
|
|
return zero, false
|
|
}
|
|
var entry runtimeAuthEntry
|
|
if json.Unmarshal(plain, &entry) != nil || entry.Container == "" || entry.Started == "" || entry.Token == "" || entry.Host == "" || entry.Container+"/"+entry.Started != key {
|
|
return zero, false
|
|
}
|
|
return entry, true
|
|
}
|
|
|
|
func (a *API) saveRuntimeAuth(id, key string, entry runtimeAuthEntry) error {
|
|
if len(a.authKey) != 32 {
|
|
return errors.New("invalid authentication key")
|
|
}
|
|
bl, err := aes.NewCipher(a.runtimeAuthKey(id))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
gcm, err := cipher.NewGCM(bl)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
nonce := make([]byte, gcm.NonceSize())
|
|
if _, err = rand.Read(nonce); err != nil {
|
|
return err
|
|
}
|
|
plain, err := json.Marshal(entry)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
disk, err := json.Marshal(runtimeAuthDisk{base64.RawStdEncoding.EncodeToString(nonce), base64.RawStdEncoding.EncodeToString(gcm.Seal(nil, nonce, plain, []byte(key)))})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dir := filepath.Dir(a.runtimeAuthPath(id, key))
|
|
if err = os.MkdirAll(dir, 0700); err != nil {
|
|
return err
|
|
}
|
|
if err = os.Chmod(dir, 0700); err != nil {
|
|
return err
|
|
}
|
|
tmp, err := os.CreateTemp(dir, ".credential-*")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
name := tmp.Name()
|
|
defer os.Remove(name)
|
|
if err = tmp.Chmod(0600); err == nil {
|
|
_, err = tmp.Write(disk)
|
|
}
|
|
if err == nil {
|
|
err = tmp.Sync()
|
|
}
|
|
closeErr := tmp.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if closeErr != nil {
|
|
return closeErr
|
|
}
|
|
if err = os.Rename(name, a.runtimeAuthPath(id, key)); err != nil {
|
|
return err
|
|
}
|
|
d, err := os.Open(dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer d.Close()
|
|
return d.Sync()
|
|
}
|
|
|
|
// Bound total work while allowing all retained runtime logs (3 x 10 MiB).
|
|
const runtimeLogLimit int64 = 64 << 20
|
|
|
|
func runtimeTokenFromStream(r io.Reader) (string, error) {
|
|
br := bufio.NewReader(io.LimitReader(r, runtimeLogLimit+1))
|
|
first, err := br.Peek(8)
|
|
if err != nil && err != io.EOF {
|
|
return "", err
|
|
}
|
|
framed := len(first) == 8 && first[0] <= 2 && first[1] == 0 && first[2] == 0 && first[3] == 0
|
|
var used int64
|
|
tail, token := "", ""
|
|
scan := func(p []byte) {
|
|
text := tail + string(p)
|
|
for {
|
|
at := strings.IndexByte(text, '\n')
|
|
if at < 0 {
|
|
break
|
|
}
|
|
if found := latestRuntimeToken(text[:at]); found != "" {
|
|
token = found
|
|
}
|
|
text = text[at+1:]
|
|
}
|
|
if len(text) > 8192 {
|
|
text = text[len(text)-8192:]
|
|
}
|
|
tail = text
|
|
}
|
|
buf := make([]byte, 32<<10)
|
|
for {
|
|
var source io.Reader = br
|
|
var frameLength int64
|
|
if framed {
|
|
var header [8]byte
|
|
n, e := io.ReadFull(br, header[:])
|
|
used += int64(n)
|
|
if e == io.EOF {
|
|
break
|
|
}
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
if header[0] > 2 || header[1] != 0 || header[2] != 0 || header[3] != 0 {
|
|
return "", errors.New("invalid logs")
|
|
}
|
|
frameLength = int64(binary.BigEndian.Uint32(header[4:]))
|
|
if frameLength > runtimeLogLimit-used {
|
|
return "", errors.New("log scan limit")
|
|
}
|
|
source = io.LimitReader(br, frameLength)
|
|
}
|
|
var read int64
|
|
for {
|
|
n, e := source.Read(buf)
|
|
used += int64(n)
|
|
read += int64(n)
|
|
if used > runtimeLogLimit {
|
|
return "", errors.New("log scan limit")
|
|
}
|
|
if n > 0 {
|
|
scan(buf[:n])
|
|
}
|
|
if e == io.EOF {
|
|
break
|
|
}
|
|
if e != nil {
|
|
return "", e
|
|
}
|
|
}
|
|
if !framed {
|
|
break
|
|
}
|
|
if read != frameLength {
|
|
return "", io.ErrUnexpectedEOF
|
|
}
|
|
}
|
|
if found := latestRuntimeToken(tail); found != "" {
|
|
token = found
|
|
}
|
|
return token, nil
|
|
}
|
|
func (a *API) proxy(w http.ResponseWriter, r *http.Request) {
|
|
u, _, err := a.user(r)
|
|
if err != nil {
|
|
destination := r.Header.Get("Sec-Fetch-Dest")
|
|
if r.Method == http.MethodGet && r.Header.Get("Sec-Fetch-Mode") == "navigate" && (destination == "document" || destination == "iframe") && r.Header.Get("Upgrade") == "" && a.runtimeOriginAllowed(r) {
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.Header().Set("Referrer-Policy", "no-referrer")
|
|
target := r.URL.EscapedPath()
|
|
if query := withoutRuntimeToken(r.URL.RawQuery); query != "" {
|
|
target += "?" + query
|
|
}
|
|
http.Redirect(w, r, "/#runtime-renew?return="+url.QueryEscape(target), http.StatusSeeOther)
|
|
return
|
|
}
|
|
fail(w, 401, "authentication required")
|
|
return
|
|
}
|
|
p := strings.SplitN(strings.TrimPrefix(r.URL.Path, "/dsh/"), "/", 2)
|
|
if p[0] == "" {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
inst := a.instanceLocks.Acquire(p[0])
|
|
locked := inst
|
|
defer func() {
|
|
if locked != nil {
|
|
locked.Release()
|
|
}
|
|
}()
|
|
i, err := a.s.GetInstance(p[0])
|
|
if err != nil || i.UID != u.ID {
|
|
fail(w, 404, "not found")
|
|
return
|
|
}
|
|
if i.Status == "error" || a.upgradeRecoveryRequired(i) {
|
|
fail(w, 409, "instance requires recovery")
|
|
return
|
|
}
|
|
if !a.runtimeOriginAllowed(r) {
|
|
fail(w, 403, "origin denied")
|
|
return
|
|
}
|
|
prefix := "/dsh/" + i.ID
|
|
if len(p) == 1 {
|
|
http.Redirect(w, r, prefix+"/", http.StatusTemporaryRedirect)
|
|
return
|
|
}
|
|
host := a.runtimeHost(r)
|
|
bridge := runtimeFreeSearchPath(strings.TrimPrefix(r.URL.Path, prefix))
|
|
if bridge {
|
|
if r.Method != http.MethodPost {
|
|
fail(w, 405, "method not allowed")
|
|
return
|
|
}
|
|
if r.Header.Get("Sec-Fetch-Site") == "cross-site" {
|
|
fail(w, 403, "origin denied")
|
|
return
|
|
}
|
|
host = "localhost"
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
|
defer cancel()
|
|
var c struct {
|
|
State struct {
|
|
Running bool
|
|
StartedAt string
|
|
}
|
|
NetworkSettings struct {
|
|
Networks map[string]struct{ IPAddress string }
|
|
}
|
|
}
|
|
err = a.docker.Do(ctx, "GET", "/containers/"+url.PathEscape(i.ContainerID)+"/json", nil, &c)
|
|
if err != nil || !c.State.Running || c.State.StartedAt == "" {
|
|
fail(w, 502, "runtime unavailable")
|
|
return
|
|
}
|
|
ip := c.NetworkSettings.Networks[runtimeNetwork(i.ID)].IPAddress
|
|
if net.ParseIP(ip) == nil {
|
|
fail(w, 502, "runtime network unavailable")
|
|
return
|
|
}
|
|
target := &url.URL{Scheme: "http", Host: net.JoinHostPort(ip, "8080")}
|
|
cookies, err := a.runtimeCookie(ctx, i, c.State.StartedAt, target, host)
|
|
if err != nil {
|
|
fail(w, 502, "runtime authentication unavailable")
|
|
return
|
|
}
|
|
cancel()
|
|
locked.Release()
|
|
locked = nil
|
|
proxy := runtimeReverseProxy(target, prefix, host, cookies)
|
|
a.invalidateRejectedRuntimeCookie(proxy, i, c.State.StartedAt, host, cookies)
|
|
if bridge {
|
|
// Only after panel authentication, ownership and Origin checks may the
|
|
// loopback-only plugin receive a matching local Origin through Caddy.
|
|
rewrite := proxy.Rewrite
|
|
proxy.Rewrite = func(p *httputil.ProxyRequest) {
|
|
rewrite(p)
|
|
p.Out.Header.Set("Origin", "http://localhost")
|
|
}
|
|
}
|
|
proxy.ServeHTTP(w, r)
|
|
}
|
|
|
|
// Do not replay mutations after an authentication failure. The next request
|
|
// exchanges a new cookie using the retained token, even after logs rotate.
|
|
func (a *API) invalidateRejectedRuntimeCookie(proxy *httputil.ReverseProxy, i store.Instance, started, host, cookies string) {
|
|
modify := proxy.ModifyResponse
|
|
proxy.ModifyResponse = func(resp *http.Response) error {
|
|
if resp.StatusCode == 401 || resp.StatusCode == 403 {
|
|
value, ok := runtimeAuthCache.Load(runtimeAuthKey{a, i.ID})
|
|
if ok {
|
|
auth := value.(*runtimeAuth)
|
|
auth.mu.Lock()
|
|
baseKey := i.ContainerID + "/" + started
|
|
key := baseKey + "/" + host
|
|
if entry, exists := auth.entries[key]; exists && entry.Cookies == cookies {
|
|
entry.Cookies = ""
|
|
delete(auth.entries, key)
|
|
if err := a.saveRuntimeAuth(i.ID, baseKey, entry); err != nil {
|
|
auth.mu.Unlock()
|
|
return errors.New("runtime authentication unavailable")
|
|
}
|
|
}
|
|
auth.mu.Unlock()
|
|
}
|
|
}
|
|
return modify(resp)
|
|
}
|
|
}
|
|
|
|
func runtimeFreeSearchPath(path string) bool {
|
|
switch path {
|
|
case "/api/dsh-free-search-settings/describe", "/api/dsh-free-search-settings/mutate",
|
|
"/api/dsh-free-search-settings/credentials-status", "/api/dsh-free-search-settings/credentials-set",
|
|
"/api/dsh-free-search-settings/credentials-unset", "/api/dsh-free-search-settings/check-update",
|
|
"/api/dsh-free-search-settings/update", "/api/dsh-free-search-settings/raw-search":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func exchangeRuntimeCookie(ctx context.Context, target *url.URL, host, token string) (string, error) {
|
|
if target == nil {
|
|
return "", errors.New("exchange target unavailable")
|
|
}
|
|
endpoint := *target
|
|
endpoint.Path = "/"
|
|
endpoint.RawQuery = url.Values{"token": {token}}.Encode()
|
|
req, err := http.NewRequestWithContext(ctx, "GET", endpoint.String(), nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Host = host
|
|
client := &http.Client{Transport: runtimeTransport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
return "", errors.New("exchange failed")
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusSeeOther {
|
|
return "", errors.New("exchange rejected")
|
|
}
|
|
var cookies []string
|
|
for _, cookie := range resp.Cookies() {
|
|
if cookie.Value != "" && cookie.MaxAge >= 0 {
|
|
cookies = append(cookies, (&http.Cookie{Name: cookie.Name, Value: cookie.Value}).String())
|
|
}
|
|
}
|
|
if len(cookies) == 0 {
|
|
return "", errors.New("exchange missing cookie")
|
|
}
|
|
return strings.Join(cookies, "; "), nil
|
|
}
|
|
|
|
func withoutRuntimeToken(rawQuery string) string {
|
|
parts := strings.Split(rawQuery, "&")
|
|
kept := parts[:0]
|
|
for _, part := range parts {
|
|
key, _, _ := strings.Cut(part, "=")
|
|
decoded, err := url.QueryUnescape(key)
|
|
if err == nil && decoded == "token" {
|
|
continue
|
|
}
|
|
kept = append(kept, part)
|
|
}
|
|
return strings.Join(kept, "&")
|
|
}
|
|
|
|
func runtimeReverseProxy(target *url.URL, prefix, host, cookies string) *httputil.ReverseProxy {
|
|
return &httputil.ReverseProxy{
|
|
Transport: runtimeTransport, FlushInterval: -1, ErrorLog: log.New(io.Discard, "", 0),
|
|
Rewrite: func(p *httputil.ProxyRequest) {
|
|
p.SetURL(target)
|
|
p.Out.Host = host
|
|
p.Out.URL.Path = strings.TrimPrefix(p.In.URL.Path, prefix)
|
|
p.Out.URL.RawPath = ""
|
|
// DSH combo assets use an opaque ?module1,module2 query prefix.
|
|
p.Out.URL.RawQuery = withoutRuntimeToken(p.In.URL.RawQuery)
|
|
p.Out.Header.Del("Cookie")
|
|
p.Out.Header.Del("Authorization")
|
|
p.Out.Header.Del("Proxy-Authorization")
|
|
p.Out.Header.Del("X-CSRF-Token")
|
|
p.Out.Header.Set("Accept-Encoding", "identity")
|
|
p.Out.Header.Set("Cookie", cookies)
|
|
p.SetXForwarded()
|
|
p.Out.Header.Set("X-Forwarded-Prefix", prefix)
|
|
},
|
|
ModifyResponse: func(resp *http.Response) error {
|
|
resp.Header.Del("Set-Cookie")
|
|
if resp.StatusCode == http.StatusOK && strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") && resp.Header.Get("Content-Encoding") == "" {
|
|
data, err := io.ReadAll(io.LimitReader(resp.Body, (8<<20)+1))
|
|
resp.Body.Close()
|
|
if err != nil || len(data) > 8<<20 {
|
|
return fmt.Errorf("runtime document unavailable")
|
|
}
|
|
data = bytes.Replace(data, []byte("<head>"), []byte(`<head><script src="/api/runtime-session.js"></script>`), 1)
|
|
resp.Body = io.NopCloser(bytes.NewReader(data))
|
|
resp.ContentLength = int64(len(data))
|
|
resp.Header.Set("Content-Length", fmt.Sprint(len(data)))
|
|
resp.Header.Del("ETag")
|
|
resp.Header.Set("Cache-Control", "no-store")
|
|
}
|
|
if location := resp.Header.Get("Location"); location != "" {
|
|
u, err := url.Parse(location)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid redirect")
|
|
}
|
|
u.RawQuery = withoutRuntimeToken(u.RawQuery)
|
|
if u.Host == target.Host || u.Host == host {
|
|
u.Scheme = ""
|
|
u.Host = ""
|
|
}
|
|
if u.Host == "" && strings.HasPrefix(u.Path, "/") {
|
|
u.Path = prefix + u.Path
|
|
u.RawPath = ""
|
|
}
|
|
resp.Header.Set("Location", u.String())
|
|
}
|
|
return nil
|
|
},
|
|
ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { fail(w, 502, "runtime proxy unavailable") },
|
|
}
|
|
}
|