Compare commits
20
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 m1saka
|
169e1ffdc2 | ||
|
m1saka
|
0389555d76 | ||
|
m1saka
|
9961d42a2f | ||
|
m1saka
|
e36f3c1fdd | ||
|
m1saka
|
605a48adb0 | ||
|
m1saka
|
e84906652e | ||
|
m1saka
|
b13b71b27c | ||
|
m1saka
|
b2d97dfda2 | ||
|
m1saka
|
2e31e74972 | ||
|
m1saka
|
eb7e809999 | ||
|
m1saka
|
bbdee632f6 | ||
|
m1saka
|
65d118a145 | ||
|
m1saka
|
39cd470f92 | ||
|
m1saka
|
1fbb2a6fed | ||
|
m1saka
|
a6bf83f037 | ||
|
m1saka
|
ed6467d132 | ||
|
m1saka
|
1fda59fb09 | ||
|
m1saka
|
05ecfc88c0 | ||
|
m1saka
|
e72d73c497 | ||
|
m1saka
|
ab76be422d |
@@ -1,13 +0,0 @@
|
|||||||
.mimocode
|
|
||||||
.git
|
|
||||||
**/node_modules
|
|
||||||
web/dist
|
|
||||||
data
|
|
||||||
bin
|
|
||||||
.env
|
|
||||||
output
|
|
||||||
**/output
|
|
||||||
**/*.tsbuildinfo
|
|
||||||
.playwright-cli
|
|
||||||
**/*.sqlite*
|
|
||||||
image/*.test.mjs
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
# Copy to .env (ignored by git). Use an absolute dedicated data directory.
|
|
||||||
DSH_HOST_DATA_ROOT=/var/lib/dsh-mgr
|
|
||||||
DSH_ADMIN_USERNAME=admin
|
|
||||||
# Supply a unique strong password before FIRST start; the panel reads it only
|
|
||||||
# while the account table is empty. Remove the value after the administrator
|
|
||||||
# account exists, then run `docker compose up -d --force-recreate panel`
|
|
||||||
# to remove it from the container environment and `docker inspect` output.
|
|
||||||
# Restarting the existing container does not update its environment.
|
|
||||||
DSH_ADMIN_PASSWORD=
|
|
||||||
# Numeric group from: stat -c %g /var/run/docker.sock
|
|
||||||
DOCKER_GID=
|
|
||||||
# Exact docker info --format {{.DockerRootDir}}; mounted read-only for statfs.
|
|
||||||
DSH_DOCKER_ROOT_DIR=
|
|
||||||
# Optional exact extension allowlist, e.g. .txt,.md,.json,.yaml,.go
|
|
||||||
# Empty permits arbitrary workspace files; this is not malware/content scanning.
|
|
||||||
UPLOAD_ALLOWED_EXTENSIONS=
|
|
||||||
# Public origin of the panel exactly as browsers see it (scheme://host, no
|
|
||||||
# path); controls Origin/CSRF validation and Secure cookies.
|
|
||||||
PANEL_PUBLIC_URL=http://127.0.0.1:8088
|
|
||||||
# Optional proxy networks. X-Forwarded-For is ignored unless the peer is in these CIDRs.
|
|
||||||
TRUSTED_PROXY_CIDRS=
|
|
||||||
+46
-7
@@ -1,9 +1,48 @@
|
|||||||
.mimocode/
|
# Local tooling and browser artifacts
|
||||||
node_modules/
|
/.mimocode/
|
||||||
web/dist/
|
|
||||||
data/
|
|
||||||
bin/
|
|
||||||
.env
|
|
||||||
output/
|
|
||||||
.playwright-cli/
|
.playwright-cli/
|
||||||
|
|
||||||
|
# Dependencies and generated files, including leftovers when switching branches
|
||||||
|
node_modules/
|
||||||
|
.vite/
|
||||||
|
/web/dist/
|
||||||
|
*.tsbuildinfo
|
||||||
|
/bin/
|
||||||
|
output/
|
||||||
|
|
||||||
|
# Local runtime data and credentials
|
||||||
|
/data/
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
*.sqlite*
|
*.sqlite*
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Local IDE files
|
||||||
|
.idea/
|
||||||
|
*.iml
|
||||||
|
|
||||||
|
# Generated files left when switching from android-app
|
||||||
|
/.dart_tool/
|
||||||
|
/.flutter-plugins-dependencies
|
||||||
|
/build/
|
||||||
|
/android/.gradle/
|
||||||
|
/android/.kotlin/
|
||||||
|
/android/build/
|
||||||
|
/android/app/build/
|
||||||
|
/android/local.properties
|
||||||
|
/android/gradlew
|
||||||
|
/android/gradlew.bat
|
||||||
|
/android/gradle/wrapper/gradle-wrapper.jar
|
||||||
|
/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java
|
||||||
|
|
||||||
|
# Android packages and signing credentials
|
||||||
|
*.apk
|
||||||
|
*.aab
|
||||||
|
*.jks
|
||||||
|
*.keystore
|
||||||
|
*.p12
|
||||||
|
*.pfx
|
||||||
|
keystore.properties
|
||||||
|
signing.properties
|
||||||
|
/android/key.properties
|
||||||
|
|||||||
-251
@@ -1,251 +0,0 @@
|
|||||||
# DESIGN.md 验收记录
|
|
||||||
|
|
||||||
## 恢复宽限期与轮询缓存修复（独立复审后）
|
|
||||||
|
|
||||||
范围：本轮为对上述工作树的独立代码审计与全量重测后的修复。审计确认八项修复中六项正确，发现三处残留：创建结果不确定的挂起标志永不过期导致实例永久无法删除（高）、构建校验不确定行无限重试并持续新增审计（中）、意图轮询每秒对每个成功镜像发起 Docker inspect（中）。修复未改变凭据文件管理边界或准入策略；未部署、未操作真实 Docker 实例。
|
|
||||||
|
|
||||||
- `runtime_creates` 增加 `container_pending_at`/`network_pending_at` 置位时间（列名白名单，禁用未校验列名拼接）。确定性名称 404 在宽限窗口（默认1分钟）内仍视为不确定；窗口过后判定资源从未创建，标志落账清除，实例可正常删除，删除成功后总是清理 `runtime_creates`/`runtime_networks` 行。无时间戳的遗留行视为已过期立即解析。
|
|
||||||
- `image_build_checks` 增加 `created_at`。无已记录容器 ID 且名称 404 的不确定行在宽限窗口过后删除行停止重试；窗口内保持降频重试与 `build.cleanup_error` 审计（≤1/分钟不变）。
|
|
||||||
- 意图轮询改用 `verifiedImagesPoll`：成功构建行每秒重读，逐镜像准入评估按 `imageDeleteGeneration`+5秒TTL缓存；镜像删除或准入写入使缓存立即失效。创建/升级入口仍走未缓存的权威校验。
|
|
||||||
- 前端：无 Web Locks 浏览器的会话恢复重试窗口从约0.85秒延长至约3.85秒；`useResource` 对60秒不返回的请求自动放弃并重新武装轮询（序号守卫防止迟到结果覆盖新状态）；`runtimeSession` 测试改为 TypeScript 并纳入 tsc 检查；`test-panel.mjs` 构建详情轮询对瞬时 5xx 容错续轮。
|
|
||||||
|
|
||||||
回归覆盖：宽限窗口内删除被拒、窗口过后删除成功且无残留行（TestUncertainPendingResolvesAfterGrace）；校验行窗口内保持不确定、窗口过后与遗留行均解析（TestBuildCheckUncertainRowResolvesAfterGrace）；轮询三次仅一次 Docker inspect、代际提升后缓存失效（TestIntentPollerCachesImageEvaluations）；悬挂请求逃生阀（ui.test.ts）。Go共有192个顶层测试（另有子测试）。前端15个测试文件、95项测试通过；`@types/node` 新增为 devDependency，官方源 npm audit 0 漏洞。
|
|
||||||
|
|
||||||
## 完整home与镜像准入策略实施
|
|
||||||
|
|
||||||
本节落实用户确认的完整home所有者访问、兼容清单及管理员实验构建策略，取代下方上一轮“尚未改变这两项策略”的历史边界。未部署，未操作真实Docker实例或为实际镜像写入准入。
|
|
||||||
|
|
||||||
- 所有者可读取/打包凭据，普通文件管理不强制使用专用Profile流程；匿名、跨用户和路径穿越仍拒绝。下载及敏感修改增加确认，取消不触发操作；上传同名拒绝不覆盖。专用Profile页面仍要求停机，文案与通用文件管理明确区分。
|
|
||||||
- 默认普通构建支持linux/amd64、模板7的rc.1/rc.2；未知版本只能管理员实验构建。实验完整执行原检查且success不自动可选。准入绑定精确imageID/version/recipe/platform与成功验证记录，管理员填写依据，策略和审计同事务，审计失败回滚。
|
|
||||||
- 历史模板3–6默认仅存量，经明确验收可准入并使用旧双挂载；未知布局拒绝正向准入，模板7向旧共享布局降级拒绝。已准入未知版本只通过本地imageID使用，不自动扩大版本构建白名单。
|
|
||||||
- blocked拒绝新建、升级目标、start/restart及受控配置非GET操作，避免恢复/备份自动启动绕过；不自动停止既有容器，普通文件管理保持可用。队列恢复/执行重查版本、平台及实验请求者管理员权限。
|
|
||||||
- 独立复核修正了准入与生命周期锁顺序、受控配置启动遗漏及两处提示矛盾；最终静态复核未发现剩余阻塞项。
|
|
||||||
|
|
||||||
回归覆盖精确版本/模板/平台匹配、旧模板实际创建挂载、实验success仍不可选、未知请求零预留、普通用户和CSRF拒绝、审计回滚、并发降级等待创建完成、blocked启停/配置拒绝，以及前端准入确认/取消和选择器过滤。Go共有189个顶层测试（另有子测试）；本轮Go普通全量和全量race禁缓存通过，API包分别约17秒与97秒。前端15个测试文件、94项测试通过。浏览器使用模拟API验证实验警告、准入表单、创建筛选及unknown latest禁止预留，临时服务和浏览器已关闭；非真实Docker或新DSH版本兼容验收。
|
|
||||||
|
|
||||||
完整 `scripts/check-trial.sh` 最终退出码0；`go vet`、Go构建、TypeScript与Vite生产构建、profile初始化3项Node测试、验收脚本自身11项Node测试和脚本语法检查均通过。仍有既有Vite大chunk提示，非构建失败；`git diff --check`通过。
|
|
||||||
|
|
||||||
## 2026-10-14 真实环境补充验收与构建拆包
|
|
||||||
|
|
||||||
- 全量 race：`go test -race -count=1 -timeout=180s ./...` 退出 0，全部包通过、无 race 报告；API 包 166.645 秒，总耗时约 170 秒。
|
|
||||||
- 真实 Docker：新增默认跳过、需 `DSH_DOCKER_LIVE_TEST=1` 启用的 `TestDockerLiveRecovery`。经故障代理向真实 `/var/run/docker.sock` 转发，真实 create/delete 成功后丢失响应，或断开客户端并延迟创建，先取得真实 404 再放行。六个子场景连续两轮通过，另一次 race 模式通过。覆盖 runtime/network/build-check 恢复、面板未连接/完整 ID/名称连接及删除响应丢失。仅使用专用唯一标签资源、本地 node:24-alpine；按精确 ID 清理后标签查询零残留。测试调用恢复方法，不模拟整机断电，也不重启共享 daemon。
|
|
||||||
- 真实浏览器续期：当前 Go 源码、隔离临时数据库、真实 DSH rc.2 模板7容器、Chromium 独立 Cookie 上下文，原始 5 分钟 access / 4 分钟续期间隔。2026-10-14T03:49:07.767Z 至 03:59:22.828Z 共 615.061 秒；初次、第 240.023 秒及第 480.020 秒 refresh 均 200；每 15 秒原生 fetch 检查 `/api/me` 和真实 runtime，共 41 轮全部 200（含第 301 秒和第 601 秒）。没有加速时钟、手动续期或共享 Cookie 的其他页面。证据 `output/playwright/renewal-acceptance.json`；不代表休眠/后台节流或多小时压力验收。
|
|
||||||
- Monaco 改为 React.lazy 按需加载，依模块分层拆包；主入口从约 2589 kB 降至 284.92 kB，最大 chunk 411.21 kB，worker 232.35 kB，保留默认 500 kB 阈值且最终构建无大 chunk 或循环 chunk 告警。新增构建测试约束体积、入口不静态依赖 Monaco 及 chunk 无环；16 文件/114 项前端测试通过。真实浏览器验证编辑输入、运行中保存禁用、只读预览及 worker 初始化成功，截图 `output/playwright/editor-chunks-verified.png`。
|
|
||||||
- 最终 `go vet ./...`、`go build ./...`、TypeScript/Vite 构建和 `git diff --check` 通过。专用面板、两个浏览器已关闭，验收容器/网络及临时数据和凭据已清理；原四个运行容器仍在，未部署到现有面板、未更改共享镜像。
|
|
||||||
|
|
||||||
复现真实故障测试：`DSH_DOCKER_LIVE_TEST=1 go test ./internal/api -run TestDockerLiveRecovery -v -count=2 -timeout=3m`；需可访问 Docker socket 和本地 node:24-alpine，不会自动拉取镜像。
|
|
||||||
|
|
||||||
## 再次审查修复（本地回归）
|
|
||||||
|
|
||||||
- runtime 注入脚本每 4 分钟主动续期，不再因 `/api/me` 仍有效而跳过；保留 Web Locks 串行刷新、同页请求合并及冲突复验。模拟时钟覆盖连续续期，补测锁等待期间 Cookie 更新。
|
|
||||||
- 资源加载超时会结束调用方等待链、清除 loading 并提示重试；覆盖无轮询、旧请求和后续请求均悬挂、迟到成功/失败、超时配置变化，不再只清空 in-flight 引用。
|
|
||||||
- Docker 创建结果不确定时，不凭一分钟后的名称 404 清除 pending/验证容器记录；覆盖旧/缺失/非法时间戳及迟到资源回收。网络清理只断开确实连接的面板端点，保留身份校验和真实错误处理。持续未确认的创建需按 OPERATIONS.md 人工确认。
|
|
||||||
- 准入依据前端按实际提交内容计算 UTF-8 字节数，超出 4096 字节时提示并禁止提交；后端保持字节上限及请求体限制，长度错误明确返回。覆盖中文、ASCII、多字节、表情及边界。
|
|
||||||
|
|
||||||
本轮通过：`go test -count=1 -timeout=180s ./...`、准入与资源恢复相关 API 测试的禁缓存 race 检查、`go vet ./...`、`go build ./...`；前端 15 文件/113 测试及 TypeScript/Vite 生产构建；profile 初始化与验收脚本自身共 14 项 Node 测试、集成脚本语法检查、`git diff --check`。本轮未执行全量 race、真实浏览器长时续期或真实 Docker 故障注入；未部署、未操作现有实例。Vite 仍有既有大 chunk 提示。
|
|
||||||
|
|
||||||
## 八项审查问题闭环（本地回归）
|
|
||||||
|
|
||||||
范围：普通创建响应丢失、验证容器回收、构建终态持久化、多页面续期、慢请求轮询、当前密码重认证、配额更新竞态、构建历史分页与按需日志。未改变凭据文件管理边界或历史镜像兼容策略；未部署、未操作真实 Docker 实例。
|
|
||||||
|
|
||||||
- 资源恢复：创建前保存恢复依据；校验资源身份后按持久 ID 清理。覆盖迟到资源（首次404随后出现）、网络创建/删除响应丢失、验证容器重命名、外部同名网络拒删、清理失败重试、重开数据库后无排队任务自动回收。个别预约待恢复不会阻止面板启动。
|
|
||||||
- 构建终态：任务与镜像索引事务提交，故障注入验证无半提交；独立重试不阻挡后续构建，Store关闭后退出。持续不确定创建保留恢复记录，不凭单次404释放资源依据；人工确认流程见 OPERATIONS.md。
|
|
||||||
- 认证：备份、升级、改密共用失败计数与锁定；旧备份密码不参与登录失败计数。并发refresh冲突返回409且不签发cookie，前端及runtime复验后保留可重试错误，真正撤销仍退出；覆盖慢升级不阻塞其他账号登录。
|
|
||||||
- 文件与前端：上传提交锁内读取最新额度；管理员修改额度共用用户锁。慢轮询不重叠、不废弃有效慢响应，显式刷新保证额外读取，key切换隔离旧结果。构建列表默认20条、最多100条且不读取日志正文，详情按需加载；参数、路由鉴权、整页边界和当前页镜像对账均有回归。
|
|
||||||
|
|
||||||
最终验证：
|
|
||||||
|
|
||||||
- 完整 `scripts/check-trial.sh` 通过，最终退出码0；最终Go测试通过 `GOFLAGS=-count=1` 禁用测试结果缓存。Go共有182个顶层测试（另含子测试），API普通测试约18秒，全量race中API包约110秒；无race报告。
|
|
||||||
- `go vet ./...`、`go build ./...`、前端类型检查与生产构建通过；Vite仍提示原有大chunk，不影响构建成功。
|
|
||||||
- 前端10个文件、57项测试通过；profile初始化3项Node测试、验收脚本自身11项Node测试通过；真实runtime注入脚本的401/403/409/500行为已自动测试。
|
|
||||||
- 浏览器配合模拟API验证每页20条、末页和禁用下一页、日志打开/关闭及终态停止轮询。临时浏览器和开发服务器已关闭。
|
|
||||||
- `git diff --check`通过。Docker故障场景使用httptest模拟，并非真实daemon断电/断网验收；未运行指向真实部署的test-panel脚本，仅更新其单任务查询并检查语法。
|
|
||||||
|
|
||||||
升级注意：已打开的旧页面需刷新；外部 `/api/builds` 消费者需迁移分页和单任务日志接口。持续数据库故障或身份冲突不能承诺自动成功，保留记录并按运维流程处理。
|
|
||||||
|
|
||||||
## 2026-10-12 可信团队试运行加固（本地自动化验收）
|
|
||||||
|
|
||||||
范围：本机或仅可信账号的团队，可由管理员通过公网 HTTPS 反代访问。未修改 Nginx/Caddy，未部署面板或操作现有实例。本节仅说明当前代码的本地自动化验收，不将历史部署证据当作本次公网实测。
|
|
||||||
|
|
||||||
已修复：
|
|
||||||
|
|
||||||
- 登录来源30次/分钟、有界1024来源桶、8个并发槽；Origin/严格JSON校验先于额度消耗；保留账号失败锁定。默认不信任XFF，仅配置的可信代理CIDR可提供来源链，非法链回退peer。非法公网URL/CIDR在创建数据目录前拒绝；URL规范化大小写与默认端口。
|
|
||||||
- 实例文件传输改为短锁登记lease，不再持全局生命周期锁传输；同实例启停/删除/修改/升级/恢复冲突返回409。配置JSON/multipart锁外有界接收（并发4、30秒读deadline），JSON响应在解锁后发送。
|
|
||||||
- runtime token按容器ID/StartedAt加密持久化，Host分别缓存；日志流式扫描64MiB上限、完整帧校验、逐行选最新启动token。上游401/403失效cookie，下次请求重新交换，不重放写请求。
|
|
||||||
- 恢复快照复制失败清理本次半成品，保留有效快照；清理失败明确500及审计。目录列表10000项上限，用量和快照分批遍历。
|
|
||||||
- 构建容量测试注入可控探测，生产2GiB阈值不变；更新模板7文档、实例范围集成脚本和一键验收入口。
|
|
||||||
|
|
||||||
实际通过：
|
|
||||||
|
|
||||||
- `bash scripts/check-trial.sh` 退出0：Go全量测试、全量race、vet、build；前端32测试及生产构建；初始化3测试；集成脚本语法检查。全量race在此脚本中使用刚刚分包成功运行的Go测试缓存；API包单独完整race约54秒通过。此前一次全量并行race被工具60秒上限终止，不计通过。
|
|
||||||
- `node --test scripts/check-trial.test.mjs`：11项通过（包括每阶段失败即停）。
|
|
||||||
- `TestSecurityTLSCookieJarLifecycle`：真实本地TLS HTTP服务/cookiejar登录、me、refresh轮换、logout、旧会话拒绝；来源隔离、伪造头、非法配置、并发限流回归通过。不是公网证书/反代验收。
|
|
||||||
- 慢上传/下载/配置body/错误JSON不持锁、同实例busy、失败恢复三次不耗名额回归及race通过。
|
|
||||||
- runtime专项race覆盖9MiB raw/framed日志、逐字节分片、截断帧拒绝、面板对象重建、Host切换、启动时间变化、磁盘权限/加密/篡改、写盘失败不缓存、cookie拒绝后重交换。Docker及上游由测试HTTP服务模拟。
|
|
||||||
- Compose用占位环境变量只读解析，公网URL及可信代理CIDR正确展开。
|
|
||||||
|
|
||||||
剩余边界与部署前检查：
|
|
||||||
|
|
||||||
- 未重新构建/启动真实runtime镜像，未执行真实Docker生命周期、浏览器或实际公网域名验收；未调用模型、搜索或收费服务。必须在部署入口确认登录/刷新、iframe、WS/SSE和插件设置工作。
|
|
||||||
- 必须设置实际 `PANEL_PUBLIC_URL` 和准确的 `TRUSTED_PROXY_CIDRS`；代理及后端可达范围由管理员负责。公网并不意味着支持不可信租户。
|
|
||||||
- 旧实例首次成功连接前若启动token已轮转丢失，需维护窗口重启该实例再连接；任意上游token主动轮转不保证自动恢复。软配额、同源插件、已建立WS/SSE不即时撤销等既有边界保留。
|
|
||||||
- 前端仍有Monaco大chunk警告。本轮未补许可证、依赖漏洞全量扫描或开源发布材料，不作开源发布就绪声明。
|
|
||||||
|
|
||||||
结论：本地代码与自动化验收通过，可进入本机/可信团队受控试运行的部署验证；不宣称实际公网部署已验收或长期生产SLA已达标。
|
|
||||||
|
|
||||||
## Free Search 模板5（部署及真实iframe验收完成）
|
|
||||||
|
|
||||||
- 最终面板已部署：6067f9e8bd12aff649c3b8cc662c37a7a14eebec2d686b84a0d965ee599a4dc5，镜像sha256:6f397fba50a9c3469c1e5ae1ed685e6a4fe90f08ed91d947f38eca9c0b3a4932；原配置/挂载保留，旧面板停止留作回滚。正常API构建记录617457ffdd4c168c58f9cb21e7a591cc成功，模板5可用。
|
|
||||||
- 专用新实例5dfabbdde3339c8df7bb616e1c65dce9真实iframe：Installed2、Free Search0.8.5 Running、Configure打开完整设置；前缀describe/credentials-status/mutate均200。Bing地区及缓存时长修改后保存，再恢复zh-CN/5并确认；普通DSH页面重新加载200。截图output/playwright/free-search-settings.png。未执行搜索/引擎测试/在线更新/密钥配置。
|
|
||||||
- 实际HTTP匿名401、恶意Origin403、缺失Origin403，根路径请求拒绝；跨用户/public Host由代理自动化测试覆盖，本轮未另做实际公开域名或真实跨用户浏览器验证。全量race TEST_EXIT=0（/tmp/dsh-free-search-race.log）。
|
|
||||||
- 上游0.8.5已确认UI限制：单改Bing地区不会置dirty，保存按钮不启用；修改缓存时长后可一起保存。未擅改上游发布包，不把此缺陷视为已修复。
|
|
||||||
- 测试实例已经正常API删除，bind数据保留。旧63b实例仍为recipe4，容器ID/StartedAt13:01:01.657475588Z/RestartCount0保持，未安装或启用free-search。
|
|
||||||
|
|
||||||
- 固定内置dsh-free-search0.8.5及三项普通依赖原始tarball，SHA-512校验；新profile正式依赖及bundle默认启用，旧profile不注入，mobile旧路径保留。runtime镜像dsh-web:0.2.0-rc.2-5，ID sha256:aea9e1ed8614b30457c1d60274a68175e85c1716734e4bca396205bffa885eb6。
|
|
||||||
- 断网/非root/只读rootfs启动、两个插件inventory及listBundles、settings describe200、无重复宿主peers或绝对symlink、离线恢复和禁用状态保留、recipe4 mobile-only恢复通过。证据output/recipe5-build.log及recipe5-smoke.log。未调用搜索、引擎测试、收费服务或模型。
|
|
||||||
- 面板精确适配8个settings路径，先登录/实例归属/Origin检查再转换loopback Host及Origin，并重新交换Host绑定cookie；客户端fetch仅重写同源精确路径。真实HTTP upstream测试涵盖公开Host、body/query、普通资源、匿名/跨用户/恶意或缺失Origin拒绝，Node测试覆盖Request/URL/string及流语义。Go全量/vet、初始化3测试和聚焦代理race通过。
|
|
||||||
- 上游会接管等于deepseek-official的搜索及默认http抓取（包含显式同名值）；其他自定义provider保留。默认Bing但自动回退可能使用已有收费API凭据，不承诺零费用。
|
|
||||||
- 早期部署曾被权限规则拒绝，后续用户授权继续，主会话接手完成上述部署和验收；实际公网域名未单独验收。
|
|
||||||
|
|
||||||
## 管理员重设与用户自助修改密码验收
|
|
||||||
|
|
||||||
- 已部署 `dsh-mgr:password-complete`（sha256:83189ad42e35b5a7db3ba74e27f62adcd35f4c45635be35d4f5e568b37c54c75）至127.0.0.1:40411。面板Config除Image外、HostConfig及Mounts一致，旧面板停止保留供回滚；DSH runtime ID/启动时间/重启计数未变。
|
|
||||||
- 管理员入口：设置→用户与审计→用户列表→重设密码。普通用户及管理员自助入口：设置→账号→修改密码，必须验证当前密码。
|
|
||||||
- 专用成员pwdqa_1791643434738真实Playwright CLI验收：管理员重设、确认不一致错误、新密码登录；成员错误当前密码保留登录、确认不一致错误、成功修改返回登录页及新密码登录均通过。没有修改既有用户密码。
|
|
||||||
- 两次改密后旧密码及旧access/session请求401，携带原CSRF的旧refresh请求403（session已删除），新session刷新200。审计显示user.reset_password和user.change_password，不含密码。
|
|
||||||
- Go全量race通过（/tmp/dsh-password-race-final.log，TEST_EXIT=0），vet、前端21测试及生产构建通过；既有Monaco大chunk警告仍在。
|
|
||||||
- 证据：output/playwright/password-member-empty-form.png、password-admin-reset-empty-form.png；/tmp/dsh-password-browser-evidence.txt。测试会话已退出；因无删除用户API，专用成员保留，随机测试密码未持久化。
|
|
||||||
- 限制：已建立的WS/SSE不会被主动即时断开，后续请求及重连拒绝旧会话。保留的旧面板是部署回滚副本，不是数据库时间点备份。
|
|
||||||
|
|
||||||
## 模板4：修复插件列表不可见（取代模板3管理可用性结论）
|
|
||||||
|
|
||||||
- 根因：仅host依赖和bundle可使插件运行，但DSH管理UI按profile dependencies判断installed，模板3未登记而被过滤。模板4固定tarball正式依赖，补缺失workspace的no-auto-peers配置，保留已有manifest/bundle禁用状态。
|
|
||||||
- runtime `dsh-web:0.2.0-rc.2-4`：sha256:d967800c73888cefba67b8ea14b923c14032f96058311630a37210219877fcdd。面板已部署模板4并经正常API登记镜像。
|
|
||||||
- 当前实例63b337c848bf23156270068ff9e5cf38已备份、升级并补mobile依赖，容器2e2d6c75946a65eb7b39a11d72e2e3d23394c3d92d6508cfe25a8fa827a450fb，配置/数据保留。备份73e27b58f421868de00b07958f88113d及此前备份保留。
|
|
||||||
- 实际面板iframe触屏浏览器：Plugins新增Installed1，dsh-web-mobile v3.0.5，详情组件Running；真实开关关闭显示Off、再次开启恢复运行，最终保持启用。截图output/playwright/plugin-touch/recipe4-installed-detail.png。UI显示Uninstall但未执行卸载；无独立参数配置表单。
|
|
||||||
- 初始化3测试、Go全量/race/vet通过。断网非root只读冒烟验证正式inventory、上游listBundles、无重复peers/绝对symlink，以及仅manifest/patch/lockfile恢复后离线安装并保留disabled状态。当前实例未另做disabled后容器重启测试。
|
|
||||||
- 迁移曾因旧package.json.lock及.plugin-manager/run.json阻塞并回滚；锁PID16与新容器安装进程PID16相同。确认实例完全停止后将两份记录改名保留，停机直接升级成功。未删除配置数据；该上游纯PID锁风险未通过自动删锁规避，后续遇到需停所有profile写入者再人工核验恢复。
|
|
||||||
|
|
||||||
## 默认内置 dsh-web-mobile 增量验收
|
|
||||||
|
|
||||||
- 模板修订3固定内置 `[email protected]`，校验发布包SHA-512与宿主peer版本，不执行插件生命周期脚本。当前限定DSH rc.1/rc.2；真实镜像测试使用rc.2。
|
|
||||||
- 新建空web profile默认加入mobile bundle；已有manifest或其他profile文件不覆盖、不强制启用。初始化拒绝符号链接目录，幂等/已有配置保留及符号链接测试均通过。
|
|
||||||
- 发布包原始权限导致非root加载失败，已通过镜像内 `chmod -R a+rX` 修复。修复后 `node scripts/test-mobile-image.mjs` 通过：断网、非root、只读rootfs启动，manifest与实际dump-config均含mobile插件。专用冒烟容器及其tmpfs测试数据已删除。
|
|
||||||
- 验收面板 `127.0.0.1:40411` 已更新至 `dsh-mgr:plugin3-active`（sha256:06bca1aae39accbac0721af723082e8a35a0c9cb2129e4cb942015b48fbffd85），HTTP 200。正常构建API成功登记rc.2模板3，镜像API状态available。旧面板停止保留供回滚，现有模板2实例未重启或自动加入插件。
|
|
||||||
- 本轮通过：初始化2测试、Go全量测试/race/vet、前端19测试及生产构建（仅既有Monaco大chunk警告）。
|
|
||||||
- 后续浏览器验收发现单纯放入node_modules仍会导入失败：DSH运行时只解析安装依赖图。已将内置包加入宿主package.json dependencies，且冒烟增加stdout/stderr激活失败检查（不输出token）。最终runtime ID：sha256:30ca773ce5d70a3a540a8f4aae149c83733102f8dfcd6a77ef9794bd992bde8e。此前只检查dump-config的成功结果不等于运行时激活成功，以下复验取代它。
|
|
||||||
- Playwright CLI独立loopback runtime触屏复验：coarse pointer=true、maxTouchPoints=1；含dsh-web-mobile/client.js的实际模块请求200；四个data-mobile-nav标记生效。首页及tap展开目录后viewport/client/scroll/body宽度均390。截图：output/playwright/plugin-touch/rc2-3-fixed-default-390.png、rc2-3-fixed-drawer-390.png。临时runtime已删除。
|
|
||||||
- 边界：本次插件复验未覆盖面板iframe、实体手机或模型交互；额外目录关闭tap超时，未计通过。前轮专用临时runtime token曾在工具输出回显，已删除该容器使服务不可访问，未涉及旧实例凭据。插件还包含响应压缩、会话删除入口和推理档位补全，不是纯CSS；未执行模型调用。
|
|
||||||
|
|
||||||
## 2026-10-10 移动端内嵌工作台增量验收
|
|
||||||
|
|
||||||
用户追加需求：登录默认 iframe 内嵌 DSH，设置切换容器，移动端底部导航。已实现并部署至原验收地址，镜像 dsh-mgr:mobile-renew（sha256:392c8bb64c822572005625df5d662774e1f780a1c83121caa249fafe6a16d01f）。
|
|
||||||
|
|
||||||
- 真实浏览器390×844及320×640：外层scrollWidth等于viewport宽度；iframe底边分别780/576，与底栏顶边一致，无底栏遮挡。桌面1440×900保留侧栏。
|
|
||||||
- 设置切换第二个真实容器、整页刷新后选择保留；停止所选实例显示启动提示而非自动切换；删除测试实例后回到可用实例。测试实例已删除，bind数据保留。
|
|
||||||
- 设置往返iframe DOM标记保留，未重载；退出登录iframe卸载；重新登录默认iframe已验证。
|
|
||||||
- 清除dsh_access模拟过期，真实iframe续期refresh200并回到原DSH地址，无加载错误。修复iframe导航续期及SPA X-Frame-Options冲突；使用SAMEORIGIN与frame-ancestors self保留跨源嵌入限制。
|
|
||||||
- Go全量测试、race及vet通过；前端19测试/生产构建通过。截图：output/playwright/design-final/mobile-settings.png及page-2026-10-10T09-29-51-590Z.png、page-2026-10-10T09-34-19-112Z.png。
|
|
||||||
- 验证为Chrome小屏视口，不代表iOS/Android真机软键盘兼容已全测。同源iframe不是安全沙箱；保留受信任团队部署边界。
|
|
||||||
|
|
||||||
以 DESIGN.md 为目标，不通过修改设计降低标准。以下最新结论覆盖后文早期进度记录。
|
|
||||||
|
|
||||||
## 2026-10-10 最终验收结案
|
|
||||||
|
|
||||||
**结论：按用户确认的默认路径代理、软配额及功能等价替代范围，验收通过并结案。** 用户在最终结构化确认中明确选择“接受功能等价替代”：标准库 Docker HTTP 客户端替代 SDK、每实例独立 bridge 替代共享 bridge、服务端保管 DSH cookie 替代下发浏览器、React+CSS 替代 shadcn（保留 Monaco）、部分模块集中于 internal/api。上述选型不再是待确认项；并未修改 DESIGN.md 来降低要求。
|
|
||||||
|
|
||||||
结案时再次确认 /healthz 返回 ok，dsh-design-check 正在运行 dsh-mgr:design-final，用户1000:1000、只读根文件系统、内存限制1073741824字节。此次结案仅更新验收文档，没有新增代码变更；沿用下列已完成测试证据。
|
|
||||||
|
|
||||||
默认路径代理、软配额方案的 M1–M7 功能已实现，真实主要流程通过；故障边界结合自动化注入测试验收，不声称所有故障均在宿主实际制造。可选 XFS/ZFS、socket-proxy、公网 TLS 未实测。
|
|
||||||
|
|
||||||
| 范围 | 最新证据 |
|
|
||||||
|---|---|
|
|
||||||
| M1 镜像 | rc.1/rc.2 recipe2实际构建，rc.2 smoke通过，精确版本/非root/只读/pnpm10.11.0 |
|
|
||||||
| M2 生命周期 | 最终面板完整API通过；test-intent-live.mjs验证预约自动创建、两个真实运行实例、第三创建与预约409、删除释放 |
|
|
||||||
| M3 代理 | 新浏览器DSH/空白会话/插件资源/SSE200；最终WS 1连接、5发送、5接收；面板替换自动重新接入网络；独立DSH页超过5分钟refresh200且PTY可操作 |
|
|
||||||
| M4 工作区 | 33项真实HTTP+43项多用户断言；ZIP/Range/穿越；PTY读取上传文件；直接调用镜像内DSH read/write实现在/workspace往返（无模型调用）；扩展名白名单.txt成功/.exe415 |
|
|
||||||
| M5 配置备份 | Monaco键盘编辑、保存启动、bundle/脱敏dump-config；浏览器备份创建/下载/错误密码拒绝/正确密码导入恢复 |
|
|
||||||
| M6 升级回滚 | rc.1→rc.2；manifest加入is-number@7.0.0验证依赖安装/恢复/迁移；不可达依赖触发失败，原ID/版本/停止状态恢复，数据marker保留。fixture验证安装机制，不代表任意第三方插件兼容 |
|
|
||||||
| M7 管理安全 | JWT/refresh/隔离/配额/审计；最终面板1000:1000、只读、1GiB、socket组989；管理员显式回收停止孤儿容器通过且bind目录保留 |
|
|
||||||
|
|
||||||
故障注入回归覆盖：构建去重/限频、磁盘不足/未知磁盘、npm精确版本、镜像缺失、重启恢复、失败释放预约、镜像引用保护、备份资源与旧包、路径/符号链接/配额并发和回滚。见internal/api/*_test.go及internal/backup/*_test.go；没有填满宿主磁盘或删除共享镜像。
|
|
||||||
|
|
||||||
真实验收发现并修复：dump-config顶层YAML列表误拒绝；代理日志旧token污染启动token提取。新增测试并容器复验通过。
|
|
||||||
|
|
||||||
最终镜像：
|
|
||||||
- panel dsh-mgr:design-final：sha256:d2da2f35e00e15fc0da7afc5183c49e40be6215bd4c5d89145e48f1034617cc5
|
|
||||||
- rc.2 recipe2：sha256:7bd09f12b4b60fa9d13358e99bae14caafc79d3a0936e70cf590fdd1af64998e
|
|
||||||
- rc.1 recipe2：sha256:1bb553deb3cef25c4529f93526d8cd3282d218df22787be2dc5d5762defaf0c4
|
|
||||||
|
|
||||||
通过命令：go test ./... -timeout 45s；go test -race ./... -timeout 90s；go vet ./...；npm --prefix web test（12项）；npm --prefix web run build。前端有Monaco大chunk警告。
|
|
||||||
|
|
||||||
证据：output/playwright/design-final/；/tmp/dsh-design-check.8ekWaC/{api-check-results.txt,isolation-results.json}；/tmp/dsh-design-plugin-rollback-verified.log；/tmp/dsh-design-cross-version.log。保留专用面板及一个浏览器实例；临时实例删除但数据保留，未推送镜像、未调用付费模型。
|
|
||||||
|
|
||||||
用户确认的孤儿策略：管理员二次确认回收身份/挂载核验通过、无实例/升级引用的停止容器；不自动纳管，不触碰运行中或来源不明容器，不删数据/网络。其他架构替代见下表；不声称逐字遵循SDK、组件库或目录示意。
|
|
||||||
|
|
||||||
## 早期进度记录（以下由上方最新结论覆盖）
|
|
||||||
|
|
||||||
## 当前实现与待验收
|
|
||||||
|
|
||||||
| 设计章节 | 代码已接入 | 尚需真实证据 |
|
|
||||||
|---|---|---|
|
|
||||||
| §1–6 | sibling路径映射、每用户两实例、持久预约/取消/失败恢复、生命周期与身份对账 | 模板2非root全流程、刷新/重启预约恢复、并发名额 |
|
|
||||||
| §5 | 精确npm版本、模板/平台身份、构建去重/限频/磁盘预检、镜像usage/missing及管理员清理 | 真实构建与删除、外部删镜像、空间/网络故障及重启对账 |
|
|
||||||
| §7–8 | 路径反代、归属鉴权、服务端token/cookie桥接、WS/SSE | 新部署的新浏览器完整流程、重启重新认证、长时续期 |
|
|
||||||
| §9 | 文件管理、目录ZIP、写前配额检查/串行化、周期巡检、管理员额度更新 | 新UI文件流程、agent实际读写、面板各写入路径配额覆盖 |
|
|
||||||
| §10 | 密码加密tar.gz v2、配置/资源目录/manifest、外部包导入、旧key包兼容、停启/健康检查/回滚 | 真实资源往返、原密码/错误密码、无key新包、旧key包、插件恢复失败回滚 |
|
|
||||||
| §11 | 本地Monaco、YAML校验、bundle与脱敏dump-config预览、保存后启动验证 | 新浏览器编辑/预览/错误路径；当前仍需先停止再保存 |
|
|
||||||
| §12 | 预升级密码备份、home快照、成功删旧、失败回滚及恢复原运行状态 | 不同DSH版本、真实插件重建/离线失败、故障矩阵与数据保留 |
|
|
||||||
| §13–16 | 短JWT/refresh轮换/失败锁定、审计、资源限制、非root部署配置 | 模板2非root数据写入和socket访问、新浏览器及DSH独立页长JWT续期 |
|
|
||||||
| §17–19 | 默认路径反代与软配额；已有回归测试及旧版集成记录 | 必需流程未关闭，不标记M1–M7整体完成；硬quota为可选增强 |
|
|
||||||
|
|
||||||
备份包含六个固定配置文件及skins、pets、agent-presets、.agent-presets资源树和manifest，不包含node_modules、会话或工作区。新密码包不依赖backup.key；保留旧密钥包时仍须保管原key。备份/恢复/升级不再统一要求用户手动停机；操作中断或回滚失败不保证自动灾难恢复。
|
|
||||||
|
|
||||||
## 合理替代方案（不改变验收目标）
|
|
||||||
|
|
||||||
| DESIGN选型 | 当前实现 | 等价目标与边界 |
|
|
||||||
|---|---|---|
|
|
||||||
| 官方Docker SDK | stdlib net/http Docker Engine client，API v1.47 | 仍须验证编排、构建流、错误处理和daemon兼容性，不能以未用SDK认定功能缺失 |
|
|
||||||
| 共享bridge | 每实例独立bridge，面板接入 | 减少实例直接互通；不等于完整出站或宿主网络隔离 |
|
|
||||||
| DSH cookie改写后下发浏览器 | 服务端交换/缓存cookie，反代注入并移除上游Set-Cookie | 保留面板登录与实例归属鉴权；WS/SSE、重启失效与续期仍需实测 |
|
|
||||||
| shadcn/ui | 普通React组件及CSS，保留本地Monaco | 不要求替换组件库，但交互、错误反馈和可用性仍须浏览器验收 |
|
|
||||||
|
|
||||||
代码主要按internal/api、internal/docker、internal/backup、internal/store组织，与设计目录示意不同；不宣称逐字一致。默认软配额符合§9/§18路线，XFS/ZFS硬quota依赖运维选择，不列为默认必过项。npm 0.2.0-rc.2不支持--public-url，当前使用路径代理而不传该参数。
|
|
||||||
|
|
||||||
## 历史证据与本轮边界
|
|
||||||
|
|
||||||
本轮已取得真实证据：
|
|
||||||
- 非root面板 `dsh-design-check`（1000:1000、只读rootfs、socket附加组989）已启动；33项HTTP认证/刷新/配额/ZIP/Range检查通过，报告 `/tmp/dsh-design-check.8ekWaC/api-check-results.txt`。
|
|
||||||
- 两普通用户隔离与预约43项断言通过，包括越权拒绝、两名额及取消释放；报告同目录 `isolation-results.json`。未把预约名额测试算作真实runtime并发测试。
|
|
||||||
- 浏览器基础UI登录、配额、ZIP、镜像状态、版本确认与预约页面通过，证据 `output/playwright/design-final/`。实例相关Monaco/备份/长JWT待runtime模板2就绪。
|
|
||||||
- 模板2实际构建进行中，详细安装日志持续有进度；尚无runtime2启动通过结论。
|
|
||||||
|
|
||||||
- 既有记录包含旧版真实Docker文件/实例/备份恢复及同镜像替换流程；同镜像替换不能证明跨DSH版本迁移。
|
|
||||||
- 旧浏览器证据目录output/playwright/browsercheck18091/记录插件加载、空白会话及WS/SSE，不代表模型生成、PTY或真实插件安装通过，也不能替代模板2和新增UI。
|
|
||||||
- 上轮记录的go test -race ./... -timeout 90s、go vet ./...、前端12项测试及前后端构建通过，仅为当时检查结果；冻结后须重跑，不能据此宣称全完成。
|
|
||||||
- **仍需关闭：模板2非root真实部署、新浏览器新增功能、插件跨版本/离线失败、长时JWT续期**。主代理进行中，结论与证据待回填；本文件不将此前权限拒绝误写为当前持续阻塞。
|
|
||||||
- 真实测试仅限已授权的本项目测试资源；工具再次拒绝时不绕过。不得隐式使用实际模型密钥或付费调用，不自动删除未知孤儿容器。
|
|
||||||
# 默认工作区 /workspace（模板 6，已由下述模板 7 数据根方案取代）
|
|
||||||
|
|
||||||
- 当前用户实例已通过 DSH 界面添加并选择 `/workspace`，刷新后仍保留；原 `normal` 工作区和历史会话未移动。
|
|
||||||
- 空实例初始化通过 DSH workspaceRegistry 注册准确的 `/workspace`，重复启动复用默认 ID；有历史工作区或会话时不强制迁移。
|
|
||||||
- 真实断网非 root 容器首次初始化、插件加载、备份恢复及旧模板兼容验证通过，详见 `/tmp/dsh-recipe6-test.log`。
|
|
||||||
- 面板已部署 `dsh-mgr:workspace6`；正常构建 API 记录 `e9a368bbb66e51e6c8f2439a8e94698a` 成功，recipe_rev=6，镜像 `sha256:38e04308da602da22e8df49d54166da1835922d50a3fa485711b7c5f6479d287`。
|
|
||||||
- 面板部署保留配置和挂载，旧面板停止留作回滚；现有 DSH 实例未重启，构建前后实例记录一致。
|
|
||||||
|
|
||||||
## 模板 7：完整 DSH_HOME 改为 /workspace
|
|
||||||
|
|
||||||
### 管理交互优化验收
|
|
||||||
|
|
||||||
- 配额静态展示与模态编辑、常用预设、未改变禁提交；密码明确按钮、原生模态、显示隐藏、即时UTF-8校验、取消清空及焦点恢复。账号创建规则对齐，移动用户卡片和退出分区。
|
|
||||||
- 前端26测试、生产构建通过。Playwright真实测试账号 uxqa_1791711946466：配额10→20→10 GiB；管理员重设→登录→自助修改→退出→新密码登录均通过；取消清空、焦点恢复、确认不一致禁提交、320/390px对话框不越界通过。
|
|
||||||
- 测试账号最终重新随机化密码、撤销会话，账号保留；未修改真实账号配额或密码。
|
|
||||||
- 正式面板 dsh-mgr:ux-polish 已部署，挂载集合核验相同（Docker返回顺序不同），DSH容器953ed34d034f未重启。正式入口重新加载后配额/密码对话框检查通过。
|
|
||||||
- 截图：output/playwright/ux-before-settings.png、ux-before-admin.png、ux-deployed-settings.png、ux-deployed-quota.png、ux-mobile-quota.png、ux-mobile-password-320.png。浏览器密码管理器用户名提示尚存；不声称全面无bug或完整无障碍合规。
|
|
||||||
|
|
||||||
- 依用户澄清，实例 home 直接挂载 /workspace，四个 HOME 环境变量同步；/data/dsh-home 为兼容软链接。旧共享目录保留为 /shared，不合并、不覆盖历史文件。
|
|
||||||
- 面板文件接口以 instance 参数校验归属、跟随所选实例；旧共享文件保留显式入口。账号总配额不变，恢复期间禁止写入，实例文件请求与升级互斥。
|
|
||||||
- 当前实例 63b337c848bf23156270068ff9e5cf38 正常升级成功，容器 953ed34d034faed9e0f8c7373964dd0156450b1d65c2965c73a539a83a226934，运行镜像 sha256:bd541770aaaf5a846ecb898ec80863c8a356524eb7f4b8053b0eb646d6e10fad。
|
|
||||||
- 构建 6c3589998c431acc809b5307c30e8e1e 成功，真实离线 smoke 验证环境变量、旧路径链接及同文件读写。升级中修复宿主路径转换和 pnpm 非交互日志边界；早期失败自动回滚，快照和备份保留。
|
|
||||||
- 最终验证：DSH 代理 GET 200；实例文件 API 可见 profiles/storages，storages/normal 可见四项历史报告/笔记；旧共享接口可访问。未调用模型或搜索。
|
|
||||||
- 前端 23 测试/build 通过，后端全量及 vet 曾通过，race 通过。最终定向 Workspace/Upgrade/Orphan 测试通过；最后一次全量受 Docker 磁盘不足 2 GiB 策略影响，3 个构建测试失败，不记为全绿。浏览器交互验证被权限拒绝，未绕过；HTTP 验证不替代浏览器验收。
|
|
||||||
@@ -1,100 +0,0 @@
|
|||||||
# DSH Web 面板 API（MVP）
|
|
||||||
|
|
||||||
默认监听 127.0.0.1:8088。JSON 错误统一为 {error:string}。登录使用 HttpOnly session/access/refresh cookie；写请求需 X-CSRF-Token 和精确同源 Origin（未配置 PANEL_PUBLIC_URL 时也校验）。access JWT 有效期5分钟，refresh轮换且受24小时session限制。
|
|
||||||
|
|
||||||
## 配置
|
|
||||||
- UPLOAD_ALLOWED_EXTENSIONS：可选逗号分隔扩展名白名单（例如 .txt,.md）；空表示不限制。大小写不敏感，非内容/恶意软件检测。
|
|
||||||
|
|
||||||
## 管理员孤儿回收
|
|
||||||
|
|
||||||
GET /api/orphans 返回可回收的完整容器ID数组。DELETE /api/orphans?id=<id>&confirm=<相同完整id> 二次核验后非强制删除，保留bind数据及网络。仅接受本项目身份/挂载匹配、已停止、无实例或升级引用的容器；普通用户403，确认不匹配400，状态不安全409。
|
|
||||||
|
|
||||||
## 其他配置
|
|
||||||
- LISTEN_ADDR（默认 127.0.0.1:8088）
|
|
||||||
- DATA_ROOT（默认 ./data）
|
|
||||||
- HOST_DATA_ROOT（默认等于 DATA_ROOT 的绝对路径，用于 Docker bind source；容器部署必须显式设置宿主绝对路径）
|
|
||||||
- CONTAINER_DATA_ROOT（默认 DATA_ROOT）
|
|
||||||
- DOCKER_HOST（默认 unix:///var/run/docker.sock）
|
|
||||||
- PANEL_PUBLIC_URL（可选）
|
|
||||||
- ADMIN_USERNAME / ADMIN_PASSWORD（首次启动创建管理员；密码必须 12–72 字节，不设置则拒绝启动，不打印密码）
|
|
||||||
- WEB_DIR（可选静态前端目录）
|
|
||||||
- PANEL_CONTAINER_ID（面板容器名/ID，用于加入实例网络；宿主进程运行时留空）
|
|
||||||
- DEFAULT_QUOTA_BYTES（默认10GiB）
|
|
||||||
- DSH_DOCKER_ROOT_VISIBLE（与DockerRootDir相同路径的明确映射，用于构建前容量检查；未知容量拒绝新构建）
|
|
||||||
|
|
||||||
## 认证
|
|
||||||
- POST /api/login {username,password} → {user,csrf_token}；user 字段为 id,username,admin,quota_bytes,created_at
|
|
||||||
- POST /api/logout → {ok:true}
|
|
||||||
- POST /api/refresh → {user,csrf_token}（刷新HttpOnly cookie，需Origin和CSRF；并发轮换冲突且session仍有效时返回409，不签发cookie，客户端可复验 `/api/me` 后重试原请求；会话撤销/过期返回401，存储异常不能当作注销）
|
|
||||||
- GET /api/me → {user,csrf_token}
|
|
||||||
- POST /api/me/password {current_password,password} → {ok:true,reauthenticate:true}（普通用户及管理员仅修改本人；不接收目标 ID，未知 JSON 字段返回 400）。需有效 access/session、同源 Origin 和 X-CSRF-Token；新密码为 UTF-8 12–72 字节，非法 JSON、缺少当前密码或长度不符返回 400。错误当前密码返回 400 而非 401，客户端不应因此退出登录。未登录 401，Origin/CSRF 不符 403（失效 session 在 guard 中也可能返回 403），已认证的非 POST 请求返回 405。
|
|
||||||
- 与登录、管理员重设共用 authLoginMu，等待后复验会话与当前密码；沿用账户登录失败计数，累计 5 次错误锁定 15 分钟，锁定期间返回 429（即使当前密码正确），已登录不能绕过。认证存储故障返回 503。
|
|
||||||
- 同一事务保存 bcrypt hash、撤销本人全部 session/refresh、清除失败计数，并记录 user.change_password 审计（操作者和目标均为本人 ID，detail 为空，无密码/hash/token）；事务失败返回 500 并完整回滚，其他用户不受影响。成功后客户端应清理登录状态并重新登录；旧密码、旧 access/session/refresh 不再可用。已建立的 runtime WS/SSE 不主动断开，下一次请求或重连重新鉴权。
|
|
||||||
|
|
||||||
## 管理
|
|
||||||
- GET /api/users（管理员）
|
|
||||||
- POST /api/users {username,password,admin?,quota_bytes?}（管理员）
|
|
||||||
- PATCH /api/users {id,quota_bytes}（管理员修改正数配额）
|
|
||||||
- POST /api/users/reset-password {id,password} → {ok:true,reauthenticate:boolean}（仅管理员；需有效 access/session、同源 Origin 和 X-CSRF-Token）。密码按 UTF-8 字节计为 12–72 字节，以 bcrypt 保存；缺少 ID、非法 JSON 或长度不符返回 400，目标不存在 404，未认证 401，非管理员或 Origin/CSRF 不符 403（失效 session 在 guard 中也可能返回 403），已认证的非 POST 请求返回 405。
|
|
||||||
- 同一事务更新密码、删除目标全部 session/refresh、清除目标登录失败锁定，并记录 user.reset_password 审计（操作者 ID、目标 ID、时间，detail 为空，不含密码/hash/token）。任何数据库失败返回 500 并回滚，不影响其他用户会话。与登录串行，等待后重新检查管理员会话和权限。
|
|
||||||
- 允许重设自身，此时 reauthenticate:true，当前会话也立即失效，客户端应清理登录状态并回到登录页；重设他人返回 false。旧密码不能再登录，旧 access/refresh 不能发起新的认证请求。已建立的 runtime WS/SSE 不会被主动立即断开：代理仅在请求/握手时鉴权，下一次请求或重连会被拒绝；现有 runtime 页面刷新认证脚本不是即时断连保证。
|
|
||||||
- GET /api/audit（管理员）
|
|
||||||
|
|
||||||
## 镜像与构建
|
|
||||||
- GET /api/images（每项含 admission、admission_reason、selectable；只有 status=available 且 selectable=true 才可新建/升级）
|
|
||||||
- GET /api/compatibility（登录后可读；返回 recipe_rev、platform、entries[{version,recipe_rev,platform,state,build_allowed,reason}]、unknown{state,build_allowed,reason}、experimental_admin_only。此为普通版本构建清单，不等于每个实际镜像的准入结果）
|
|
||||||
- POST /api/images/admission {image_id,state,evidence}（管理员、同源和CSRF；state为recommended/compatible/existing_only/blocked，evidence必填。正向准入要求精确镜像身份、成功验证记录、受支持挂载布局与平台；策略与审计持久化。单镜像准入不自动扩大普通版本构建白名单）
|
|
||||||
- DELETE /api/images?image_id=sha256:...（管理员；保护实例、活动任务、升级及全部Docker容器引用，不force）
|
|
||||||
- GET /api/latest（查询 npm dist-tags.latest；附带 supported、build_allowed、admission、admission_reason，最新版不一定受支持）
|
|
||||||
- GET /api/builds?limit=20&offset=0（摘要数组，不含日志正文；limit默认20、最大100，offset非负；X-Has-More响应头表示后续是否有记录）
|
|
||||||
- GET /api/builds/{id}（按需获取单个任务及脱敏日志；仅已登录用户，日志最多64 KiB）
|
|
||||||
- POST /api/builds {version,experimental?}（固定版本，异步；experimental=true仅管理员可用。未知版本普通构建会预先拒绝；实验仍执行完整性、依赖和启动验证，success不自动准入。返回任务含experimental标记。版本/模板/平台去重，新任务每用户每小时最多3个，共享复用不扣次数）
|
|
||||||
|
|
||||||
镜像策略：recommended/compatible可在实际验证与身份检查通过后用于新建/升级；existing_only允许已有实例运行和启停，不能新建/作为升级目标；blocked还拒绝新的start/restart及受控配置非GET操作（含Profile保存、备份和恢复），但不会自动停止已运行容器或断开工作台，通用文件管理仍允许。experimental是成功实验镜像未准入时的展示状态，不是兼容性结论。missing/unverified不可选。
|
|
||||||
|
|
||||||
内置普通构建清单为 linux/amd64、模板7、0.2.0-rc.1（compatible）和0.2.0-rc.2（recommended）。其他平台/版本不继承这些验收结论。模板3–6的历史成功镜像默认仅存量，具有精确验证记录并经管理员验收可准入；未知模板布局不可正向准入，模板7向旧共享布局降级仍拒绝。
|
|
||||||
|
|
||||||
## 实例
|
|
||||||
- GET /api/instances → Instance[]；字段 id,uid,name,version,image_id,status,container_id,created_at
|
|
||||||
- POST /api/instances {name,image_id,version} → Instance（每用户最多 2 个）
|
|
||||||
- POST /api/instances/{id}/start|stop|restart
|
|
||||||
- DELETE /api/instances/{id} → {ok:true}
|
|
||||||
- POST /api/intents {name,version}（固定精确版本，持久预留构建后创建名额）
|
|
||||||
- GET /api/intents（当前用户意图）
|
|
||||||
- DELETE /api/intents/{id}（仅取消等待，不取消共享构建）
|
|
||||||
|
|
||||||
所有成功 JSON 列表均为顶层数组（不是 data 包装）；错误均为 {error:string}。实现与验收仍在进行，不应仅凭接口列在此处认定已验收。
|
|
||||||
- GET /api/instances/{id}/logs
|
|
||||||
- /dsh/{id}/...（登录和 owner 校验；需同 Docker 网络）
|
|
||||||
|
|
||||||
## 工作区
|
|
||||||
|
|
||||||
指定实例时文件根为该所有者实例的完整home；允许下载凭据及目录ZIP，不承诺write-only。下载提示仅在界面防误操作，API不按敏感文件名过滤。通用文件修改不强制走专用Profile的停机/YAML校验流程；归属、路径安全、配额、恢复状态和现有文件操作并发保护仍有效。上传不覆盖同名文件（409）。
|
|
||||||
|
|
||||||
- GET /api/ws/tree?path=.
|
|
||||||
- GET /api/ws/usage
|
|
||||||
- POST /api/ws/mkdir {path}
|
|
||||||
- POST /api/ws/rename {path,new_path}
|
|
||||||
- DELETE /api/ws/delete?path=...
|
|
||||||
- POST /api/ws/upload multipart file,path
|
|
||||||
- GET /api/ws/download?path=...：普通文件保持原始流与 Range 支持；目录返回 `application/zip` 附件（目录名.zip），空 path 或 `.` 下载整个工作区（workspace.zip）。ZIP 条目相对所选目录，保留空目录，不额外包裹顶层目录，使用 Store 模式（不压缩），不支持 ZIP Range。
|
|
||||||
- ZIP 先校验有界文件清单，再逐文件流式输出，不生成临时归档，不持 workspace/global 锁跨网络传输。每次最多 10,000 个文件/目录条目、64 层、10 GiB 文件总大小；进程最多同时 4 个目录下载，超出分别返回 413 / 429；扫描和传输合计限时 2 分钟。
|
|
||||||
- 路径必须位于工作区内；下载拒绝任意路径分量的符号链接、跨挂载点、特殊文件和多硬链接普通文件。目录含上述条目或含反斜杠/冒号的不安全 ZIP 文件名时整体拒绝（400），不静默漏文件。依赖 Linux openat2，内核不支持时安全失败。
|
|
||||||
- 下载不是原子快照：流式读取前复核文件身份、大小、类型及修改时间；并发修改、取消或传输错误可能导致连接中断/不完整 ZIP，请停止写入后重试。单文件下载不受 ZIP 数量、大小、并发和时限约束。
|
|
||||||
|
|
||||||
## profile / 备份
|
|
||||||
- GET /api/instances/{id}/profile
|
|
||||||
- PUT /api/instances/{id}/profile {content,restart?}（停机原子写；restart=true启动并检查就绪）
|
|
||||||
- GET /api/instances/{id}/profile-preview（bundle清单和脱敏dump-config，字符串值隐藏；运行中才能dump）
|
|
||||||
- POST /api/instances/{id}/backups {password}（当前登录密码加密；自动恢复原运行状态）
|
|
||||||
- POST /api/instances/{id}/backups multipart file,password（导入外部包，password为原包密码；不立即覆盖home）
|
|
||||||
- GET /api/instances/{id}/backups
|
|
||||||
- GET /api/instances/{id}/backups?download={backup_id}
|
|
||||||
- POST /api/instances/{id}/restore {backup_id,password}（原备份密码；旧平台key包可留空）
|
|
||||||
- POST /api/instances/{id}/upgrade {image_id,password}（当前登录密码，加密pre-upgrade备份）
|
|
||||||
|
|
||||||
Profile写入要求实例先停止。备份/还原/升级会受控停启；恢复保留原运行状态，升级成功运行新版，失败恢复旧状态。模板2入口先准备插件再启动应用。删除实例仅删除容器与网络并移除索引，保留数据目录。升级失败恢复实例home，不撤销共享workspace或外部系统副作用。
|
|
||||||
|
|
||||||
创建恢复：Docker 创建响应丢失时，首次 inspect 404 不代表原请求已结束。实例保留为可见的 error 状态并继续占用名额，用户可重试删除；后台不会仅凭名称删除身份不匹配资源。不确定创建记录不会自动过期：若资源始终不存在，需管理员确认 Docker 原创建请求已终止后再处理保留记录，不能仅凭等待时长判定安全。构建验证容器在创建前持久登记任务身份，启动时即使没有排队构建也会回收并重试失败清理；持续不存在的未知结果仍保留登记以捕获晚到容器。构建终态和镜像索引事务失败会独立重试并记录日志，不阻塞后续构建；数据库关闭停止重试，重启将未完成构建标记失败。
|
|
||||||
|
|
||||||
已实现并通过核心集成检查：镜像复用验证、Docker 生命周期、token/cookie 桥接、加密备份恢复、同镜像替换升级。WS/SSE 代理有单元测试，真实完整会话待验证。硬配额与跨公网租户域名隔离未实现；当前 dsh 发行版不支持 --public-url。默认仅 loopback 监听，不应直接暴露公网。完整限制见 README.md。
|
|
||||||
@@ -1,418 +0,0 @@
|
|||||||
# DeepSeek Harness 多租户容器管理平台 — 设计文档
|
|
||||||
|
|
||||||
> 状态：本机管理面板核心流程已实现并通过真实 Docker 集成检查；生产隔离、硬配额等仍有差距，以 README.md 的实现边界与验证范围为准。下文是目标设计，不代表所有条目已实现。
|
|
||||||
> 目标项目：`deepseek-ai/deepseek-harness`（npm CLI `@deepseek-ai/dsh`，命令 `dsh web`）
|
|
||||||
> 控制面技术栈：Go + 前端 SPA
|
|
||||||
> 部署形态：单机 Docker daemon；面板自身亦运行于容器中
|
|
||||||
> 鉴权/配额：内置账号 + 目录配额
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 0. 调研结论与硬约束
|
|
||||||
|
|
||||||
> 实现阶段实测更正：本地 npm `0.2.0-rc.2` 支持 `--trusted-host`，但拒绝 `--public-url`；下文基于 upstream master 的 public-url 描述不能直接用于该发行版。实际启动仅传 web/no-open/port/trusted-host，代理负责外部路径。`scripts/probe-runtime.mjs` 已验证默认 token 交换为 303 + cookie、认证首页 200；完整代理以集成测试为准。
|
|
||||||
|
|
||||||
针对本体 `deepseek-ai/deepseek-harness`（npm CLI `@deepseek-ai/dsh`，命令 `dsh web`）确认：
|
|
||||||
|
|
||||||
| 事实 | 来源 | 对设计的影响 |
|
|
||||||
|---|---|---|
|
|
||||||
| **无官方 Docker 镜像/Dockerfile** | repo 根、`apps/`、`.gitlab-ci.yml` 均无 | 必须自建镜像包装 `dsh web` |
|
|
||||||
| **`dsh web` 默认监听 `127.0.0.1:3080`，且启动时拒绝 `--host 0.0.0.0`**（安全原因） | `packages/bundle/web-app/README.md` | 容器内必须跑反向代理绑 `0.0.0.0` 转发到 loopback |
|
|
||||||
| 端口/主机是 **CLI flag**（`--port/--host/--public-url/--trusted-host/--no-open`），**无 `DSH_PORT/DSH_HOST` 环境变量** | 同上 | 容器启动参数化靠 CMD args，不靠 env |
|
|
||||||
| 数据根 = `$DSH_HOME`（默认 `~/.dsh`） | `docs/user/guide/providers.md`、CLI README | 每实例独立 `DSH_HOME`，bind-mount 即可保留数据 |
|
|
||||||
| **多实例硬约束：每个容器必须独立 `DSH_HOME` + 独立端口**（task-board 单写者） | architecture.md | 一用户两容器 ⇒ 两个 `DSH_HOME`，不能共享 |
|
|
||||||
| 模型/API Key 存储：密钥在 `$DSH_HOME/.credentials.yaml`（YAML，上游配置接口的 write-only 语义不限制本面板所有者文件访问），模型/provider 配置在 `$DSH_HOME/profiles/<profile>/cordis.patch.yml`；`cordis.patch.yml` 只存 `apiKeyEnv` 引用 | `providers.md` | 备份"模型"= 备份这两个文件 |
|
|
||||||
| **"profile 文件" = `profiles/web/cordis.patch.yml`（YAML，用户可编辑的 patch 层）**；profile 目录还有 `package.json`（含插件依赖 + `dsh.profile` bundle 清单）+ `node_modules`（pnpm 装的插件，可重建） | `apps/cli/README.md` | 编辑 profile = 编辑 `cordis.patch.yml`；插件清单在 `package.json` |
|
|
||||||
| 升级：用户数据全在 `DSH_HOME`（npm 安装树之外），升级 `@deepseek-ai/dsh` 后数据保留；`node_modules` 由 pnpm 重建 | README、CLI README | "保留数据 + 重启新版容器"可行 |
|
|
||||||
| 公网部署：`--public-url <url/mount/>` + `--trusted-host`；代理必须 **剥前缀、保留 Host、把 `Path=/` 的 cookie 改写为 mount + 加 Secure** | `docs/user/guide/public-deployments.md` | 面板反代要处理 cookie/前缀，是最复杂点 |
|
|
||||||
| 启动 URL 带 **per-process token**，首次访问换签名 cookie | 同上 | 面板需从容器日志取 token，服务端代换 cookie |
|
|
||||||
|
|
||||||
> 结论：DSH 不是为容器多租户设计，但靠"自建镜像 + 容器内反代 + 面板反代桥接 token/cookie + 每实例独立 DSH_HOME"可稳态支撑。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 1. 目标与范围
|
|
||||||
|
|
||||||
### 已确认的访问与镜像策略（覆盖下文早期假设）
|
|
||||||
|
|
||||||
- 面向可信团队，实例所有者可通过文件管理读取、下载、修改完整 DSH_HOME，包括自己的凭据。面板不承诺凭据 write-only；跨用户隔离、路径安全、配额与恢复状态保护保持不变。下载文件/打包目录会提示泄密风险，提示属于交互防误操作，不是阻止所有者访问的安全边界。
|
|
||||||
- 专用 Profile 编辑器提供 YAML 校验与受控停启流程，但通用文件操作、DSH 终端和 agent 不强制经过该流程；修改运行中配置存在不一致风险，建议先停机并备份。
|
|
||||||
- 镜像准入分为推荐、仍兼容、仅存量运行、禁止使用。兼容依据绑定精确版本、模板修订、平台与镜像身份；不再仅因历史模板不是当前模板就一律判为不可用，也不能仅因历史构建成功就认定兼容。
|
|
||||||
- 未验证 DSH 新版仅管理员可显式实验构建；保留既有完整性、依赖与启动检查。实验构建成功不是兼容验收，未明确准入的镜像不可新建实例或作为升级目标。管理员正向准入须填写验收依据并记录审计，不绕过实际镜像身份与验证记录检查。
|
|
||||||
|
|
||||||
- 通过 docker.sock 在单机上**创建/启停/删除**多个 `dsh web` 容器，**每用户最多 2 个**。
|
|
||||||
- 每用户一个**独立工作区**，支持文件**上传/下载/删除/重命名**，并有**容量配额**。
|
|
||||||
- 工作区**映射进每个 dsh 容器**（agent 在工作区内干活）。
|
|
||||||
- **备份/还原 dsh 配置**（含模型与插件清单）。
|
|
||||||
- **编辑 profile 文件**（`cordis.patch.yml`）。
|
|
||||||
- **升级**：保留数据，用新版本镜像重启容器。
|
|
||||||
- **本地镜像管理**：面板查询 DSH 最新版本，用户手动触发本机构建；创建实例时选择本地已有版本，或手动构建最新版后创建。不需要私有镜像仓库，不推送生成的镜像。
|
|
||||||
- 控制面：Go 后端 + SPA 前端；内置账号 + 目录配额；**面板自身容器化运行**。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 2. 总体架构
|
|
||||||
|
|
||||||
面板不是宿主进程，而是挂了 `docker.sock` 的**兄弟容器**（sibling），通过 sock 编排 dsh 兄弟容器，并通过同一 bridge 网络反代它们。
|
|
||||||
|
|
||||||
```
|
|
||||||
┌────────────── 宿主 ──────────────────┐
|
|
||||||
│ docker.sock ──┐ │
|
|
||||||
│ /var/lib/dsh-mgr ──┐ (bind) │
|
|
||||||
│ │ │
|
|
||||||
│ ┌─────────────────▼─────────────▼───┐ │ dsh-net (bridge)
|
|
||||||
│ │ 面板容器 dsh-mgr │ │ :80/:443 发布
|
|
||||||
│ │ - /data ← /var/lib/dsh-mgr │ │
|
|
||||||
│ │ - /var/run/docker.sock │ │
|
|
||||||
│ │ - SPA + API + 反代 + 编排引擎 │ │
|
|
||||||
│ └─────────┬───────────────────────────┘ │
|
|
||||||
│ │ sock │
|
|
||||||
│ ┌─────────▼───────────┐ ┌───────────────┐ │
|
|
||||||
│ │ dsh-web:<v> 容器1 │ │ 容器2 ... │ │ 不发布端口
|
|
||||||
│ │ caddy:8080 → 3080 │ │ │ │ /data/dsh-home、/workspace ← 宿主 bind
|
|
||||||
│ └─────────────────────┘ └───────────────┘ │
|
|
||||||
└─────────────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
组件：
|
|
||||||
1. **面板服务**（Go 容器）：REST API + 静态 SPA + 反向代理子系统 + docker client + 配额/备份/升级引擎。
|
|
||||||
2. **dsh-web 镜像**（自建）：Node + `@deepseek-ai/dsh` + caddy + 入口脚本。
|
|
||||||
3. **dsh-net**：内部 bridge 网络，容器不发布主机端口，仅面板可达。
|
|
||||||
4. **（可选）socket-proxy**：面板与 sock 之间的白名单代理。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 3. 宿主路径 ↔ 面板内路径映射（关键）
|
|
||||||
|
|
||||||
Docker bind 的 `Source` 必须是**守护进程所在宿主路径**，不是面板容器内路径。面板容器把宿主 `/var/lib/dsh-mgr` 挂到 `/data`，于是同一文件树有两个视角：
|
|
||||||
|
|
||||||
| 用途 | 面板直接读写（容器视角） | 创建 dsh 容器 bind.Source（宿主视角） |
|
|
||||||
|---|---|---|
|
|
||||||
| 用户工作区 | `/data/workspaces/<uid>/files/...` | `/var/lib/dsh-mgr/workspaces/<uid>/files` |
|
|
||||||
| 实例 DSH_HOME | `/data/workspaces/<uid>/instances/<id>/home/...` | `/var/lib/dsh-mgr/workspaces/<uid>/instances/<id>/home` |
|
|
||||||
| 备份包 | `/data/backups/<uid>/<id>/...` | （不入容器，仅面板读写） |
|
|
||||||
|
|
||||||
面板配置：
|
|
||||||
```
|
|
||||||
HOST_DATA_ROOT=/var/lib/dsh-mgr # 传给 docker API 的 bind 源
|
|
||||||
CONTAINER_DATA_ROOT=/data # 面板自己 fs 操作的根
|
|
||||||
```
|
|
||||||
|
|
||||||
- 所有"面板直接动文件"（文件 API、备份 tar、profile 读写）用 `CONTAINER_DATA_ROOT`。
|
|
||||||
- 所有"喂给 docker API 的 bind.Source / build context"用 `HOST_DATA_ROOT` 拼接。
|
|
||||||
- 封装 `toHostPath(containerPath)` 翻译函数集中调用，避免散落拼接出错——这是 sibling 容器最常见的 bug。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 4. 数据与目录布局（宿主机）
|
|
||||||
|
|
||||||
```
|
|
||||||
/var/lib/dsh-mgr/
|
|
||||||
db.sqlite # 用户、实例、配额、备份索引
|
|
||||||
workspaces/<uid>/
|
|
||||||
files/ # 用户工作区（面板文件 API + 各 dsh 容器共享挂载）
|
|
||||||
└─ <用户自由目录结构>
|
|
||||||
instances/<instId>/
|
|
||||||
home/ # DSH_HOME（bind → 容器 /data/dsh-home）
|
|
||||||
.credentials.yaml # API Key（机密）
|
|
||||||
cordis.patch.yml # home 级 patch
|
|
||||||
profiles/web/
|
|
||||||
package.json # 插件依赖 + dsh.profile bundle 清单
|
|
||||||
cordis.patch.yml # profile 级 patch（= "profile 文件"）
|
|
||||||
node_modules/ # 插件实体（可重建，不入备份）
|
|
||||||
skins/ pets/ agent-presets/ .agent-presets/
|
|
||||||
dsh-ssh.json # 0600
|
|
||||||
sessions/ worktrees/ # 运行态（不入配置备份）
|
|
||||||
backups/<uid>/<instId>/
|
|
||||||
<timestamp>.tar.gz # 配置备份包
|
|
||||||
# 镜像层由本机 Docker daemon 管理，不作为此目录下的文件保存；DB 记录镜像索引
|
|
||||||
```
|
|
||||||
|
|
||||||
**配额作用域**：`workspaces/<uid>/` 整棵树（files + 所有 instances 的 home）。每用户一个配额单位。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 5. 自建 DSH 镜像设计
|
|
||||||
|
|
||||||
```
|
|
||||||
FROM node:22-slim
|
|
||||||
# 1) 装 dsh CLI（版本参数化）
|
|
||||||
ARG DSH_VERSION=0.2.0-rc.2
|
|
||||||
RUN npm i -g @deepseek-ai/dsh@${DSH_VERSION}
|
|
||||||
# 2) 装 caddy（容器内反代，绕过 dsh 拒绝 0.0.0.0）
|
|
||||||
COPY caddy /usr/bin/caddy
|
|
||||||
COPY Caddyfile /etc/caddy/Caddyfile # :8080 { reverse_proxy 127.0.0.1:3080 }
|
|
||||||
COPY entrypoint.sh /entrypoint.sh
|
|
||||||
ENV DSH_HOME=/data/dsh-home
|
|
||||||
WORKDIR /workspace
|
|
||||||
ENTRYPOINT ["/entrypoint.sh"]
|
|
||||||
# CMD 由面板按实例注入：dsh web --no-open --port 3080 --public-url <url> --trusted-host <host>
|
|
||||||
```
|
|
||||||
|
|
||||||
- `entrypoint.sh`：`tini` 接管信号 → 起 caddy（后台）→ `exec dsh "$@"`。dsh 绑 loopback:3080，caddy 绑 0.0.0.0:8080。
|
|
||||||
- **存储方式**：生成的镜像仅保存在本机 Docker daemon，不推送 registry，也不从 registry 拉取成品 `dsh-web` 镜像。所有用户共享面板管理的本地镜像目录。
|
|
||||||
- **版本查询**：面板从 npm 查询 `@deepseek-ai/dsh` 的 `dist-tags.latest`，展示精确版本和查询时间；查询不触发构建或升级。不通过版本字符串排序猜测最新版。查询失败时保留本地版本列表，缓存结果需标明过期。
|
|
||||||
- **版本固定**：用户确认构建时锁定页面展示的精确版本，后端校验其确实存在于 npm 元数据中；构建过程中不再次解析 `latest`，Dockerfile 不安装浮动的 `@latest`。
|
|
||||||
- **本机构建**：Docker API `POST /build` 接受 tar 流；面板将内置 `/image/` 的 Dockerfile+caddy+entrypoint+Caddyfile 打包后提交给 daemon，并通过 Docker API 的 buildargs 传入 `DSH_VERSION`。用户不能提交任意 Dockerfile、构建上下文、镜像地址或其他构建参数。
|
|
||||||
- **镜像身份**：tag 使用 `dsh-web:<dshver>-<recipeRev>`，其中 `recipeRev` 标识镜像模板修订；记录精确 DSH 版本、模板修订、架构、本地 image ID 和构建时间。创建实例使用已确认的 image ID，不依赖可能被覆盖的 tag。
|
|
||||||
- **构建任务**：手动触发后异步执行，展示排队、构建中、验证中、成功、失败状态及脱敏日志；任务持久化。同版本、模板修订和架构的并发请求复用同一任务，已有可用镜像直接复用。默认全局串行构建，并限制用户触发频率、任务时长及日志大小，构建前检查磁盘空间。
|
|
||||||
- **可用性判定**：Docker build 成功后，用临时隔离容器验证版本和 DSH 启动就绪；验证通过才加入可选择列表。失败保留错误记录，不自动替换已有可用镜像。面板重启后对账未完成任务及本地镜像，不盲目重复构建。
|
|
||||||
- **本地清单**：以 Docker inspect/list 和面板管理标签对账，展示精确版本、构建时间、可用状态及使用情况。外部删除导致 image ID 不存在时标记缺失，不再允许新建；管理员清理时保护被实例、待执行任务或回滚流程引用的镜像，不自动运行全局 prune。
|
|
||||||
- **网络要求**：仅本地保存不代表离线构建；首次构建仍可能需要拉取基础镜像及下载 npm 包。网络故障不影响选择已存在的本地可用镜像。
|
|
||||||
|
|
||||||
**创建实例交互**：
|
|
||||||
1. **选择本地版本**：从本地可用列表选择版本，校验 image ID 仍存在后创建。
|
|
||||||
2. **构建最新版并创建**：查询并展示精确版本 → 用户确认 → 事务预留实例名额 → 构建或复用任务 → 验证成功 → 用该 image ID 创建实例。等待构建的实例也计入每用户 2 个名额；构建/创建失败或取消时清理未完成资源并释放名额，不回退到其他版本。取消某个用户的等待不取消其他用户共享的构建任务。
|
|
||||||
3. 面板另提供独立的“构建最新版”按钮，仅生成本地镜像，不创建或升级实例，也不占实例名额。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 6. 容器生命周期（docker.sock + socket-proxy）
|
|
||||||
|
|
||||||
Go 端用 `github.com/docker/docker/client`：
|
|
||||||
|
|
||||||
- **创建**：
|
|
||||||
- Image = 本地可用镜像的固定 image ID（对应精确 DSH 版本及模板修订，见 §5）
|
|
||||||
- Env：`DSH_HOME=/data/dsh-home`（密钥不入 env；用户在 dsh UI 里设 key，落到 `.credentials.yaml`）
|
|
||||||
- Binds（Source 用 **HOST 路径**）：
|
|
||||||
- `workspaces/<uid>/instances/<instId>/home` → `/data/dsh-home`
|
|
||||||
- `workspaces/<uid>/files` → `/workspace`
|
|
||||||
- Network = `dsh-net`（不发布端口）
|
|
||||||
- Cmd = `web --no-open --port 3080 --public-url https://<panel>/dsh/<instId>/ --trusted-host <panel>`
|
|
||||||
- Labels：`dsh-mgr.uid`、`dsh-mgr.instId`、`dsh-mgr.version`、`dsh-mgr.role=dsh`
|
|
||||||
- 资源：`--memory=2g --cpus=2`（可配）；`--tmpfs /tmp`；只读 rootfs + 必要写入挂载
|
|
||||||
- **操作**：start/stop/restart/remove/logs/inspect/exec，全走 SDK。
|
|
||||||
- **2 实例上限**：按逻辑实例在 DB 事务中预留名额，等待构建/创建中的实例也占名额；失败清理后释放。Docker labels 用于对账，不作为并发名额控制。
|
|
||||||
- **对账**：面板启动时按 label 枚举容器与 DB 对账（孤儿容器回收或纳管）。
|
|
||||||
- **sock 安全**：面板挂裸 sock = 等效宿主 root。缓解：
|
|
||||||
1. **socket-proxy**（推荐）：只放行白名单 API（containers/exec/images/networks/volumes/logs/build），关 plugins/daemon/swarm。面板连 proxy 而非裸 sock。
|
|
||||||
2. 面板自身非 root、只读 rootfs + tmpfs /tmp、资源限制。dsh 容器不 `--privileged`、去 cap。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 7. 网络与访问方式
|
|
||||||
|
|
||||||
- 面板**启动时 ensure** `dsh-net`（bridge）存在，并把自身加入（compose 声明或运行时 `network connect`）。
|
|
||||||
- dsh 容器挂 `dsh-net`，**不发布主机端口**。
|
|
||||||
- 面板反代 `/dsh/<instId>/` → `http://<container-name>.dsh-net:8080`，容器名 = instId。
|
|
||||||
- **访问方式（待定，见 §18）**：默认推荐"路径前缀反代 + token 桥接"（§8），面板天然为 ingress。备选：子域名（需泛 DNS+TLS）/ 直发端口（MVP，隔离弱）。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 8. 访问与认证（token 桥接 + cookie 改写）
|
|
||||||
|
|
||||||
> 此为最复杂点。推荐方案：路径前缀反代 + token 桥接。
|
|
||||||
|
|
||||||
- 面板路由 `/dsh/<instId>/*` → 反代到 `http://<instId>.dsh-net:8080`，**剥掉 `/dsh/<instId>` 前缀**，**保留浏览器 Host**，加 `X-Forwarded-*`。
|
|
||||||
- **token 桥接**：容器启动后，面板从 `docker logs` 流式抓 dsh 打印的 `http://127.0.0.1:3080/?token=xxx` 里的 token；用户首次打开 `/dsh/<instId>/` 时，面板服务端用该 token 打一次容器换 cookie，再把 `Set-Cookie` 的 `Path=/` 改写为 `Path=/dsh/<instId>`、加 `Secure`/`HttpOnly`/`SameSite=Lax` 下发给浏览器。后续请求带该 cookie 透传。
|
|
||||||
- **鉴权**：面板先校验用户登录 + 该 instId 归属当前 uid，再放行反代。dsh 自身 token cookie 为第二层。
|
|
||||||
- **WebSocket/SSE**：反代需支持 WS 升级；cookie 改写后需实测 WS 握手是否正常（关键风险）。
|
|
||||||
|
|
||||||
## 9. 工作区文件管理 + 容量配额
|
|
||||||
|
|
||||||
**文件 API**（path 校验：解析后必须在 `files/` 内，拒 `..`/绝对路径）：
|
|
||||||
- `GET /api/ws/tree?path=` 列目录（带 size/mtime）
|
|
||||||
- `POST /api/ws/mkdir`、`POST /api/ws/rename`、`DELETE /api/ws/delete`
|
|
||||||
- `POST /api/ws/upload`（分片/流式，**写前查配额**）
|
|
||||||
- `GET /api/ws/download?path=`（单文件流；目录自动 zip）
|
|
||||||
- `GET /api/ws/usage`（当前用量/配额）
|
|
||||||
|
|
||||||
**配额实现**（容器化面板的约束）：
|
|
||||||
- **默认：软配额**（无特权，随处可跑）。面板写前 `du` 校验 + 周期巡检 + 写入并发限流。并发大文件可能短暂超限。
|
|
||||||
- **可选硬配额**（二选一，需运维确认）：
|
|
||||||
- **XFS prjquota**：面板 `--privileged` + `cap_add:[SYS_ADMIN,CHOWN]` + 把宿主 XFS 挂载点 bind 进来，面板内对 `/data` 跑 `xfs_quota`/`chattr -p`。每用户唯一 project ID（uid+1000）。
|
|
||||||
- **ZFS refquota**（若宿主用 ZFS）：面板 `--privileged` + zfsutils，每用户 `zfs create pool/users/<uid> && zfs set refquota=10G`，再 bind 该 dataset 到 `/data/workspaces/<uid>`。
|
|
||||||
- 或在宿主跑**极小特权配额 sidecar**（gRPC/HTTP），面板远程调它做 quota 操作——面板自身保持非特权。
|
|
||||||
- 配额值存 DB，按用户可配（默认 10GiB）。面板展示用量。
|
|
||||||
|
|
||||||
> 具体选型见 §18 待拍板。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 10. 配置备份与还原（含模型与插件）
|
|
||||||
|
|
||||||
**备份内容**（tar.gz，**排除可重建/运行态**）：
|
|
||||||
|
|
||||||
| 包含 | 说明 |
|
|
||||||
|---|---|
|
|
||||||
| `.credentials.yaml` | 模型 API Key（机密）→ 备份包加密 |
|
|
||||||
| `cordis.patch.yml`（home 级） | home patch |
|
|
||||||
| `profiles/web/package.json` | **插件清单**（含 dsh.profile bundle 列表） |
|
|
||||||
| `profiles/web/cordis.patch.yml` | **模型/provider 配置** + apiKeyEnv 引用 |
|
|
||||||
| `profiles/web/pnpm-lock.yaml`（若有） | 插件版本锁 |
|
|
||||||
| `skins/ pets/ agent-presets/ .agent-presets/` | 用户资源 |
|
|
||||||
| `dsh-ssh.json` | SSH 凭据（0600，加密入包） |
|
|
||||||
| `manifest.json` | dsh 版本、备份 schema 版、时间戳 |
|
|
||||||
|
|
||||||
**排除**：`node_modules/`（pnpm 重建）、`worktrees/`、`sessions/`、日志。
|
|
||||||
|
|
||||||
**备份包加密**：用用户密码派生密钥（Argon2id → AES-GCM）加密整包；密钥不落库。下载/上传均加密包。
|
|
||||||
|
|
||||||
**备份流程**：停容器（或仅停写）→ tar 指定文件 → 加密 → 落 `backups/<uid>/<instId>/` + 索引入库 → 重启容器。支持立即下载。
|
|
||||||
|
|
||||||
**还原流程**：校验+解密包 → 停容器 → 备份当前 home 为 `.pre-restore.bak` → 覆盖配置文件（不动 node_modules）→ **按 `package.json` 重建插件**（`dsh plugin --profile web install` 或 entrypoint 首启时 pnpm reconcile）→ 启容器 → 健康检查 → 失败则回滚 `.pre-restore.bak`。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 11. profile 文件编辑
|
|
||||||
|
|
||||||
- 编辑对象：`profiles/web/cordis.patch.yml`（YAML）。
|
|
||||||
- 前端用 Monaco 编辑器；后端 `GET` 返回原文、`POST` 校验（YAML 合法 + 可选 `--dump-config-schema` 比对）后原子写（临时文件 + rename，`dsh-atomic-write` 风格）。
|
|
||||||
- 保存后**重启容器**使生效（dsh 有 `dsh-hmr`，但配置类变更稳妥起见重启；后续可探热更）。
|
|
||||||
- 只读展示 `dsh.profile`（bundle 清单）与 `--dump-config` 预览（面板 `docker exec` 跑 `dsh --profile web --dump-config` 取回渲染）。
|
|
||||||
- 专用配置预览保持脱敏；完整 home 文件管理允许实例所有者读取 `.credentials.yaml` 原文，不承诺 write-only。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 12. 升级流程（保留数据 + 重启新版容器）
|
|
||||||
|
|
||||||
```
|
|
||||||
1. 用户选择本地可用目标版本；若需最新版，先手动触发 §5 构建并等待验证成功。锁定本地 image ID 后才进入备份/停机流程，不自动 pull 成品镜像
|
|
||||||
2. 自动备份当前配置（§10）→ 标记 pre-upgrade 备份
|
|
||||||
3. docker stop <old>（不 remove，保留作回滚）
|
|
||||||
4. docker create <new>：同名 instId、同 binds（home + files）、同 labels、新 image、
|
|
||||||
Cmd 按新版本要求重算 --public-url/--trusted-host
|
|
||||||
5. entrypoint 首启：若 profiles/web/node_modules 缺失/与 package.json 不一致 → pnpm install 重建插件
|
|
||||||
6. docker start <new> → 面板抓 token、健康检查（轮询 caddy :8080）
|
|
||||||
7. 成功：docker rm <old>；失败：stop <new>、start <old>、标记回滚、提示用户
|
|
||||||
```
|
|
||||||
|
|
||||||
关键：**bind 不变 ⇒ 数据零丢失**；`node_modules` 在新镜像里按 `package.json` 重建（跨版本兼容性由 pnpm/dsh-plugin-manager 保证）。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 13. 安全与资源
|
|
||||||
|
|
||||||
- **docker.sock**：面板容器能访问 sock 即等效宿主 root；socket-proxy 白名单 + 面板非 root + 只读 rootfs 缓解。
|
|
||||||
- dsh 容器：**不 `--privileged`**、去 CAP_NET_RAW 等、只读 rootfs、tmpfs /tmp、mem/cpu 限制、不挂 sock。
|
|
||||||
- 面板账号：bcrypt + JWT（短期 access + refresh），失败锁定。
|
|
||||||
- 配置备份包加密；实例所有者通过文件 API 下载的凭据或目录 ZIP 不属于加密配置备份，应按机密数据保管。
|
|
||||||
- 路径穿越校验、上传大小/类型白名单（可配）。
|
|
||||||
- 审计日志：容器操作/备份/还原/升级/profile 改动入库。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 14. 技术栈与项目结构
|
|
||||||
|
|
||||||
- **后端**：Go 1.22+，`net/http`（或 chi/gin），`github.com/docker/docker/client`，`github.com/docker/go-connections`，sqlite（`modernc.org/sqlite` 纯 Go 驱动，免 CGO），Argon2id、AES-GCM。
|
|
||||||
- **前端**：Vite + React + TypeScript，Monaco（profile 编辑），shadcn/ui。
|
|
||||||
|
|
||||||
```
|
|
||||||
dsh-mgr/
|
|
||||||
cmd/server/ # Go 入口
|
|
||||||
internal/
|
|
||||||
auth/ # 账号/JWT
|
|
||||||
docker/ # 容器编排（镜像、生命周期、对账、路径翻译）
|
|
||||||
workspace/ # 文件 API、配额（软 du + 可选 xfs/zfs）
|
|
||||||
proxy/ # /dsh/<id> 反代 + token 桥接 + cookie 改写
|
|
||||||
backup/ # 备份/还原/加密
|
|
||||||
upgrade/ # 升级编排
|
|
||||||
profile/ # cordis.patch.yml 编辑
|
|
||||||
store/ # sqlite schema/查询
|
|
||||||
api/ # HTTP handlers
|
|
||||||
web/ # SPA
|
|
||||||
image/ # 自建 dsh-web 镜像（Dockerfile + Caddyfile + entrypoint.sh + caddy 二进制）
|
|
||||||
deploy/ # 面板自身 docker-compose / socket-proxy / systemd
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 15. 数据模型（sqlite 概要）
|
|
||||||
|
|
||||||
- `users(id, username, pass_hash, quota_bytes, created_at)`
|
|
||||||
- `instances(id, uid, name, dsh_version, image_id, build_id, status, container_id, created_at, ...)`
|
|
||||||
- `backups(id, uid, inst_id, path, size, schema_ver, created_at, kind=auto|manual|pre-upgrade|pre-restore)`
|
|
||||||
- `images(image_id, version, recipe_rev, platform, tag, status, built_at)`（本地镜像索引，status=validating|available|failed|missing；与 Docker 实际状态对账）
|
|
||||||
- `image_builds(id, requested_by, version, recipe_rev, platform, status, image_id, error, created_at, finished_at)`（持久化构建任务；同版本+模板修订+架构的活动任务去重）
|
|
||||||
- `audit(id, uid, action, target, detail, at)`
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 16. 面板自身部署（compose 示例）
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
services:
|
|
||||||
dsh-mgr:
|
|
||||||
image: dsh-mgr:latest
|
|
||||||
restart: unless-stopped
|
|
||||||
ports:
|
|
||||||
- "80:80" # 前置 TLS 反代或直接 443
|
|
||||||
user: "65532:65532" # 非 root
|
|
||||||
environment:
|
|
||||||
HOST_DATA_ROOT: /var/lib/dsh-mgr
|
|
||||||
CONTAINER_DATA_ROOT: /data
|
|
||||||
PANEL_PUBLIC_URL: https://dsh.example.com
|
|
||||||
DSH_NET_NAME: dsh-net
|
|
||||||
DEFAULT_QUOTA_BYTES: 10737418240
|
|
||||||
DOCKER_HOST: unix:///var/run/docker.sock # 或指向 socket-proxy
|
|
||||||
volumes:
|
|
||||||
- /var/lib/dsh-mgr:/data
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro # 或 socket-proxy
|
|
||||||
networks: [dsh-net]
|
|
||||||
read_only: true
|
|
||||||
tmpfs: [/tmp]
|
|
||||||
deploy: { resources: { limits: { memory: 1g } } }
|
|
||||||
# 硬配额(XFS)可选: privileged: true; cap_add: [SYS_ADMIN,CHOWN]; volumes: [xfs mount]
|
|
||||||
|
|
||||||
# 可选 socket-proxy
|
|
||||||
socket-proxy:
|
|
||||||
image: tecnativa/docker-socket-proxy
|
|
||||||
environment:
|
|
||||||
CONTAINERS: 1; EXEC: 1; IMAGES: 1; NETWORKS: 1; VOLUMES: 1; LOGS: 1; BUILD: 1
|
|
||||||
POST: 1
|
|
||||||
volumes: [/var/run/docker.sock:/var/run/docker.sock:ro]
|
|
||||||
networks: [dsh-net]
|
|
||||||
restart: unless-stopped
|
|
||||||
|
|
||||||
networks:
|
|
||||||
dsh-net: { driver: bridge }
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 17. 关键风险与待验证
|
|
||||||
|
|
||||||
1. **token/cookie 桥接**（§8）最脆——需实测 dsh token 换 cookie 在反代+前缀下的行为，确认 cookie 改写后 **WS/SSE 正常**（dsh web 可能有 WebSocket）。
|
|
||||||
2. **插件首启重建**：跨 dsh 版本 `pnpm install` 在 entrypoint 无网时失败——可能需升级时由面板 `docker exec` 联网重建后再切流量。
|
|
||||||
3. **XFS/ZFS 硬配额**依赖宿主 FS 与特权/agent；否则退化为软配额。
|
|
||||||
4. **dsh agent 工作目录**：`dsh-fs-local`/`dsh-fs-sandbox` 的根是否等于 cwd——需确认 `/workspace` 作为 cwd 即可让 agent 在工作区操作，或需额外配置 fs root。
|
|
||||||
5. dsh 版本仍为 `0.2.0-rc.*`（RC），API/flag 可能变动——需维护版本兼容矩阵。
|
|
||||||
6. **sibling bind 路径翻译**：`toHostPath` 出错会导致 bind 挂错目录或挂空目录，需单测覆盖。
|
|
||||||
7. **本地镜像构建**：验证 npm 查询失败仍可选本地版本、并发构建去重、失败释放实例名额、面板重启对账、镜像被外部删除，以及构建磁盘不足等路径；构建最新版不保证兼容，必须通过启动验证。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 18. 待拍板决策点（open）
|
|
||||||
|
|
||||||
**已确认：镜像仅保存在本机；面板支持查询并手动构建最新版；创建实例可选本地已有版本，或构建最新版后创建。不需要 registry/CI 镜像分发。**
|
|
||||||
|
|
||||||
> 以下访问方式和配额选型仍待确认；此前审查发现的隔离、文件安全和升级回滚问题仍需另行修订。
|
|
||||||
|
|
||||||
1. **访问方式**：路径前缀反代 + token 桥接（推荐，面板天然 ingress）／子域名（需泛 DNS+TLS）／直发端口（MVP，隔离弱）。
|
|
||||||
- **推荐默认**：路径前缀反代。
|
|
||||||
2. **配额**：宿主 FS 是 XFS / ZFS / 其他？是否接受"特权面板"或"宿主配额 sidecar"做硬配额？还是软配额默认即可？
|
|
||||||
- **推荐默认**：软配额（du）起步，硬配额作为后续增强；若宿主为 XFS/ZFS 且可接受特权，则上硬配额。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 19. 里程碑
|
|
||||||
|
|
||||||
1. **M1 镜像 + 单容器手动跑通**：自建本地 `dsh-web` 镜像，验证精确版本、caddy+dsh 启动与就绪。
|
|
||||||
2. **M2 面板骨架**：账号/鉴权 + npm 最新版查询 + 本地镜像列表/手动构建任务 + 创建时选本地版本或构建最新版 + docker 编排（创建/启停/删除/2 上限）+ 对账 + 路径翻译。
|
|
||||||
3. **M3 反代 + token 桥接**：`/dsh/<id>/` 可用（含 WS）。
|
|
||||||
4. **M4 工作区文件 API + 软配额**。
|
|
||||||
5. **M5 备份/还原 + profile 编辑**。
|
|
||||||
6. **M6 升级流程 + 回滚**。
|
|
||||||
7. **M7 SPA 完善 + 审计 + 资源限制 + 可选硬配额**。
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## 附：调研依据（关键 URL）
|
|
||||||
|
|
||||||
- 仓库：https://github.com/deepseek-ai/deepseek-harness
|
|
||||||
- README：https://raw.githubusercontent.com/deepseek-ai/deepseek-harness/master/README.md
|
|
||||||
- CLI README：https://raw.githubusercontent.com/deepseek-ai/deepseek-harness/master/apps/cli/README.md
|
|
||||||
- web-app README：https://raw.githubusercontent.com/deepseek-ai/deepseek-harness/master/packages/bundle/web-app/README.md
|
|
||||||
- providers 指南：https://raw.githubusercontent.com/deepseek-ai/deepseek-harness/master/docs/user/guide/providers.md
|
|
||||||
- 公网部署指南：https://raw.githubusercontent.com/deepseek-ai/deepseek-harness/master/docs/user/guide/public-deployments.md
|
|
||||||
- npm 包：https://registry.npmjs.org/@deepseek-ai/dsh/latest
|
|
||||||
-26
@@ -1,26 +0,0 @@
|
|||||||
FROM node:24-bookworm-slim AS web
|
|
||||||
WORKDIR /src/web
|
|
||||||
COPY web/package*.json ./
|
|
||||||
RUN npm ci
|
|
||||||
COPY web/ ./
|
|
||||||
RUN npm run build
|
|
||||||
|
|
||||||
FROM golang:1.26-bookworm AS backend
|
|
||||||
WORKDIR /src
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
COPY cmd/ ./cmd/
|
|
||||||
COPY internal/ ./internal/
|
|
||||||
COPY image/ ./image/
|
|
||||||
RUN CGO_ENABLED=0 go build -trimpath -o /out/dsh-mgr ./cmd/server
|
|
||||||
|
|
||||||
FROM debian:bookworm-slim
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*
|
|
||||||
WORKDIR /app
|
|
||||||
COPY --from=backend /out/dsh-mgr /usr/local/bin/dsh-mgr
|
|
||||||
COPY --from=web /src/web/dist ./web/dist
|
|
||||||
COPY image/ ./image/
|
|
||||||
USER 1000:1000
|
|
||||||
EXPOSE 8088
|
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 CMD ["/usr/local/bin/dsh-mgr", "-healthcheck"]
|
|
||||||
ENTRYPOINT ["/usr/local/bin/dsh-mgr"]
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
# 本机或可信团队试运行运维
|
|
||||||
|
|
||||||
本版本面向本机或受信任团队，不应直接用于公网不可信租户。DSH 与面板使用同源路径代理；Docker socket 权限等价于宿主 root。
|
|
||||||
|
|
||||||
## 启动
|
|
||||||
|
|
||||||
1. 复制 `.env.example` 为 `.env`，填写专用绝对目录 `DSH_HOST_DATA_ROOT` 和 12–72 字节的随机管理员密码。限制 `.env` 的读取权限，不要提交。管理员账号创建后该密码不再被读取，应从 `.env` 中移除，再运行 `docker compose up -d --force-recreate panel` 重建面板容器，清除容器环境变量中保留的明文密码（`docker inspect` 可见）。仅修改 `.env` 或重启原容器不会更新其环境变量；重建面板会短暂中断面板访问。
|
|
||||||
2. 面板以 UID/GID 1000 运行。首次部署前由管理员创建专用数据目录并赋予 1000:1000 所有权、0700 权限；不要对已有未知目录递归改权限。将 `stat -c %g /var/run/docker.sock` 的结果填入 `.env` 的 `DOCKER_GID`，以附加组访问 socket。然后运行 `docker compose up --build -d`。默认访问 `http://127.0.0.1:8088`；远程管理优先使用 SSH 本地端口转发，保留此访问地址。Compose 定义了 `dsh-mgr -healthcheck` 健康检查（探测本机 `/healthz`）；进程无响应时容器会被标记为 unhealthy，配合 `restart: unless-stopped` 在退出时自动拉起。
|
|
||||||
3. 登录后在镜像页提交精确 DSH 版本或查询最新版再手动构建。已有本地镜像会复用，但必须通过启动验证才允许创建实例。镜像不推送至仓库。
|
|
||||||
4. 创建实例后从面板进入 DSH。不要给 runtime 发布宿主端口或挂载 Docker socket。
|
|
||||||
|
|
||||||
首次启动创建管理员。修改环境变量不会重设已有账号密码。容器内 `/data` 必须对应 `HOST_DATA_ROOT` 指向的同一宿主目录，否则 sibling 容器会挂载错误数据。
|
|
||||||
|
|
||||||
新镜像构建前必须检查 Docker 存储空间：将 `docker info --format '{{.DockerRootDir}}'` 返回的绝对目录填入 `DSH_DOCKER_ROOT_DIR`。Compose 将该目录以相同路径只读挂载，面板只用 statfs 检查至少 2 GiB 可用空间；这不是授权浏览或修改 Docker 内部文件。映射无法确认、不可访问或空间不足时，新构建拒绝，已有本地镜像复用仍可进行。不要用面板数据盘冒充 Docker 存储盘。
|
|
||||||
|
|
||||||
## 公网 HTTPS 入口（仅可信账号）
|
|
||||||
|
|
||||||
反向代理与证书由部署管理员管理，面板仍建议只监听代理可达的私有入口。设置 `PANEL_PUBLIC_URL=https://你的域名`（仅 origin，不含业务子路径）；Compose 默认值仍为本机 HTTP。此值决定 Cookie 的 Secure 属性及 Origin/CSRF 校验，不从客户端的 X-Forwarded-Proto/Host 推断。
|
|
||||||
|
|
||||||
默认只按直连来源 IP 限流，不信任 X-Forwarded-For。经反代部署时，设置 `TRUSTED_PROXY_CIDRS` 为面板实际看到的代理来源地址/CIDR，且确保只有这些代理能连接后端、代理正确维护转发链；不要填写全网 CIDR。未设置时，代理后所有用户将共享该代理来源的每分钟登录额度。公网开放前应实际验证不同客户端来源识别、登录/刷新/退出、DSH iframe、WS/SSE及插件设置接口。
|
|
||||||
|
|
||||||
账号由管理员创建，不支持不可信租户。软配额、同源插件与 Docker socket 权限边界不因 HTTPS 而改变；本版本不承诺抵御分布式拒绝服务。
|
|
||||||
|
|
||||||
## 配置与全量备份
|
|
||||||
|
|
||||||
实例所有者有完整 home 读写权限，包括自己的凭据。通用文件管理、DSH 终端及 agent 不受专用 Profile 编辑器的 YAML 校验或停机流程限制；这不是 write-only 凭据系统。文件下载和目录 ZIP 可包含明文密钥，不等同于下述加密配置备份。提示可帮助避免误操作，但无法阻止所有者导出凭据。共享下载文件前应自行检查，修改运行中配置前建议先停机和备份。
|
|
||||||
|
|
||||||
面板提供加密配置与资源备份：固定配置、插件清单/锁文件、skins、pets、agent-presets、.agent-presets及manifest；不包含会话、node_modules或工作区。新包用创建时的登录密码经Argon2id派生密钥、AES-GCM加密，不依赖或创建 `backup.key`。导入/恢复使用原备份密码；修改登录密码不会重加密旧包。旧版密钥包仍兼容，只有这类包依赖原 `backup.key`，应单独安全保管，不能因升级删除。所有包均可能含凭据。
|
|
||||||
|
|
||||||
配置备份自动停止原运行实例并恢复运行；恢复先校验包，再快照、写入、启动验证，失败尝试回滚，成功保留原运行/停止状态。当前模板7启动前按清单协调插件；历史跨版本及离线失败验证范围见 ACCEPTANCE.md，不保证任意第三方插件兼容。配置内容上限32 MiB/4096文件，备份入口最多保留20个包；快照和保留上限需管理员监控清理。此流程不能代替全量备份。
|
|
||||||
|
|
||||||
要获得一致的全量备份，先通过面板停止所有实例，再停止面板，使用宿主备份工具复制专用数据目录（包含数据库、可能存在的 WAL 文件、工作区、实例 home、快照和密钥）。只停止面板不会自动停止 sibling 实例。恢复时保留权限和所有权，将整个目录恢复到同一部署的配置路径；不要将运行中的数据库文件单独覆盖。备份应在隔离环境定期验证。
|
|
||||||
|
|
||||||
## 升级与故障
|
|
||||||
|
|
||||||
### 镜像兼容验收与准入
|
|
||||||
|
|
||||||
管理员实验构建只负责生成并执行基础验证，不代表上线批准。准入前应在独立测试环境核验精确镜像的启动、认证/代理、终端与流式连接、文件读写、插件协调、配置备份恢复，以及所需升级/回滚路径；记录版本、模板、平台、镜像ID、测试范围和证据位置。不要在验收依据中粘贴凭据，也不要仅填写“构建成功”。本次实现不自动启动待验收镜像供普通用户使用，不自动运行真实部署测试。
|
|
||||||
|
|
||||||
在镜像页“准入管理”选择推荐或兼容并填写依据，后端还会检查精确镜像身份、成功验证记录和可支持的挂载布局。审批只作用于该镜像；新镜像需要自己的验证与审批。退为仅存量后不允许新建/升级目标；标为禁止使用还阻止新的启动/重启及受控配置修改、备份/恢复写操作，避免其自动重启绕过禁用。已有配置读取和通用home文件管理仍保留，不会自动停止正在运行的实例。若需要立即隔离已运行实例，管理员必须另行确认并执行停止操作。
|
|
||||||
|
|
||||||
历史home布局不同：模板7使用完整home作为 `/workspace`，受支持旧模板使用独立 `/data/dsh-home` 与用户共享 `/workspace`。不要把旧模板准入理解为数据布局可任意互换；不支持的反向迁移仍会拒绝，先做好完整备份。
|
|
||||||
|
|
||||||
运行时认证材料由面板首次成功连接实例时提取，并使用 `auth-signing.key` 派生的密钥加密保存在数据根的 `runtime-auth/`（不在用户home内）。按容器 ID 和启动时间绑定，不会将旧启动周期凭据用于新容器；不同 Host 分别缓存 cookie。全量备份应连同签名密钥和该目录保存，不要向用户公开。
|
|
||||||
|
|
||||||
运行时返回401/403时会使该Host的cookie缓存失效，下次请求重新交换。不会自动重放写请求，避免插件操作重复执行；遇到认证失败请重新加载页面后确认操作结果。若runtime自行轮转了启动token且旧token也不可交换，需要管理员处理，不能保证任意上游认证机制变更自动恢复。
|
|
||||||
|
|
||||||
升级到本版本后应在维护窗口逐个访问现有实例，完成首次认证持久化。若在首次成功连接前，启动token已被Docker日志轮转删除，本版本无法凭空恢复；先保存工作，再由管理员安排重启相关实例并访问验证。不要为此删除或重建实例。初次日志扫描上限64 MiB，常规每文件10 MiB/3份保留在此范围内。
|
|
||||||
|
|
||||||
- 实例升级选择已经启动验证的本地镜像并提供当前登录密码，面板自动停机、创建密码配置备份与home快照、启动验证及失败回滚，无需先手动停止。home快照最大1 GiB；允许受约束的内部相对链接以支持pnpm，拒绝不安全链接/特殊文件。模板7的 `/workspace` 是实例home，包含在home回滚内；旧共享数据不回滚。跨DSH版本历史验证范围见 ACCEPTANCE.md。
|
|
||||||
- Profile是例外：当前需先停止实例，再使用“保存并启动”；页面包含Monaco和脱敏配置预览。不要把备份/升级自动停启理解为Profile可在线改写。
|
|
||||||
- 若实例进入 `error`，不要直接修改数据库状态强制启动。保留原容器、实例目录与 `upgrade_journal` 记录，检查实际容器与 journal 的旧/新容器 ID、阶段、快照路径是否一致。
|
|
||||||
- 人工恢复前停止相关容器并另做完整备份。中断点不同，不能统一用一次目录覆盖或状态修改处理；当前无自动灾难恢复命令。
|
|
||||||
- 删除实例会删除其容器与网络，但保留数据目录。升级快照暂无自动清理；不要删除仍被未完成 journal 引用的快照。
|
|
||||||
|
|
||||||
## 创建与构建恢复记录
|
|
||||||
|
|
||||||
创建失败不代表 Docker 没有创建资源。面板在发请求前记录 `runtime_creates`（实例 ID、容器/网络结果待确认标志及其置位时间）及 `image_build_checks`（构建 ID、镜像 ID、已确认的验证容器 ID、记录创建时间），已确认网络身份保存在 `runtime_networks`。资源可见后按身份校验再清理；已记录 ID 时按 ID 查询，不能把名称消失当成资源删除。身份不一致时拒绝删除。构建验证残留在启动时及后台重试回收，无需新建构建任务触发。
|
|
||||||
|
|
||||||
创建请求失联后，确定性名称的 404 仍视为结果不确定，不会仅因经过一分钟而清除恢复依据：实例保持 `error` 并占用名额，资源迟到出现后可重试删除并按身份清理；验证容器的不确定记录保留并后台降频重试、审计记录 `build.cleanup_error`，不会每秒新增审计。即使恢复记录的时间戳缺失或损坏，也不能据此推断创建已结束。一直未出现的资源需要按下述人工确认流程处理，不能保证所有失联创建自动释放预约。删除实例成功后总是清理其 `runtime_creates`/`runtime_networks` 行。数据库暂时不可写时，构建终态事务独立重试，不阻挡后续构建；面板退出后未提交的任务按中断失败处理，不承诺恢复为成功。
|
|
||||||
|
|
||||||
人工确认流程（仅部署管理员，不能因单次404直接改库）：
|
|
||||||
|
|
||||||
1. 停止面板接收新操作并停止面板进程，备份数据库（含 WAL）及相关恢复记录。确认 Docker daemon 健康，并结合 daemon 日志确认原创建请求已结束；必要的 daemon 维护会影响其他容器，应单独安排。
|
|
||||||
2. 根据精确实例/构建 ID 核查 `dsh-inst-<instance-id>` 网络与容器，或 `dsh-build-check-<build-id>` 验证容器；已记录容器 ID 时还必须按 ID 查询，不能仅按名称判断不存在。核对归属标签、镜像及名称，不运行全局 prune。
|
|
||||||
3. 资源存在且身份一致时，优先恢复面板并走正常删除/自动回收；身份冲突则保留记录，先调查冲突来源。
|
|
||||||
4. 只有确认原请求终结、对应资源确实不存在后，才可在维护备份基础上用 SQLite 事务清除该精确 ID 的待确认标志，或删除该精确 build ID 的 `image_build_checks` 行。不得批量清空恢复表，不修改实例为 running。重新启动面板后重试正常删除，并核对实例名额、构建状态与 Docker 资源。
|
|
||||||
|
|
||||||
## 容量与网络
|
|
||||||
|
|
||||||
实例文件请求只在检查归属和登记使用状态时取得生命周期锁，上传/下载期间不持全局生命周期锁。同一实例存在文件操作时，启停、删除、配置修改、恢复及升级返回 409，等待传输结束后重试。文件树单目录最多列出 10000 项，超限返回 413；用量与快照遍历仍有总条目/深度上限。此限制不是文件系统硬配额。
|
|
||||||
|
|
||||||
用户软配额覆盖工作区与实例目录，已接入上传、Profile增长、恢复/升级快照的写前检查及写入串行化，并每5分钟巡检记录超额/恢复审计；覆盖范围见 ACCEPTANCE.md。它不能阻止DSH直接写盘或插件安装增长。监控数据目录和Docker存储空间，快照也消耗容量。硬quota需宿主文件系统和管理员配置，是可选增强，不是默认路线必做项。
|
|
||||||
|
|
||||||
每个实例配置独立bridge、非root、只读根文件系统和资源上限，但并非完整网络安全边界：未限制访问宿主、局域网或互联网。公网不可信租户需另外设计独立origin、可信TLS入口与出站策略，并按威胁模型评估硬配额。不要给runtime发布端口；server-held DSH cookie不下发浏览器，不能代替面板归属鉴权。
|
|
||||||
|
|
||||||
## 验收
|
|
||||||
|
|
||||||
按版本分层记录的验收结果见 [ACCEPTANCE.md](ACCEPTANCE.md)。以下命令是检查入口；历史通过记录不能替代当前版本回归，自动化测试不能替代实际公网域名与浏览器验收。
|
|
||||||
|
|
||||||
```sh
|
|
||||||
bash scripts/check-trial.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
该脚本执行本地自动化测试、race、静态检查及前后端构建，不启动容器、不安装依赖、不部署。需提前准备 Go/Node/npm 和前端依赖。可通过 `TEST_TMPDIR` 指定已存在的临时目录。镜像构建单测已隔离真实磁盘空闲量；生产磁盘检查仍保留。
|
|
||||||
|
|
||||||
`scripts/test-panel.mjs` 必须指向独立测试部署，通过 `TEST_PANEL_URL`、`TEST_USERNAME`、`TEST_PASSWORD` 指定访问参数；`TEST_UPGRADE=1` 包含同镜像替换升级。它会创建及删除测试实例、留下保留策略要求的数据，并可能构建或复用本地 runtime 镜像。不要对生产账号运行。
|
|
||||||
@@ -1,208 +1,171 @@
|
|||||||
# DSH Web Manager
|
# DSH Runtime Release
|
||||||
|
|
||||||
面向本机或受信任团队的DSH容器管理面板，使用Go、React和SQLite。目标见 [DESIGN.md](DESIGN.md)，接口见 [API.md](API.md)，部署和恢复见 [OPERATIONS.md](OPERATIONS.md)。不适合直接作为公网不可信多租户平台。
|
## runtime-7.11：DSH rc.5 与内置插件更新（待运行验收）
|
||||||
|
|
||||||
本项目定位为本机或可信团队试运行，可由管理员通过 HTTPS 反代提供公网入口；不支持不可信租户。选型差异及真实验证边界见 [ACCEPTANCE.md](ACCEPTANCE.md)。
|
DSH 从 `0.2.1-rc.3` 更新为 `0.2.1-rc.5`（查询时 npm `latest` 和 `next` 均为此版本，不切换 alpha 渠道）。free-search 从 `0.9.15` 更新为 `0.9.21`，Chat Manager 从 `1.5.18-dsh-mgr.1` 更新为 `1.5.20-dsh-mgr.1`。mobile `3.2.1` 已是最新；自有 workspace `1.0.0`、preview-download `1.0.1` 及运行布局 7 不变。
|
||||||
|
|
||||||
## DSH 工作台与移动端
|
同步安装归档 SHA-512、新 profile、已有内置归档迁移目标及精确宿主版本门禁，保留自定义来源、启用选择和备份机制。真实发布包的 Chat Manager 客户端补丁、原生设置接口及 Office converter 导入检查通过。48 项单元测试通过，新增 runtime-7.10 插件引用迁移、备份和幂等回归。完整镜像与运行验收结果另行记录；未提交、推送或修改线上实例。
|
||||||
|
|
||||||
登录后默认进入内嵌 DSH 工作台。在“设置”中选择当前账号的实例；选择仅保存在当前浏览器并按账号区分，不保存登录凭据。没有实例时先到“实例”创建，停止或异常的实例需先启动或处理。切换实例会重新加载内嵌页面，但不会停止原容器。
|
AMD64 镜像 `dsh-web:runtime-7.11-amd64` 已成功构建；新版本真实依赖、插件 SHA-512 和安装期兼容性检查全部通过。禁网 UID 1000 新 profile 离线安装、重复执行启动入口（`--version`）及锁文件 SHA-256 不变通过；默认 Web 服务启动后，3080/8080 未认证请求均为 HTTP 401。测试容器已清理，未挂载现有工作区。尚未验收浏览器交互、真实搜索、设置保存、Office 实际转换、终端原生模块及 ARM64；本机没有 runtime-7.10 镜像，旧镜像锁文件迁移集成测试未执行（配置迁移单元回归已通过）。
|
||||||
|
|
||||||
移动端使用底部导航：DSH、实例、文件、设置；桌面保留侧栏。镜像管理和管理员功能从设置进入。切换面板栏目会保留内嵌 DSH 页面，提供重新加载及独立窗口入口。此为同源 iframe，不是安全隔离边界，仍仅适合本机或受信任团队。
|
## runtime-7.10：DSH rc.3 与内置插件更新（待运行验收）
|
||||||
|
|
||||||
## 当前实现
|
沿用此前的 npm `next` 渠道，DSH 精确固定为 `0.2.1-rc.3`；查询时 `latest` 为 `0.2.1-rc.2`。mobile 更新为 `3.2.1`，free-search 为 `0.9.15`，Chat Manager 为 `1.5.18-dsh-mgr.1`（基于上游 `1.5.18`）。workspace `1.0.0`、preview-download `1.0.1` 和运行布局 7 保持不变。三个上游插件归档的 SHA-512 已核对并更新。
|
||||||
|
|
||||||
实例所有者可管理完整 home，包括读取自己的配置和凭据；不承诺密钥 write-only，跨用户隔离保持不变。通用文件操作不受专用 Profile 编辑器的 YAML 校验和停机要求约束。下载、打包与敏感修改提示用于防误操作，不是权限限制；文件下载和目录 ZIP 不会自动加密，修改配置前建议先停止实例并备份。
|
同步新 profile 和已有内置归档引用的迁移目标，保留自定义来源、插件启用选择、备份及锁文件同步机制。设置和 Office 补丁版本门禁对齐 rc.3，真实发布包的原生设置接口与 Office converter 导入入口检查通过。Chat Manager 的三处客户端适配仍精确匹配，新版真实源码补丁及语法检查通过；没有修改认证机制。mobile/free-search 的预发布 peer 例外仍只允许精确 rc.3 且满足 includePrerelease 范围。
|
||||||
|
|
||||||
- 内置账号、管理员配额、短JWT与refresh轮换、失败锁定及审计。
|
本次 `node --test image/*.test.mjs` 共 46 项通过，包含新增的 runtime-7.9 实际插件版本引用迁移、备份及幂等回归。单元测试和源码核查不等同于完整运行兼容性证明；浏览器交互、真实搜索、设置保存、Office 实际转换、终端原生模块、容器离线启动及 ARM64 仍需本版本验收。未提交、推送或修改线上实例。
|
||||||
- 精确版本查询、本地镜像异步构建/验证/清理、持久创建预约与事务名额（每用户最多两个实例）、启动对账。预约不再仅保存在当前页面。
|
|
||||||
- 实例启停删除、独立home（模板7挂载为 `/workspace`）、每实例bridge、归属鉴权及路径反代；DSH认证cookie由服务端保管。
|
|
||||||
- 文件上传/下载/重命名/删除及目录ZIP；软配额写前检查、写入串行化与周期巡检。上传最大32 MiB且不覆盖已有文件，删除目录须先清空；删除实例保留数据。
|
|
||||||
- 密码加密配置备份、资源目录与manifest、外部加密包导入、恢复健康检查及回滚。备份不再限于六个固定文件；不包含会话、工作区及node_modules。
|
|
||||||
- 备份/恢复/升级自动编排停启；升级保存密码配置备份和home快照，失败尝试回滚。Profile使用本地Monaco及脱敏预览；目前仍需先停止实例，页面“保存并启动”随后验证就绪。
|
|
||||||
|
|
||||||
新备份使用创建时的登录密码经Argon2id派生密钥、AES-GCM加密，不依赖或创建 `backup.key`。导入/恢复须使用原备份密码，修改登录密码不会重加密旧包。旧版密钥包仍兼容，保留这些包时必须保管原 `backup.key`。
|
锁文件集成脚本默认验证 `dsh-web:runtime-7.9-amd64` → `dsh-web:runtime-7.10-amd64`，也可使用 `DSH_OLD_IMAGE` / `DSH_NEW_IMAGE` 指定已构建的镜像：`node image/lockfile-upgrade-smoke.mjs`。须先准备两个镜像；本次尚未完成该集成检查，不沿用旧发布的通过结论。
|
||||||
|
|
||||||
## 启动面板
|
后续运行验收发现 preview-download 安装器仍限定 rc.2，已核对 rc.3 的操作插槽及 readBytes 接口并修正精确宿主门禁。新增安装器回归直接校验发布清单中的宿主版本可以完成打包调用、未经验证版本在写入前被拒绝；目前全部 47 项回归通过。
|
||||||
|
|
||||||
需要Linux 5.6+（openat2/renameat2）、Go 1.25+、Node/npm及本机Docker daemon。Docker socket权限等价宿主root。
|
AMD64 镜像 `dsh-web:runtime-7.10-amd64` 已成功构建（ID `sha256:57d99cad32380dfc837fd1365814943b53f96abc07e84df154d02b7baa95f0bf`）。MiSans 下载首次超时，原来源重试后校验通过。禁网 UID 1000 新 profile 离线依赖安装、默认启动、未认证 HTTP 401、重启及锁文件哈希不变均通过。Office 实际 DOCX/PPTX/DOC/PPT 并发转换、已安装 DSH convertBytes 集成、输出/ZIP 限制、超时和清理通过。以上结果更新此前待验收状态；浏览器完整交互、真实搜索、设置保存、终端原生模块、旧镜像锁文件迁移和 ARM64 尚未验收，不代表可直接发布。
|
||||||
|
|
||||||
```sh
|
## runtime-7.9：DSH next 与启动配置迁移
|
||||||
npm --prefix web ci
|
|
||||||
npm --prefix web run build
|
|
||||||
go build -o bin/dsh-mgr ./cmd/server
|
|
||||||
# 首次启动设置ADMIN_USERNAME及12–72字节强密码ADMIN_PASSWORD
|
|
||||||
./bin/dsh-mgr
|
|
||||||
```
|
|
||||||
|
|
||||||
默认访问 http://127.0.0.1:8088，数据在 `./data`。修改环境密码不会重置已有账号。容器部署使用 `.env.example` 和 `compose.yaml`，按运维文档准备UID/GID 1000数据目录、Docker socket附加组及Docker存储目录映射，再执行 `docker compose up --build -d`。Compose仅发布loopback。
|
锁文件同步修复：内置依赖迁移或补建 workspace 配置前，先持久化 `.dsh-lockfile-pending` 标记。仅带标记的启动安装显式传入 `--no-frozen-lockfile`，同步 `pnpm-lock.yaml`；成功后清除标记，后续启动恢复 CI 默认冻结检查。安装失败或超时保留标记并阻止应用启动，下次启动继续同步，不自动恢复旧 manifest。已有 manifest 备份机制保持不变。若旧版失败迁移已经留下“新 manifest＋旧锁文件”且没有标记，应先恢复匹配的旧 profile 备份再升级；不要直接删除锁文件。
|
||||||
|
|
||||||
## 公网 HTTPS 反向代理（Nginx / Caddy）
|
本次修复验证：45 项自动化测试通过；`node image/lockfile-upgrade-smoke.mjs` 使用本地 `dsh-upgrade-test:runtime-7.8` 与 `dsh-web:runtime-7.9-amd64` 镜像、独立临时卷、禁网非 root 容器，挂载当前修复脚本，验证真实旧锁文件迁移、随后冻结安装成功及重启锁文件哈希不变。脚本结束删除测试卷。本次未重跑面板 HTTP 升级接口或首页验收，未发布新镜像。
|
||||||
|
|
||||||
以下两种方案任选一种，假设反代运行在面板所在宿主机，域名为 `dsh.example.com`，后端为 `http://127.0.0.1:8088`。将示例域名、证书路径替换为实际值。仅支持独立域名根路径部署，不要挂在 `/panel/` 等子路径下；所有路径（包括 `/api/` 和 `/dsh/`）统一转发到面板，不直接代理或公开 runtime 的端口。公网入口仍仅供可信账号使用，不是多租户安全隔离方案。
|
DSH 固定为 `0.2.1-rc.2`（本次明确选择 npm `next`，查询时 `latest` 仍为 `0.2.0-rc.2`），mobile 升级为 `3.1.0`，Chat Manager 升级为明确声明支持新宿主的 `1.5.16-dsh-mgr.1`。free-search 升级为 `0.9.2`，workspace `1.0.0`、preview-download `1.0.1` 保持不变。
|
||||||
|
|
||||||
### 面板环境变量
|
现有 runtime 每次启动时检查 `profiles/web/package.json`，不另加容器、不联网追踪最新版。仅将 dependencies/devDependencies/optionalDependencies 中同名 `file:/usr/local/lib/dsh-mgr/<包名>-<版本>.tgz` 引用对齐镜像内置清单，并补齐相关缺失的 pnpm overrides。保留自定义安装来源、自定义 overrides、bundles、已删除的插件和其他配置。目标归档缺失、无效配置或不安全的符号链接会阻止启动。
|
||||||
|
|
||||||
使用 Compose 时在项目 `.env` 中设置；直接运行二进制时需通过进程环境传入（二进制不会自动读取 `.env`）：
|
只有配置变化才写入，原文保存在同目录 `package.json.backup-<UUID>`，新文件通过原子替换写入，权限 `0600`；重复启动不会反复改写。恢复时先停止实例并备份现状，再恢复对应备份；要回退旧内置版本必须同时使用匹配的旧镜像，否则下一次启动会再次对齐当前镜像。迁移不提供针对同 UID 恶意并发修改目录的沙箱保护。
|
||||||
|
|
||||||
```dotenv
|
新版设置 UI 已原生使用 host settings describe/mutate，构建只验证接口、不再应用旧 persistence 替换。Office 转换补丁仍限定精确宿主及插件版本；下载 slot/readBytes 和 workspaceRegistry 接口已核对上游源码。mobile/free-search 的预发布 peer 例外仅允许精确 `0.2.1-rc.2` 的 DSH 包且仍要求满足原版本范围（includePrerelease），其他依赖照常严格校验。
|
||||||
PANEL_PUBLIC_URL=https://dsh.example.com
|
|
||||||
# 仅适用于反代和面板二进制均在宿主机、通过127.0.0.1连接的情况
|
|
||||||
TRUSTED_PROXY_CIDRS=127.0.0.1/32
|
|
||||||
```
|
|
||||||
|
|
||||||
`PANEL_PUBLIC_URL` 必须与浏览器访问的 origin 一致，不带路径；非标准 HTTPS 端口需包含端口号。它控制 Origin/CSRF 校验和 Secure Cookie，不依赖 `X-Forwarded-Proto` 推断。修改环境变量后需重启面板进程；Compose 部署应执行 `docker compose up -d panel` 使容器读取新环境，单纯 `docker compose restart` 不会更新环境。
|
当前自动化回归 40 项通过；AMD64 镜像已构建，禁网 UID 1000 新 profile 安装、HTTP 未认证 401、旧 mobile/free-search/chat-manager 引用迁移、备份与幂等检查通过。Office 的已安装 DSH convertBytes 集成及 DOCX/PPTX/DOC/PPT 并发、限制、超时、清理测试通过。pnpm 会提示由宿主提供的 peer 缺失/范围冲突；这不是浏览器交互兼容性证明。浏览器完整交互、真实搜索结果、设置保存、终端原生模块和 ARM64 尚未验收。以下旧发布记录中的“已有 profile 不自动改写”仅适用于旧版本。
|
||||||
|
|
||||||
注意 Docker 网络差异：宿主机反代访问 Compose 发布端口时，面板看到的来源可能是 Docker 网桥网关，而不是 `127.0.0.1`。此时应将 `TRUSTED_PROXY_CIDRS` 改为确认过的实际代理来源 IP `/32`（IPv6 用 `/128`），不要照抄上述值，也不要信任 `0.0.0.0/0`、`::/0` 或整个不受控内网。未配置信任时，转发头会被忽略，所有用户可能共享代理 IP 的登录额度。可通过后端连接观测确认来源，Docker 网络 inspect 只能提供候选网关信息，不能替代实际确认。
|
## runtime-7.8：修复下载按钮服务依赖
|
||||||
|
|
||||||
如果反代也运行在容器内，`127.0.0.1` 指向反代容器自身，不能使用下面的后端地址。应让反代与面板加入专用 Docker 网络，将后端改为 `panel:8088`（实际服务名），并按面板看到的代理容器地址设置可信来源；不要在同一受信网络放入不可信容器。后端8088和runtime端口不应直接暴露公网。
|
下载插件升级为 dsh-preview-download@1.0.1，补齐 remote 服务声明，修复点击时在发送请求前抛出 cannot get property "remote" without inject 的问题。同步更新内置包和新 profile 引用，新增注册回调依赖回归测试。此修复不处理面板 401、Office 504 或 plugins/events HTTP/2 连接错误。
|
||||||
|
|
||||||
### Nginx
|
已有实例升级镜像时，必须把 profile 中 dsh-preview-download 依赖的归档路径从 dsh-preview-download-1.0.0.tgz 改为 dsh-preview-download-1.0.1.tgz，保留其他配置和 bundles，重启后刷新页面；已有 profile 不自动改写。尚未完成线上浏览器下载验收。
|
||||||
|
|
||||||
示例可放入由 `http {}` 引入的 `/etc/nginx/conf.d/dsh.conf`；`map` 必须位于 `http` 上下文，不能放在 `server` 或 `location` 内。证书需提前申请并配置续期；示例不负责签发证书。
|
## runtime-7.7：预览下载插件
|
||||||
|
|
||||||
```nginx
|
内置独立插件 dsh-preview-download@1.0.0，在文档预览工具栏提供「下载原文件」。使用正式 sidebar.right.tab.document.actions 插槽，不修改上游 UI、不扫描 DOM、不增加服务端下载接口。通过当前会话 workspaceFiles.readBytes 获取原始字节，沿用服务端权限和完整文件大小上限；支持中文文件名、下载中状态、失败提示及切换文件/卸载时取消读取。下载的是原始文件，不是 Office 转换出的 PDF。文件元数据无法读取时不会显示工具栏按钮。
|
||||||
map $http_upgrade $dsh_connection_upgrade {
|
|
||||||
default upgrade;
|
|
||||||
'' close;
|
|
||||||
}
|
|
||||||
|
|
||||||
server {
|
新建空 profile 默认启用。已有实例升级镜像后，在 /workspace/profiles/web/package.json 的 dependencies 中添加以下条目，并在 dsh.profile.bundles 中追加一次 dsh-preview-download，保留其他配置，然后重启：
|
||||||
listen 80;
|
|
||||||
server_name dsh.example.com;
|
|
||||||
return 308 https://dsh.example.com$request_uri;
|
|
||||||
}
|
|
||||||
|
|
||||||
server {
|
~~~json
|
||||||
listen 443 ssl;
|
"dsh-preview-download": "file:/usr/local/lib/dsh-mgr/dsh-preview-download-1.0.1.tgz"
|
||||||
server_name dsh.example.com;
|
~~~
|
||||||
|
|
||||||
ssl_certificate /etc/letsencrypt/live/dsh.example.com/fullchain.pem;
|
插件源码位于 image/preview-download，也可单独打包安装。无第三方运行时依赖，React 由 DSH 浏览器提供。当前仅针对 DSH 0.2.0-rc.2 验证。25 项离线回归测试通过，已在现有 AMD64 镜像中验证插件打包、新 profile 离线安装及服务启动；完整 runtime-7.7 构建在 MiSans 下载阶段超时，尚未完成浏览器实际点击下载、移动端和 ARM64 验收。不应把单元测试中的拒绝响应模拟当作真实权限端到端验收。
|
||||||
ssl_certificate_key /etc/letsencrypt/live/dsh.example.com/privkey.pem;
|
|
||||||
ssl_protocols TLSv1.2 TLSv1.3;
|
|
||||||
|
|
||||||
# 为multipart封装留余量；应用仍执行自己的上传/备份上限
|
## runtime-7.6：原生 Office 文档预览
|
||||||
client_max_body_size 64m;
|
|
||||||
client_body_timeout 60s;
|
|
||||||
|
|
||||||
location / {
|
Linux 文档预览通过构建期补丁将 Office-to-PDF 转换入口接到系统 LibreOffice，不再使用该入口原先默认的 WASM 后端。补丁限定 DSH 和 dsh-office-to-pdf 为 0.2.0-rc.2，目标 import 不匹配或重复应用时构建失败。保留上游授权、队列、缓存和 PDF 验证；不修改独立 dsoffice CLI 或其他插件。
|
||||||
proxy_pass http://127.0.0.1:8088;
|
|
||||||
proxy_http_version 1.1;
|
|
||||||
proxy_set_header Host $http_host;
|
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
|
||||||
proxy_set_header Connection $dsh_connection_upgrade;
|
|
||||||
|
|
||||||
# 单层公网边缘代理：覆盖客户端提供的转发来源，不能原样信任
|
每次转换使用私有临时目录、独立 profile、最高宏安全级别和独立进程组；保留默认 60 秒超时，取消时终止进程组。检查输入大小、OOXML 条目数、解压总量和 CRC、输出大小及完整性。子进程采用最小环境变量；校验器地址空间上限 512 MiB，LibreOffice 为 2 GiB，单文件写入上限为 max(maxOutputBytes, 16 MiB)，最终 PDF 仍严格检查 maxOutputBytes。进程资源限制不是进程树总量或临时目录总量限制，部署时仍须容器内存、CPU、PID 和临时磁盘限制。
|
||||||
proxy_set_header X-Forwarded-For $remote_addr;
|
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
|
||||||
proxy_set_header X-Forwarded-Proto $scheme;
|
|
||||||
proxy_set_header X-Forwarded-Host $http_host;
|
|
||||||
|
|
||||||
proxy_buffering off;
|
安全边界：原生 LibreOffice 与 DSH 使用同一 UID，独立 profile 不是文件系统或网络沙箱；不应将此功能视为隔离恶意文档的服务。容器内敏感文件及网络访问仍需部署侧隔离，尤其是处理不可信外部文档时。
|
||||||
proxy_request_buffering off;
|
|
||||||
proxy_cache off;
|
|
||||||
proxy_connect_timeout 10s;
|
|
||||||
proxy_read_timeout 3600s;
|
|
||||||
proxy_send_timeout 3600s;
|
|
||||||
send_timeout 3600s;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
`Upgrade/Connection` 支持 WebSocket，关闭响应缓冲可及时传送 SSE；读写超时指相邻读写的空闲时间，不是保证连接永久存活。不要改写或删除 `Origin`、Cookie、`X-CSRF-Token`，不要额外添加阻止同源 iframe 的 `X-Frame-Options: DENY`。
|
字体由系统解析；显式 fontDirectories/fontFallbacks 配置会被拒绝，应移除并改用系统字体配置。kit 的 maxFontFiles/maxFontFileBytes/maxLoadedFontBytes 不再约束系统字体。missingFonts 为空仅表示不提供缺失字体诊断，不代表没有字体替换。
|
||||||
|
|
||||||
保存后先检查，再由管理员重载：
|
已完成 AMD64 构建、22 项回归测试，以及非 root/禁网容器中 DOCX/PPTX/DOC/PPT 并发转换、已安装 DSH convertBytes 集成、输出/压缩包限制、超时及清理验证。小型单页样本约 3 秒/文件，不代表实际文档性能保证。ARM64、真实失败样本、浏览器端到端、XLS/XLSX 和视觉版式尚未验收。
|
||||||
|
|
||||||
```sh
|
构建后在仓库根目录运行：
|
||||||
sudo nginx -t
|
|
||||||
# 仅在检查成功后执行
|
|
||||||
sudo systemctl reload nginx
|
|
||||||
```
|
|
||||||
|
|
||||||
### Caddy
|
~~~sh
|
||||||
|
docker run --rm --network none --init \
|
||||||
|
--mount type=bind,src="$PWD/image",dst=/test,readonly \
|
||||||
|
--entrypoint node dsh-web:runtime-7.6-amd64 /test/office-preview-smoke.mjs
|
||||||
|
~~~
|
||||||
|
|
||||||
将以下内容加入 `/etc/caddy/Caddyfile`。域名 DNS 应指向该主机，80/443 需按证书签发方式可达；Caddy 默认自动申请/续期 HTTPS 证书并将 HTTP 重定向到 HTTPS。
|
现有实例需升级新镜像才生效，不需要清空工作区。不自动提交、推送或升级线上实例。
|
||||||
|
|
||||||
```caddyfile
|
## runtime-7.5：精简 MiSans 字重
|
||||||
dsh.example.com {
|
|
||||||
request_body {
|
|
||||||
max_size 64MB
|
|
||||||
}
|
|
||||||
|
|
||||||
reverse_proxy 127.0.0.1:8088 {
|
runtime-7.5 修订仅保留 MiSans Regular 和 Bold，从第三方 GitHub 仓库 boyan01/mi_sans_font 的固定提交下载独立 TTF 并逐个校验 SHA-256，合计约 15.73 MB，不再下载官方完整 ZIP。构建环境需要能够访问 raw.githubusercontent.com。
|
||||||
# 单层公网边缘代理，覆盖外部传入的XFF
|
|
||||||
header_up X-Forwarded-For {http.request.remote.host}
|
|
||||||
flush_interval -1
|
|
||||||
transport http {
|
|
||||||
dial_timeout 10s
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Caddy 自动支持 WebSocket，默认保留请求 Host，并设置转发协议信息；`flush_interval -1` 用于及时刷新流式响应。此处不设置较短的响应读取/流生命周期超时，避免中断长时间 SSE/终端连接。Caddy 重载可能关闭现有 WebSocket，客户端需重连。
|
## runtime-7.4：文档工具、中文字体与 Chat Manager 修复
|
||||||
|
|
||||||
```sh
|
runtime-7.4 修订内容：新增 Python、LibreOffice、Noto CJK 与 MiSans，并内置修复版 Chat Manager。DSH 版本仍为 0.2.0-rc.2，运行布局仍为 7，继续保留 runtime-7.3 的域名设置适配。已完成本地 AMD64 构建、19 项测试、离线启动及插件接口认证拒绝检查；ARM64 与线上面板完整删除流程尚未验收。
|
||||||
sudo caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile
|
|
||||||
# 仅在检查成功后执行（使用systemd安装时）
|
|
||||||
sudo systemctl reload caddy
|
|
||||||
```
|
|
||||||
|
|
||||||
以上示例均假设客户端直接连接这层反代。若前方还有 CDN、负载均衡或第二层代理，不能直接依赖这里的 `$remote_addr` / `remote.host` 得到真实客户端 IP；需要在边缘代理单独配置严格的可信上游及来源解析，并相应调整转发策略，否则用户会共享上一层代理的限流额度。
|
## 内置修复版 Chat Manager
|
||||||
|
|
||||||
### 部署后检查
|
镜像内置 `[email protected]`（基于官方 1.5.12，保留 MIT 许可及服务端安全校验）。补丁仅针对客户端：删除、恢复、归档搜索在面板内使用当前 `/dsh/<实例ID>/` 前缀，独立 DSH 根路径部署保持原行为；非 JSON 响应显示 HTTP 状态和代理排查提示，不再直接暴露 JSON 解析异常。不会修改全局 fetch、绕过 Cookie/Origin/可信主机校验或自动重试删除请求。
|
||||||
|
|
||||||
- 确认 HTTP 跳转 HTTPS，证书有效，公网只能到达反代而不能直接访问面板8088或runtime端口。
|
新建空 web profile 默认启用此插件，内置包和 clsx 依赖均支持离线安装。已有 profile 不会被覆盖，也不会自动替换用户已安装的官方版本。升级镜像后，如需迁移已有实例，请保留其他设置，将 profile 中同名依赖改为 `file:/usr/local/lib/dsh-mgr/dsh-chat-manager-1.5.12-dsh-mgr.1.tgz`，并设置 pnpm override `dsh-chat-manager>clsx` 为 `file:/usr/local/lib/dsh-mgr/clsx-2.1.1.tgz`，确保 bundles 中只有一次 `dsh-chat-manager`，再重启实例。不要同时启用重复的手动 patch 插件条目。
|
||||||
- 通过实际域名完成登录、刷新、退出，确认无 Origin/CSRF 403，认证 Cookie 带 Secure；直接使用后端HTTP测试不能替代HTTPS验收。
|
|
||||||
- 打开内嵌 DSH、终端/WebSocket、SSE及插件设置；空闲后重连、登录续期均需验证。
|
|
||||||
- 验证文件上传/下载及备份导入；应用限制仍有效，反代64MB上限不代表单文件可上传64MB。
|
|
||||||
- 使用不同真实客户端确认登录限流来源隔离；测试应使用专用账号，避免锁定实际账号。
|
|
||||||
|
|
||||||
这些是配置模板，不代表已在你的公网域名部署或验收；应用侧测试和运维边界见 [ACCEPTANCE.md](ACCEPTANCE.md) 与 [OPERATIONS.md](OPERATIONS.md)。
|
域名访问仍要求 DSH 配置相应 trusted host，且 runtime 必须经过面板认证与实例归属校验。此修复不开放面板根路径的插件接口。
|
||||||
|
|
||||||
## 用户密码重设
|
## 内置 Python 与 LibreOffice
|
||||||
|
|
||||||
所有已登录用户（包括管理员）可在「设置 → 账号 → 修改密码」输入当前密码、新密码及确认新密码。当前密码必须验证通过；成功后撤销该账号全部旧会话并返回登录页。忘记当前密码时仍需联系管理员，不提供匿名找回入口。
|
Runtime 镜像包含 Debian Bookworm 的 Python 3、pip、venv（`python` 指向 `python3`），以及 LibreOffice 和 Noto CJK 中文字体。LibreOffice 可通过 `libreoffice --headless` 无界面运行。Python 项目依赖请安装到工作区的虚拟环境中，不要覆盖系统 Python 包。
|
||||||
|
|
||||||
管理员可在「设置 → 用户与审计 → 用户列表 → 重设密码」为用户设置新密码（12–72 UTF-8字节，二次输入及确认）。重设会撤销目标账号全部登录会话和refresh令牌、清除登录失败锁定；旧密码不能再登录，审计仅记录操作者和目标，不记录密码。重设管理员自身会返回登录页。已建立的DSH WebSocket/SSE连接不保证在重设瞬间断开，后续HTTP请求、连接重建与会话续期会被拒绝。
|
这些依赖在镜像构建时安装；已有实例需要重建镜像并升级后才生效。
|
||||||
|
|
||||||
## 本地runtime构建
|
另包含小米 MiSans 简体中文字体的 Regular 和 Bold 两个静态 TTF 字重，保留 Noto CJK，不修改默认字体优先级。构建时从第三方镜像 boyan01/mi_sans_font 的固定提交 68f3ec3fee5830ce8a1d955d462fa0df401a5341 下载两个 TTF 并校验各自 SHA-256；镜像文件与当前官网 ZIP 不同，不保证为最新版本，GitHub 仓库许可不替代小米字体许可。字体及声明位于 `/usr/local/share/fonts/misans`。MiSans 使用小米专有免费商用许可，并非开源字体许可；安装使用及对外分发镜像前应阅读[官方许可协议](https://hyperos.mi.com/font/zh/download/)，不得单独分发或售卖字体。
|
||||||
|
|
||||||
### 兼容清单与实验构建
|
## runtime-7.3：域名访问设置适配
|
||||||
|
|
||||||
镜像的构建验证与使用准入分开管理：推荐/兼容镜像还必须通过精确身份与实际验证检查才可选择；仅存量镜像允许原实例继续运行和启停，不可新建或作为升级目标；禁止使用镜像还拒绝新的启动/重启，但不会自动停止现有容器或断开工作台。
|
自 runtime-7.3 起，构建时对 DSH 0.2.0-rc.2 的设置 UI 应用精确补丁，使设置视图与表单使用 host 持久化；不修改 connection.isLoopback、可信主机、Origin 或认证逻辑。版本或目标语句不匹配时构建失败，升级上游版本必须重新审核补丁。
|
||||||
|
|
||||||
未知版本仅管理员可在镜像页显式提交实验构建。实验保留镜像模板的完整性、依赖和启动检查，可能因不兼容而失败；不会为让新版通过而跳过检查。实验成功后仍不可创建实例，管理员需完成独立兼容验收，再在“准入管理”填写可追溯依据并确认。准入绑定精确镜像ID、版本、模板和平台，不自动信任同tag的新镜像，也不扩大普通版本构建白名单；已准入的未知版本请通过本地镜像选择创建。
|
这是构建期适配，不是普通插件。补丁本身不识别面板登录状态，该镜像的设置 UI 会对所有访问地址尝试远程设置操作，因此只能部署在受认证、校验实例归属的面板入口之后，不得绕过认证直接公开 runtime 端口。权限由服务端保证。
|
||||||
|
|
||||||
历史模板按其自身验证记录与挂载布局判断，不因模板号不是当前版本就标为缺失。有成功验证记录、布局受支持的旧镜像可经管理员验收准入；无法识别的布局不会仅凭一段说明放行。界面展示实际准入原因，以后端校验为准。
|
发布前须在非 localhost 域名验证提供商目录、配置与凭据保存、刷新和重启持久化，以及未登录、跨用户和非法 Origin 的拒绝行为。单元测试不代替这些验收；现有实例需升级镜像才生效，不覆盖用户 profile 或创建空 patch。
|
||||||
|
|
||||||
当前模板为修订7，同时内置 `[email protected]` 与 `[email protected]`（MIT，https://github.com/DDDMUC/dsh-free-search）。Free Search及其普通依赖固定版本、校验SHA-512并保留镜像内原始tarball；新profile默认正式安装并启用，已有profile不自动注入。模板4的mobile恢复路径继续保留。仅包含内置固定依赖的profile在启动前离线安装，不限制运行期插件管理器联网。
|
`dockerfile` 分支只维护发布清单和 runtime 镜像构建文件，不包含管理面板源码或部署配置。
|
||||||
|
|
||||||
Free Search默认引擎为Bing，注册的统一provider ID为ddg；未设置或等于deepseek-official的搜索provider会被它接管（包括显式选择同名官方provider），默认http抓取也会被接管，其他自定义provider保留。名称含free不等于保证零费用：自动回退可能读取已有API凭据并调用计费服务。请在插件设置中按需禁用引擎；fallback顺序不是引擎白名单。内置集成不自动测试搜索、配置密钥或执行在线更新。
|
## 发布清单
|
||||||
|
|
||||||
插件设置页的八个固定API路径由面板在实例归属、登录和同源检查后适配前缀及loopback Host，不开放匿名根路径接口。用户在插件内更新到其他版本可能改变接口或离线依赖，需重新验证兼容性。旧实例需显式安装/启用；升级镜像不会覆盖其已有插件选择。
|
`release.json` 是发布入口：`schema_version` 为清单格式版本，`release_id` 标识一次发布，`version` 为精确 DSH 版本，`deployment_schema` 为运行布局版本，`platforms` 为支持的平台。当前待验收配置为 `runtime-7.11`、DSH `0.2.1-rc.5`、布局 7、`linux/amd64` 和 `linux/arm64`（aarch64，不含 32 位 ARM）。
|
||||||
|
|
||||||
模板修订4默认内置 `[email protected]`（MIT，来源 https://github.com/mexiaosqwq/dsh-web-mobile）。构建校验固定SHA-512、不执行插件安装脚本；新建web profile使用镜像内tarball作为正式依赖并默认启用bundle，可在Plugins的Installed列表查看及启停。为兼容旧profile仍保留宿主副本；UI可能显示Uninstall，但本轮未验证卸载行为，不承诺移除镜像内置副本。已有profile不会被覆盖或强制启用；模板3旧实例须在升级后补正式依赖登记，仅升级镜像不会自动出现在Installed列表。该插件按触屏能力启用移动布局，仅缩窄桌面窗口不一定触发。
|
ARM64 验收状态：已确认 Node/Caddy 基础镜像提供 ARM64 版本，但尚未完成 ARM64 runtime 构建及运行验收。清单声明会开放面板的对应平台构建入口，不等于兼容性证明；验收完成前不要推送发布。
|
||||||
|
|
||||||
依赖固定引用 `file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.0.5.tgz`，首次安装与备份恢复无需下载该包。备份包含依赖和bundle清单，恢复目标必须提供相同tarball路径（模板4及兼容后续镜像）；初始化会在此依赖存在且workspace缺失时补建官方no-auto-peers配置，保留已有workspace及禁用状态。
|
此分支是可信代码来源，不是仅供读取的数据源：Dockerfile 和安装脚本会在构建时执行。只使用经过审核的提交，不要将不可信分支、PR 或任意远端内容直接用于构建。清单本身不代表镜像已经完成运行验收。
|
||||||
|
|
||||||
需要Docker、Bash、npm及基础镜像/npm网络访问；不推送镜像。
|
## 本地构建与检查
|
||||||
|
|
||||||
注意：dsh-web-mobile还包含响应压缩、移动会话删除入口和自定义模型推理档位补全，并非纯CSS。内置版本随镜像固定，不跟踪GitHub main或npm latest。
|
需要 Docker、Node.js，以及拉取基础镜像和构建依赖的网络。从仓库根目录执行，DSH 版本取自清单，平台按 Docker daemon 架构选择，不跟踪 latest：
|
||||||
模板4目前限定DSH 0.2.0-rc.1/rc.2，并校验已安装宿主的Cordis/DSH peer版本；其他版本需先更新兼容验证，不会自动安装第二套宿主依赖。插件没有独立参数配置页；它修改DSH原有移动界面与设置页。
|
|
||||||
|
|
||||||
```sh
|
~~~sh
|
||||||
bash scripts/build-image.sh # 查询latest并锁定精确版本
|
set -eu
|
||||||
bash scripts/build-image.sh 0.2.0-rc.2 # 或指定精确版本
|
DSH_VERSION="$(node -p "require('./release.json').version")"
|
||||||
bash scripts/smoke-image.sh dsh-web:0.2.0-rc.2-7
|
case "$(docker info --format '{{.Architecture}}')" in
|
||||||
```
|
aarch64|arm64) PLATFORM=linux/arm64 ;;
|
||||||
|
x86_64|amd64) PLATFORM=linux/amd64 ;;
|
||||||
|
*) echo 'Unsupported Docker daemon architecture' >&2; exit 1 ;;
|
||||||
|
esac
|
||||||
|
RELEASE_ID="$(node -p "require('./release.json').release_id")"
|
||||||
|
node --test image/*.test.mjs
|
||||||
|
docker build --platform "$PLATFORM" \
|
||||||
|
--build-arg DSH_VERSION="$DSH_VERSION" \
|
||||||
|
-t "dsh-web:$RELEASE_ID-${PLATFORM#linux/}" ./image
|
||||||
|
~~~
|
||||||
|
|
||||||
当前tag为 `dsh-web:<version>-7`（模板修订7）。runtime以UID/GID 1000运行，home和workspace需允许该身份读写。DSH监听loopback:3080，Caddy监听容器内8080；不要发布runtime端口或挂载socket。原生模块需要可执行的 `/tmp`，保留nosuid/nodev和容量限制。存在插件清单时先协调插件，失败不启动应用；真实验证范围见 ACCEPTANCE.md。
|
默认按 Docker daemon 架构原生构建，不再固定取清单第一个平台。在交叉构建环境可手动将 PLATFORM 设置为 linux/arm64 或 linux/amd64；需要事先配置对应原生 builder 或 QEMU/binfmt，仅设置 --platform 不会自动提供模拟能力。
|
||||||
|
|
||||||
## 设计替代与边界
|
Dockerfile 两个阶段均继承目标平台，无需写死架构；不要将 Caddy 阶段固定为构建主机平台，也不要复用另一架构的 node_modules。
|
||||||
|
|
||||||
构建历史接口现默认分页（20条，最大100条），列表不再携带日志正文；日志通过 `/api/builds/{id}` 按需读取。升级面板后请刷新已打开的旧页面；外部脚本需按 API.md 更新，不能再依赖 `/api/builds` 返回全部历史或日志。
|
发布前分别验证两个架构的镜像构建、非 root 启动、代理认证、终端及原生模块、内置插件离线协调、工作区写入和重启持久化，以及升级/恢复路径。迁移工作区时需重新安装其中的架构相关依赖。
|
||||||
|
|
||||||
合理替代方案为：stdlib HTTP Docker client而非官方SDK；每实例独立bridge而非共享bridge；服务端保管DSH cookie而非下发浏览器；普通React组件而非shadcn/ui（保留Monaco）。这些是选型差异，不降低功能、安全或验收标准，详见验收表。
|
`image/` 是构建上下文。构建只生成本地镜像，不推送镜像，也不自动发布分支。保留 `image/init-profile.test.mjs` 作为独立 runtime 回归测试，覆盖初始化幂等性、保留已有配置和拒绝符号链接目录；它不依赖面板，且被 `image/.dockerignore` 排除在构建上下文之外。此测试不能替代真实容器启动、插件和持久化验收。
|
||||||
|
|
||||||
同源路径代理仅适用于可信用户/插件，尚无独立origin及宿主/局域网出站防护。默认软配额不能阻止agent直接写满磁盘；硬quota是依赖宿主的可选增强，不是默认路线必做项。升级/恢复快照需额外容量，home快照上限1 GiB，模板7的 `/workspace` 就是实例home，会随home快照回滚；旧共享目录不随回滚恢复；中断及回滚失败仍可能需要管理员恢复。
|
## 布局 7 合约
|
||||||
|
|
||||||
|
- 以 UID/GID 1000 运行；实例持久目录挂载到 `/workspace`，必须允许该身份读写。`DSH_HOME`、`HOME` 和工作目录均为 `/workspace`，XDG 数据与配置也位于其中。
|
||||||
|
- `/data/dsh-home` 是指向 `/workspace` 的兼容符号链接；不要用旧 home 挂载覆盖该路径。`/shared` 不是布局 7 的持久化入口。
|
||||||
|
- DSH 监听 loopback 3080，Caddy 在容器 8080 转发；只通过受信入口访问，不直接公开 runtime 端口，不挂载 Docker socket。原生模块需要可执行的 `/tmp`，可保留 nosuid/nodev 与容量限制。
|
||||||
|
- 新 profile 默认启用固定版本的 mobile、free-search 和 workspace bundle，默认工作区为 `/workspace`。已有非空 profile 不自动注入或覆盖；升级镜像不会强制改变已有插件选择。
|
||||||
|
- 启动前协调插件依赖，失败则不启动应用；仅含内置固定依赖的 profile 使用离线安装。用户添加插件后可能需要网络。
|
||||||
|
|
||||||
|
## 手动发布
|
||||||
|
|
||||||
|
1. 审核 runtime 资产变更及其可信来源，选择精确 DSH 版本，更新 `release.json`。每次发布变更都必须使用新的 `release_id`；布局合约变化时同步更新 `deployment_schema`、镜像及调用方适配，不能只修改数字。
|
||||||
|
2. 运行上述测试与本地构建，并完成目标平台的容器启动、插件、持久化与兼容验收；检查 `git diff --check` 和最终差异。
|
||||||
|
3. 维护者确认后，**手动提交并手动推送** `dockerfile` 分支到已核实的发布远端。修改清单或本地构建不会自动 commit/push，也不会自动更新运行中的实例。
|
||||||
|
|
||||||
|
## 文件范围
|
||||||
|
|
||||||
|
- `release.json`：发布清单。
|
||||||
|
- `README.md`：本分支说明。
|
||||||
|
- `image/Dockerfile`、`image/Caddyfile`、`image/entrypoint.sh`、`image/.dockerignore`：镜像构建与启动。
|
||||||
|
- `image/install-*.mjs`、`image/init-profile.mjs`：内置 runtime 资产安装与 profile 初始化。
|
||||||
|
- `image/init-profile.test.mjs`：保留的 profile 初始化回归测试。
|
||||||
|
- `image/entrypoint.test.mjs`：从面板测试迁移的 runtime 启动失败保护与镜像约束测试，需要 Bash；不依赖 Go 或面板。
|
||||||
|
|||||||
@@ -1,167 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/api"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
if len(os.Args) > 1 && os.Args[1] == "-healthcheck" {
|
|
||||||
if err := healthcheck(); err != nil {
|
|
||||||
log.Printf("healthcheck failed: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := run(); err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// healthcheck probes the local /healthz endpoint for container health
|
|
||||||
// reporting; it avoids shipping curl into the runtime image.
|
|
||||||
func healthcheck() error {
|
|
||||||
addr := getenv("LISTEN_ADDR", "127.0.0.1:8088")
|
|
||||||
if host, port, err := net.SplitHostPort(addr); err == nil && (host == "" || host == "0.0.0.0" || host == "::") {
|
|
||||||
addr = net.JoinHostPort("127.0.0.1", port)
|
|
||||||
}
|
|
||||||
client := &http.Client{Timeout: 5 * time.Second}
|
|
||||||
resp, err := client.Get("http://" + addr + "/healthz")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Errorf("healthz returned %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func run() error {
|
|
||||||
if err := api.ValidateAuthConfig(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
root, err := filepath.Abs(getenv("CONTAINER_DATA_ROOT", getenv("DATA_ROOT", "./data")))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err = os.MkdirAll(root, 0700); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if os.Getenv("HOST_DATA_ROOT") == "" {
|
|
||||||
os.Setenv("HOST_DATA_ROOT", root)
|
|
||||||
}
|
|
||||||
s, err := store.Open(filepath.Join(root, "db.sqlite"))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer s.Close()
|
|
||||||
var count int
|
|
||||||
if err = s.DB.QueryRow("SELECT COUNT(*) FROM users").Scan(&count); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if count == 0 {
|
|
||||||
password := os.Getenv("ADMIN_PASSWORD")
|
|
||||||
if len(password) < 12 || len(password) > 72 {
|
|
||||||
return fmt.Errorf("first startup requires ADMIN_PASSWORD of 12–72 bytes")
|
|
||||||
}
|
|
||||||
hash, e := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
|
||||||
if e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
if err = s.CreateUser(store.User{ID: store.ID(), Username: getenv("ADMIN_USERNAME", "admin"), Admin: true, Quota: api.DefaultQuota()}, string(hash)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
panel := api.New(s, root)
|
|
||||||
if err := panel.ConfigError(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
checkCtx, checkCancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
||||||
if err := panel.Reconcile(checkCtx); err != nil {
|
|
||||||
log.Printf("startup reconciliation unavailable; Docker connectivity must be checked: %v", err)
|
|
||||||
}
|
|
||||||
checkCancel()
|
|
||||||
if err := panel.StartImageBuilds(); err != nil {
|
|
||||||
return fmt.Errorf("image build recovery: %w", err)
|
|
||||||
}
|
|
||||||
backend := panel.Handler()
|
|
||||||
static := http.FileServer(http.Dir(getenv("WEB_DIR", "web/dist")))
|
|
||||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
||||||
w.Header().Set("Referrer-Policy", "no-referrer")
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/api/") || strings.HasPrefix(r.URL.Path, "/dsh/") {
|
|
||||||
backend.ServeHTTP(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.URL.Path == "/healthz" {
|
|
||||||
w.Write([]byte("ok"))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet && r.Method != http.MethodHead {
|
|
||||||
http.Error(w, "method not allowed", 405)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// The embedded runtime visits the same-origin SPA renewal bridge.
|
|
||||||
// Keep cross-origin framing blocked without blocking that bridge.
|
|
||||||
w.Header().Set("X-Frame-Options", "SAMEORIGIN")
|
|
||||||
w.Header().Set("Content-Security-Policy", "frame-ancestors 'self'")
|
|
||||||
static.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
srv := &http.Server{Addr: getenv("LISTEN_ADDR", "127.0.0.1:8088"), Handler: handler, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 60 * time.Second, MaxHeaderBytes: 1 << 20}
|
|
||||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
||||||
defer stop()
|
|
||||||
if err := panel.StartIntents(ctx); err != nil {
|
|
||||||
return fmt.Errorf("creation intent recovery: %w", err)
|
|
||||||
}
|
|
||||||
go panel.RunQuotaMonitor(ctx)
|
|
||||||
go panel.RunSessionCleanup(ctx)
|
|
||||||
return serve(ctx, srv)
|
|
||||||
}
|
|
||||||
|
|
||||||
func serve(ctx context.Context, srv *http.Server) error {
|
|
||||||
drained := make(chan struct{})
|
|
||||||
finished := make(chan struct{})
|
|
||||||
defer close(finished)
|
|
||||||
go func() {
|
|
||||||
defer close(drained)
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
case <-finished:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := srv.Shutdown(c); err != nil {
|
|
||||||
log.Printf("graceful shutdown deadline reached: %v", err)
|
|
||||||
_ = srv.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
log.Printf("dsh panel listening on %s", srv.Addr)
|
|
||||||
err := srv.ListenAndServe()
|
|
||||||
if errors.Is(err, http.ErrServerClosed) {
|
|
||||||
<-drained
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
func getenv(k, d string) string {
|
|
||||||
if v := os.Getenv(k); v != "" {
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
@@ -1,117 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestInvalidAuthConfigFailsBeforeDataCreation(t *testing.T) {
|
|
||||||
for _, key := range []string{"PANEL_PUBLIC_URL", "TRUSTED_PROXY_CIDRS"} {
|
|
||||||
t.Run(key, func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_PUBLIC_URL", "")
|
|
||||||
t.Setenv("TRUSTED_PROXY_CIDRS", "")
|
|
||||||
t.Setenv(key, "invalid")
|
|
||||||
root := filepath.Join(t.TempDir(), "not-created")
|
|
||||||
t.Setenv("CONTAINER_DATA_ROOT", root)
|
|
||||||
if err := run(); err == nil || !strings.Contains(err.Error(), key) {
|
|
||||||
t.Fatalf("unexpected error: %v", err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(root); !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("data directory created: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestServeWaitsForActiveHandler(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
address := make(chan string, 1)
|
|
||||||
entered := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
srv := &http.Server{Addr: "127.0.0.1:0", BaseContext: func(l net.Listener) context.Context {
|
|
||||||
address <- l.Addr().String()
|
|
||||||
return context.Background()
|
|
||||||
}, Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
close(entered)
|
|
||||||
<-release
|
|
||||||
_, _ = w.Write([]byte("finished"))
|
|
||||||
})}
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- serve(ctx, srv) }()
|
|
||||||
var addr string
|
|
||||||
select {
|
|
||||||
case addr = <-address:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("server did not listen")
|
|
||||||
}
|
|
||||||
requestDone := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
resp, err := http.Get("http://" + addr)
|
|
||||||
if err == nil {
|
|
||||||
_, err = io.Copy(io.Discard, resp.Body)
|
|
||||||
resp.Body.Close()
|
|
||||||
}
|
|
||||||
requestDone <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-entered:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("handler did not start")
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
close(release)
|
|
||||||
t.Fatalf("returned before handler completed: %v", err)
|
|
||||||
case <-time.After(100 * time.Millisecond):
|
|
||||||
}
|
|
||||||
close(release)
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("shutdown stalled")
|
|
||||||
}
|
|
||||||
if err := <-requestDone; err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestHealthcheckProbesLocalEndpoint(t *testing.T) {
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/healthz" {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Write([]byte("ok"))
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
_, port, err := net.SplitHostPort(srv.Listener.Addr().String())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Setenv("LISTEN_ADDR", "127.0.0.1:"+port)
|
|
||||||
if err := healthcheck(); err != nil {
|
|
||||||
t.Fatalf("healthy server rejected: %v", err)
|
|
||||||
}
|
|
||||||
// The container listens on 0.0.0.0; the probe must target loopback.
|
|
||||||
t.Setenv("LISTEN_ADDR", "0.0.0.0:"+port)
|
|
||||||
if err := healthcheck(); err != nil {
|
|
||||||
t.Fatalf("wildcard address rejected: %v", err)
|
|
||||||
}
|
|
||||||
t.Setenv("LISTEN_ADDR", "127.0.0.1:1")
|
|
||||||
if err := healthcheck(); err == nil {
|
|
||||||
t.Fatal("dead endpoint accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,49 +0,0 @@
|
|||||||
services:
|
|
||||||
panel:
|
|
||||||
container_name: dsh-mgr-panel
|
|
||||||
build: .
|
|
||||||
image: dsh-mgr:local
|
|
||||||
restart: unless-stopped
|
|
||||||
stop_grace_period: 20s
|
|
||||||
user: "1000:1000"
|
|
||||||
group_add:
|
|
||||||
- ${DOCKER_GID:?Set the numeric group ID of the Docker socket}
|
|
||||||
ports:
|
|
||||||
- '127.0.0.1:8088:8088'
|
|
||||||
environment:
|
|
||||||
LISTEN_ADDR: 0.0.0.0:8088
|
|
||||||
DATA_ROOT: /data
|
|
||||||
CONTAINER_DATA_ROOT: /data
|
|
||||||
HOST_DATA_ROOT: ${DSH_HOST_DATA_ROOT:?Set an absolute host data directory}
|
|
||||||
PANEL_PUBLIC_URL: ${PANEL_PUBLIC_URL:-http://127.0.0.1:8088}
|
|
||||||
TRUSTED_PROXY_CIDRS: ${TRUSTED_PROXY_CIDRS:-}
|
|
||||||
WEB_DIR: /app/web/dist
|
|
||||||
UPLOAD_ALLOWED_EXTENSIONS: ${UPLOAD_ALLOWED_EXTENSIONS:-}
|
|
||||||
ADMIN_USERNAME: ${DSH_ADMIN_USERNAME:-admin}
|
|
||||||
# Read only on first startup while the user table is empty; remove the
|
|
||||||
# value from .env afterwards so the plaintext password is not retained.
|
|
||||||
ADMIN_PASSWORD: ${DSH_ADMIN_PASSWORD:-}
|
|
||||||
DOCKER_HOST: unix:///var/run/docker.sock
|
|
||||||
DSH_DOCKER_ROOT_VISIBLE: ${DSH_DOCKER_ROOT_DIR:?Set the exact DockerRootDir reported by docker info}
|
|
||||||
PANEL_CONTAINER_ID: dsh-mgr-panel
|
|
||||||
volumes:
|
|
||||||
- ${DSH_HOST_DATA_ROOT}:/data
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
|
||||||
- ${DSH_DOCKER_ROOT_DIR}:${DSH_DOCKER_ROOT_DIR}:ro
|
|
||||||
read_only: true
|
|
||||||
tmpfs:
|
|
||||||
- /tmp:size=256m
|
|
||||||
mem_limit: 1g
|
|
||||||
cpus: 2
|
|
||||||
pids_limit: 512
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD", "/usr/local/bin/dsh-mgr", "-healthcheck"]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 3
|
|
||||||
start_period: 15s
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: 10m
|
|
||||||
max-file: '3'
|
|
||||||
@@ -1,20 +0,0 @@
|
|||||||
module git.misaka.ren/m1saka/dsh-web
|
|
||||||
|
|
||||||
go 1.25
|
|
||||||
|
|
||||||
require modernc.org/sqlite v1.34.5
|
|
||||||
|
|
||||||
require golang.org/x/crypto v0.28.0
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
|
||||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
|
||||||
golang.org/x/sys v0.26.0 // indirect
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
|
||||||
modernc.org/libc v1.55.3 // indirect
|
|
||||||
modernc.org/mathutil v1.6.0 // indirect
|
|
||||||
modernc.org/memory v1.8.0 // indirect
|
|
||||||
)
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
|
||||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
|
|
||||||
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
|
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
|
||||||
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
|
|
||||||
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
|
||||||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
|
|
||||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
|
||||||
golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
|
|
||||||
golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|
||||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
|
||||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
|
||||||
golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
|
|
||||||
golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
|
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
|
||||||
modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ=
|
|
||||||
modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
|
|
||||||
modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y=
|
|
||||||
modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s=
|
|
||||||
modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
|
|
||||||
modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
|
|
||||||
modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
|
|
||||||
modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
|
|
||||||
modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U=
|
|
||||||
modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w=
|
|
||||||
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
|
|
||||||
modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
|
|
||||||
modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
|
|
||||||
modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU=
|
|
||||||
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
|
|
||||||
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
|
|
||||||
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
|
|
||||||
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
|
|
||||||
modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g=
|
|
||||||
modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE=
|
|
||||||
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
|
|
||||||
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
|
|
||||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
|
||||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
.git
|
||||||
|
**/.git
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
**/.env
|
||||||
|
**/.env.*
|
||||||
|
node_modules
|
||||||
|
**/node_modules
|
||||||
|
*.test.mjs
|
||||||
|
embed.go
|
||||||
+19
-2
@@ -3,16 +3,25 @@ FROM node:24-bookworm-slim
|
|||||||
ARG DSH_VERSION
|
ARG DSH_VERSION
|
||||||
# Keep the historical dependency-layer cache key; the template revision is set below.
|
# Keep the historical dependency-layer cache key; the template revision is set below.
|
||||||
ARG RECIPE_REV=3
|
ARG RECIPE_REV=3
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends tini ca-certificates git procps && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
tini ca-certificates git procps \
|
||||||
|
python3 python3-pip python3-venv python-is-python3 \
|
||||||
|
libreoffice fonts-noto-cjk \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
RUN python --version && python3 -m pip --version && libreoffice --headless --version
|
||||||
RUN test -n "$DSH_VERSION" && npm install --global --foreground-scripts --loglevel=verbose --registry=https://registry.npmjs.org "@deepseek-ai/dsh@${DSH_VERSION}"
|
RUN test -n "$DSH_VERSION" && npm install --global --foreground-scripts --loglevel=verbose --registry=https://registry.npmjs.org "@deepseek-ai/dsh@${DSH_VERSION}"
|
||||||
RUN npm install --global --ignore-scripts --registry=https://registry.npmjs.org [email protected] && test "$(pnpm --version)" = "10.11.0"
|
RUN npm install --global --ignore-scripts --registry=https://registry.npmjs.org [email protected] && test "$(pnpm --version)" = "10.11.0"
|
||||||
COPY install-mobile.mjs /usr/local/lib/dsh-mgr/install-mobile.mjs
|
COPY install-mobile.mjs /usr/local/lib/dsh-mgr/install-mobile.mjs
|
||||||
RUN node /usr/local/lib/dsh-mgr/install-mobile.mjs
|
RUN node /usr/local/lib/dsh-mgr/install-mobile.mjs
|
||||||
COPY install-free-search.mjs /usr/local/lib/dsh-mgr/install-free-search.mjs
|
COPY install-free-search.mjs /usr/local/lib/dsh-mgr/install-free-search.mjs
|
||||||
RUN node /usr/local/lib/dsh-mgr/install-free-search.mjs
|
RUN node /usr/local/lib/dsh-mgr/install-free-search.mjs
|
||||||
|
COPY install-chat-manager.mjs patch-chat-manager.mjs /usr/local/lib/dsh-mgr/
|
||||||
|
RUN node /usr/local/lib/dsh-mgr/install-chat-manager.mjs
|
||||||
COPY install-workspace.mjs /usr/local/lib/dsh-mgr/install-workspace.mjs
|
COPY install-workspace.mjs /usr/local/lib/dsh-mgr/install-workspace.mjs
|
||||||
RUN node /usr/local/lib/dsh-mgr/install-workspace.mjs
|
RUN node /usr/local/lib/dsh-mgr/install-workspace.mjs
|
||||||
COPY init-profile.mjs /usr/local/lib/dsh-mgr/init-profile.mjs
|
COPY init-profile.mjs reconcile-profile.mjs /usr/local/lib/dsh-mgr/
|
||||||
|
COPY install-host-settings.mjs /usr/local/lib/dsh-mgr/install-host-settings.mjs
|
||||||
|
RUN node /usr/local/lib/dsh-mgr/install-host-settings.mjs
|
||||||
COPY --from=proxy /usr/bin/caddy /usr/bin/caddy
|
COPY --from=proxy /usr/bin/caddy /usr/bin/caddy
|
||||||
# Strip upstream file capabilities without granting any runtime capabilities.
|
# Strip upstream file capabilities without granting any runtime capabilities.
|
||||||
# A fresh file copy omits security.capability xattrs; port 8080 needs none.
|
# A fresh file copy omits security.capability xattrs; port 8080 needs none.
|
||||||
@@ -20,6 +29,14 @@ RUN cp /usr/bin/caddy /usr/bin/caddy.clean && mv /usr/bin/caddy.clean /usr/bin/c
|
|||||||
COPY Caddyfile /etc/caddy/Caddyfile
|
COPY Caddyfile /etc/caddy/Caddyfile
|
||||||
COPY entrypoint.sh /usr/local/bin/dsh-entrypoint
|
COPY entrypoint.sh /usr/local/bin/dsh-entrypoint
|
||||||
RUN chmod 755 /usr/local/bin/dsh-entrypoint && mkdir -p /data /workspace /shared && ln -s /workspace /data/dsh-home && chown -R node:node /data /workspace /shared
|
RUN chmod 755 /usr/local/bin/dsh-entrypoint && mkdir -p /data /workspace /shared && ln -s /workspace /data/dsh-home && chown -R node:node /data /workspace /shared
|
||||||
|
COPY install-misans.mjs /usr/local/lib/dsh-mgr/install-misans.mjs
|
||||||
|
RUN node /usr/local/lib/dsh-mgr/install-misans.mjs
|
||||||
|
COPY MiSans-NOTICE.txt /usr/local/share/fonts/misans/NOTICE.txt
|
||||||
|
COPY install-office-preview.mjs native-office.mjs validate-office.py office-worker.py /usr/local/lib/dsh-mgr/
|
||||||
|
RUN node /usr/local/lib/dsh-mgr/install-office-preview.mjs
|
||||||
|
COPY preview-download /usr/local/lib/dsh-mgr/preview-download
|
||||||
|
COPY install-preview-download.mjs /usr/local/lib/dsh-mgr/install-preview-download.mjs
|
||||||
|
RUN node /usr/local/lib/dsh-mgr/install-preview-download.mjs
|
||||||
ARG RECIPE_REV=7
|
ARG RECIPE_REV=7
|
||||||
LABEL dsh-mgr.role=runtime dsh-mgr.version=$DSH_VERSION dsh-mgr.recipe=$RECIPE_REV
|
LABEL dsh-mgr.role=runtime dsh-mgr.version=$DSH_VERSION dsh-mgr.recipe=$RECIPE_REV
|
||||||
ENV DSH_HOME=/workspace HOME=/workspace XDG_DATA_HOME=/workspace/.local/share XDG_CONFIG_HOME=/workspace/.config
|
ENV DSH_HOME=/workspace HOME=/workspace XDG_DATA_HOME=/workspace/.local/share XDG_CONFIG_HOME=/workspace/.config
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
This runtime uses MiSans fonts by Xiaomi.
|
||||||
|
Official source: https://hyperos.mi.com/font/zh/download/
|
||||||
|
Download mirror (third-party, not an official Xiaomi repository):
|
||||||
|
https://github.com/boyan01/mi_sans_font/tree/68f3ec3fee5830ce8a1d955d462fa0df401a5341/lib/fonts
|
||||||
|
Files: MiSans-Regular.ttf and MiSans-Bold.ttf
|
||||||
|
These mirrored files differ from the current official ZIP; they are not claimed
|
||||||
|
to be the latest release. The mirror repository's license does not replace the
|
||||||
|
Xiaomi font license.
|
||||||
|
|
||||||
|
MiSans is subject to the MiSans Fonts Intellectual Property License Agreement
|
||||||
|
published on the official download page. It is not an open-source font license.
|
||||||
|
Installation and use indicate acceptance of that agreement. Do not modify the
|
||||||
|
fonts or distribute/sell them separately. Review the complete agreement before
|
||||||
|
redistributing a runtime image containing these fonts.
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
package image
|
|
||||||
|
|
||||||
import "embed"
|
|
||||||
|
|
||||||
//go:embed Dockerfile Caddyfile entrypoint.sh install-mobile.mjs install-free-search.mjs install-workspace.mjs init-profile.mjs
|
|
||||||
var Files embed.FS
|
|
||||||
+13
-2
@@ -8,16 +8,27 @@ if [[ -f "${DSH_HOME}/profiles/web/package.json" ]]; then
|
|||||||
# pnpm otherwise probes registry metadata for the overlapping optional peer.
|
# pnpm otherwise probes registry metadata for the overlapping optional peer.
|
||||||
# Restrict offline mode to profiles containing only our fixed archives.
|
# Restrict offline mode to profiles containing only our fixed archives.
|
||||||
install_env=()
|
install_env=()
|
||||||
if node -e 'const m=require(process.env.DSH_HOME+"/profiles/web/package.json"); const pins={"dsh-mgr-workspace":"file:/usr/local/lib/dsh-mgr/dsh-mgr-workspace-1.0.0.tgz","dsh-web-mobile":"file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.0.5.tgz","dsh-free-search":"file:/usr/local/lib/dsh-mgr/dsh-free-search-0.8.5.tgz"}; process.exit(Object.entries({...m.dependencies,...m.devDependencies,...m.optionalDependencies}).every(([n,v])=>pins[n]===v)?0:1)'; then
|
if node -e 'import("/usr/local/lib/dsh-mgr/reconcile-profile.mjs").then(({BUILTIN_DEPENDENCIES:pins,BUILTIN_OVERRIDES:overrides})=>{const m=require(process.env.DSH_HOME+"/profiles/web/package.json"); process.exit([m.dependencies,m.devDependencies,m.optionalDependencies].every(section=>Object.entries(section||{}).every(([n,v])=>pins[n]===v))&&Object.entries(m.pnpm?.overrides||{}).every(([n,v])=>overrides[n]===v)?0:1)})'; then
|
||||||
install_env=(npm_config_offline=true)
|
install_env=(npm_config_offline=true)
|
||||||
fi
|
fi
|
||||||
if timeout --signal=TERM --kill-after=5s 120s env CI=true "${install_env[@]}" dsh plugin --profile web install --ignore-scripts; then
|
# Only migration installs may refresh the lockfile. Keep the marker on failure
|
||||||
|
# so a restart retries even when package.json already contains the new pins.
|
||||||
|
pending_lockfile="${DSH_HOME}/profiles/web/.dsh-lockfile-pending"
|
||||||
|
install_args=()
|
||||||
|
if [[ -f "$pending_lockfile" ]]; then
|
||||||
|
install_args+=(--no-frozen-lockfile)
|
||||||
|
fi
|
||||||
|
# Hide routine progress/peer warnings, but replay diagnostics on any failure.
|
||||||
|
if install_output=$(timeout --signal=TERM --kill-after=5s 120s env CI=true "${install_env[@]}" dsh plugin --profile web install --ignore-scripts "${install_args[@]}" 2>&1); then
|
||||||
|
rm -f -- "$pending_lockfile"
|
||||||
echo "dsh-mgr: plugin prestart reconciliation complete" >&2
|
echo "dsh-mgr: plugin prestart reconciliation complete" >&2
|
||||||
else
|
else
|
||||||
status=$?
|
status=$?
|
||||||
|
printf '%s\n' "$install_output" >&2
|
||||||
echo "dsh-mgr: plugin prestart reconciliation failed (exit $status); application not started" >&2
|
echo "dsh-mgr: plugin prestart reconciliation failed (exit $status); application not started" >&2
|
||||||
exit "$status"
|
exit "$status"
|
||||||
fi
|
fi
|
||||||
|
unset install_output
|
||||||
else
|
else
|
||||||
echo "dsh-mgr: no web plugin manifest; using bundled plugins" >&2
|
echo "dsh-mgr: no web plugin manifest; using bundled plugins" >&2
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, existsSync } from 'node:fs';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
|
||||||
|
for (const status of ['0', '23', '124', '137', 'empty']) {
|
||||||
|
for (const pending of [false, true]) {
|
||||||
|
test('runtime prestart status ' + status + ', pending migration ' + pending, () => {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), 'dsh-prestart-'));
|
||||||
|
try {
|
||||||
|
if (status !== 'empty') {
|
||||||
|
mkdirSync(join(dir, 'profiles/web'), { recursive: true, mode: 0o700 });
|
||||||
|
writeFileSync(join(dir, 'profiles/web/package.json'), '{}', { mode: 0o600 });
|
||||||
|
if (pending) writeFileSync(join(dir, 'profiles/web/.dsh-lockfile-pending'), 'pending');
|
||||||
|
}
|
||||||
|
const scripts = {
|
||||||
|
node: `#!/bin/bash
|
||||||
|
exit 0
|
||||||
|
`,
|
||||||
|
timeout: `#!/bin/bash
|
||||||
|
printf 'timeout %s\n' "$*" >> "$TRACE"
|
||||||
|
test "$1 $2 $3" = "--signal=TERM --kill-after=5s 120s" || exit 90
|
||||||
|
shift 3
|
||||||
|
exec "$@"
|
||||||
|
`,
|
||||||
|
dsh: `#!/bin/bash
|
||||||
|
printf 'dsh %s\n' "$*" >> "$TRACE"
|
||||||
|
test "$npm_config_ignore_scripts:$npm_config_ignore_pnpmfile" = true:true || exit 91
|
||||||
|
if [ "$1" = plugin ]; then
|
||||||
|
echo 'install progress and peer warnings'
|
||||||
|
echo 'install stderr diagnostics' >&2
|
||||||
|
test "$*" = "plugin --profile web install --ignore-scripts$EXPECTED_ARGS" || exit 92
|
||||||
|
exit "$PLUGIN_STATUS"
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
|
`,
|
||||||
|
caddy: `#!/bin/bash
|
||||||
|
echo caddy >> "$TRACE"
|
||||||
|
sleep 0.1
|
||||||
|
`,
|
||||||
|
};
|
||||||
|
for (const [name, script] of Object.entries(scripts)) {
|
||||||
|
writeFileSync(join(dir, name), script, { mode: 0o700 });
|
||||||
|
}
|
||||||
|
const result = spawnSync('bash', [fileURLToPath(new URL('./entrypoint.sh', import.meta.url)), 'web', '--no-open', '--port', '3080'], {
|
||||||
|
env: { ...process.env, PATH: dir + ':' + process.env.PATH, DSH_HOME: dir, TRACE: join(dir, 'trace'), PLUGIN_STATUS: status, EXPECTED_ARGS: pending ? ' --no-frozen-lockfile' : '' },
|
||||||
|
encoding: 'utf8', timeout: 5000,
|
||||||
|
});
|
||||||
|
assert.ifError(result.error);
|
||||||
|
const output = result.stdout + result.stderr;
|
||||||
|
const trace = readFileSync(join(dir, 'trace'), 'utf8');
|
||||||
|
assert.equal(result.status, status === 'empty' ? 0 : Number(status), output);
|
||||||
|
if (status === 'empty') {
|
||||||
|
assert.doesNotMatch(trace, /plugin/);
|
||||||
|
assert.match(trace, /dsh web/);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
assert.match(trace, /dsh plugin --profile web install --ignore-scripts/);
|
||||||
|
assert.equal(existsSync(join(dir, 'profiles/web/.dsh-lockfile-pending')), pending && status !== '0');
|
||||||
|
if (status === '0') {
|
||||||
|
assert.doesNotMatch(output, /install progress|install stderr/);
|
||||||
|
assert.match(output, /reconciliation complete/);
|
||||||
|
assert.match(trace, /dsh web/);
|
||||||
|
} else {
|
||||||
|
assert.match(result.stderr, /install progress and peer warnings/);
|
||||||
|
assert.match(result.stderr, /install stderr diagnostics/);
|
||||||
|
assert.ok(output.includes('failed (exit ' + status + ')'), output);
|
||||||
|
assert.doesNotMatch(trace, /caddy|dsh web/);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test('runtime pins pnpm and uses nonroot with tini process-group forwarding', () => {
|
||||||
|
const dockerfile = readFileSync(new URL('./Dockerfile', import.meta.url), 'utf8');
|
||||||
|
for (const required of ['[email protected]', 'test "$(pnpm --version)" = "10.11.0"', 'USER node', '"/usr/bin/tini", "-g"']) {
|
||||||
|
assert.ok(dockerfile.includes(required), 'missing ' + required);
|
||||||
|
}
|
||||||
|
});
|
||||||
+36
-33
@@ -1,35 +1,38 @@
|
|||||||
import { mkdirSync, existsSync, writeFileSync, readFileSync, readdirSync, lstatSync } from 'node:fs';
|
import { mkdirSync, existsSync, writeFileSync, readFileSync, readdirSync } from 'node:fs';
|
||||||
import { join } from 'node:path';
|
import { join, resolve } from 'node:path';
|
||||||
const home = process.env.DSH_HOME;
|
import { pathToFileURL } from 'node:url';
|
||||||
if (!home) throw new Error('DSH_HOME required');
|
import { BUILTIN_DEPENDENCIES, BUILTIN_OVERRIDES, assertSafeProfile, reconcileProfile, markLockfilePending } from './reconcile-profile.mjs';
|
||||||
const profile = join(home, 'profiles/web');
|
export function initProfile(home, options) {
|
||||||
for (const dir of [home, join(home, 'profiles'), profile]) {
|
if (!home) throw new Error('DSH_HOME required');
|
||||||
try {
|
const profile = join(home, 'profiles/web');
|
||||||
const stat = lstatSync(dir);
|
assertSafeProfile(profile);
|
||||||
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('unsafe profile directory');
|
if (!existsSync(profile) || readdirSync(profile).length === 0) {
|
||||||
} catch (error) { if (error.code !== 'ENOENT') throw error; }
|
mkdirSync(profile, { recursive: true });
|
||||||
}
|
writeFileSync(join(profile, 'package.json'), JSON.stringify({
|
||||||
// Never rewrite an existing user manifest or patch.
|
name: 'dsh-profile-web', private: true,
|
||||||
if (!existsSync(profile) || readdirSync(profile).length === 0) {
|
dependencies: BUILTIN_DEPENDENCIES,
|
||||||
mkdirSync(profile, { recursive: true });
|
pnpm: { overrides: BUILTIN_OVERRIDES },
|
||||||
writeFileSync(join(profile, 'package.json'), JSON.stringify({
|
dsh: { profile: { bundles: ['@deepseek-ai/dsh-base', '@deepseek-ai/dsh-web-app', 'dsh-web-mobile', 'dsh-free-search', 'dsh-mgr-workspace', 'dsh-chat-manager', 'dsh-preview-download'] } },
|
||||||
name: 'dsh-profile-web', private: true,
|
}, null, 2), { flag: 'wx', mode: 0o600 });
|
||||||
dependencies: { 'dsh-mgr-workspace': 'file:/usr/local/lib/dsh-mgr/dsh-mgr-workspace-1.0.0.tgz', 'dsh-web-mobile': 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.0.5.tgz', 'dsh-free-search': 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.8.5.tgz' },
|
console.log('dsh-mgr: new web profile includes dsh-web-mobile@3.2.1 and [email protected] and [email protected]');
|
||||||
pnpm: { overrides: {
|
}
|
||||||
'dsh-free-search>@deepseek-ai/schemastery': 'file:/usr/local/lib/dsh-mgr/schemastery-3.18.4.tgz',
|
const migration = reconcileProfile(profile, options);
|
||||||
'@deepseek-ai/schemastery>@deepseek-ai/cosmokit': 'file:/usr/local/lib/dsh-mgr/cosmokit-1.8.5.tgz',
|
console.log(migration.changed
|
||||||
'@deepseek-ai/schemastery>@standard-schema/spec': 'file:/usr/local/lib/dsh-mgr/spec-1.1.0.tgz',
|
? "dsh-mgr: builtin profile updated; original saved to " + migration.backupPath
|
||||||
} },
|
: "dsh-mgr: builtin profile check complete; no changes");
|
||||||
dsh: { profile: { bundles: ['@deepseek-ai/dsh-base', '@deepseek-ai/dsh-web-app', 'dsh-web-mobile', 'dsh-free-search', 'dsh-mgr-workspace'] } },
|
const manifestPath = join(profile, 'package.json');
|
||||||
}, null, 2), { flag: 'wx', mode: 0o600 });
|
const workspacePath = join(profile, 'pnpm-workspace.yaml');
|
||||||
console.log('dsh-mgr: new web profile includes [email protected] and [email protected]');
|
// Backups omit workspace settings; restore host-provided peers before pnpm runs.
|
||||||
}
|
if (existsSync(manifestPath) && !existsSync(workspacePath)) {
|
||||||
const manifestPath = join(profile, 'package.json');
|
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
||||||
const workspacePath = join(profile, 'pnpm-workspace.yaml');
|
if (['dependencies', 'devDependencies', 'optionalDependencies'].some(section =>
|
||||||
// Backups omit workspace settings; restore host-provided peers before pnpm runs.
|
['dsh-chat-manager', 'dsh-web-mobile', 'dsh-free-search'].some(name => manifest[section]?.[name] === BUILTIN_DEPENDENCIES[name]))) {
|
||||||
if (existsSync(manifestPath) && !existsSync(workspacePath)) {
|
markLockfilePending(profile);
|
||||||
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
|
writeFileSync(workspacePath, 'packages:\n - .\n\nnodeLinker: hoisted\nautoInstallPeers: false\n', { flag: 'wx', mode: 0o600 });
|
||||||
if (manifest.dependencies?.['dsh-web-mobile'] === 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.0.5.tgz' || manifest.dependencies?.['dsh-free-search'] === 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.8.5.tgz') {
|
}
|
||||||
writeFileSync(workspacePath, 'packages:\n - .\n\nnodeLinker: hoisted\nautoInstallPeers: false\n', { flag: 'wx', mode: 0o600 });
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||||
|
initProfile(process.env.DSH_HOME);
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,33 +2,53 @@ import { test } from 'node:test';
|
|||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, symlinkSync, existsSync } from 'node:fs';
|
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, symlinkSync, existsSync } from 'node:fs';
|
||||||
import { tmpdir } from 'node:os';
|
import { tmpdir } from 'node:os';
|
||||||
import { join } from 'node:path';
|
import { join, basename } from 'node:path';
|
||||||
import { execFileSync } from 'node:child_process';
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { initProfile } from './init-profile.mjs';
|
||||||
|
import { BUILTIN_DEPENDENCIES, BUILTIN_OVERRIDES } from './reconcile-profile.mjs';
|
||||||
|
function archives(home) {
|
||||||
|
const archiveDirectory = join(home, 'archives');
|
||||||
|
mkdirSync(archiveDirectory, { recursive: true });
|
||||||
|
for (const value of [...Object.values(BUILTIN_DEPENDENCIES), ...Object.values(BUILTIN_OVERRIDES)]) {
|
||||||
|
writeFileSync(join(archiveDirectory, basename(value)), 'fixture');
|
||||||
|
}
|
||||||
|
return { archiveDirectory };
|
||||||
|
}
|
||||||
test('new profile enables bundle once and preserves existing configuration', () => {
|
test('new profile enables bundle once and preserves existing configuration', () => {
|
||||||
const home = mkdtempSync(join(tmpdir(), 'dsh-plugin-init-'));
|
const home = mkdtempSync(join(tmpdir(), 'dsh-plugin-init-'));
|
||||||
try {
|
try {
|
||||||
const run = () => execFileSync(process.execPath, [new URL('./init-profile.mjs', import.meta.url).pathname], {env:{...process.env, DSH_HOME:home}});
|
const options = archives(home);
|
||||||
|
const run = () => initProfile(home, options);
|
||||||
run();
|
run();
|
||||||
const file=join(home,'profiles/web/package.json');
|
const file=join(home,'profiles/web/package.json');
|
||||||
const original=readFileSync(file,'utf8');
|
const original=readFileSync(file,'utf8');
|
||||||
|
assert.equal(JSON.parse(original).dependencies['dsh-preview-download'], 'file:/usr/local/lib/dsh-mgr/dsh-preview-download-1.0.1.tgz');
|
||||||
|
assert.equal(JSON.parse(original).dsh.profile.bundles.filter(name => name === 'dsh-preview-download').length, 1);
|
||||||
|
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-chat-manager'));
|
||||||
|
assert.equal(JSON.parse(original).dependencies['dsh-chat-manager'], 'file:/usr/local/lib/dsh-mgr/dsh-chat-manager-1.5.20-dsh-mgr.1.tgz');
|
||||||
|
assert.equal(JSON.parse(original).pnpm.overrides['dsh-chat-manager>clsx'], 'file:/usr/local/lib/dsh-mgr/clsx-2.1.1.tgz');
|
||||||
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-web-mobile'));
|
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-web-mobile'));
|
||||||
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-free-search'));
|
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-free-search'));
|
||||||
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-mgr-workspace'));
|
assert.ok(JSON.parse(original).dsh.profile.bundles.includes('dsh-mgr-workspace'));
|
||||||
assert.equal(JSON.parse(original).dependencies['dsh-mgr-workspace'], 'file:/usr/local/lib/dsh-mgr/dsh-mgr-workspace-1.0.0.tgz');
|
assert.equal(JSON.parse(original).dependencies['dsh-mgr-workspace'], 'file:/usr/local/lib/dsh-mgr/dsh-mgr-workspace-1.0.0.tgz');
|
||||||
assert.equal(JSON.parse(original).dependencies['dsh-free-search'], 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.8.5.tgz');
|
assert.equal(JSON.parse(original).dependencies['dsh-free-search'], 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.9.21.tgz');
|
||||||
assert.equal(JSON.parse(original).pnpm.overrides['dsh-free-search>@deepseek-ai/schemastery'], 'file:/usr/local/lib/dsh-mgr/schemastery-3.18.4.tgz');
|
assert.equal(JSON.parse(original).pnpm.overrides['dsh-free-search>@deepseek-ai/schemastery'], 'file:/usr/local/lib/dsh-mgr/schemastery-3.18.4.tgz');
|
||||||
assert.equal(JSON.parse(original).dependencies['dsh-web-mobile'], 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.0.5.tgz');
|
assert.equal(JSON.parse(original).dependencies['dsh-web-mobile'], 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.2.1.tgz');
|
||||||
const workspace = join(home, 'profiles/web/pnpm-workspace.yaml');
|
const workspace = join(home, 'profiles/web/pnpm-workspace.yaml');
|
||||||
assert.match(readFileSync(workspace, 'utf8'), /autoInstallPeers: false/);
|
assert.match(readFileSync(workspace, 'utf8'), /autoInstallPeers: false/);
|
||||||
assert.match(readFileSync(workspace, 'utf8'), /nodeLinker: hoisted/);
|
assert.match(readFileSync(workspace, 'utf8'), /nodeLinker: hoisted/);
|
||||||
run();assert.equal(readFileSync(file,'utf8'),original);
|
run();assert.equal(readFileSync(file,'utf8'),original);
|
||||||
const legacy = JSON.parse(original);
|
const legacy = JSON.parse(original);
|
||||||
delete legacy.dependencies['dsh-mgr-workspace'];
|
delete legacy.dependencies['dsh-mgr-workspace'];
|
||||||
|
delete legacy.dependencies['dsh-chat-manager'];
|
||||||
|
delete legacy.pnpm.overrides['dsh-chat-manager>clsx'];
|
||||||
|
legacy.dsh.profile.bundles = legacy.dsh.profile.bundles.filter(name => name !== 'dsh-chat-manager');
|
||||||
legacy.dsh.profile.bundles = legacy.dsh.profile.bundles.filter(name => name !== 'dsh-mgr-workspace');
|
legacy.dsh.profile.bundles = legacy.dsh.profile.bundles.filter(name => name !== 'dsh-mgr-workspace');
|
||||||
writeFileSync(file, JSON.stringify(legacy));
|
writeFileSync(file, JSON.stringify(legacy));
|
||||||
run();
|
run();
|
||||||
assert.deepEqual(JSON.parse(readFileSync(file)), legacy);
|
assert.deepEqual(JSON.parse(readFileSync(file)), legacy);
|
||||||
const disabled = JSON.parse(original);
|
const disabled = JSON.parse(original);
|
||||||
|
disabled.dsh.profile.bundles = disabled.dsh.profile.bundles.filter(name => name !== 'dsh-chat-manager');
|
||||||
disabled.dsh.profile.bundles = disabled.dsh.profile.bundles.filter(name => name !== 'dsh-web-mobile');
|
disabled.dsh.profile.bundles = disabled.dsh.profile.bundles.filter(name => name !== 'dsh-web-mobile');
|
||||||
writeFileSync(file, JSON.stringify(disabled));
|
writeFileSync(file, JSON.stringify(disabled));
|
||||||
writeFileSync(workspace, 'custom workspace');
|
writeFileSync(workspace, 'custom workspace');
|
||||||
@@ -42,13 +62,37 @@ test('new profile enables bundle once and preserves existing configuration', ()
|
|||||||
writeFileSync(file,custom);run();assert.equal(readFileSync(file,'utf8'),custom);
|
writeFileSync(file,custom);run();assert.equal(readFileSync(file,'utf8'),custom);
|
||||||
} finally { rmSync(home,{recursive:true,force:true}); }
|
} finally { rmSync(home,{recursive:true,force:true}); }
|
||||||
});
|
});
|
||||||
|
test('chat-manager-only restored profile gets overrides and offline peer settings', () => {
|
||||||
|
const home = mkdtempSync(join(tmpdir(), 'dsh-chat-restore-'));
|
||||||
|
try {
|
||||||
|
const profile = join(home, 'profiles/web');
|
||||||
|
mkdirSync(profile, { recursive: true });
|
||||||
|
const manifest = JSON.stringify({ dependencies: { 'dsh-chat-manager': 'file:/usr/local/lib/dsh-mgr/dsh-chat-manager-1.5.20-dsh-mgr.1.tgz' } });
|
||||||
|
writeFileSync(join(profile, 'package.json'), manifest);
|
||||||
|
initProfile(home, archives(home));
|
||||||
|
assert.deepEqual(JSON.parse(readFileSync(join(profile, 'package.json'))), { ...JSON.parse(manifest), pnpm: { overrides: { 'dsh-chat-manager>clsx': BUILTIN_OVERRIDES['dsh-chat-manager>clsx'] } } });
|
||||||
|
assert.match(readFileSync(join(profile, 'pnpm-workspace.yaml'), 'utf8'), /autoInstallPeers: false/);
|
||||||
|
const entrypoint = readFileSync(new URL('./entrypoint.sh', import.meta.url), 'utf8');
|
||||||
|
const predicate = entrypoint.match(/if node -e '([^']+)'/)[1].replace('/usr/local/lib/dsh-mgr/reconcile-profile.mjs', new URL('./reconcile-profile.mjs', import.meta.url).href);
|
||||||
|
execFileSync(process.execPath, ['-e', predicate], { env: { ...process.env, DSH_HOME: home } });
|
||||||
|
for (const custom of [
|
||||||
|
{ dependencies: { 'dsh-chat-manager': BUILTIN_DEPENDENCIES['dsh-chat-manager'] }, pnpm: { overrides: { 'dsh-chat-manager>clsx': '^2.0.0' } } },
|
||||||
|
{ dependencies: { 'dsh-chat-manager': '^1.5.16' }, optionalDependencies: { 'dsh-chat-manager': BUILTIN_DEPENDENCIES['dsh-chat-manager'] } },
|
||||||
|
]) {
|
||||||
|
writeFileSync(join(profile, 'package.json'), JSON.stringify(custom));
|
||||||
|
assert.throws(() => execFileSync(process.execPath, ['-e', predicate], { env: { ...process.env, DSH_HOME: home }, stdio: 'pipe' }));
|
||||||
|
}
|
||||||
|
writeFileSync(join(profile, 'package.json'), JSON.stringify({ dependencies: { 'dsh-chat-manager': '^1.5.16' } }));
|
||||||
|
assert.throws(() => execFileSync(process.execPath, ['-e', predicate], { env: { ...process.env, DSH_HOME: home }, stdio: 'pipe' }));
|
||||||
|
} finally { rmSync(home, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
test('nonempty legacy profiles are not automatically injected', () => {
|
test('nonempty legacy profiles are not automatically injected', () => {
|
||||||
const home = mkdtempSync(join(tmpdir(), 'dsh-plugin-legacy-'));
|
const home = mkdtempSync(join(tmpdir(), 'dsh-plugin-legacy-'));
|
||||||
try {
|
try {
|
||||||
const profile = join(home, 'profiles/web');
|
const profile = join(home, 'profiles/web');
|
||||||
mkdirSync(profile, { recursive: true });
|
mkdirSync(profile, { recursive: true });
|
||||||
writeFileSync(join(profile, 'cordis.patch.yml'), '[]');
|
writeFileSync(join(profile, 'cordis.patch.yml'), '[]');
|
||||||
execFileSync(process.execPath, [new URL('./init-profile.mjs', import.meta.url).pathname], { env: { ...process.env, DSH_HOME: home } });
|
initProfile(home, archives(home));
|
||||||
assert.equal(existsSync(join(profile, 'package.json')), false);
|
assert.equal(existsSync(join(profile, 'package.json')), false);
|
||||||
assert.equal(existsSync(join(profile, 'pnpm-workspace.yaml')), false);
|
assert.equal(existsSync(join(profile, 'pnpm-workspace.yaml')), false);
|
||||||
} finally { rmSync(home, { recursive: true, force: true }); }
|
} finally { rmSync(home, { recursive: true, force: true }); }
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { mkdirSync, writeFileSync, readFileSync, readdirSync } from 'node:fs';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
|
import { patchClient } from './patch-chat-manager.mjs';
|
||||||
|
|
||||||
|
const root = '/usr/local/lib/dsh-mgr';
|
||||||
|
const version = '1.5.20';
|
||||||
|
const patchedVersion = '1.5.20-dsh-mgr.1';
|
||||||
|
const hostManifest = '/usr/local/lib/node_modules/@deepseek-ai/dsh/package.json';
|
||||||
|
const destination = '/usr/local/lib/node_modules/@deepseek-ai/dsh/node_modules/dsh-chat-manager';
|
||||||
|
const packages = [
|
||||||
|
['dsh-chat-manager', version, 'https://registry.npmjs.org/dsh-chat-manager/-/dsh-chat-manager-1.5.20.tgz', 'V/7N48lJioxBr2h+hmVnNoZB5jjoT/EGC6Pk8pSbNJveQoHye+zBj3tUX1Xf520YyopVk5fSVpt+aeCkBBVxdA=='],
|
||||||
|
['clsx', '2.1.1', 'https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz', 'eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA=='],
|
||||||
|
];
|
||||||
|
mkdirSync(root, { recursive: true });
|
||||||
|
for (const [name, release, url, integrity] of packages) {
|
||||||
|
const response = await fetch(url, { signal: AbortSignal.timeout(120000) });
|
||||||
|
if (!response.ok) throw new Error('download failed: ' + name);
|
||||||
|
const data = Buffer.from(await response.arrayBuffer());
|
||||||
|
if (createHash('sha512').update(data).digest('base64') !== integrity) throw new Error('integrity mismatch: ' + name);
|
||||||
|
writeFileSync(`${root}/${name}-${release}.tgz`, data, { mode: 0o644 });
|
||||||
|
}
|
||||||
|
mkdirSync(destination, { recursive: true });
|
||||||
|
execFileSync('tar', ['-xzf', `${root}/dsh-chat-manager-${version}.tgz`, '--strip-components=1', '--no-same-owner', '-C', destination]);
|
||||||
|
const pkg = JSON.parse(readFileSync(destination + '/package.json'));
|
||||||
|
if (pkg.name !== 'dsh-chat-manager' || pkg.version !== version || pkg.license !== 'MIT' || JSON.stringify(pkg.dependencies) !== JSON.stringify({ clsx: '^2.0.0' })) throw new Error('unexpected chat-manager metadata');
|
||||||
|
if (!readFileSync(destination + '/LICENSE', 'utf8').includes('MIT License')) throw new Error('missing chat-manager MIT license');
|
||||||
|
const require = createRequire(hostManifest);
|
||||||
|
const host = JSON.parse(readFileSync(hostManifest));
|
||||||
|
const semver = createRequire('/usr/local/lib/node_modules/npm/package.json')('semver');
|
||||||
|
for (const [name, range] of Object.entries(pkg.peerDependencies || {})) {
|
||||||
|
// 1.5.20 retains this optional legacy peer but its runtime no longer imports it.
|
||||||
|
if (name === '@deepseek-ai/dsh-invariants' && pkg.peerDependenciesMeta?.[name]?.optional) {
|
||||||
|
for (const file of ['src/index.js', 'src/host/archive-manager.mjs', 'src/host/delete-session.mjs', 'lib/client.js']) {
|
||||||
|
if (readFileSync(destination + '/' + file, 'utf8').includes(name)) throw new Error('legacy invariants peer is used again');
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (name === 'react') {
|
||||||
|
if (host.version !== '0.2.1-rc.5') throw new Error('unverified browser React host: ' + host.version);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// New web hosts bundle this pure UI module in the browser bootstrap.
|
||||||
|
if (['@deepseek-ai/dsh-client-ui-primitives', '@deepseek-ai/dsh-client-ui-slots'].includes(name) && host.version === '0.2.1-rc.5') {
|
||||||
|
const frontend = require.resolve('@deepseek-ai/dsh-web-frontend/package.json');
|
||||||
|
const assets = frontend.slice(0, -'package.json'.length) + 'dist/assets';
|
||||||
|
const frontendVersion = JSON.parse(readFileSync(frontend)).version;
|
||||||
|
if (frontendVersion !== host.version || !semver.satisfies(frontendVersion, range) || !readdirSync(assets).filter(file => /^index-.*\.js$/.test(file)).some(file => readFileSync(assets + '/' + file, 'utf8').includes(JSON.stringify(name) + ':'))) throw new Error('missing browser UI provider');
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const peer = require(name + '/package.json');
|
||||||
|
if (!semver.satisfies(peer.version, range)) throw new Error('incompatible chat-manager host dependency: ' + name + '@' + peer.version);
|
||||||
|
}
|
||||||
|
const client = destination + '/lib/client.js';
|
||||||
|
writeFileSync(client, patchClient(readFileSync(client, 'utf8'), version));
|
||||||
|
pkg.version = patchedVersion;
|
||||||
|
writeFileSync(destination + '/package.json', JSON.stringify(pkg, null, 2));
|
||||||
|
execFileSync('chmod', ['-R', 'a+rX', destination]);
|
||||||
|
execFileSync('tar', ['-czf', `${root}/dsh-chat-manager-${patchedVersion}.tgz`, '--transform=s,^dsh-chat-manager,package,', '-C', destination + '/..', 'dsh-chat-manager']);
|
||||||
|
execFileSync('chmod', ['644', `${root}/dsh-chat-manager-${patchedVersion}.tgz`]);
|
||||||
|
const clsx = destination + '/node_modules/clsx';
|
||||||
|
mkdirSync(clsx, { recursive: true });
|
||||||
|
execFileSync('tar', ['-xzf', `${root}/clsx-2.1.1.tgz`, '--strip-components=1', '--no-same-owner', '-C', clsx]);
|
||||||
|
execFileSync('chmod', ['-R', 'a+rX', clsx]);
|
||||||
|
host.dependencies = { ...host.dependencies, 'dsh-chat-manager': patchedVersion };
|
||||||
|
writeFileSync(hostManifest, JSON.stringify(host, null, 2));
|
||||||
|
console.log('Installed dsh-chat-manager@' + patchedVersion + ' and [email protected] (SHA-512 verified; no lifecycle scripts; MIT retained)');
|
||||||
@@ -4,7 +4,7 @@ import { execFileSync } from 'node:child_process';
|
|||||||
import { createRequire } from 'node:module';
|
import { createRequire } from 'node:module';
|
||||||
const root = '/usr/local/lib/dsh-mgr';
|
const root = '/usr/local/lib/dsh-mgr';
|
||||||
const packages = [
|
const packages = [
|
||||||
['dsh-free-search', '0.8.5', 'https://registry.npmjs.org/dsh-free-search/-/dsh-free-search-0.8.5.tgz', 'MT00Xtypw9uk9QsqaiRmtVbYwnPON+HJY1lZjYritEvuwSe9ahgG9NVO43lUI+Eje2umrqUapaeqLRYf34DrZw=='],
|
['dsh-free-search', '0.9.21', 'https://registry.npmjs.org/dsh-free-search/-/dsh-free-search-0.9.21.tgz', '3RtN+hphVn5WcqjbwPBkG+QGgz7YO6KtkAKQFmWmsVl3lmDEhLENuno95JheKahllt0IXCLFCTHsI2HSzttg4g=='],
|
||||||
['@deepseek-ai/schemastery', '3.18.4', 'https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.4.tgz', 'SSXO6tYuyrIqKVbmOnIq0s+riUywYzouFMcnBluHF9n4KMo2G8HvEzhCYj6pj2617/glOjbJuVInJ9hfONsjkg=='],
|
['@deepseek-ai/schemastery', '3.18.4', 'https://registry.npmjs.org/@deepseek-ai/schemastery/-/schemastery-3.18.4.tgz', 'SSXO6tYuyrIqKVbmOnIq0s+riUywYzouFMcnBluHF9n4KMo2G8HvEzhCYj6pj2617/glOjbJuVInJ9hfONsjkg=='],
|
||||||
['@deepseek-ai/cosmokit', '1.8.5', 'https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.5.tgz', 'LXsrlem9z8dq4sLflj2yuCuYX7KqhdzF5hZly3eZyuTo8d6oDSOXuEgC5DbQWKW+lksm6zmOutQLsP/ma6wR6A=='],
|
['@deepseek-ai/cosmokit', '1.8.5', 'https://registry.npmjs.org/@deepseek-ai/cosmokit/-/cosmokit-1.8.5.tgz', 'LXsrlem9z8dq4sLflj2yuCuYX7KqhdzF5hZly3eZyuTo8d6oDSOXuEgC5DbQWKW+lksm6zmOutQLsP/ma6wR6A=='],
|
||||||
['@standard-schema/spec', '1.1.0', 'https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz', 'l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=='],
|
['@standard-schema/spec', '1.1.0', 'https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz', 'l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w=='],
|
||||||
@@ -20,21 +20,21 @@ for (const [name, version, url, integrity] of packages) {
|
|||||||
const hostManifest = '/usr/local/lib/node_modules/@deepseek-ai/dsh/package.json';
|
const hostManifest = '/usr/local/lib/node_modules/@deepseek-ai/dsh/package.json';
|
||||||
const destination = '/usr/local/lib/node_modules/@deepseek-ai/dsh/node_modules/dsh-free-search';
|
const destination = '/usr/local/lib/node_modules/@deepseek-ai/dsh/node_modules/dsh-free-search';
|
||||||
mkdirSync(destination, { recursive: true });
|
mkdirSync(destination, { recursive: true });
|
||||||
execFileSync('tar', ['-xzf', `${root}/dsh-free-search-0.8.5.tgz`, '--strip-components=1', '--no-same-owner', '-C', destination]);
|
execFileSync('tar', ['-xzf', `${root}/dsh-free-search-0.9.21.tgz`, '--strip-components=1', '--no-same-owner', '-C', destination]);
|
||||||
execFileSync('chmod', ['-R', 'a+rX', destination]);
|
execFileSync('chmod', ['-R', 'a+rX', destination]);
|
||||||
const pkg = JSON.parse(readFileSync(destination + '/package.json'));
|
const pkg = JSON.parse(readFileSync(destination + '/package.json'));
|
||||||
if (pkg.name !== 'dsh-free-search' || pkg.version !== '0.8.5' || JSON.stringify(pkg.dependencies) !== JSON.stringify({ '@deepseek-ai/schemastery': '^3.18.4' })) throw new Error('unexpected free-search metadata');
|
if (pkg.name !== 'dsh-free-search' || pkg.version !== '0.9.21' || JSON.stringify(pkg.dependencies) !== JSON.stringify({ '@deepseek-ai/schemastery': '^3.18.4' })) throw new Error('unexpected free-search metadata');
|
||||||
const require = createRequire(hostManifest);
|
const require = createRequire(hostManifest);
|
||||||
const semver = createRequire('/usr/local/lib/node_modules/npm/package.json')('semver');
|
const semver = createRequire('/usr/local/lib/node_modules/npm/package.json')('semver');
|
||||||
for (const [name, range] of Object.entries({ ...pkg.peerDependencies, ...pkg.dependencies })) {
|
for (const [name, range] of Object.entries({ ...pkg.peerDependencies, ...pkg.dependencies })) {
|
||||||
if (name === 'react') {
|
if (name === 'react') {
|
||||||
if (!['0.2.0-rc.1', '0.2.0-rc.2'].includes(require('./package.json').version)) throw new Error('unverified browser React host');
|
if (require('./package.json').version !== '0.2.1-rc.5') throw new Error('unverified browser React host');
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
const peer = require(name + '/package.json');
|
const peer = require(name + '/package.json');
|
||||||
if (!semver.satisfies(peer.version, range)) throw new Error('incompatible free-search host dependency: ' + name);
|
if (!(semver.satisfies(peer.version, range) || (peer.version === "0.2.1-rc.5" && name.startsWith("@deepseek-ai/dsh-") && semver.satisfies(peer.version, range, { includePrerelease: true })))) throw new Error('incompatible free-search host dependency: ' + name);
|
||||||
}
|
}
|
||||||
const host = JSON.parse(readFileSync(hostManifest));
|
const host = JSON.parse(readFileSync(hostManifest));
|
||||||
host.dependencies = { ...host.dependencies, 'dsh-free-search': '0.8.5' };
|
host.dependencies = { ...host.dependencies, 'dsh-free-search': '0.9.21' };
|
||||||
writeFileSync(hostManifest, JSON.stringify(host, null, 2));
|
writeFileSync(hostManifest, JSON.stringify(host, null, 2));
|
||||||
console.log('Installed dsh-free-search@0.8.5 and pinned offline dependency archives (SHA-512 verified; no lifecycle scripts)');
|
console.log('Installed dsh-free-search@0.9.21 and pinned offline dependency archives (SHA-512 verified; no lifecycle scripts)');
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { readFileSync, writeFileSync } from 'node:fs';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
|
||||||
|
export const target = 'const persistence = ctx.remote.$host.isLoopback ? "host" : "memory";';
|
||||||
|
const replacement = 'const persistence = "host"; /* dsh-mgr: authenticated remote settings */';
|
||||||
|
|
||||||
|
// Deliberately scoped to the settings UI, not connection.isLoopback or auth.
|
||||||
|
// Fail closed on upstream changes instead of silently shipping a broken UI.
|
||||||
|
export function patchSettings(source, version) {
|
||||||
|
// Native host settings supersede the old persistence patch.
|
||||||
|
if (version === "0.2.1-rc.5") {
|
||||||
|
if (source.includes(target) || !source.includes("this.ctx.remote.settings.describe()") || !source.includes("this.ctx.remote.settings.mutate(")) throw new Error("expected native host settings API");
|
||||||
|
return source;
|
||||||
|
}
|
||||||
|
if (version !== '0.2.0-rc.2') throw new Error('unsupported DSH version for host settings patch');
|
||||||
|
if (source.split(target).length !== 2 || source.includes(replacement)) {
|
||||||
|
throw new Error('expected exactly one unpatched settings persistence selector');
|
||||||
|
}
|
||||||
|
return source.replace(target, replacement);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function install(root = '/usr/local/lib/node_modules/@deepseek-ai/dsh') {
|
||||||
|
const { version } = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'));
|
||||||
|
const file = join(root, 'node_modules/@deepseek-ai/dsh-client-ui-settings/lib/client.js');
|
||||||
|
const source = readFileSync(file, 'utf8');
|
||||||
|
writeFileSync(file, patchSettings(source, version));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||||
|
install();
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { runInNewContext } from 'node:vm';
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { patchSettings, target } from './install-host-settings.mjs';
|
||||||
|
|
||||||
|
test("next host settings API remains unchanged", () => {
|
||||||
|
const source = "this.ctx.remote.settings.describe(); this.ctx.remote.settings.mutate(namespace, ops, revision);";
|
||||||
|
assert.equal(patchSettings(source, "0.2.1-rc.5"), source);
|
||||||
|
assert.throws(() => patchSettings(target, "0.2.1-rc.5"), /native host settings/);
|
||||||
|
assert.throws(() => patchSettings("", "0.2.1-rc.5"), /native host settings/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('settings use host persistence on domains without changing loopback facts', () => {
|
||||||
|
for (const isLoopback of [false, true]) {
|
||||||
|
const ctx = { remote: { $host: { isLoopback } } };
|
||||||
|
const source = '(function(ctx) { ' + target + ' return persistence; })(ctx)';
|
||||||
|
assert.equal(runInNewContext(patchSettings(source, '0.2.0-rc.2'), { ctx }), 'host');
|
||||||
|
assert.equal(ctx.remote.$host.isLoopback, isLoopback);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('reject unsupported versions, changed selectors, duplicates and repeat installation', () => {
|
||||||
|
assert.throws(() => patchSettings(target, '0.2.0-rc.3'), /unsupported DSH version/);
|
||||||
|
for (const source of ['', target.replace('isLoopback', 'isLocal'), target + target,
|
||||||
|
patchSettings(target, '0.2.0-rc.2')]) {
|
||||||
|
assert.throws(() => patchSettings(source, '0.2.0-rc.2'), /exactly one/);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('patch leaves surrounding code unchanged and is wired into the image', () => {
|
||||||
|
const source = 'before; ' + target + ' after;';
|
||||||
|
const result = patchSettings(source, '0.2.0-rc.2');
|
||||||
|
assert.ok(result.startsWith('before; '));
|
||||||
|
assert.ok(result.endsWith(' after;'));
|
||||||
|
const dockerfile = readFileSync(new URL('./Dockerfile', import.meta.url), 'utf8');
|
||||||
|
assert.ok(dockerfile.includes('COPY install-host-settings.mjs /usr/local/lib/dsh-mgr/install-host-settings.mjs'));
|
||||||
|
assert.ok(dockerfile.indexOf('RUN node /usr/local/lib/dsh-mgr/install-host-settings.mjs') > dockerfile.indexOf('npm install --global --foreground-scripts'));
|
||||||
|
});
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import { writeFileSync, mkdirSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
|
||||||
|
// Third-party mirror, pinned to a reviewed commit; Xiaomi's font license applies.
|
||||||
|
const base = 'https://raw.githubusercontent.com/boyan01/mi_sans_font/68f3ec3fee5830ce8a1d955d462fa0df401a5341/lib/fonts/';
|
||||||
|
const fonts = [
|
||||||
|
['MiSans-Regular.ttf', '7172aa1b5c703780ee12df3fe4c50481dde01f3b125a045a398000a88d032ae5', 'Regular'],
|
||||||
|
['MiSans-Bold.ttf', '250fb5c8d4657f103e332b0b948a995e45f2c159beb8d0528424e2d32ae421be', 'Bold'],
|
||||||
|
];
|
||||||
|
const destination = '/usr/local/share/fonts/misans';
|
||||||
|
mkdirSync(destination, { recursive: true });
|
||||||
|
for (const [name, sha256] of fonts) {
|
||||||
|
console.log('Downloading ' + name);
|
||||||
|
const response = await fetch(base + name, { signal: AbortSignal.timeout(120000) });
|
||||||
|
if (!response.ok) throw new Error('MiSans download failed: ' + name + ' HTTP ' + response.status);
|
||||||
|
const font = Buffer.from(await response.arrayBuffer());
|
||||||
|
if (createHash('sha256').update(font).digest('hex') !== sha256) {
|
||||||
|
throw new Error('MiSans checksum mismatch: ' + name);
|
||||||
|
}
|
||||||
|
writeFileSync(join(destination, name), font, { mode: 0o644 });
|
||||||
|
}
|
||||||
|
execFileSync('fc-cache', ['-f', destination], { stdio: 'inherit' });
|
||||||
|
for (const [name, , style] of fonts) {
|
||||||
|
const file = execFileSync('fc-match', ['-f', '%{file}', 'MiSans:style=' + style], { encoding: 'utf8' });
|
||||||
|
if (file !== join(destination, name)) throw new Error('MiSans font lookup failed: ' + style + ': ' + file);
|
||||||
|
}
|
||||||
|
console.log('Installed MiSans Regular and Bold (SHA-256 verified)');
|
||||||
@@ -2,15 +2,15 @@ import { createHash } from 'node:crypto';
|
|||||||
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
|
import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
|
||||||
import { execFileSync } from 'node:child_process';
|
import { execFileSync } from 'node:child_process';
|
||||||
import { createRequire } from 'node:module';
|
import { createRequire } from 'node:module';
|
||||||
const version = '3.0.5';
|
const version = '3.2.1';
|
||||||
const integrity = 'N3wmegFHSd23J0+xjotKyBq0ZbLXUbkWZ65E+aSw5pzB/nudhaNHWT2V3CSNaW/LX3s6AxAWB2APpfA0dLXJZQ==';
|
const integrity = '2XHbebwr78ujeJVt3wqffSp7/LEnLqUajawnL0TWNGG6LcAimAm8q8GJdh28c7gdRs6BYAGxfF/TatA5E6xZ+A==';
|
||||||
const response = await fetch('https://registry.npmjs.org/dsh-web-mobile/-/dsh-web-mobile-3.0.5.tgz', { signal: AbortSignal.timeout(120000) });
|
const response = await fetch('https://registry.npmjs.org/dsh-web-mobile/-/dsh-web-mobile-3.2.1.tgz', { signal: AbortSignal.timeout(120000) });
|
||||||
if (!response.ok) throw new Error('mobile plugin download failed');
|
if (!response.ok) throw new Error('mobile plugin download failed');
|
||||||
const data = Buffer.from(await response.arrayBuffer());
|
const data = Buffer.from(await response.arrayBuffer());
|
||||||
if (createHash('sha512').update(data).digest('base64') !== integrity) throw new Error('mobile plugin integrity mismatch');
|
if (createHash('sha512').update(data).digest('base64') !== integrity) throw new Error('mobile plugin integrity mismatch');
|
||||||
const destination = '/usr/local/lib/node_modules/@deepseek-ai/dsh/node_modules/dsh-web-mobile';
|
const destination = '/usr/local/lib/node_modules/@deepseek-ai/dsh/node_modules/dsh-web-mobile';
|
||||||
mkdirSync(destination, { recursive: true });
|
mkdirSync(destination, { recursive: true });
|
||||||
const archive = '/usr/local/lib/dsh-mgr/dsh-web-mobile-3.0.5.tgz';
|
const archive = '/usr/local/lib/dsh-mgr/dsh-web-mobile-3.2.1.tgz';
|
||||||
writeFileSync(archive, data, { mode: 0o644 });
|
writeFileSync(archive, data, { mode: 0o644 });
|
||||||
execFileSync('tar', ['-xzf', archive, '--strip-components=1', '--no-same-owner', '-C', destination]);
|
execFileSync('tar', ['-xzf', archive, '--strip-components=1', '--no-same-owner', '-C', destination]);
|
||||||
// Published tar modes can be owner-only; runtime is UID 1000, not build root.
|
// Published tar modes can be owner-only; runtime is UID 1000, not build root.
|
||||||
@@ -24,7 +24,7 @@ for (const [name, range] of Object.entries(pkg.peerDependencies || {})) {
|
|||||||
// Limit that exception to the upstream-declared, tested RC hosts.
|
// Limit that exception to the upstream-declared, tested RC hosts.
|
||||||
if (name === 'react') {
|
if (name === 'react') {
|
||||||
const host = require('./package.json').version;
|
const host = require('./package.json').version;
|
||||||
if (!['0.2.0-rc.1', '0.2.0-rc.2'].includes(host)) throw new Error('unverified browser React host: ' + host);
|
if (host !== '0.2.1-rc.5') throw new Error('unverified browser React host: ' + host);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let peer;
|
let peer;
|
||||||
@@ -33,7 +33,7 @@ for (const [name, range] of Object.entries(pkg.peerDependencies || {})) {
|
|||||||
if (pkg.peerDependenciesMeta?.[name]?.optional) continue;
|
if (pkg.peerDependenciesMeta?.[name]?.optional) continue;
|
||||||
throw new Error('missing required mobile peer: ' + name);
|
throw new Error('missing required mobile peer: ' + name);
|
||||||
}
|
}
|
||||||
if (!semver.satisfies(peer.version, range)) throw new Error('incompatible mobile peer: ' + name + '@' + peer.version);
|
if (!(semver.satisfies(peer.version, range) || (peer.version === "0.2.1-rc.5" && name.startsWith("@deepseek-ai/dsh-") && semver.satisfies(peer.version, range, { includePrerelease: true })))) throw new Error('incompatible mobile peer: ' + name + '@' + peer.version);
|
||||||
}
|
}
|
||||||
// DSH runtime resolution traverses the installation manifest, not arbitrary
|
// DSH runtime resolution traverses the installation manifest, not arbitrary
|
||||||
// directories in node_modules. Declare the baked package in that graph.
|
// directories in node_modules. Declare the baked package in that graph.
|
||||||
@@ -41,4 +41,4 @@ const hostManifest = '/usr/local/lib/node_modules/@deepseek-ai/dsh/package.json'
|
|||||||
const host = JSON.parse(readFileSync(hostManifest));
|
const host = JSON.parse(readFileSync(hostManifest));
|
||||||
host.dependencies = { ...host.dependencies, 'dsh-web-mobile': version };
|
host.dependencies = { ...host.dependencies, 'dsh-web-mobile': version };
|
||||||
writeFileSync(hostManifest, JSON.stringify(host, null, 2));
|
writeFileSync(hostManifest, JSON.stringify(host, null, 2));
|
||||||
console.log('Installed dsh-web-mobile@3.0.5 (SHA-512 verified; no lifecycle scripts)');
|
console.log('Installed dsh-web-mobile@3.2.1 (SHA-512 verified; no lifecycle scripts)');
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
import { readFileSync, writeFileSync, copyFileSync } from 'node:fs';
|
||||||
|
import { join, resolve } from 'node:path';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
|
||||||
|
export const target = 'import { createConverter } from "@deepseek-ai/libreoffice-kit";';
|
||||||
|
export function patchOffice(source, hostVersion, packageVersion) {
|
||||||
|
if (hostVersion !== '0.2.1-rc.5' || packageVersion !== '0.2.1-rc.5') throw new Error('unsupported Office preview version');
|
||||||
|
if (source.split(target).length !== 2) throw new Error('expected exactly one original converter import');
|
||||||
|
return source.replace(target, 'import { createConverter } from "./dsh-native-office.mjs";');
|
||||||
|
}
|
||||||
|
export function install(root = '/usr/local/lib/node_modules/@deepseek-ai/dsh') {
|
||||||
|
const dir = join(root, 'node_modules/@deepseek-ai/dsh-office-to-pdf');
|
||||||
|
const version = path => JSON.parse(readFileSync(join(path, 'package.json'), 'utf8')).version;
|
||||||
|
const file = join(dir, 'lib/index.js');
|
||||||
|
const patched = patchOffice(readFileSync(file, 'utf8'), version(root), version(dir));
|
||||||
|
for (const [source, name] of [['native-office.mjs', 'dsh-native-office.mjs'], ['validate-office.py', 'validate-office.py'], ['office-worker.py', 'office-worker.py']]) {
|
||||||
|
copyFileSync(new URL(source, import.meta.url), join(dir, 'lib', name));
|
||||||
|
}
|
||||||
|
writeFileSync(file, patched);
|
||||||
|
}
|
||||||
|
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) install();
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
import { cpSync, mkdirSync, readFileSync } from 'node:fs';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
const root = '/usr/local/lib/dsh-mgr';
|
||||||
|
const host = JSON.parse(readFileSync('/usr/local/lib/node_modules/@deepseek-ai/dsh/package.json'));
|
||||||
|
if (host.version !== '0.2.1-rc.5') throw new Error('Unverified preview download host');
|
||||||
|
mkdirSync(root + '/preview-download-package', { recursive: true });
|
||||||
|
cpSync(new URL('./preview-download', import.meta.url), root + '/preview-download-package/package', { recursive: true });
|
||||||
|
execFileSync('tar', ['-czf', root + '/dsh-preview-download-1.0.1.tgz', '-C', root + '/preview-download-package', 'package']);
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
// Local Docker integration: no published ports or existing workspaces are used.
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { execFileSync } from 'node:child_process';
|
||||||
|
import { fileURLToPath } from 'node:url';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
const oldImage = process.env.DSH_OLD_IMAGE || 'dsh-web:runtime-7.10-amd64';
|
||||||
|
const newImage = process.env.DSH_NEW_IMAGE || 'dsh-web:runtime-7.11-amd64';
|
||||||
|
const volume = `dsh-lockfile-smoke-${randomUUID()}`;
|
||||||
|
const docker = args => execFileSync('docker', args, { encoding: 'utf8', timeout: 180000, maxBuffer: 8 * 1024 * 1024 });
|
||||||
|
const run = (image, args, mounts = []) => docker(['run', '--rm', '--network', 'none', '-v', `${volume}:/workspace`, ...mounts, '--entrypoint', 'bash', image, '-ec', args]);
|
||||||
|
docker(['volume', 'create', volume]);
|
||||||
|
try {
|
||||||
|
run(oldImage, 'node /usr/local/lib/dsh-mgr/init-profile.mjs; CI=true npm_config_offline=true dsh plugin --profile web install --ignore-scripts; test -s /workspace/profiles/web/pnpm-lock.yaml');
|
||||||
|
const mounts = ['init-profile.mjs', 'reconcile-profile.mjs'].flatMap(name => ['-v', `${fileURLToPath(new URL(name, import.meta.url))}:/usr/local/lib/dsh-mgr/${name}:ro`]);
|
||||||
|
mounts.push('-v', `${fileURLToPath(new URL('entrypoint.sh', import.meta.url))}:/usr/local/bin/dsh-entrypoint:ro`);
|
||||||
|
const output = run(newImage, 'bash /usr/local/bin/dsh-entrypoint --version; test ! -e /workspace/profiles/web/.dsh-lockfile-pending; CI=true npm_config_offline=true dsh plugin --profile web install --ignore-scripts', mounts);
|
||||||
|
assert.doesNotMatch(output, /ERR_PNPM_OUTDATED_LOCKFILE/);
|
||||||
|
const again = run(newImage, 'sha256sum /workspace/profiles/web/pnpm-lock.yaml; bash /usr/local/bin/dsh-entrypoint --version; sha256sum /workspace/profiles/web/pnpm-lock.yaml', mounts);
|
||||||
|
const hashes = again.match(/^[a-f0-9]{64} .*pnpm-lock.yaml$/gm);
|
||||||
|
assert.equal(hashes?.length, 2, again);
|
||||||
|
assert.equal(hashes[0], hashes[1]);
|
||||||
|
console.log(`PASS: ${oldImage} → ${newImage} migration, frozen reinstall, unchanged lockfile on restart`);
|
||||||
|
} finally {
|
||||||
|
docker(['volume', 'rm', volume]);
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { constants } from 'node:fs';
|
||||||
|
import { open, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, extname, isAbsolute } from 'node:path';
|
||||||
|
import { pathToFileURL, fileURLToPath } from 'node:url';
|
||||||
|
|
||||||
|
const failure = (code, message) => Object.assign(new Error(message), { code });
|
||||||
|
|
||||||
|
// No shell; terminate the whole group, including soffice.bin, on cancellation.
|
||||||
|
export function runProcess(executable, args, signal, env = process.env) {
|
||||||
|
signal.throwIfAborted();
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(executable, args, { detached: true, stdio: 'ignore', env });
|
||||||
|
let error;
|
||||||
|
const kill = () => {
|
||||||
|
if (child.pid) {
|
||||||
|
try { process.kill(-child.pid, 'SIGKILL'); }
|
||||||
|
catch (cause) { if (cause.code !== 'ESRCH') error ??= cause; }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const abort = () => { error = signal.reason; kill(); };
|
||||||
|
signal.addEventListener('abort', abort, { once: true });
|
||||||
|
if (signal.aborted) abort();
|
||||||
|
child.once('error', cause => { error ??= failure('unavailable', 'Cannot start helper: ' + cause.code); });
|
||||||
|
child.once('close', code => {
|
||||||
|
signal.removeEventListener('abort', abort);
|
||||||
|
kill();
|
||||||
|
if (error) reject(error);
|
||||||
|
else if (code !== 0) reject(failure('invalid-document', 'Conversion helper exit: ' + code));
|
||||||
|
else resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readBounded(path, maximum, kind, signal) {
|
||||||
|
const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||||
|
try {
|
||||||
|
const stat = await file.stat();
|
||||||
|
if (!stat.isFile()) throw failure('invalid-' + (kind === 'input' ? 'document' : 'output'), 'Regular file required');
|
||||||
|
if (stat.size > maximum) throw failure(kind + '-too-large', 'File exceeds limit');
|
||||||
|
const bytes = Buffer.alloc(stat.size + 1);
|
||||||
|
let length = 0;
|
||||||
|
while (length < bytes.length) {
|
||||||
|
signal.throwIfAborted();
|
||||||
|
const read = await file.read(bytes, length, bytes.length - length, null);
|
||||||
|
if (!read.bytesRead) break;
|
||||||
|
length += read.bytesRead;
|
||||||
|
}
|
||||||
|
if (length !== stat.size) throw failure('invalid-' + (kind === 'input' ? 'document' : 'output'), 'File changed');
|
||||||
|
return bytes.subarray(0, length);
|
||||||
|
} finally { await file.close(); }
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createConverter(options = {}) {
|
||||||
|
if (process.platform !== 'linux') throw failure('unavailable', 'Native preview requires Linux');
|
||||||
|
if (options.fontDirectories !== undefined || options.fontFallbacks !== undefined) {
|
||||||
|
throw failure('unavailable', 'Native preview uses system fonts; remove kit-only font overrides');
|
||||||
|
}
|
||||||
|
const lifetime = new AbortController();
|
||||||
|
const active = new Set();
|
||||||
|
const limits = { timeoutMs: 60000, maxInputBytes: 50 * 1024 ** 2, maxOutputBytes: 100 * 1024 ** 2, maxArchiveEntries: 10000, maxUncompressedBytes: 250 * 1024 ** 2, maxImageResolution: 192, ...options };
|
||||||
|
for (const key of ['timeoutMs', 'maxInputBytes', 'maxOutputBytes', 'maxArchiveEntries', 'maxUncompressedBytes', 'maxImageResolution']) {
|
||||||
|
if (!Number.isSafeInteger(limits[key]) || limits[key] < 1) throw new TypeError('Invalid ' + key);
|
||||||
|
}
|
||||||
|
async function render({ inputPath, outputPath }, external = new AbortController().signal) {
|
||||||
|
const deadline = new AbortController();
|
||||||
|
const signal = AbortSignal.any([external, lifetime.signal, deadline.signal]);
|
||||||
|
const timer = setTimeout(() => deadline.abort(failure('timeout', 'Native Office conversion timed out')), limits.timeoutMs);
|
||||||
|
let scratch;
|
||||||
|
let output;
|
||||||
|
let success = false;
|
||||||
|
try {
|
||||||
|
signal.throwIfAborted();
|
||||||
|
const extension = extname(inputPath).slice(1).toLowerCase();
|
||||||
|
if (!['doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx'].includes(extension)) throw failure('unsupported-format', 'Unsupported Office extension');
|
||||||
|
if (!isAbsolute(inputPath) || !isAbsolute(outputPath)) throw failure('invalid-document', 'Absolute paths required');
|
||||||
|
output = await open(outputPath, 'wx', 0o600);
|
||||||
|
scratch = await mkdtemp(join(tmpdir(), 'dsh-native-office-'));
|
||||||
|
const source = join(scratch, 'source.' + extension);
|
||||||
|
await writeFile(source, await readBounded(inputPath, limits.maxInputBytes, 'input', signal), { flag: 'wx', mode: 0o600, signal });
|
||||||
|
const launcher = fileURLToPath(new URL('./office-worker.py', import.meta.url));
|
||||||
|
const env = { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', HOME: scratch, TMPDIR: scratch, SAL_USE_VCLPLUGIN: 'svp', SAL_DISABLE_PRINTERLIST: '1', SAL_DISABLE_DEFAULTPRINTER: '1' };
|
||||||
|
await runProcess('/usr/bin/python3', ['-I', launcher, String(512 * 1024 ** 2), String(limits.maxOutputBytes), '/usr/bin/python3', '-I', fileURLToPath(new URL('./validate-office.py', import.meta.url)), source, extension, String(limits.maxArchiveEntries), String(limits.maxUncompressedBytes)], signal, env);
|
||||||
|
const profile = join(scratch, 'profile');
|
||||||
|
await mkdir(join(profile, 'user'), { recursive: true, mode: 0o700 });
|
||||||
|
await writeFile(join(profile, 'user/registrymodifications.xcu'), '<?xml version="1.0"?><oor:items xmlns:oor="http://openoffice.org/2001/registry"><item oor:path="/org.openoffice.Office.Common/Security/Scripting"><prop oor:name="MacroSecurityLevel" oor:op="fuse"><value>3</value></prop></item></oor:items>');
|
||||||
|
const filter = extension.startsWith('ppt') ? 'impress_pdf_Export' : extension.startsWith('xls') ? 'calc_pdf_Export' : 'writer_pdf_Export';
|
||||||
|
const pdfOptions = JSON.stringify({ ReduceImageResolution: { type: 'boolean', value: 'true' }, MaxImageResolution: { type: 'long', value: String(limits.maxImageResolution) } });
|
||||||
|
await runProcess('/usr/bin/python3', ['-I', launcher, String(2 * 1024 ** 3), String(Math.max(limits.maxOutputBytes, 16 * 1024 ** 2)), '/usr/bin/libreoffice', '-env:UserInstallation=' + pathToFileURL(profile).href, '--headless', '--nologo', '--nodefault', '--nofirststartwizard', '--convert-to', 'pdf:' + filter + ':' + pdfOptions, '--outdir', scratch, source], signal, env);
|
||||||
|
let bytes;
|
||||||
|
try { bytes = await readBounded(join(scratch, 'source.pdf'), limits.maxOutputBytes, 'output', signal); }
|
||||||
|
catch (cause) { if (cause.code === 'ENOENT') throw failure('invalid-output', 'LibreOffice produced no PDF'); throw cause; }
|
||||||
|
if (!/^%PDF-[0-9][.][0-9]/.test(bytes.subarray(0, 8).toString()) || !bytes.subarray(-1024).toString().trimEnd().endsWith('%%EOF')) throw failure('invalid-output', 'Incomplete PDF');
|
||||||
|
signal.throwIfAborted();
|
||||||
|
await output.writeFile(bytes);
|
||||||
|
signal.throwIfAborted();
|
||||||
|
success = true;
|
||||||
|
// System LibreOffice has no kit missing-font diagnostic API.
|
||||||
|
return { backend: 'native-system', missingFonts: [] };
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
try { await output?.close(); }
|
||||||
|
finally {
|
||||||
|
try { if (output && !success) await rm(outputPath, { force: true }); }
|
||||||
|
finally { if (scratch) await rm(scratch, { recursive: true, force: true }); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
render(request, signal) {
|
||||||
|
const task = render(request, signal);
|
||||||
|
active.add(task);
|
||||||
|
task.then(() => active.delete(task), () => active.delete(task));
|
||||||
|
return task;
|
||||||
|
},
|
||||||
|
async dispose() { lifetime.abort(failure('unavailable', 'Converter disposed')); await Promise.allSettled([...active]); }
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { mkdtemp, writeFile, readFile, rm, access } from 'node:fs/promises';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { createConverter, runProcess } from './native-office.mjs';
|
||||||
|
import { patchOffice, target } from './install-office-preview.mjs';
|
||||||
|
|
||||||
|
test('patch is exact, version gated and refuses repeat application', () => {
|
||||||
|
const patched = patchOffice(target, '0.2.1-rc.5', '0.2.1-rc.5');
|
||||||
|
assert.match(patched, /dsh-native-office/);
|
||||||
|
for (const source of ['', target + target, patched]) assert.throws(() => patchOffice(source, '0.2.1-rc.5', '0.2.1-rc.5'));
|
||||||
|
assert.throws(() => patchOffice(target, 'next', '0.2.1-rc.5'));
|
||||||
|
assert.throws(() => patchOffice(target, '0.2.1-rc.5', 'next'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('process cancellation, missing binary and failure are classified', async () => {
|
||||||
|
const signal = new AbortController().signal;
|
||||||
|
await assert.rejects(runProcess('/not-an-office-binary', [], signal), { code: 'unavailable' });
|
||||||
|
await assert.rejects(runProcess(process.execPath, ['-e', 'process.exit(2)'], signal), { code: 'invalid-document' });
|
||||||
|
const controller = new AbortController();
|
||||||
|
const reason = new Error('cancel test');
|
||||||
|
const pending = runProcess(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], controller.signal);
|
||||||
|
controller.abort(reason);
|
||||||
|
await assert.rejects(pending, error => error === reason);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('input bounds, corruption, exclusive output, abort and disposal', async () => {
|
||||||
|
const directory = await mkdtemp(join(tmpdir(), 'office-test-'));
|
||||||
|
const inputPath = join(directory, 'source.docx');
|
||||||
|
const outputPath = join(directory, 'output.pdf');
|
||||||
|
const converter = await createConverter();
|
||||||
|
try {
|
||||||
|
await writeFile(inputPath, 'invalid');
|
||||||
|
await assert.rejects(converter.render({ inputPath, outputPath }), { code: 'invalid-document' });
|
||||||
|
await assert.rejects(access(outputPath));
|
||||||
|
const small = await createConverter({ maxInputBytes: 1 });
|
||||||
|
await assert.rejects(small.render({ inputPath, outputPath }), { code: 'input-too-large' });
|
||||||
|
await small.dispose();
|
||||||
|
await writeFile(outputPath, 'preserve');
|
||||||
|
await assert.rejects(converter.render({ inputPath, outputPath }), { code: 'EEXIST' });
|
||||||
|
assert.equal(await readFile(outputPath, 'utf8'), 'preserve');
|
||||||
|
const controller = new AbortController();
|
||||||
|
controller.abort(new Error('cancelled'));
|
||||||
|
await assert.rejects(converter.render({ inputPath, outputPath }, controller.signal), /cancelled/);
|
||||||
|
await converter.dispose();
|
||||||
|
await assert.rejects(converter.render({ inputPath, outputPath }), { code: 'unavailable' });
|
||||||
|
await assert.rejects(createConverter({ fontDirectories: [] }), { code: 'unavailable' });
|
||||||
|
} finally { await converter.dispose(); await rm(directory, { recursive: true, force: true }); }
|
||||||
|
});
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
// Run inside the built image as UID 1000. No downloads or fixture dependencies.
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { mkdtemp, writeFile, readFile, rm, readdir } from 'node:fs/promises';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { pathToFileURL } from 'node:url';
|
||||||
|
import { runProcess } from './native-office.mjs';
|
||||||
|
const installed = '/usr/local/lib/node_modules/@deepseek-ai/dsh/node_modules/@deepseek-ai/dsh-office-to-pdf/lib/';
|
||||||
|
const { createConverter } = await import(installed + 'dsh-native-office.mjs');
|
||||||
|
const { OfficeToPdf } = await import(installed + 'index.js');
|
||||||
|
|
||||||
|
assert.notEqual(process.getuid(), 0, 'smoke test must run as non-root');
|
||||||
|
const dir = await mkdtemp(join(tmpdir(), 'office-smoke-'));
|
||||||
|
const signal = new AbortController().signal;
|
||||||
|
const ns = 'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"';
|
||||||
|
const converter = await createConverter();
|
||||||
|
try {
|
||||||
|
await writeFile(join(dir, 'word.fodt'), '<?xml version="1.0"?><office:document ' + ns + ' office:version="1.2" office:mimetype="application/vnd.oasis.opendocument.text"><office:body><office:text><text:p>中文预览 Word conversion test</text:p></office:text></office:body></office:document>');
|
||||||
|
await writeFile(join(dir, 'slides.fodp'), '<?xml version="1.0"?><office:document ' + ns + ' office:version="1.2" office:mimetype="application/vnd.oasis.opendocument.presentation"><office:body><office:presentation><draw:page draw:name="slide1"><draw:frame svg:x="2cm" svg:y="2cm" svg:width="20cm" svg:height="5cm"><draw:text-box><text:p>中文预览 PPT conversion test</text:p></draw:text-box></draw:frame></draw:page></office:presentation></office:body></office:document>');
|
||||||
|
for (const [source, format] of [['word.fodt', 'docx'], ['slides.fodp', 'pptx'], ['word.fodt', 'doc'], ['slides.fodp', 'ppt']]) {
|
||||||
|
await runProcess('/usr/bin/libreoffice', ['-env:UserInstallation=' + pathToFileURL(join(dir, 'fixture-profile')).href, '--headless', '--convert-to', format, '--outdir', dir, join(dir, source)], signal);
|
||||||
|
}
|
||||||
|
await Promise.all(['word.docx', 'slides.pptx', 'word.doc', 'slides.ppt'].map(async name => {
|
||||||
|
const start = performance.now();
|
||||||
|
const outputPath = join(dir, name + '.pdf');
|
||||||
|
const result = await converter.render({ inputPath: join(dir, name), outputPath });
|
||||||
|
assert.equal(result.backend, 'native-system');
|
||||||
|
assert.match((await readFile(outputPath)).subarray(0, 8).toString(), /^%PDF-/);
|
||||||
|
console.log(name + ': native PDF in ' + Math.round(performance.now() - start) + 'ms');
|
||||||
|
}));
|
||||||
|
const small = await createConverter({ maxOutputBytes: 1 });
|
||||||
|
await assert.rejects(small.render({ inputPath: join(dir, 'word.docx'), outputPath: join(dir, 'small.pdf') }), { code: 'output-too-large' });
|
||||||
|
await small.dispose();
|
||||||
|
const timeout = await createConverter({ timeoutMs: 1 });
|
||||||
|
await assert.rejects(timeout.render({ inputPath: join(dir, 'word.docx'), outputPath: join(dir, 'timeout.pdf') }), { code: 'timeout' });
|
||||||
|
await timeout.dispose();
|
||||||
|
const limited = await createConverter({ maxArchiveEntries: 1 });
|
||||||
|
await assert.rejects(limited.render({ inputPath: join(dir, 'word.docx'), outputPath: join(dir, 'limit.pdf') }), { code: 'invalid-document' });
|
||||||
|
await limited.dispose();
|
||||||
|
const provider = { slots: [], options: {}, config: { maxOutputBytes: 100 * 1024 ** 2 } };
|
||||||
|
const converted = await OfficeToPdf.prototype.convertBytes.call(provider, await readFile(join(dir, 'slides.pptx')), 'pptx', signal);
|
||||||
|
assert.match(Buffer.from(converted.pdf).subarray(0, 8).toString(), /^%PDF-/);
|
||||||
|
await Promise.all(provider.slots.map(async slot => (await slot.converter).dispose()));
|
||||||
|
console.log('installed DSH convertBytes integration passed');
|
||||||
|
assert.equal((await readdir(tmpdir())).filter(name => name.startsWith('dsh-native-office-')).length, 0);
|
||||||
|
console.log('concurrent DOCX/PPTX/DOC/PPT, output/ZIP limits, timeout and cleanup passed');
|
||||||
|
} finally { await converter.dispose(); await rm(dir, { recursive: true, force: true }); }
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
"""Apply process limits before executing a trusted conversion helper."""
|
||||||
|
import os
|
||||||
|
import resource
|
||||||
|
import sys
|
||||||
|
|
||||||
|
memory, file_bytes, *command = sys.argv[1:]
|
||||||
|
resource.setrlimit(resource.RLIMIT_AS, (int(memory), int(memory)))
|
||||||
|
resource.setrlimit(resource.RLIMIT_FSIZE, (int(file_bytes), int(file_bytes)))
|
||||||
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||||
|
os.execv(command[0], command)
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
// Scoped to this plugin; host authentication and global fetch are unchanged.
|
||||||
|
async function dshChatFetch(path, init) {
|
||||||
|
const parts = window.location.pathname.split('/');
|
||||||
|
const prefix = parts[1] === 'dsh' && parts[2] ? '/dsh/' + parts[2] : '';
|
||||||
|
return fetch(prefix + path, init);
|
||||||
|
}
|
||||||
|
async function dshChatJson(response) {
|
||||||
|
const text = await response.text();
|
||||||
|
try { return JSON.parse(text); } catch {
|
||||||
|
throw new Error('Chat Manager: HTTP ' + response.status + ' returned a non-JSON response; check the instance proxy path and authentication.');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export const clientHelpers = dshChatFetch.toString() + String.fromCharCode(10) + dshChatJson.toString() + String.fromCharCode(10);
|
||||||
|
|
||||||
|
export function patchClient(source, version) {
|
||||||
|
if (version !== '1.5.20') throw new Error('unsupported chat-manager version');
|
||||||
|
if (source.includes('dshChatFetch') || source.split('await fetch(').length !== 4
|
||||||
|
|| source.split('await response.json()').length !== 4
|
||||||
|
|| source.split('/plugins/dsh-session-delete/').length !== 4) {
|
||||||
|
throw new Error('unexpected chat-manager client; review upstream changes');
|
||||||
|
}
|
||||||
|
return clientHelpers + source.replaceAll('await fetch(', 'await dshChatFetch(')
|
||||||
|
.replaceAll('await response.json()', 'await dshChatJson(response)');
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { runInNewContext } from 'node:vm';
|
||||||
|
import { clientHelpers, patchClient } from './patch-chat-manager.mjs';
|
||||||
|
|
||||||
|
test('chat paths preserve instance prefix and POST options', async () => {
|
||||||
|
for (const pathname of ['/dsh/demo/', '/dsh/demo', '/dsh/demo/session/123', '/', '/other/']) {
|
||||||
|
let received;
|
||||||
|
const context = { window: { location: { pathname } }, fetch: (...args) => { received = args; } };
|
||||||
|
runInNewContext(clientHelpers, context);
|
||||||
|
for (const action of ['delete', 'restore', 'archive-search']) {
|
||||||
|
const path = '/plugins/dsh-session-delete/' + action;
|
||||||
|
const options = { method: 'POST', body: '{}', headers: { 'x-dsh-session-delete-confirmation': 'delete-session' } };
|
||||||
|
await context.dshChatFetch(path, options);
|
||||||
|
assert.equal(received[0], (pathname.startsWith('/dsh/demo') ? '/dsh/demo' : '') + path);
|
||||||
|
assert.equal(received[1], options);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
test('non-JSON errors report HTTP status without exposing response content', async () => {
|
||||||
|
const context = {};
|
||||||
|
runInNewContext(clientHelpers, context);
|
||||||
|
await assert.rejects(context.dshChatJson(new Response('method not allowed', { status: 405 })), /HTTP 405.*non-JSON/);
|
||||||
|
const body = await context.dshChatJson(Response.json({ ok: false, error: { code: 'forbidden' } }, { status: 403 }));
|
||||||
|
assert.equal(body.error.code, 'forbidden');
|
||||||
|
});
|
||||||
|
test('patch rejects unsupported or changed clients and double patching', () => {
|
||||||
|
const source = [
|
||||||
|
"await fetch('/plugins/dsh-session-delete/delete', options); await response.json()",
|
||||||
|
'await fetch(`/plugins/dsh-session-delete/${action}`, options); await response.json()',
|
||||||
|
"await fetch('/plugins/dsh-session-delete/archive-search', options); await response.json()",
|
||||||
|
].join(';');
|
||||||
|
const patched = patchClient(source, '1.5.20');
|
||||||
|
assert.equal(patched.split('await dshChatFetch(').length, 4);
|
||||||
|
assert.equal(patched.slice(clientHelpers.length).replaceAll('await dshChatFetch(', 'await fetch(')
|
||||||
|
.replaceAll('await dshChatJson(response)', 'await response.json()'), source);
|
||||||
|
assert.throws(() => patchClient(source, '1.5.16'), /unsupported/);
|
||||||
|
assert.throws(() => patchClient(source, '1.5.13'), /unsupported/);
|
||||||
|
assert.throws(() => patchClient('', '1.5.20'), /unexpected/);
|
||||||
|
assert.throws(() => patchClient(source.replace('await fetch(', 'fetch('), '1.5.20'), /unexpected/);
|
||||||
|
assert.throws(() => patchClient(source.replace('await response.json()', 'response.json()'), '1.5.20'), /unexpected/);
|
||||||
|
assert.throws(() => patchClient(source.replace('/plugins/dsh-session-delete/', '/plugins/changed/'), '1.5.20'), /unexpected/);
|
||||||
|
assert.throws(() => patchClient(patched, '1.5.20'), /unexpected/);
|
||||||
|
});
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
|
||||||
|
test('release targets AMD64 and ARM64 with layout 7', () => {
|
||||||
|
const release = JSON.parse(readFileSync(new URL('../release.json', import.meta.url), 'utf8'));
|
||||||
|
assert.deepEqual(release.platforms, ['linux/amd64', 'linux/arm64']);
|
||||||
|
assert.equal(release.deployment_schema, 7);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Node and Caddy stages inherit the target platform', () => {
|
||||||
|
const dockerfile = readFileSync(new URL('./Dockerfile', import.meta.url), 'utf8');
|
||||||
|
const stages = dockerfile.split(String.fromCharCode(10)).filter(line => line.startsWith('FROM '));
|
||||||
|
assert.deepEqual(stages, ['FROM caddy:2.10.2 AS proxy', 'FROM node:24-bookworm-slim']);
|
||||||
|
assert.ok(dockerfile.includes('COPY --from=proxy /usr/bin/caddy /usr/bin/caddy'));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runtime includes Python tooling and headless LibreOffice with CJK fonts', () => {
|
||||||
|
const dockerfile = readFileSync(new URL('./Dockerfile', import.meta.url), 'utf8');
|
||||||
|
const install = dockerfile.split(String.fromCharCode(92, 10)).join(' ').split(String.fromCharCode(10)).find(line => line.startsWith('RUN apt-get update && apt-get install'));
|
||||||
|
for (const pkg of ['python3', 'python3-pip', 'python3-venv', 'python-is-python3', 'libreoffice', 'fonts-noto-cjk']) {
|
||||||
|
assert.ok(install.split(/\s+/).includes(pkg), `missing runtime package: ${pkg}`);
|
||||||
|
}
|
||||||
|
assert.ok(dockerfile.includes('RUN python --version && python3 -m pip --version && libreoffice --headless --version'));
|
||||||
|
});
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { runInNewContext } from 'node:vm';
|
||||||
|
|
||||||
|
test('installer accepts the release host and rejects unverified hosts before writing', () => {
|
||||||
|
const source = readFileSync(new URL('./install-preview-download.mjs', import.meta.url), 'utf8')
|
||||||
|
.replace(/^import .*;$/gm, '').replaceAll('import.meta.url', JSON.stringify(import.meta.url));
|
||||||
|
const release = JSON.parse(readFileSync(new URL('../release.json', import.meta.url)));
|
||||||
|
for (const version of [release.version, '0.2.1-rc.2', '0.2.1-rc.4', 'next']) {
|
||||||
|
const calls = [];
|
||||||
|
const run = () => runInNewContext(source, {
|
||||||
|
URL, readFileSync: () => JSON.stringify({ version }),
|
||||||
|
mkdirSync: (...args) => calls.push(['mkdir', ...args]),
|
||||||
|
cpSync: (...args) => calls.push(['copy', ...args]),
|
||||||
|
execFileSync: (...args) => calls.push(['exec', ...args]),
|
||||||
|
});
|
||||||
|
if (version === release.version) {
|
||||||
|
run();
|
||||||
|
assert.deepEqual(calls.map(call => call[0]), ['mkdir', 'copy', 'exec']);
|
||||||
|
assert.equal(calls[2][1], 'tar');
|
||||||
|
assert.equal(calls[2][2][1], '/usr/local/lib/dsh-mgr/dsh-preview-download-1.0.1.tgz');
|
||||||
|
} else {
|
||||||
|
assert.throws(run, /Unverified preview download host/);
|
||||||
|
assert.equal(calls.length, 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
function load() {
|
||||||
|
let plugin;
|
||||||
|
runInNewContext(readFileSync(new URL('./preview-download/client.js', import.meta.url), 'utf8'), {
|
||||||
|
Uint8Array, window: { __ModuleLoader__: { load({ id, factory }) { assert.equal(id, 'dsh-preview-download'); plugin = factory(() => ({})); } } }
|
||||||
|
});
|
||||||
|
return plugin;
|
||||||
|
}
|
||||||
|
test('download passes exact session/path/signal to authorized complete-file RPC', async () => {
|
||||||
|
const plugin = load();
|
||||||
|
const data = new Uint8Array([0, 255, 10]);
|
||||||
|
const signal = new AbortController().signal;
|
||||||
|
const remote = { workspaceFiles: { async readBytes(...args) {
|
||||||
|
assert.equal(args[0], 'session-A'); assert.equal(args[1], '/workspace/中文.pptx');
|
||||||
|
assert.equal(Object.keys(args[2]).length, 0); assert.equal(args[3], signal);
|
||||||
|
return { ok: true, value: { offset: 0, eof: true, bytes: 3, data } };
|
||||||
|
} } };
|
||||||
|
assert.equal(await plugin.readOriginal(remote, 'session-A', '/workspace/中文.pptx', signal), data);
|
||||||
|
assert.equal(plugin.filename('/workspace/中文.pptx'), '中文.pptx');
|
||||||
|
assert.equal(plugin.filename('C:' + String.fromCharCode(92) + 'folder' + String.fromCharCode(92) + 'file.docx'), 'file.docx');
|
||||||
|
});
|
||||||
|
test('denied, partial, size mismatch and cancelled reads never produce download data', async () => {
|
||||||
|
const plugin = load();
|
||||||
|
for (const result of [
|
||||||
|
{ ok: false, error: { code: 'gateway/forbidden' } },
|
||||||
|
{ ok: true, value: { offset: 0, eof: false, data: new Uint8Array() } },
|
||||||
|
{ ok: true, value: { offset: 0, eof: true, bytes: 5, data: new Uint8Array(2) } }
|
||||||
|
]) {
|
||||||
|
await assert.rejects(plugin.readOriginal({ workspaceFiles: { readBytes: async () => result } }, 's', '/file', new AbortController().signal));
|
||||||
|
}
|
||||||
|
const controller = new AbortController(); controller.abort();
|
||||||
|
await assert.rejects(plugin.readOriginal({ workspaceFiles: { readBytes: async () => ({ ok: true, value: {} }) } }, 's', '/file', controller.signal));
|
||||||
|
});
|
||||||
|
test('plugin registers only the supported preview action slot', () => {
|
||||||
|
const plugin = load();
|
||||||
|
let registration;
|
||||||
|
plugin.apply({ effect: fn => fn(), locale: { register: () => () => {} }, remote: {}, slots: {
|
||||||
|
inject(name, fn) { assert.equal(name, 'sidebar.right.tab.document.actions'); return fn(); },
|
||||||
|
register(options, component) { registration = options; assert.equal(typeof component, 'function'); return () => {}; }
|
||||||
|
} });
|
||||||
|
assert.equal(registration.id, 'dsh-preview-download');
|
||||||
|
assert.equal(typeof registration.inject('session-A').download, 'function');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('registered download declares every service used by its callback', async () => {
|
||||||
|
const plugin = load();
|
||||||
|
assert.ok(plugin.inject.includes('remote'));
|
||||||
|
assert.ok(plugin.inject.includes('remote.workspaceFiles'));
|
||||||
|
let registration;
|
||||||
|
let calls = 0;
|
||||||
|
const data = new Uint8Array([1, 2, 255]);
|
||||||
|
const ctx = {
|
||||||
|
effect: fn => fn(),
|
||||||
|
locale: { register: () => () => {} },
|
||||||
|
slots: { inject: (_name, fn) => fn(), register(options) { registration = options; return () => {}; } },
|
||||||
|
get remote() {
|
||||||
|
assert.ok(plugin.inject.includes('remote'), 'Cordis requires explicit remote injection');
|
||||||
|
return { workspaceFiles: { async readBytes(session, path) {
|
||||||
|
assert.equal(session, 'session-A'); assert.equal(path, '/file.bin'); calls++;
|
||||||
|
return { ok: true, value: { offset: 0, eof: true, bytes: 3, data } };
|
||||||
|
} } };
|
||||||
|
}
|
||||||
|
};
|
||||||
|
plugin.apply(ctx);
|
||||||
|
assert.equal(await registration.inject('session-A').download('/file.bin', new AbortController().signal), data);
|
||||||
|
assert.equal(calls, 1);
|
||||||
|
});
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
window.__ModuleLoader__.load({ id: 'dsh-preview-download', factory: require => {
|
||||||
|
const React = require('react');
|
||||||
|
const { createElement: h, useState, useRef, useEffect } = React;
|
||||||
|
const namespace = 'previewDownload';
|
||||||
|
|
||||||
|
function filename(path) {
|
||||||
|
const name = path.replaceAll('\\', '/').split('/').pop() || 'download';
|
||||||
|
return Array.from(name, char => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127 || '<>:"|?*'.includes(char) ? '_' : char).join('');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readOriginal(remote, sessionId, path, signal) {
|
||||||
|
signal.throwIfAborted();
|
||||||
|
const result = await remote.workspaceFiles.readBytes(sessionId, path, {}, signal);
|
||||||
|
signal.throwIfAborted();
|
||||||
|
if (!result.ok) throw Object.assign(new Error('Download refused'), { code: result.error.code });
|
||||||
|
const file = result.value;
|
||||||
|
if (!file.eof || file.offset !== 0 || !(file.data instanceof Uint8Array) || (file.bytes !== undefined && file.bytes !== file.data.byteLength)) {
|
||||||
|
throw new Error('Incomplete file response');
|
||||||
|
}
|
||||||
|
return file.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
function Download({ absolutePath, download, t }) {
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState('');
|
||||||
|
const pending = useRef(null);
|
||||||
|
useEffect(() => {
|
||||||
|
setBusy(false);
|
||||||
|
setError('');
|
||||||
|
return () => { pending.current?.abort(); pending.current = null; };
|
||||||
|
}, [absolutePath, download]);
|
||||||
|
async function start() {
|
||||||
|
if (pending.current) return;
|
||||||
|
const controller = new AbortController();
|
||||||
|
pending.current = controller;
|
||||||
|
setBusy(true);
|
||||||
|
setError('');
|
||||||
|
try {
|
||||||
|
const bytes = await download(absolutePath, controller.signal);
|
||||||
|
controller.signal.throwIfAborted();
|
||||||
|
const url = URL.createObjectURL(new Blob([bytes], { type: 'application/octet-stream' }));
|
||||||
|
const link = document.createElement('a');
|
||||||
|
link.href = url;
|
||||||
|
link.download = filename(absolutePath);
|
||||||
|
link.hidden = true;
|
||||||
|
document.body.appendChild(link);
|
||||||
|
try { link.click(); }
|
||||||
|
finally { link.remove(); setTimeout(() => URL.revokeObjectURL(url), 60000); }
|
||||||
|
} catch (cause) {
|
||||||
|
if (!controller.signal.aborted) setError(t(cause?.code === 'workspace-file/too-large' ? 'tooLarge' : 'failed'));
|
||||||
|
} finally {
|
||||||
|
if (pending.current === controller) { pending.current = null; setBusy(false); }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return h('span', { style: { display: 'inline-flex', alignItems: 'center', gap: '6px', flexShrink: 0 } },
|
||||||
|
h('button', { type: 'button', disabled: busy, onClick: start, title: t('download'), 'aria-label': t('download'), 'aria-busy': busy, 'data-preview-download': '',
|
||||||
|
style: { color: 'inherit', background: 'transparent', border: '1px solid currentColor', borderRadius: '6px', padding: '4px 8px', font: 'inherit', cursor: busy ? 'wait' : 'pointer' }
|
||||||
|
}, busy ? t('loading') : t('download')),
|
||||||
|
error && h('span', { role: 'alert', style: { fontSize: '12px', maxWidth: '180px' } }, error));
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
inject: ['slots', 'locale', 'remote', 'remote.workspaceFiles'],
|
||||||
|
apply(ctx) {
|
||||||
|
ctx.effect(() => ctx.locale.register(namespace, {
|
||||||
|
zh: { download: '下载原文件', loading: '下载中…', failed: '下载失败，请检查文件权限或重试。', tooLarge: '文件超过下载大小限制。' },
|
||||||
|
en: { download: 'Download original', loading: 'Downloading…', failed: 'Download failed. Check access or retry.', tooLarge: 'File exceeds the download size limit.' }
|
||||||
|
}));
|
||||||
|
ctx.effect(() => ctx.slots.inject('sidebar.right.tab.document.actions', () => ctx.slots.register({
|
||||||
|
name: 'sidebar.right.tab.document.actions', id: 'dsh-preview-download', locale: namespace,
|
||||||
|
inject: sessionId => ({ download: (path, signal) => readOriginal(ctx.remote, sessionId, path, signal) })
|
||||||
|
}, Download)));
|
||||||
|
},
|
||||||
|
// Pure helpers are also exercised by the offline regression suite.
|
||||||
|
filename, readOriginal
|
||||||
|
};
|
||||||
|
} });
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
- insert:
|
||||||
|
- id: dsh-preview-download
|
||||||
|
name: dsh-preview-download
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
// Client-only extension: no new server endpoints or authorization changes.
|
||||||
|
export const name = 'dsh-preview-download';
|
||||||
|
export function apply() {}
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"name": "dsh-preview-download",
|
||||||
|
"version": "1.0.1",
|
||||||
|
"private": true,
|
||||||
|
"type": "module",
|
||||||
|
"main": "index.js",
|
||||||
|
"exports": { ".": "./index.js", "./client": "./client.js" },
|
||||||
|
"dsh": {
|
||||||
|
"bundle": { "patch": "cordis.patch.yml" },
|
||||||
|
"client": {
|
||||||
|
"platform": "web",
|
||||||
|
"inject": ["@deepseek-ai/dsh-client-ui-sidebar-documentpreview", "@deepseek-ai/dsh-client-ui-slots", "@deepseek-ai/dsh-client-locale", "@deepseek-ai/dsh-api-workspace-files"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,133 @@
|
|||||||
|
import { lstatSync, readFileSync, writeFileSync, openSync, closeSync, fsyncSync, renameSync, unlinkSync } from 'node:fs';
|
||||||
|
import { dirname, join, resolve, basename } from 'node:path';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
|
||||||
|
export const BUILTIN_DEPENDENCIES = Object.freeze({
|
||||||
|
'dsh-preview-download': 'file:/usr/local/lib/dsh-mgr/dsh-preview-download-1.0.1.tgz',
|
||||||
|
'dsh-chat-manager': 'file:/usr/local/lib/dsh-mgr/dsh-chat-manager-1.5.20-dsh-mgr.1.tgz',
|
||||||
|
'dsh-mgr-workspace': 'file:/usr/local/lib/dsh-mgr/dsh-mgr-workspace-1.0.0.tgz',
|
||||||
|
'dsh-web-mobile': 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.2.1.tgz',
|
||||||
|
'dsh-free-search': 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.9.21.tgz',
|
||||||
|
});
|
||||||
|
export const BUILTIN_OVERRIDES = Object.freeze({
|
||||||
|
'dsh-chat-manager>clsx': 'file:/usr/local/lib/dsh-mgr/clsx-2.1.1.tgz',
|
||||||
|
'dsh-free-search>@deepseek-ai/schemastery': 'file:/usr/local/lib/dsh-mgr/schemastery-3.18.4.tgz',
|
||||||
|
'@deepseek-ai/schemastery>@deepseek-ai/cosmokit': 'file:/usr/local/lib/dsh-mgr/cosmokit-1.8.5.tgz',
|
||||||
|
'@deepseek-ai/schemastery>@standard-schema/spec': 'file:/usr/local/lib/dsh-mgr/spec-1.1.0.tgz',
|
||||||
|
});
|
||||||
|
const sections = ['dependencies', 'devDependencies', 'optionalDependencies'];
|
||||||
|
export const LOCKFILE_PENDING = '.dsh-lockfile-pending';
|
||||||
|
|
||||||
|
export function markLockfilePending(profile) {
|
||||||
|
assertSafeProfile(profile);
|
||||||
|
const fd = openSync(join(profile, LOCKFILE_PENDING), 'w', 0o600);
|
||||||
|
try { writeFileSync(fd, 'builtin profile migration\n'); fsyncSync(fd); }
|
||||||
|
finally { closeSync(fd); }
|
||||||
|
const directoryFd = openSync(profile, 'r');
|
||||||
|
try { fsyncSync(directoryFd); } finally { closeSync(directoryFd); }
|
||||||
|
}
|
||||||
|
const overrideOwners = ['dsh-chat-manager', 'dsh-free-search', 'dsh-free-search', 'dsh-free-search'];
|
||||||
|
const semver = '(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)\\.(?:0|[1-9]\\d*)(?:-(?:0|[1-9]\\d*|\\d*[A-Za-z-][0-9A-Za-z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?';
|
||||||
|
|
||||||
|
function object(value, label) {
|
||||||
|
if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error(`invalid ${label}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reconcileManifest(input) {
|
||||||
|
object(input, 'manifest');
|
||||||
|
const manifest = structuredClone(input);
|
||||||
|
const active = new Set();
|
||||||
|
const archives = new Set();
|
||||||
|
let changed = false;
|
||||||
|
for (const section of sections) {
|
||||||
|
if (manifest[section] === undefined) continue;
|
||||||
|
object(manifest[section], section);
|
||||||
|
for (const [name, target] of Object.entries(BUILTIN_DEPENDENCIES)) {
|
||||||
|
const current = manifest[section][name];
|
||||||
|
if (typeof current !== 'string' || !new RegExp(`^file:/usr/local/lib/dsh-mgr/${name}-${semver}\\.tgz$`).test(current)) continue;
|
||||||
|
active.add(name);
|
||||||
|
archives.add(target);
|
||||||
|
if (current !== target) {
|
||||||
|
manifest[section][name] = target;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (manifest.pnpm !== undefined) {
|
||||||
|
object(manifest.pnpm, 'pnpm');
|
||||||
|
if (manifest.pnpm.overrides !== undefined) object(manifest.pnpm.overrides, 'pnpm.overrides');
|
||||||
|
}
|
||||||
|
Object.entries(BUILTIN_OVERRIDES).forEach(([key, target], index) => {
|
||||||
|
if (!active.has(overrideOwners[index])) return;
|
||||||
|
manifest.pnpm ??= {};
|
||||||
|
manifest.pnpm.overrides ??= {};
|
||||||
|
if (!Object.hasOwn(manifest.pnpm.overrides, key)) {
|
||||||
|
manifest.pnpm.overrides[key] = target;
|
||||||
|
changed = true;
|
||||||
|
}
|
||||||
|
if (manifest.pnpm.overrides[key] === target) archives.add(target);
|
||||||
|
});
|
||||||
|
return { manifest, changed, archives: [...archives] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function statIfPresent(path) {
|
||||||
|
try { return lstatSync(path); } catch (error) {
|
||||||
|
if (error.code !== 'ENOENT') throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function assertSafeProfile(profile) {
|
||||||
|
for (let dir = resolve(profile); ; dir = dirname(dir)) {
|
||||||
|
const stat = statIfPresent(dir);
|
||||||
|
if (stat && (!stat.isDirectory() || stat.isSymbolicLink())) throw new Error(`unsafe profile directory: ${dir}`);
|
||||||
|
if (dirname(dir) === dir) break;
|
||||||
|
}
|
||||||
|
for (const name of ['package.json', 'pnpm-workspace.yaml', 'pnpm-lock.yaml', LOCKFILE_PENDING]) {
|
||||||
|
const path = join(profile, name);
|
||||||
|
const stat = statIfPresent(path);
|
||||||
|
if (stat && (!stat.isFile() || stat.isSymbolicLink())) throw new Error(`unsafe profile file: ${path}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function reconcileProfile(profile, { archiveDirectory = '/usr/local/lib/dsh-mgr' } = {}) {
|
||||||
|
assertSafeProfile(profile);
|
||||||
|
const path = join(profile, 'package.json');
|
||||||
|
if (!statIfPresent(path)) return { changed: false };
|
||||||
|
const original = readFileSync(path);
|
||||||
|
const result = reconcileManifest(JSON.parse(original.toString('utf8')));
|
||||||
|
for (const reference of result.archives) {
|
||||||
|
const archive = join(archiveDirectory, basename(reference.slice(5)));
|
||||||
|
const stat = lstatSync(archive);
|
||||||
|
if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`unsafe builtin archive: ${archive}`);
|
||||||
|
}
|
||||||
|
if (!result.changed) return { changed: false };
|
||||||
|
const backupPath = `${path}.backup-${randomUUID()}`;
|
||||||
|
const temporaryPath = `${path}.tmp-${randomUUID()}`;
|
||||||
|
let temporaryCreated = false;
|
||||||
|
try {
|
||||||
|
const backupFd = openSync(backupPath, 'wx', 0o600);
|
||||||
|
try { writeFileSync(backupFd, original); fsyncSync(backupFd); }
|
||||||
|
finally { closeSync(backupFd); }
|
||||||
|
const syncDirectory = () => {
|
||||||
|
const directoryFd = openSync(profile, 'r');
|
||||||
|
try { fsyncSync(directoryFd); } finally { closeSync(directoryFd); }
|
||||||
|
};
|
||||||
|
// Make the recovery copy durable before replacing the active manifest.
|
||||||
|
syncDirectory();
|
||||||
|
const fd = openSync(temporaryPath, 'wx', 0o600);
|
||||||
|
temporaryCreated = true;
|
||||||
|
try {
|
||||||
|
writeFileSync(fd, JSON.stringify(result.manifest, null, 2) + '\n');
|
||||||
|
fsyncSync(fd);
|
||||||
|
} finally { closeSync(fd); }
|
||||||
|
assertSafeProfile(profile);
|
||||||
|
// Persist before changing the manifest, including across failed installs/restarts.
|
||||||
|
markLockfilePending(profile);
|
||||||
|
renameSync(temporaryPath, path);
|
||||||
|
temporaryCreated = false;
|
||||||
|
syncDirectory();
|
||||||
|
} finally {
|
||||||
|
if (temporaryCreated) unlinkSync(temporaryPath);
|
||||||
|
}
|
||||||
|
return { changed: true, backupPath };
|
||||||
|
}
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
import { test } from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs, { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync, symlinkSync, readdirSync, statSync, linkSync } from 'node:fs';
|
||||||
|
import { syncBuiltinESMExports } from 'node:module';
|
||||||
|
import { tmpdir } from 'node:os';
|
||||||
|
import { join, basename } from 'node:path';
|
||||||
|
import { BUILTIN_DEPENDENCIES, BUILTIN_OVERRIDES, reconcileManifest, reconcileProfile } from './reconcile-profile.mjs';
|
||||||
|
import { initProfile } from './init-profile.mjs';
|
||||||
|
|
||||||
|
const old = name => 'file:/usr/local/lib/dsh-mgr/' + name + '-0.0.1-rc.1+build.2.tgz';
|
||||||
|
test('runtime-7.9 plugin references migrate to runtime-7.11 without changing bundle choices', t => {
|
||||||
|
const bundles = ['dsh-web-mobile', 'dsh-chat-manager'];
|
||||||
|
const f = fixture(t, {
|
||||||
|
dependencies: {
|
||||||
|
'dsh-web-mobile': 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.1.0.tgz',
|
||||||
|
'dsh-free-search': 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.9.2.tgz',
|
||||||
|
'dsh-chat-manager': 'file:/usr/local/lib/dsh-mgr/dsh-chat-manager-1.5.16-dsh-mgr.1.tgz',
|
||||||
|
},
|
||||||
|
dsh: { profile: { bundles } },
|
||||||
|
});
|
||||||
|
const result = f.run();
|
||||||
|
assert.equal(result.changed, true);
|
||||||
|
assert.equal(readFileSync(result.backupPath, 'utf8'), f.original);
|
||||||
|
const current = JSON.parse(readFileSync(f.path));
|
||||||
|
for (const name of Object.keys(current.dependencies)) assert.equal(current.dependencies[name], BUILTIN_DEPENDENCIES[name]);
|
||||||
|
assert.deepEqual(current.dsh.profile.bundles, bundles);
|
||||||
|
assert.deepEqual(f.run(), { changed: false });
|
||||||
|
});
|
||||||
|
test('runtime-7.10 plugin references migrate to runtime-7.11 without changing bundle choices', t => {
|
||||||
|
const bundles = ['dsh-web-mobile', 'dsh-chat-manager'];
|
||||||
|
const f = fixture(t, {
|
||||||
|
dependencies: {
|
||||||
|
'dsh-web-mobile': 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-3.2.1.tgz',
|
||||||
|
'dsh-free-search': 'file:/usr/local/lib/dsh-mgr/dsh-free-search-0.9.15.tgz',
|
||||||
|
'dsh-chat-manager': 'file:/usr/local/lib/dsh-mgr/dsh-chat-manager-1.5.18-dsh-mgr.1.tgz',
|
||||||
|
},
|
||||||
|
dsh: { profile: { bundles } },
|
||||||
|
});
|
||||||
|
const result = f.run();
|
||||||
|
assert.equal(result.changed, true);
|
||||||
|
assert.equal(readFileSync(result.backupPath, 'utf8'), f.original);
|
||||||
|
const current = JSON.parse(readFileSync(f.path));
|
||||||
|
for (const name of Object.keys(current.dependencies)) assert.equal(current.dependencies[name], BUILTIN_DEPENDENCIES[name]);
|
||||||
|
assert.deepEqual(current.dsh.profile.bundles, bundles);
|
||||||
|
assert.deepEqual(f.run(), { changed: false });
|
||||||
|
});
|
||||||
|
function fixture(t, manifest) {
|
||||||
|
const root = mkdtempSync(join(tmpdir(), 'dsh-reconcile-'));
|
||||||
|
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||||
|
const profile = join(root, 'profiles/web');
|
||||||
|
const archiveDirectory = join(root, 'archives');
|
||||||
|
mkdirSync(profile, { recursive: true });
|
||||||
|
mkdirSync(archiveDirectory);
|
||||||
|
for (const reference of [...Object.values(BUILTIN_DEPENDENCIES), ...Object.values(BUILTIN_OVERRIDES)]) {
|
||||||
|
writeFileSync(join(archiveDirectory, basename(reference)), 'fixture');
|
||||||
|
}
|
||||||
|
const path = join(profile, 'package.json');
|
||||||
|
const original = typeof manifest === 'string' ? manifest : JSON.stringify(manifest, null, 4) + '\n';
|
||||||
|
writeFileSync(path, original);
|
||||||
|
return { root, profile, path, original, archiveDirectory, run: () => reconcileProfile(profile, { archiveDirectory }) };
|
||||||
|
}
|
||||||
|
|
||||||
|
test('migrates all sections, backs up exact bytes, replaces atomically with 0600, and is idempotent', t => {
|
||||||
|
const f = fixture(t, {
|
||||||
|
dependencies: { 'dsh-web-mobile': old('dsh-web-mobile') },
|
||||||
|
devDependencies: { 'dsh-chat-manager': old('dsh-chat-manager') },
|
||||||
|
optionalDependencies: { 'dsh-free-search': old('dsh-free-search') },
|
||||||
|
dsh: { profile: { bundles: [] } },
|
||||||
|
});
|
||||||
|
const originalLink = join(f.root, 'original-link');
|
||||||
|
linkSync(f.path, originalLink);
|
||||||
|
const first = f.run();
|
||||||
|
assert.equal(first.changed, true);
|
||||||
|
assert.equal(readFileSync(first.backupPath, 'utf8'), f.original);
|
||||||
|
assert.equal(readFileSync(originalLink, 'utf8'), f.original);
|
||||||
|
assert.equal(statSync(f.path).mode & 0o777, 0o600);
|
||||||
|
assert.equal(statSync(first.backupPath).mode & 0o777, 0o600);
|
||||||
|
const current = JSON.parse(readFileSync(f.path));
|
||||||
|
for (const section of ['dependencies', 'devDependencies', 'optionalDependencies']) {
|
||||||
|
for (const name of Object.keys(current[section])) assert.equal(current[section][name], BUILTIN_DEPENDENCIES[name]);
|
||||||
|
}
|
||||||
|
assert.deepEqual(current.pnpm.overrides, BUILTIN_OVERRIDES);
|
||||||
|
assert.deepEqual(current.dsh.profile.bundles, []);
|
||||||
|
const before = statSync(f.path);
|
||||||
|
assert.deepEqual(f.run(), { changed: false });
|
||||||
|
assert.equal(statSync(f.path).ino, before.ino);
|
||||||
|
assert.equal(statSync(f.path).mtimeMs, before.mtimeMs);
|
||||||
|
assert.equal(readdirSync(f.profile).length, 3);
|
||||||
|
assert.equal(readFileSync(join(f.profile, '.dsh-lockfile-pending'), 'utf8'), 'builtin profile migration\n');
|
||||||
|
writeFileSync(f.path, f.original);
|
||||||
|
assert.notEqual(f.run().backupPath, first.backupPath);
|
||||||
|
assert.equal(readFileSync(first.backupPath, 'utf8'), f.original);
|
||||||
|
});
|
||||||
|
|
||||||
|
function withSyncTrace(f, callback, failAt) {
|
||||||
|
const originals = Object.fromEntries(
|
||||||
|
['openSync', 'writeFileSync', 'fsyncSync', 'closeSync', 'renameSync'].map(name => [name, fs[name]]),
|
||||||
|
);
|
||||||
|
const descriptors = new Map();
|
||||||
|
const events = [];
|
||||||
|
let directoryOpens = 0;
|
||||||
|
fs.openSync = (path, ...args) => {
|
||||||
|
const fd = originals.openSync(path, ...args);
|
||||||
|
const target = path === f.profile ? `directory-${++directoryOpens}`
|
||||||
|
: String(path).startsWith(`${f.path}.backup-`) ? 'backup'
|
||||||
|
: String(path).startsWith(`${f.path}.tmp-`) ? 'temporary' : undefined;
|
||||||
|
if (target) {
|
||||||
|
descriptors.set(fd, target);
|
||||||
|
events.push(`open:${target}`);
|
||||||
|
}
|
||||||
|
return fd;
|
||||||
|
};
|
||||||
|
for (const [name, operation] of [['writeFileSync', 'write'], ['fsyncSync', 'fsync'], ['closeSync', 'close']]) {
|
||||||
|
fs[name] = (fd, ...args) => {
|
||||||
|
const target = descriptors.get(fd);
|
||||||
|
if (target) {
|
||||||
|
const event = `${operation}:${target}`;
|
||||||
|
events.push(event);
|
||||||
|
if (event === failAt) throw new Error(`injected ${event} failure`);
|
||||||
|
}
|
||||||
|
const result = originals[name](fd, ...args);
|
||||||
|
if (name === 'closeSync') descriptors.delete(fd);
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
}
|
||||||
|
fs.renameSync = (source, destination) => {
|
||||||
|
assert.ok(source.startsWith(`${f.path}.tmp-`));
|
||||||
|
assert.equal(destination, f.path);
|
||||||
|
events.push('rename:manifest');
|
||||||
|
return originals.renameSync(source, destination);
|
||||||
|
};
|
||||||
|
syncBuiltinESMExports();
|
||||||
|
try {
|
||||||
|
callback(events);
|
||||||
|
} finally {
|
||||||
|
Object.assign(fs, originals);
|
||||||
|
syncBuiltinESMExports();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const durableWriteOrder = [
|
||||||
|
'open:backup', 'write:backup', 'fsync:backup', 'close:backup',
|
||||||
|
'open:directory-1', 'fsync:directory-1', 'close:directory-1',
|
||||||
|
'open:temporary', 'write:temporary', 'fsync:temporary', 'close:temporary',
|
||||||
|
'open:directory-2', 'fsync:directory-2', 'close:directory-2',
|
||||||
|
'rename:manifest',
|
||||||
|
'open:directory-3', 'fsync:directory-3', 'close:directory-3',
|
||||||
|
];
|
||||||
|
|
||||||
|
test('syncs backup and directory before replacement, then syncs the renamed manifest directory', t => {
|
||||||
|
const f = fixture(t, { dependencies: { 'dsh-web-mobile': old('dsh-web-mobile') } });
|
||||||
|
withSyncTrace(f, events => {
|
||||||
|
const result = f.run();
|
||||||
|
assert.equal(result.changed, true);
|
||||||
|
assert.deepEqual(events, durableWriteOrder);
|
||||||
|
assert.equal(readFileSync(result.backupPath, 'utf8'), f.original);
|
||||||
|
assert.equal(JSON.parse(readFileSync(f.path)).dependencies['dsh-web-mobile'], BUILTIN_DEPENDENCIES['dsh-web-mobile']);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
for (const target of ['backup', 'directory-1', 'temporary']) {
|
||||||
|
test(`${target} fsync failure preserves the original manifest and does not rename`, t => {
|
||||||
|
const f = fixture(t, { dependencies: { 'dsh-web-mobile': old('dsh-web-mobile') } });
|
||||||
|
const before = statSync(f.path);
|
||||||
|
const failAt = `fsync:${target}`;
|
||||||
|
withSyncTrace(f, events => {
|
||||||
|
assert.throws(f.run, { message: `injected ${failAt} failure` });
|
||||||
|
assert.deepEqual(events, [
|
||||||
|
...durableWriteOrder.slice(0, durableWriteOrder.indexOf(failAt) + 1),
|
||||||
|
`close:${target}`,
|
||||||
|
]);
|
||||||
|
}, failAt);
|
||||||
|
assert.equal(readFileSync(f.path, 'utf8'), f.original);
|
||||||
|
assert.equal(statSync(f.path).ino, before.ino);
|
||||||
|
assert.equal(statSync(f.path).mtimeMs, before.mtimeMs);
|
||||||
|
const backups = readdirSync(f.profile).filter(name => name.startsWith('package.json.backup-'));
|
||||||
|
assert.equal(backups.length, 1);
|
||||||
|
assert.equal(readFileSync(join(f.profile, backups[0]), 'utf8'), f.original);
|
||||||
|
assert.deepEqual(readdirSync(f.profile).sort(), ['package.json', ...backups].sort());
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
test('preserves deleted plugins, disabled bundles, custom dependencies and overrides', t => {
|
||||||
|
const custom = {
|
||||||
|
dependencies: { 'dsh-chat-manager': old('dsh-chat-manager'), 'dsh-web-mobile': '^3.0.0', other: old('other') },
|
||||||
|
devDependencies: { 'dsh-free-search': 'file:/custom/dsh-free-search-0.1.0.tgz' },
|
||||||
|
optionalDependencies: { 'dsh-preview-download': 'file:/usr/local/lib/dsh-mgr/not-the-same-1.0.0.tgz' },
|
||||||
|
pnpm: { overrides: { 'dsh-chat-manager>clsx': '^2.0.0', unrelated: 'custom' }, other: true },
|
||||||
|
dsh: { profile: { bundles: ['user-plugin'] } },
|
||||||
|
};
|
||||||
|
const f = fixture(t, custom);
|
||||||
|
writeFileSync(join(f.profile, 'pnpm-workspace.yaml'), 'custom workspace');
|
||||||
|
writeFileSync(join(f.profile, 'cordis.patch.yml'), 'custom patch');
|
||||||
|
f.run();
|
||||||
|
const expected = structuredClone(custom);
|
||||||
|
expected.dependencies['dsh-chat-manager'] = BUILTIN_DEPENDENCIES['dsh-chat-manager'];
|
||||||
|
assert.deepEqual(JSON.parse(readFileSync(f.path)), expected);
|
||||||
|
assert.equal(readFileSync(join(f.profile, 'pnpm-workspace.yaml'), 'utf8'), 'custom workspace');
|
||||||
|
assert.equal(readFileSync(join(f.profile, 'cordis.patch.yml'), 'utf8'), 'custom patch');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('strict archive matching leaves non-semver references untouched', () => {
|
||||||
|
for (const version of ['latest', '01.0.0', '1.0', '1.0.0-01', '1.0.0.tgz/extra']) {
|
||||||
|
const input = { dependencies: { 'dsh-web-mobile': 'file:/usr/local/lib/dsh-mgr/dsh-web-mobile-' + version + '.tgz' } };
|
||||||
|
assert.equal(reconcileManifest(input).changed, false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('current defaults and user-only manifests remain byte-identical', t => {
|
||||||
|
for (const manifest of [
|
||||||
|
{ dependencies: BUILTIN_DEPENDENCIES, pnpm: { overrides: BUILTIN_OVERRIDES } },
|
||||||
|
{ dependencies: { 'dsh-chat-manager': '^1.0.0' }, custom: true },
|
||||||
|
{ dsh: { profile: { bundles: [] } } },
|
||||||
|
]) {
|
||||||
|
const f = fixture(t, manifest);
|
||||||
|
assert.equal(f.run().changed, false);
|
||||||
|
assert.equal(readFileSync(f.path, 'utf8'), f.original);
|
||||||
|
assert.deepEqual(readdirSync(f.profile), ['package.json']);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('missing archives fail before writes, even when already current', t => {
|
||||||
|
for (const reference of [old('dsh-web-mobile'), BUILTIN_DEPENDENCIES['dsh-web-mobile']]) {
|
||||||
|
const f = fixture(t, { dependencies: { 'dsh-web-mobile': reference } });
|
||||||
|
rmSync(join(f.archiveDirectory, basename(BUILTIN_DEPENDENCIES['dsh-web-mobile'])));
|
||||||
|
assert.throws(f.run, /ENOENT/);
|
||||||
|
assert.equal(readFileSync(f.path, 'utf8'), f.original);
|
||||||
|
assert.deepEqual(readdirSync(f.profile), ['package.json']);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('invalid JSON and malformed sections fail without changing original', t => {
|
||||||
|
for (const input of ['{ broken', 'null', '[]', JSON.stringify({ dependencies: [] }), JSON.stringify({ pnpm: { overrides: null } })]) {
|
||||||
|
const f = fixture(t, input);
|
||||||
|
assert.throws(f.run);
|
||||||
|
assert.equal(readFileSync(f.path, 'utf8'), input);
|
||||||
|
assert.deepEqual(readdirSync(f.profile), ['package.json']);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rejects manifest/workspace symlinks, dangling links and directories', t => {
|
||||||
|
for (const name of ['package.json', 'pnpm-workspace.yaml', 'pnpm-lock.yaml', '.dsh-lockfile-pending']) {
|
||||||
|
for (const kind of ['symlink', 'dangling', 'directory']) {
|
||||||
|
const f = fixture(t, { dependencies: { 'dsh-web-mobile': old('dsh-web-mobile') } });
|
||||||
|
const target = join(f.root, 'outside');
|
||||||
|
if (kind === 'symlink') writeFileSync(target, f.original);
|
||||||
|
const path = join(f.profile, name);
|
||||||
|
rmSync(path, { force: true });
|
||||||
|
if (kind === 'directory') mkdirSync(path);
|
||||||
|
else symlinkSync(target, path);
|
||||||
|
const before = readdirSync(f.profile);
|
||||||
|
assert.throws(f.run, /unsafe profile file/);
|
||||||
|
assert.deepEqual(readdirSync(f.profile), before);
|
||||||
|
if (kind === 'symlink') assert.equal(readFileSync(target, 'utf8'), f.original);
|
||||||
|
if (name !== 'package.json') assert.equal(readFileSync(f.path, 'utf8'), f.original);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('rejects symlinked profile and ancestors', t => {
|
||||||
|
const f = fixture(t, '{}');
|
||||||
|
const link = join(f.root, 'linked');
|
||||||
|
symlinkSync(join(f.root, 'profiles'), link);
|
||||||
|
assert.throws(() => reconcileProfile(join(link, 'web'), { archiveDirectory: f.archiveDirectory }), /unsafe profile directory/);
|
||||||
|
const direct = join(f.root, 'direct');
|
||||||
|
symlinkSync(f.profile, direct);
|
||||||
|
assert.throws(() => reconcileProfile(direct), /unsafe profile directory/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('init migrates before restoring workspace and rejects unsafe workspace before writes', t => {
|
||||||
|
const f = fixture(t, { optionalDependencies: { 'dsh-web-mobile': old('dsh-web-mobile') } });
|
||||||
|
const run = () => initProfile(f.root, { archiveDirectory: f.archiveDirectory });
|
||||||
|
symlinkSync(join(f.root, 'missing'), join(f.profile, 'pnpm-workspace.yaml'));
|
||||||
|
assert.throws(run, /unsafe profile file/);
|
||||||
|
assert.equal(readFileSync(f.path, 'utf8'), f.original);
|
||||||
|
rmSync(join(f.profile, 'pnpm-workspace.yaml'));
|
||||||
|
run();
|
||||||
|
assert.equal(JSON.parse(readFileSync(f.path)).optionalDependencies['dsh-web-mobile'], BUILTIN_DEPENDENCIES['dsh-web-mobile']);
|
||||||
|
assert.match(readFileSync(join(f.profile, 'pnpm-workspace.yaml'), 'utf8'), /autoInstallPeers: false/);
|
||||||
|
});
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""Bounded, non-extracting Office container validation; Python stdlib only."""
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
path, ext, entries, maximum = sys.argv[1:]
|
||||||
|
if ext in ('doc', 'xls', 'ppt'):
|
||||||
|
with open(path, 'rb') as source:
|
||||||
|
if source.read(8) != bytes.fromhex('d0cf11e0a1b11ae1'):
|
||||||
|
raise ValueError('not OLE')
|
||||||
|
else:
|
||||||
|
expected = {'docx': 'word/document.xml', 'xlsx': 'xl/workbook.xml', 'pptx': 'ppt/presentation.xml'}[ext]
|
||||||
|
with zipfile.ZipFile(path) as archive:
|
||||||
|
info = archive.infolist()
|
||||||
|
names = [entry.filename for entry in info]
|
||||||
|
if len(info) > int(entries) or len(set(names)) != len(names):
|
||||||
|
raise ValueError('entry limit or duplicate')
|
||||||
|
if expected not in names or '[Content_Types].xml' not in names:
|
||||||
|
raise ValueError('format mismatch')
|
||||||
|
if sum(entry.file_size for entry in info) > int(maximum):
|
||||||
|
raise ValueError('uncompressed size limit')
|
||||||
|
total = 0
|
||||||
|
for entry in info:
|
||||||
|
if entry.flag_bits & 1:
|
||||||
|
raise ValueError('encrypted')
|
||||||
|
with archive.open(entry) as stream:
|
||||||
|
while chunk := stream.read(65536):
|
||||||
|
total += len(chunk)
|
||||||
|
if total > int(maximum):
|
||||||
|
raise ValueError('uncompressed size limit')
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
# Authentication boundary (§13)
|
|
||||||
|
|
||||||
- Real logins retain `dsh_session` as a database session binding and require a
|
|
||||||
five-minute HS256 access JWT in an additional HttpOnly cookie. JWT verification
|
|
||||||
fixes the algorithm and binds subject/session/issuer/audience/expiry. Bare
|
|
||||||
session IDs never authenticate. Internal callers and test fixtures must also
|
|
||||||
attach a signed access cookie.
|
|
||||||
- Refresh is a random 256-bit HttpOnly cookie, scoped to `/api`, with only its
|
|
||||||
SHA-256 hash stored in SQLite. The session has a 24-hour absolute lifetime.
|
|
||||||
Conditional SQL update makes rotation single-use even across concurrent requests.
|
|
||||||
Logout deletes the database session and its refresh binding. Both refresh and
|
|
||||||
logout retain mandatory exact Origin and session-bound CSRF checks, even when
|
|
||||||
access has expired. Other operations require valid access.
|
|
||||||
- Signing key is generated once under the configured data root, private mode
|
|
||||||
0600, and reused across restarts. Initialization errors fail authentication closed.
|
|
||||||
Tokens never appear in JSON, URLs, audit detail, or application logs.
|
|
||||||
- Known accounts lock for 15 minutes after five failures; the counter/lock survive
|
|
||||||
restart. Successful login resets the counter; expired locks reset the counter.
|
|
||||||
Existing global throttling remains additional protection.
|
|
||||||
- Tests cover missing/tampered/expired access, database binding, refresh rotation
|
|
||||||
and replay, CSRF/Origin rejection, logout revocation, key reuse/permissions,
|
|
||||||
account isolation, restart persistence, lock expiry, and runtime compatibility.
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var usernamePattern = regexp.MustCompile("^[a-zA-Z0-9_-]{3,40}$")
|
|
||||||
|
|
||||||
func DefaultQuota() int64 {
|
|
||||||
if n, err := strconv.ParseInt(os.Getenv("DEFAULT_QUOTA_BYTES"), 10, 64); err == nil && n > 0 {
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
return 10 << 30
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) users(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method {
|
|
||||||
case "GET":
|
|
||||||
rows, err := a.s.DB.Query("SELECT id,username,admin,quota,created FROM users ORDER BY created")
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
out := []store.User{}
|
|
||||||
for rows.Next() {
|
|
||||||
var v store.User
|
|
||||||
var created int64
|
|
||||||
if err := rows.Scan(&v.ID, &v.Username, &v.Admin, &v.Quota, &created); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
v.Created = time.Unix(created, 0)
|
|
||||||
out = append(out, v)
|
|
||||||
}
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, out)
|
|
||||||
case "POST":
|
|
||||||
var in struct {
|
|
||||||
Username string
|
|
||||||
Password string
|
|
||||||
Admin bool
|
|
||||||
Quota int64 `json:"quota_bytes"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(r.Body).Decode(&in) != nil || !usernamePattern.MatchString(in.Username) || len(in.Password) < 12 || len(in.Password) > 72 || in.Quota < 0 {
|
|
||||||
fail(w, 400, "username must be 3–40 letters/digits/_/-, password 12–72 bytes, quota nonnegative")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if in.Quota == 0 {
|
|
||||||
in.Quota = DefaultQuota()
|
|
||||||
}
|
|
||||||
h, e := bcrypt.GenerateFromPassword([]byte(in.Password), bcrypt.DefaultCost)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "password error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
v := store.User{ID: store.ID(), Username: in.Username, Admin: in.Admin, Quota: in.Quota, Created: time.Now()}
|
|
||||||
if e = a.s.CreateUser(v, string(h)); e != nil {
|
|
||||||
fail(w, 409, "username already exists or database unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.s.AddAudit(u.ID, "user.create", v.ID, "")
|
|
||||||
writeJSON(w, v)
|
|
||||||
case "PATCH":
|
|
||||||
var in struct {
|
|
||||||
ID string
|
|
||||||
Quota int64 `json:"quota_bytes"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&in) != nil || in.ID == "" || in.Quota <= 0 {
|
|
||||||
fail(w, 400, "positive quota and user ID required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
user := a.userLocks.Acquire(in.ID)
|
|
||||||
defer user.Release()
|
|
||||||
result, err := a.s.DB.Exec("UPDATE users SET quota=? WHERE id=?", in.Quota, in.ID)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "quota update failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
count, _ := result.RowsAffected()
|
|
||||||
if count != 1 {
|
|
||||||
fail(w, 404, "user not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.s.AddAudit(u.ID, "user.quota", in.ID, strconv.FormatInt(in.Quota, 10))
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
default:
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (a *API) resetPassword(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct{ ID, Password string }
|
|
||||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
|
|
||||||
if decoder.Decode(&in) != nil || decoder.Decode(new(any)) != io.EOF || in.ID == "" || len(in.Password) < 12 || len(in.Password) > 72 {
|
|
||||||
fail(w, 400, "user ID and password 12–72 bytes required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
hash, err := bcrypt.GenerateFromPassword([]byte(in.Password), bcrypt.DefaultCost)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "password error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Login must not issue a session using a hash read before this reset.
|
|
||||||
a.authLoginMu.Lock()
|
|
||||||
defer a.authLoginMu.Unlock()
|
|
||||||
u, _, err = a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tx, err := a.s.DB.BeginTx(r.Context(), nil)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "password reset failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
var username string
|
|
||||||
err = tx.QueryRowContext(r.Context(), "SELECT username FROM users WHERE id=?", in.ID).Scan(&username)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
fail(w, 404, "user not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "UPDATE users SET pass_hash=? WHERE id=?", string(hash), in.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "DELETE FROM auth_refresh WHERE sid IN (SELECT id FROM sessions WHERE uid=?)", in.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "DELETE FROM sessions WHERE uid=?", in.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "DELETE FROM auth_failures WHERE username=?", username)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "INSERT INTO audit(uid,action,target,detail,at) VALUES(?,?,?,?,?)", u.ID, "user.reset_password", in.ID, "", time.Now().Unix())
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = tx.Commit()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "password reset failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]bool{"ok": true, "reauthenticate": u.ID == in.ID})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) audit(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, e := a.user(r)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != "GET" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
rows, e := a.s.DB.Query("SELECT id,uid,action,target,detail,at FROM audit ORDER BY id DESC LIMIT 500")
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
out := []map[string]any{}
|
|
||||||
for rows.Next() {
|
|
||||||
var id, at int64
|
|
||||||
var uid, action, target, detail string
|
|
||||||
if rows.Scan(&id, &uid, &action, &target, &detail, &at) != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
out = append(out, map[string]any{"id": id, "user_id": uid, "action": action, "target": target, "detail": detail, "created_at": time.Unix(at, 0)})
|
|
||||||
}
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, out)
|
|
||||||
}
|
|
||||||
@@ -1,303 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const resetPath = "/api/users/reset-password"
|
|
||||||
|
|
||||||
func TestResetPasswordConcurrentLogin(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
admin := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
start := make(chan struct{})
|
|
||||||
results := make(chan *httptest.ResponseRecorder, 8)
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
go func() { <-start; results <- req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil) }()
|
|
||||||
}
|
|
||||||
close(start)
|
|
||||||
w := req("POST", resetPath, resetBody("bob", resetSecret), cookieValue(admin, "dsh_csrf"), admin)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
w := <-results
|
|
||||||
if w.Code != 200 && w.Code != 401 && w.Code != 429 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w.Code == 200 {
|
|
||||||
if check := req("GET", "/api/me", "", "", w.Result().Cookies()); check.Code != 401 {
|
|
||||||
t.Fatal("old-password login retained session", check.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM sessions WHERE uid='bob'").Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatal("old login sessions", n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type resetSignalReader struct {
|
|
||||||
reader *strings.Reader
|
|
||||||
started chan struct{}
|
|
||||||
once sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *resetSignalReader) Read(p []byte) (int, error) {
|
|
||||||
r.once.Do(func() { close(r.started) })
|
|
||||||
return r.reader.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResetPasswordRechecksWaitingAdmin(t *testing.T) {
|
|
||||||
for _, revoke := range []string{"session", "admin"} {
|
|
||||||
t.Run(revoke, func(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
admin := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
reader := &resetSignalReader{reader: strings.NewReader(resetBody("bob", resetSecret)), started: make(chan struct{})}
|
|
||||||
r := httptest.NewRequest("POST", resetPath, reader)
|
|
||||||
r.Header.Set("Origin", a.origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", cookieValue(admin, "dsh_csrf"))
|
|
||||||
for _, c := range admin {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
a.authLoginMu.Lock()
|
|
||||||
locked := true
|
|
||||||
defer func() {
|
|
||||||
if locked {
|
|
||||||
a.authLoginMu.Unlock()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
done := make(chan *httptest.ResponseRecorder, 1)
|
|
||||||
go func() { w := httptest.NewRecorder(); a.Handler().ServeHTTP(w, r); done <- w }()
|
|
||||||
select {
|
|
||||||
case <-reader.started:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("request did not reach decoder")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
t.Fatal("reset bypassed login lock", w.Code)
|
|
||||||
case <-time.After(150 * time.Millisecond):
|
|
||||||
}
|
|
||||||
query := "UPDATE users SET admin=0 WHERE id='alice'"
|
|
||||||
want := 403
|
|
||||||
if revoke == "session" {
|
|
||||||
query = "DELETE FROM sessions WHERE uid='alice'"
|
|
||||||
want = 401
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec(query); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.authLoginMu.Unlock()
|
|
||||||
locked = false
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
if w.Code != want {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("reset did not finish")
|
|
||||||
}
|
|
||||||
hash, err := a.s.Password("bob")
|
|
||||||
if err != nil || bcrypt.CompareHashAndPassword([]byte(hash), []byte("password-123")) != nil {
|
|
||||||
t.Fatal("unauthorized reset", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const resetSecret = "new-password-123"
|
|
||||||
|
|
||||||
func resetBody(id, password string) string {
|
|
||||||
b, _ := json.Marshal(map[string]string{"id": id, "password": password})
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResetPasswordValidation(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
admin := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
regular := req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil).Result().Cookies()
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
csrf := cookieValue(admin, "dsh_csrf")
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, method, body, csrf string
|
|
||||||
cookies []*http.Cookie
|
|
||||||
code int
|
|
||||||
}{
|
|
||||||
{"anonymous", "POST", resetBody("bob", resetSecret), "", nil, 401},
|
|
||||||
{"regular", "POST", resetBody("alice", resetSecret), cookieValue(regular, "dsh_csrf"), regular, 403},
|
|
||||||
{"csrf", "POST", resetBody("bob", resetSecret), "bad", admin, 403},
|
|
||||||
{"method", "GET", "", csrf, admin, 405},
|
|
||||||
{"missing id", "POST", resetBody("", resetSecret), csrf, admin, 400},
|
|
||||||
{"short", "POST", resetBody("bob", strings.Repeat("a", 11)), csrf, admin, 400},
|
|
||||||
{"long", "POST", resetBody("bob", strings.Repeat("a", 73)), csrf, admin, 400},
|
|
||||||
{"unicode long", "POST", resetBody("bob", strings.Repeat("密", 25)), csrf, admin, 400},
|
|
||||||
{"malformed", "POST", "{", csrf, admin, 400},
|
|
||||||
{"trailing", "POST", resetBody("bob", resetSecret) + "{}", csrf, admin, 400},
|
|
||||||
{"oversized", "POST", resetBody("bob", strings.Repeat("a", 4096)), csrf, admin, 400},
|
|
||||||
{"missing target", "POST", resetBody("missing", resetSecret), csrf, admin, 404},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
w := req(tc.method, resetPath, tc.body, tc.csrf, tc.cookies)
|
|
||||||
if w.Code != tc.code {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for _, origin := range []string{"", "https://evil.test"} {
|
|
||||||
r := httptest.NewRequest("POST", resetPath, strings.NewReader(resetBody("bob", resetSecret)))
|
|
||||||
r.Header.Set("Origin", origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
for _, c := range admin {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
if w.Code != 403 {
|
|
||||||
t.Fatal("origin accepted", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, password := range []string{strings.Repeat("a", 12), strings.Repeat("密", 24)} {
|
|
||||||
if w := req("POST", resetPath, resetBody("bob", password), csrf, admin); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
hash, err := a.s.Password("bob")
|
|
||||||
if err != nil || bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) != nil {
|
|
||||||
t.Fatal("boundary hash", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func resetLogin(username, password string) string {
|
|
||||||
b, _ := json.Marshal(map[string]string{"username": username, "password": password})
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResetPasswordLifecycle(t *testing.T) {
|
|
||||||
for _, target := range []string{"bob", "alice"} {
|
|
||||||
t.Run(target, func(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
login := func(name string) []*http.Cookie {
|
|
||||||
w := req("POST", "/api/login", resetLogin(name, "password-123"), "", nil)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
return w.Result().Cookies()
|
|
||||||
}
|
|
||||||
admin := login("alice")
|
|
||||||
targets := [][]*http.Cookie{login(target), login(target)}
|
|
||||||
other := "alice"
|
|
||||||
if target == "alice" {
|
|
||||||
other = "bob"
|
|
||||||
}
|
|
||||||
unaffected := login(other)
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO auth_failures VALUES(?,5,?)", target, 9999999999); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := req("POST", resetPath, resetBody(target, resetSecret), cookieValue(admin, "dsh_csrf"), admin)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var result map[string]bool
|
|
||||||
if json.Unmarshal(w.Body.Bytes(), &result) != nil || !result["ok"] || result["reauthenticate"] != (target == "alice") {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
for _, cs := range targets {
|
|
||||||
var remaining int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM auth_refresh WHERE sid=?", cookieValue(cs, "dsh_session")).Scan(&remaining); err != nil || remaining != 0 {
|
|
||||||
t.Fatal("old refresh row", remaining, err)
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", cs); w.Code != 401 {
|
|
||||||
t.Fatal("old access", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/refresh", "", cookieValue(cs, "dsh_csrf"), cs); w.Code != 403 {
|
|
||||||
t.Fatal("old refresh", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, query := range []string{"SELECT count(*) FROM sessions WHERE uid=?", "SELECT count(*) FROM auth_failures WHERE username=?", "SELECT count(*) FROM auth_refresh WHERE sid IN (SELECT id FROM sessions WHERE uid=?)"} {
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow(query, target).Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatal(query, n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", unaffected); w.Code != 200 {
|
|
||||||
t.Fatal("other access", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/refresh", "", cookieValue(unaffected, "dsh_csrf"), unaffected); w.Code != 200 {
|
|
||||||
t.Fatal("other refresh", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", resetLogin(target, "password-123"), "", nil); w.Code != 401 {
|
|
||||||
t.Fatal("old password", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", resetLogin(target, resetSecret), "", nil); w.Code != 200 {
|
|
||||||
t.Fatal("new password", w.Code)
|
|
||||||
}
|
|
||||||
var actor, detail string
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT uid,detail,count(*) FROM audit WHERE action='user.reset_password' AND target=?", target).Scan(&actor, &detail, &n); err != nil || actor != "alice" || detail != "" || n != 1 {
|
|
||||||
t.Fatal("audit", actor, detail, n, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResetPasswordRollback(t *testing.T) {
|
|
||||||
for _, step := range []struct{ table, event string }{{"users", "UPDATE"}, {"auth_refresh", "DELETE"}, {"sessions", "DELETE"}, {"auth_failures", "DELETE"}, {"audit", "INSERT"}} {
|
|
||||||
t.Run(step.table, func(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
admin := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
target := req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil).Result().Cookies()
|
|
||||||
before, err := a.s.Password("bob")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO auth_failures VALUES('bob',5,9999999999)"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec(fmt.Sprintf("CREATE TRIGGER reject_reset BEFORE %s ON %s BEGIN SELECT RAISE(ABORT,'forced failure'); END", step.event, step.table)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := req("POST", resetPath, resetBody("bob", resetSecret), cookieValue(admin, "dsh_csrf"), admin)
|
|
||||||
if w.Code != 500 || strings.Contains(w.Body.String(), resetSecret) {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
after, err := a.s.Password("bob")
|
|
||||||
if err != nil || before != after {
|
|
||||||
t.Fatal("hash changed", err)
|
|
||||||
}
|
|
||||||
for query, want := range map[string]int{"SELECT count(*) FROM sessions WHERE uid='bob'": 1, "SELECT count(*) FROM auth_refresh": 2, "SELECT count(*) FROM auth_failures WHERE username='bob' AND failures=5 AND locked_until=9999999999": 1, "SELECT count(*) FROM audit WHERE action='user.reset_password'": 0} {
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow(query).Scan(&n); err != nil || n != want {
|
|
||||||
t.Fatal(query, n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", target); w.Code != 200 {
|
|
||||||
t.Fatal("session changed", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/refresh", "", cookieValue(target, "dsh_csrf"), target); w.Code != 200 {
|
|
||||||
t.Fatal("refresh changed", w.Code)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAdminQuotaUpdate(t *testing.T) {
|
|
||||||
a, sid := runtimeTestAPI(t, func(http.ResponseWriter, *http.Request) {})
|
|
||||||
request := func() *httptest.ResponseRecorder {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.users(w, runtimeRequest("PATCH", "/api/users", `{"id":"owner","quota_bytes":2048}`, sid))
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
if w := request(); w.Code != 403 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='owner'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w := request(); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
u, err := a.s.UserByID("owner")
|
|
||||||
if err != nil || u.Quota != 2048 {
|
|
||||||
t.Fatalf("%+v %v", u, err)
|
|
||||||
}
|
|
||||||
t.Setenv("DEFAULT_QUOTA_BYTES", "12345")
|
|
||||||
if DefaultQuota() != 12345 {
|
|
||||||
t.Fatal("default quota ignored")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,208 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Admission writes serialize with new runtime operations, including their Docker side effects.
|
|
||||||
var admissionMu sync.RWMutex
|
|
||||||
var errBuildUnsupported = errors.New("version is not supported for normal builds; administrator experimental build required")
|
|
||||||
|
|
||||||
func defaultAdmission(version, recipe, platform string) string {
|
|
||||||
if recipe == "7" && platform == "linux/amd64" {
|
|
||||||
switch version {
|
|
||||||
case "0.2.0-rc.1":
|
|
||||||
return "compatible"
|
|
||||||
case "0.2.0-rc.2":
|
|
||||||
return "recommended"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "existing_only"
|
|
||||||
}
|
|
||||||
func supportedLayout(recipe string) bool {
|
|
||||||
switch recipe {
|
|
||||||
case "3", "4", "5", "6", "7":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
func normalBuildAllowed(version string) bool {
|
|
||||||
return positiveAdmission(defaultAdmission(version, imageRecipeRev, imagePlatform))
|
|
||||||
}
|
|
||||||
func positiveAdmission(state string) bool { return state == "recommended" || state == "compatible" }
|
|
||||||
func (a *API) compatibility(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if _, _, err := a.user(r); err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
entries := []map[string]any{}
|
|
||||||
for _, v := range []string{"0.2.0-rc.1", "0.2.0-rc.2"} {
|
|
||||||
entries = append(entries, map[string]any{"version": v, "recipe_rev": imageRecipeRev, "platform": imagePlatform, "state": defaultAdmission(v, imageRecipeRev, imagePlatform), "build_allowed": normalBuildAllowed(v), "reason": "only linux/amd64 recipe 7 is built-in; successful exact image verification still required"})
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]any{"recipe_rev": imageRecipeRev, "platform": imagePlatform, "entries": entries, "unknown": map[string]any{"state": "existing_only", "build_allowed": false, "reason": "not accepted; administrator experimental build and explicit admission required"}, "experimental_admin_only": true})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) evaluateImage(ctx context.Context, id string) (imageRecord, error) {
|
|
||||||
im := imageRecord{ID: id, ImageID: id, Status: "missing", Admission: "existing_only", AdmissionReason: "local image missing"}
|
|
||||||
actual, err := a.inspectLocalImage(ctx, id)
|
|
||||||
if err != nil || actual == nil {
|
|
||||||
return im, err
|
|
||||||
}
|
|
||||||
im.Version = actual.Config.Labels["dsh-mgr.version"]
|
|
||||||
im.RecipeRev = actual.Config.Labels["dsh-mgr.recipe"]
|
|
||||||
im.Platform = actual.platform()
|
|
||||||
im.Created = actual.Created
|
|
||||||
im.Status = "unverified"
|
|
||||||
im.AdmissionReason = "successful exact image identity verification required"
|
|
||||||
if actual.ID != id || actual.Config.Labels["dsh-mgr.role"] != "runtime" {
|
|
||||||
im.Version = ""
|
|
||||||
return im, nil
|
|
||||||
}
|
|
||||||
var count, experimental int
|
|
||||||
err = a.s.DB.QueryRowContext(ctx, "SELECT count(*),COALESCE(min(experimental),0) FROM image_builds WHERE image_id=? AND version=? AND recipe_rev=? AND platform=? AND status='success'", id, im.Version, im.RecipeRev, im.Platform).Scan(&count, &experimental)
|
|
||||||
if err != nil {
|
|
||||||
return im, err
|
|
||||||
}
|
|
||||||
var state, evidence string
|
|
||||||
err = a.s.DB.QueryRowContext(ctx, "SELECT state,evidence FROM image_admissions WHERE image_id=? AND version=? AND recipe_rev=? AND platform=?", id, im.Version, im.RecipeRev, im.Platform).Scan(&state, &evidence)
|
|
||||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return im, err
|
|
||||||
}
|
|
||||||
if state == "blocked" {
|
|
||||||
im.Admission = state
|
|
||||||
im.Status = state
|
|
||||||
im.AdmissionReason = evidence
|
|
||||||
return im, nil
|
|
||||||
}
|
|
||||||
if count == 0 {
|
|
||||||
return im, nil
|
|
||||||
}
|
|
||||||
im.Admission = defaultAdmission(im.Version, im.RecipeRev, im.Platform)
|
|
||||||
im.AdmissionReason = "no accepted compatibility evidence for this tuple"
|
|
||||||
if positiveAdmission(im.Admission) {
|
|
||||||
im.AdmissionReason = "built-in exact tuple compatibility"
|
|
||||||
}
|
|
||||||
if experimental != 0 {
|
|
||||||
im.Admission = "existing_only"
|
|
||||||
im.Status = "experimental"
|
|
||||||
im.AdmissionReason = "experimental validation succeeded; administrator acceptance required"
|
|
||||||
} else {
|
|
||||||
im.Status = "existing_only"
|
|
||||||
}
|
|
||||||
if state != "" {
|
|
||||||
im.Admission = state
|
|
||||||
im.AdmissionReason = evidence
|
|
||||||
im.Status = state
|
|
||||||
}
|
|
||||||
if !supportedLayout(im.RecipeRev) || im.Platform != imagePlatform {
|
|
||||||
im.Admission = "existing_only"
|
|
||||||
im.Status = "existing_only"
|
|
||||||
im.AdmissionReason = "historical validation retained; current mount layout or platform not supported"
|
|
||||||
}
|
|
||||||
im.Selectable = positiveAdmission(im.Admission)
|
|
||||||
if im.Selectable {
|
|
||||||
im.Status = "available"
|
|
||||||
}
|
|
||||||
return im, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) imageAdmission(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct {
|
|
||||||
ImageID string `json:"image_id"`
|
|
||||||
State string `json:"state"`
|
|
||||||
Evidence string `json:"evidence"`
|
|
||||||
}
|
|
||||||
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 8192))
|
|
||||||
dec.DisallowUnknownFields()
|
|
||||||
if dec.Decode(&in) != nil || dec.Decode(new(any)) != io.EOF || !runtimeImageID.MatchString(in.ImageID) || strings.TrimSpace(in.Evidence) == "" || (!positiveAdmission(in.State) && in.State != "existing_only" && in.State != "blocked") {
|
|
||||||
fail(w, 400, "image_id, valid state and evidence required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if len(in.Evidence) > 4096 {
|
|
||||||
fail(w, 400, "evidence must not exceed 4096 UTF-8 bytes")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
admissionMu.Lock()
|
|
||||||
defer admissionMu.Unlock()
|
|
||||||
buildQueue.Lock()
|
|
||||||
err = a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
im, err := a.evaluateImage(ctx, in.ImageID)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 502, "image inspection failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if im.Status == "missing" || im.Version == "" || im.RecipeRev == "" || im.Platform == "" {
|
|
||||||
fail(w, 400, "real runtime image identity required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if positiveAdmission(in.State) {
|
|
||||||
var count int
|
|
||||||
err = a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM image_builds WHERE image_id=? AND version=? AND recipe_rev=? AND platform=? AND status='success'", im.ImageID, im.Version, im.RecipeRev, im.Platform).Scan(&count)
|
|
||||||
if err != nil || count == 0 || !supportedLayout(im.RecipeRev) || im.Platform != imagePlatform {
|
|
||||||
fail(w, 400, "successful exact validation and supported mount layout/platform required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
tx, err := a.s.DB.BeginTx(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
_, err = tx.ExecContext(ctx, "INSERT INTO image_admissions(image_id,version,recipe_rev,platform,state,evidence,admitted_by,admitted_at) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(image_id,version,recipe_rev,platform) DO UPDATE SET state=excluded.state,evidence=excluded.evidence,admitted_by=excluded.admitted_by,admitted_at=excluded.admitted_at", im.ImageID, im.Version, im.RecipeRev, im.Platform, in.State, strings.TrimSpace(in.Evidence), u.ID, time.Now().UTC().Format(time.RFC3339Nano))
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, err = tx.ExecContext(ctx, "INSERT INTO audit(uid,action,target,detail,at) VALUES(?,?,?,?,?)", u.ID, "image.admission", im.ImageID, in.State+": "+in.Evidence, time.Now().Unix()); err != nil {
|
|
||||||
fail(w, 500, "audit persistence failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = tx.Commit(); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.imageDeleteGeneration++
|
|
||||||
im.Admission = in.State
|
|
||||||
im.AdmissionReason = strings.TrimSpace(in.Evidence)
|
|
||||||
im.Selectable = positiveAdmission(in.State)
|
|
||||||
im.Status = in.State
|
|
||||||
if im.Selectable {
|
|
||||||
im.Status = "available"
|
|
||||||
}
|
|
||||||
writeJSON(w, im)
|
|
||||||
}
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAdmissionEvidenceUTF8Limit(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("d", 64)
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { writeImageInspect(w, id, "0.2.0-rc.2") })
|
|
||||||
seedVerifiedImage(t, a, id, "0.2.0-rc.2")
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, evidence string
|
|
||||||
want int
|
|
||||||
}{
|
|
||||||
{"ascii boundary", strings.Repeat("a", 4096), 200},
|
|
||||||
{"ascii over", strings.Repeat("a", 4097), 400},
|
|
||||||
{"Chinese regression", strings.Repeat("中", 1400), 400},
|
|
||||||
{"Chinese below", strings.Repeat("中", 1365), 200},
|
|
||||||
{"Chinese boundary", strings.Repeat("中", 1365) + "a", 200},
|
|
||||||
{"Chinese over", strings.Repeat("中", 1365) + "ab", 400},
|
|
||||||
{"emoji boundary", strings.Repeat("😀", 1024), 200},
|
|
||||||
{"emoji over", strings.Repeat("😀", 1024) + "a", 400},
|
|
||||||
{"two byte boundary", strings.Repeat("é", 2048), 200},
|
|
||||||
{"raw whitespace counts", " " + strings.Repeat("a", 4096), 400},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
body, err := json.Marshal(map[string]string{"image_id": id, "state": "compatible", "evidence": tc.evidence})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", string(body), sid))
|
|
||||||
if w.Code != tc.want {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if tc.want == 400 && !strings.Contains(w.Body.String(), "4096 UTF-8 bytes") {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
if tc.want == 200 {
|
|
||||||
var evidence string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT evidence FROM image_admissions WHERE image_id=?", id).Scan(&evidence); err != nil || evidence != tc.evidence {
|
|
||||||
t.Fatal(evidence, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdmissionRoutesEnforceAuthorization(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("d", 64)
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { writeImageInspect(w, id, "0.2.0-rc.2") })
|
|
||||||
seedVerifiedImage(t, a, id, "0.2.0-rc.2")
|
|
||||||
_, csrf, err := a.s.SessionUser(sid)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body := fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":"isolated validation reference"}`, id)
|
|
||||||
request := func(method, path, payload string, auth, token bool) *httptest.ResponseRecorder {
|
|
||||||
session := ""
|
|
||||||
if auth {
|
|
||||||
session = sid
|
|
||||||
}
|
|
||||||
r := runtimeRequest(method, path, payload, session)
|
|
||||||
r.Header.Set("Origin", "http://panel.test")
|
|
||||||
if token {
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
if w := request("GET", "/api/compatibility", "", false, false); w.Code != 401 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w := request("GET", "/api/compatibility", "", true, false); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w := request("POST", "/api/images/admission", body, true, true); w.Code != 403 {
|
|
||||||
t.Fatal("nonadmin admission", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w := request("POST", "/api/builds", `{"version":"9.8.7","experimental":true}`, true, true); w.Code != 403 {
|
|
||||||
t.Fatal("nonadmin experiment", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='owner'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w := request("POST", "/api/images/admission", body, true, false); w.Code != 403 {
|
|
||||||
t.Fatal("missing CSRF", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w := request("POST", "/api/images/admission", body, true, true); w.Code != 200 {
|
|
||||||
t.Fatal("admin admission", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for _, payload := range []string{
|
|
||||||
body + "{}",
|
|
||||||
body + strings.Repeat(" ", 8192),
|
|
||||||
fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":" "}`, id),
|
|
||||||
fmt.Sprintf(`{"image_id":%q,"state":"available","evidence":"invalid state"}`, id),
|
|
||||||
fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":"test","dockerfile":"not allowed"}`, id),
|
|
||||||
} {
|
|
||||||
if w := request("POST", "/api/images/admission", payload, true, true); w.Code != 400 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var count int
|
|
||||||
if err = a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='image.admission'").Scan(&count); err != nil || count != 1 {
|
|
||||||
t.Fatal(count, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,274 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestExperimentalBuildSuccessRequiresAcceptance(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("d", 64)
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/v1.47/images/json":
|
|
||||||
writeJSON(w, []any{map[string]any{"Id": id, "Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "9.8.7", "dsh-mgr.recipe": imageRecipeRev}}})
|
|
||||||
case "/v1.47/images/" + id + "/json":
|
|
||||||
writeImageInspect(w, id, "9.8.7")
|
|
||||||
case "/v1.47/containers/create":
|
|
||||||
writeJSON(w, map[string]string{"Id": "check"})
|
|
||||||
case "/v1.47/containers/check/exec":
|
|
||||||
writeJSON(w, map[string]string{"Id": "probe"})
|
|
||||||
case "/v1.47/exec/probe/json":
|
|
||||||
writeJSON(w, map[string]any{"Running": false, "ExitCode": 0})
|
|
||||||
case "/v1.47/containers/check/json":
|
|
||||||
writeJSON(w, map[string]any{"State": map[string]bool{"Running": true}})
|
|
||||||
default:
|
|
||||||
w.WriteHeader(204)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='u'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := imageRequest(a.builds, sid, "POST", `{"version":"9.8.7","experimental":true}`)
|
|
||||||
if w.Code != 202 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var b buildRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &b); err != nil || !b.Experimental {
|
|
||||||
t.Fatal(b, err)
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
var state string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id=?", b.ID).Scan(&state); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if state == "success" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if state == "failed" || time.Now().After(deadline) {
|
|
||||||
t.Fatal(state)
|
|
||||||
}
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
im, err := a.evaluateImage(context.Background(), id)
|
|
||||||
if err != nil || im.Selectable || im.Status != "experimental" {
|
|
||||||
t.Fatal(im, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQueuedUnsupportedPlatformFailsBeforeDocker(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unsupported build contacted Docker") })
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
previous := imagePlatform
|
|
||||||
imagePlatform = "linux/arm64"
|
|
||||||
defer func() { imagePlatform = previous }()
|
|
||||||
b := buildRecord{ID: "unsupported", Version: "0.2.0-rc.2", Status: "queued", RecipeRev: imageRecipeRev, Platform: imagePlatform}
|
|
||||||
if err := a.insertBuildRequest(b, "u"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.runImageBuild(b)
|
|
||||||
var state, detail string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status,error FROM image_builds WHERE id=?", b.ID).Scan(&state, &detail); err != nil || state != "failed" || !strings.Contains(detail, "experimental") {
|
|
||||||
t.Fatal(state, detail, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdmissionDowngradeWaitsForCreate(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("e", 64)
|
|
||||||
entered, release := make(chan struct{}), make(chan struct{})
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.Contains(r.URL.Path, "/images/"):
|
|
||||||
writeImageInspect(w, id, "0.2.0-rc.1")
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/networks/create"):
|
|
||||||
close(entered)
|
|
||||||
<-release
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
|
||||||
writeJSON(w, map[string]string{"Id": "created"})
|
|
||||||
default:
|
|
||||||
w.WriteHeader(204)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, id, "0.2.0-rc.1")
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
created := make(chan int, 1)
|
|
||||||
go func() {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf(`{"name":"race","image_id":%q}`, id), sid))
|
|
||||||
created <- w.Code
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-entered:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
close(release)
|
|
||||||
t.Fatal("create did not reach network")
|
|
||||||
}
|
|
||||||
downgraded := make(chan int, 1)
|
|
||||||
go func() {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", fmt.Sprintf(`{"image_id":%q,"state":"blocked","evidence":"downgrade test"}`, id), sid))
|
|
||||||
downgraded <- w.Code
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case status := <-downgraded:
|
|
||||||
close(release)
|
|
||||||
t.Fatalf("admission raced active create: %d", status)
|
|
||||||
case <-time.After(30 * time.Millisecond):
|
|
||||||
}
|
|
||||||
close(release)
|
|
||||||
if status := <-created; status != 200 {
|
|
||||||
t.Fatal(status)
|
|
||||||
}
|
|
||||||
if status := <-downgraded; status != 200 {
|
|
||||||
t.Fatal(status)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf(`{"name":"denied","image_id":%q}`, id), sid))
|
|
||||||
if w.Code != 400 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdmissionHistoricalCreateAndBlock(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
var body map[string]any
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.Contains(r.URL.Path, "/images/"):
|
|
||||||
writeJSON(w, map[string]any{"Id": id, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "1.2.3", "dsh-mgr.recipe": "4"}}})
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]string{"Id": "created"})
|
|
||||||
default:
|
|
||||||
w.WriteHeader(204)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, id, "1.2.3")
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET recipe_rev='4'; DELETE FROM image_admissions; UPDATE users SET admin=1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
im, err := a.evaluateImage(context.Background(), id)
|
|
||||||
if err != nil || im.Status != "existing_only" || im.Selectable {
|
|
||||||
t.Fatal(im, err)
|
|
||||||
}
|
|
||||||
records := []buildRecord{{ID: "unused", ImageID: id, Version: "1.2.3", RecipeRev: "4", Platform: imagePlatform, Status: "success"}}
|
|
||||||
if err := a.reconcileBuilds(context.Background(), records); err != nil || records[0].Status != "success" {
|
|
||||||
t.Fatal(records, err)
|
|
||||||
}
|
|
||||||
admit := func(state string) *httptest.ResponseRecorder {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", fmt.Sprintf(`{"image_id":%q,"state":%q,"evidence":"legacy layout explicitly accepted"}`, id, state), sid))
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
if w := admit("compatible"); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instances(w, runtimeRequest("POST", "/api/instances", fmt.Sprintf(`{"name":"legacy","image_id":%q}`, id), sid))
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
host := body["HostConfig"].(map[string]any)
|
|
||||||
mounts := host["Mounts"].([]any)
|
|
||||||
if len(mounts) != 2 || mounts[0].(map[string]any)["Target"] != "/data/dsh-home" || mounts[1].(map[string]any)["Target"] != "/workspace" || !strings.Contains(fmt.Sprint(body["Env"]), "DSH_HOME=/data/dsh-home") {
|
|
||||||
t.Fatal(body)
|
|
||||||
}
|
|
||||||
var instance store.Instance
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &instance); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w := admit("blocked"); w.Code != 200 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
for _, action := range []string{"restore", "backups", "profile"} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.config(w, runtimeRequest("POST", "/api/instances/"+instance.ID+"/"+action, `{}`, sid), instance, action)
|
|
||||||
if w.Code != 409 {
|
|
||||||
t.Fatal(action, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, action := range []string{"start", "restart"} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instance(w, runtimeRequest("POST", "/api/instances/"+instance.ID+"/"+action, "", sid))
|
|
||||||
if w.Code != 409 {
|
|
||||||
t.Fatal(action, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("CREATE TRIGGER audit_fail BEFORE INSERT ON audit BEGIN SELECT RAISE(FAIL,'audit unavailable'); END"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w := admit("compatible"); w.Code != 500 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
im, err = a.evaluateImage(context.Background(), id)
|
|
||||||
if err != nil || im.Admission != "blocked" {
|
|
||||||
t.Fatal(im, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAdmissionExperimentalAndPreflight(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("b", 64)
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { writeImageInspect(w, id, "9.8.7") })
|
|
||||||
seedVerifiedImage(t, a, id, "9.8.7")
|
|
||||||
if _, err := a.s.DB.Exec("DELETE FROM image_admissions; UPDATE image_builds SET experimental=1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
im, err := a.evaluateImage(context.Background(), id)
|
|
||||||
if err != nil || im.Status != "experimental" || im.Selectable {
|
|
||||||
t.Fatal(im, err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
handler http.HandlerFunc
|
|
||||||
body string
|
|
||||||
want int
|
|
||||||
}{{a.builds, `{"version":"9.8.7"}`, 400}, {a.builds, `{"version":"9.8.7","experimental":true}`, 403}, {a.intents, `{"name":"unknown","version":"9.8.7"}`, 400}, {a.instances, fmt.Sprintf(`{"name":"unknown","image_id":%q}`, id), 400}} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
tc.handler(w, runtimeRequest("POST", "/api/test", tc.body, sid))
|
|
||||||
if w.Code != tc.want {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests").Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatal(n, err)
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM creation_intents").Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatal(n, err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.imageAdmission(w, runtimeRequest("POST", "/api/images/admission", fmt.Sprintf(`{"image_id":%q,"state":"compatible","evidence":"accepted after startup validation"}`, id), sid))
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
im, err = a.evaluateImage(context.Background(), id)
|
|
||||||
if err != nil || !im.Selectable {
|
|
||||||
t.Fatal(im, err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET status='failed'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
im, err = a.evaluateImage(context.Background(), id)
|
|
||||||
if err != nil || im.Selectable || im.Status != "unverified" {
|
|
||||||
t.Fatal(im, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,290 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type API struct {
|
|
||||||
s *store.Store
|
|
||||||
origin string
|
|
||||||
root string
|
|
||||||
docker *docker.Client
|
|
||||||
readersMu sync.Mutex
|
|
||||||
workspaceReaders map[string]int
|
|
||||||
instanceLocks keyedLocks
|
|
||||||
userLocks keyedLocks
|
|
||||||
loginMu sync.Mutex
|
|
||||||
loginSources map[string]loginSource
|
|
||||||
trustedProxies []*net.IPNet
|
|
||||||
loginSlots chan struct{}
|
|
||||||
loginConfigErr error
|
|
||||||
authKey []byte
|
|
||||||
authErr error
|
|
||||||
authLoginMu sync.Mutex
|
|
||||||
|
|
||||||
// Protected by buildQueue; invalidates image observations across deletions.
|
|
||||||
imageDeleteGeneration uint64
|
|
||||||
|
|
||||||
// Per-image admission evaluations cached for the intent poller; guarded
|
|
||||||
// by evalCacheMu and invalidated by imageDeleteGeneration (buildQueue)
|
|
||||||
// plus a short TTL so Docker observations stay fresh.
|
|
||||||
evalCacheMu sync.Mutex
|
|
||||||
evalCache map[string]evalCacheEntry
|
|
||||||
evalCacheGen uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(s *store.Store, root string) *API {
|
|
||||||
a := &API{s: s, root: root, docker: docker.New(os.Getenv("DOCKER_HOST")), workspaceReaders: make(map[string]int), loginSources: make(map[string]loginSource), loginSlots: make(chan struct{}, 8)}
|
|
||||||
a.origin, a.loginConfigErr = configuredOrigin(os.Getenv("PANEL_PUBLIC_URL"))
|
|
||||||
var proxyErr error
|
|
||||||
a.trustedProxies, proxyErr = configuredProxyCIDRs(os.Getenv("TRUSTED_PROXY_CIDRS"), a.loginConfigErr)
|
|
||||||
if a.loginConfigErr == nil {
|
|
||||||
a.loginConfigErr = proxyErr
|
|
||||||
}
|
|
||||||
if a.loginConfigErr != nil {
|
|
||||||
a.authErr = a.loginConfigErr
|
|
||||||
} else {
|
|
||||||
a.authErr = a.initAuth()
|
|
||||||
}
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
func (a *API) Handler() http.Handler {
|
|
||||||
m := http.NewServeMux()
|
|
||||||
m.HandleFunc("/api/login", a.login)
|
|
||||||
m.HandleFunc("/api/refresh", a.refresh)
|
|
||||||
m.HandleFunc("/api/runtime-session.js", runtimeSessionScript)
|
|
||||||
m.HandleFunc("/api/logout", a.logout)
|
|
||||||
m.HandleFunc("/api/me", a.me)
|
|
||||||
m.HandleFunc("/api/me/password", a.changePassword)
|
|
||||||
m.HandleFunc("/api/instances", a.instances)
|
|
||||||
m.HandleFunc("/api/instances/", a.instance)
|
|
||||||
m.HandleFunc("/api/images", a.images)
|
|
||||||
m.HandleFunc("/api/images/admission", a.imageAdmission)
|
|
||||||
m.HandleFunc("/api/compatibility", a.compatibility)
|
|
||||||
m.HandleFunc("/api/latest", a.latest)
|
|
||||||
m.HandleFunc("/api/builds", a.builds)
|
|
||||||
m.HandleFunc("/api/builds/", a.buildDetail)
|
|
||||||
m.HandleFunc("/api/intents", a.intents)
|
|
||||||
m.HandleFunc("/api/intents/", a.intent)
|
|
||||||
m.HandleFunc("/api/ws/", a.workspace)
|
|
||||||
m.HandleFunc("/api/users", a.users)
|
|
||||||
m.HandleFunc("/api/users/reset-password", a.resetPassword)
|
|
||||||
m.HandleFunc("/api/audit", a.audit)
|
|
||||||
m.HandleFunc("/api/orphans", a.orphans)
|
|
||||||
m.HandleFunc("/dsh/", a.proxy)
|
|
||||||
return a.guard(m)
|
|
||||||
}
|
|
||||||
func writeJSON(w http.ResponseWriter, v any) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_ = json.NewEncoder(w).Encode(v)
|
|
||||||
}
|
|
||||||
func fail(w http.ResponseWriter, n int, s string) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(n)
|
|
||||||
writeJSON(w, map[string]string{"error": s})
|
|
||||||
}
|
|
||||||
func (a *API) guard(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if a.loginConfigErr != nil {
|
|
||||||
fail(w, 503, "authentication configuration invalid")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, 64<<20)
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/dsh/") {
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// API bodies and downloads are bounded; runtime WS/SSE above remain streaming.
|
|
||||||
controller := http.NewResponseController(w)
|
|
||||||
_ = controller.SetReadDeadline(time.Now().Add(60 * time.Second))
|
|
||||||
_ = controller.SetWriteDeadline(time.Now().Add(5 * time.Minute))
|
|
||||||
defer controller.SetReadDeadline(time.Time{})
|
|
||||||
defer controller.SetWriteDeadline(time.Time{})
|
|
||||||
if r.Method != "GET" && r.URL.Path != "/api/login" {
|
|
||||||
if !a.panelOriginAllowed(r) {
|
|
||||||
fail(w, 403, "origin denied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c, e := r.Cookie("dsh_session")
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, csrf, e := a.s.SessionUser(c.Value)
|
|
||||||
if e != nil || r.Header.Get("X-CSRF-Token") != csrf {
|
|
||||||
fail(w, 403, "csrf denied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.URL.Path != "/api/refresh" && r.URL.Path != "/api/logout" {
|
|
||||||
if _, _, e := a.user(r); e != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
func (a *API) user(r *http.Request) (store.User, string, error) {
|
|
||||||
c, e := r.Cookie("dsh_session")
|
|
||||||
if e != nil {
|
|
||||||
return store.User{}, "", e
|
|
||||||
}
|
|
||||||
u, csrf, e := a.s.SessionUser(c.Value)
|
|
||||||
if e != nil {
|
|
||||||
return store.User{}, "", e
|
|
||||||
}
|
|
||||||
if e = a.verifyAccess(r, c.Value, u.ID); e != nil {
|
|
||||||
return store.User{}, "", e
|
|
||||||
}
|
|
||||||
return u, csrf, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// dummyPasswordHash keeps login latency uniform for unknown usernames so the
|
|
||||||
// response timing cannot reveal whether an account exists.
|
|
||||||
var dummyPasswordHash, _ = bcrypt.GenerateFromPassword([]byte("timing-equalization"), bcrypt.DefaultCost)
|
|
||||||
|
|
||||||
func (a *API) login(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != "POST" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !a.panelOriginAllowed(r) {
|
|
||||||
fail(w, 403, "origin denied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct{ Username, Password string }
|
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, 4096)
|
|
||||||
decoder := json.NewDecoder(r.Body)
|
|
||||||
if decoder.Decode(&in) != nil {
|
|
||||||
fail(w, 400, "invalid json")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if decoder.Decode(new(any)) != io.EOF {
|
|
||||||
fail(w, 400, "invalid json")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !a.allowLoginSource(a.loginSource(r)) {
|
|
||||||
fail(w, 429, "too many login attempts; retry in a minute")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if a.authErr != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !a.acquireLoginSlot() {
|
|
||||||
fail(w, 503, "authentication overloaded")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer func() { <-a.loginSlots }()
|
|
||||||
a.authLoginMu.Lock()
|
|
||||||
defer a.authLoginMu.Unlock()
|
|
||||||
locked, e := a.accountLocked(in.Username)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if locked {
|
|
||||||
fail(w, 429, "account temporarily locked; retry later")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
h, e := a.s.Password(in.Username)
|
|
||||||
if e != nil {
|
|
||||||
_ = bcrypt.CompareHashAndPassword(dummyPasswordHash, []byte(in.Password))
|
|
||||||
fail(w, 401, "invalid credentials")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if bcrypt.CompareHashAndPassword([]byte(h), []byte(in.Password)) != nil {
|
|
||||||
if e = a.failedLogin(in.Username); e != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fail(w, 401, "invalid credentials")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
u, e := a.s.UserByName(in.Username)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 401, "invalid credentials")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("DELETE FROM auth_failures WHERE username=?", in.Username); e != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sid, csrf, e := a.s.Session(u.ID)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
token, e := refreshToken()
|
|
||||||
if e == nil {
|
|
||||||
_, e = a.s.DB.Exec("INSERT INTO auth_refresh(sid,token_hash) VALUES(?,?)", sid, refreshHash(token))
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
a.s.DeleteSession(sid)
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.authCookies(w, r, sid, u.ID, csrf, token)
|
|
||||||
a.s.AddAudit(u.ID, "login", u.ID, "")
|
|
||||||
writeJSON(w, map[string]any{"user": u, "csrf_token": csrf})
|
|
||||||
}
|
|
||||||
func (a *API) logout(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c, e := r.Cookie("dsh_session"); e == nil {
|
|
||||||
if _, e = a.s.DB.Exec("DELETE FROM sessions WHERE id=?", c.Value); e != nil {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, name := range []string{"dsh_session", "dsh_access", "dsh_refresh", "dsh_csrf"} {
|
|
||||||
path := "/"
|
|
||||||
if name == "dsh_refresh" {
|
|
||||||
path = "/api"
|
|
||||||
}
|
|
||||||
http.SetCookie(w, &http.Cookie{Name: name, MaxAge: -1, Path: path, Secure: a.secureCookies(r), HttpOnly: name != "dsh_csrf", SameSite: http.SameSiteLaxMode})
|
|
||||||
}
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
}
|
|
||||||
func (a *API) me(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
u, c, e := a.user(r)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]any{"user": u, "csrf_token": c})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) panelOriginAllowed(r *http.Request) bool {
|
|
||||||
if a.loginConfigErr != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
expected := a.origin
|
|
||||||
if expected == "" {
|
|
||||||
scheme := "http"
|
|
||||||
if r.TLS != nil {
|
|
||||||
scheme = "https"
|
|
||||||
}
|
|
||||||
expected = scheme + "://" + r.Host
|
|
||||||
}
|
|
||||||
return r.Header.Get("Origin") == expected
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var testAccessTokens sync.Map
|
|
||||||
|
|
||||||
func registerTestAccess(t *testing.T, a *API, sid string) {
|
|
||||||
t.Helper()
|
|
||||||
if a.authErr != nil {
|
|
||||||
t.Fatal(a.authErr)
|
|
||||||
}
|
|
||||||
u, _, err := a.s.SessionUser(sid)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
testAccessTokens.Store(sid, a.accessJWT(sid, u.ID, time.Now().Add(accessLifetime)))
|
|
||||||
t.Cleanup(func() { testAccessTokens.Delete(sid) })
|
|
||||||
}
|
|
||||||
func addTestAccess(r *http.Request, sid string) {
|
|
||||||
if token, ok := testAccessTokens.Load(sid); ok {
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_access", Value: token.(string)})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,166 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type loginSource struct {
|
|
||||||
started time.Time
|
|
||||||
count int
|
|
||||||
}
|
|
||||||
|
|
||||||
func configuredOrigin(raw string) (string, error) {
|
|
||||||
if raw == "" {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
u, err := url.Parse(raw)
|
|
||||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" || u.Hostname() == "" || u.User != nil || (u.Path != "" && u.Path != "/") || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" {
|
|
||||||
return "", errors.New("invalid PANEL_PUBLIC_URL")
|
|
||||||
}
|
|
||||||
if strings.Contains(raw, "#") || strings.HasSuffix(u.Host, ":") {
|
|
||||||
return "", errors.New("invalid PANEL_PUBLIC_URL")
|
|
||||||
}
|
|
||||||
port := u.Port()
|
|
||||||
if port != "" {
|
|
||||||
n, err := strconv.Atoi(port)
|
|
||||||
if err != nil || n < 1 || n > 65535 {
|
|
||||||
return "", errors.New("invalid PANEL_PUBLIC_URL")
|
|
||||||
}
|
|
||||||
port = strconv.Itoa(n)
|
|
||||||
}
|
|
||||||
host := strings.ToLower(u.Hostname())
|
|
||||||
if net.ParseIP(host) == nil {
|
|
||||||
if len(host) > 253 {
|
|
||||||
return "", errors.New("invalid PANEL_PUBLIC_URL")
|
|
||||||
}
|
|
||||||
for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
|
|
||||||
if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' {
|
|
||||||
return "", errors.New("invalid PANEL_PUBLIC_URL")
|
|
||||||
}
|
|
||||||
for _, c := range label {
|
|
||||||
if !(c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-') {
|
|
||||||
return "", errors.New("invalid PANEL_PUBLIC_URL")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if strings.Contains(host, ":") {
|
|
||||||
host = "[" + host + "]"
|
|
||||||
}
|
|
||||||
if port != "" && !(u.Scheme == "https" && port == "443" || u.Scheme == "http" && port == "80") {
|
|
||||||
host += ":" + port
|
|
||||||
}
|
|
||||||
return u.Scheme + "://" + host, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func configuredProxyCIDRs(raw string, prior error) ([]*net.IPNet, error) {
|
|
||||||
if prior != nil || strings.TrimSpace(raw) == "" {
|
|
||||||
return nil, prior
|
|
||||||
}
|
|
||||||
var out []*net.IPNet
|
|
||||||
for _, value := range strings.Split(raw, ",") {
|
|
||||||
value = strings.TrimSpace(value)
|
|
||||||
if value == "" {
|
|
||||||
return nil, errors.New("invalid TRUSTED_PROXY_CIDRS")
|
|
||||||
}
|
|
||||||
_, network, err := net.ParseCIDR(value)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.New("invalid TRUSTED_PROXY_CIDRS")
|
|
||||||
}
|
|
||||||
out = append(out, network)
|
|
||||||
}
|
|
||||||
return out, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func ValidateAuthConfig() error {
|
|
||||||
_, err := configuredOrigin(os.Getenv("PANEL_PUBLIC_URL"))
|
|
||||||
_, err = configuredProxyCIDRs(os.Getenv("TRUSTED_PROXY_CIDRS"), err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) ConfigError() error { return a.loginConfigErr }
|
|
||||||
|
|
||||||
func (a *API) acquireLoginSlot() bool {
|
|
||||||
a.loginMu.Lock()
|
|
||||||
defer a.loginMu.Unlock()
|
|
||||||
if a.loginSlots == nil {
|
|
||||||
a.loginSlots = make(chan struct{}, 8)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case a.loginSlots <- struct{}{}:
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) loginSource(r *http.Request) string {
|
|
||||||
peer, _, err := net.SplitHostPort(r.RemoteAddr)
|
|
||||||
if err != nil {
|
|
||||||
peer = r.RemoteAddr
|
|
||||||
}
|
|
||||||
ip := net.ParseIP(peer)
|
|
||||||
if ip == nil {
|
|
||||||
return peer
|
|
||||||
}
|
|
||||||
trusted := false
|
|
||||||
for _, network := range a.trustedProxies {
|
|
||||||
if network.Contains(ip) {
|
|
||||||
trusted = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if trusted {
|
|
||||||
parts := strings.Split(r.Header.Get("X-Forwarded-For"), ",")
|
|
||||||
for i := len(parts) - 1; i >= 0; i-- {
|
|
||||||
candidate := net.ParseIP(strings.TrimSpace(parts[i]))
|
|
||||||
if candidate == nil {
|
|
||||||
return ip.String()
|
|
||||||
}
|
|
||||||
isProxy := false
|
|
||||||
for _, network := range a.trustedProxies {
|
|
||||||
if network.Contains(candidate) {
|
|
||||||
isProxy = true
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !isProxy {
|
|
||||||
return candidate.String()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ip.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) allowLoginSource(source string) bool {
|
|
||||||
now := time.Now()
|
|
||||||
a.loginMu.Lock()
|
|
||||||
defer a.loginMu.Unlock()
|
|
||||||
if a.loginSources == nil {
|
|
||||||
a.loginSources = make(map[string]loginSource)
|
|
||||||
}
|
|
||||||
for key, entry := range a.loginSources {
|
|
||||||
if now.Sub(entry.started) >= time.Minute {
|
|
||||||
delete(a.loginSources, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
entry, ok := a.loginSources[source]
|
|
||||||
if !ok {
|
|
||||||
if len(a.loginSources) >= 1024 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
entry = loginSource{started: now}
|
|
||||||
}
|
|
||||||
if entry.count >= 30 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
entry.count++
|
|
||||||
a.loginSources[source] = entry
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
@@ -1,271 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/cookiejar"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestConfiguredOriginRejectsInvalidOrigins(t *testing.T) {
|
|
||||||
for _, raw := range []string{"https://", "https://:8443", "https://panel.test?", "https://panel.test:bad", "ftp://panel.test", "https://panel.test/path"} {
|
|
||||||
if _, err := configuredOrigin(raw); err == nil {
|
|
||||||
t.Errorf("accepted invalid origin %q", raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, raw := range []string{"https://panel.test", "https://panel.test:8443", "http://127.0.0.1:8088"} {
|
|
||||||
if got, err := configuredOrigin(raw); err != nil || got != raw {
|
|
||||||
t.Errorf("origin %q => %q, %v", raw, got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLoginSourceNilMapAndInvalidForwardedChain(t *testing.T) {
|
|
||||||
a := &API{}
|
|
||||||
if !a.allowLoginSource("127.0.0.1") {
|
|
||||||
t.Fatal("nil source map rejected")
|
|
||||||
}
|
|
||||||
a.trustedProxies = mustCIDRs(t, "10.0.0.0/8")
|
|
||||||
r := httptest.NewRequest("POST", "http://panel.test/api/login", nil)
|
|
||||||
r.RemoteAddr = "10.0.0.1:1234"
|
|
||||||
r.Header.Set("X-Forwarded-For", "198.51.100.4,not-an-ip,10.0.0.2")
|
|
||||||
if got := a.loginSource(r); got != "10.0.0.1" {
|
|
||||||
t.Fatalf("invalid chain crossed trust boundary: %s", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustCIDRs(t *testing.T, raw string) []*net.IPNet {
|
|
||||||
t.Helper()
|
|
||||||
proxies, err := configuredProxyCIDRs(raw, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return proxies
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSecurityConfigValidation(t *testing.T) {
|
|
||||||
for _, raw := range []string{"https://panel.test:0", "https://panel.test:65536", "https://panel.test:", "https://panel.test#", "https://bad_host", "https://[bad]", "https://[email protected]"} {
|
|
||||||
if _, err := configuredOrigin(raw); err == nil {
|
|
||||||
t.Errorf("accepted %q", raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for raw, want := range map[string]string{"https://PANEL.EXAMPLE:443/": "https://panel.example", "http://PANEL.EXAMPLE:80": "http://panel.example", "https://[::1]:443": "https://[::1]"} {
|
|
||||||
if got, err := configuredOrigin(raw); err != nil || got != want {
|
|
||||||
t.Errorf("%q => %q, %v", raw, got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, raw := range []string{"bad", "10.0.0.1", "10.0.0.0/8,", ",", "10.0.0.0/33"} {
|
|
||||||
if _, err := configuredProxyCIDRs(raw, nil); err == nil {
|
|
||||||
t.Errorf("accepted CIDRs %q", raw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, key := range []string{"PANEL_PUBLIC_URL", "TRUSTED_PROXY_CIDRS"} {
|
|
||||||
t.Run(key, func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_PUBLIC_URL", "")
|
|
||||||
t.Setenv("TRUSTED_PROXY_CIDRS", "")
|
|
||||||
t.Setenv(key, "invalid")
|
|
||||||
if ValidateAuthConfig() == nil {
|
|
||||||
t.Fatal("configuration accepted")
|
|
||||||
}
|
|
||||||
a := New(nil, t.TempDir())
|
|
||||||
if a.ConfigError() == nil {
|
|
||||||
t.Fatal("missing config error")
|
|
||||||
}
|
|
||||||
for _, path := range []string{"/api/login", "/api/me", "/api/refresh", "/dsh/example/"} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, httptest.NewRequest("POST", path, strings.NewReader("{}")))
|
|
||||||
if w.Code != 503 {
|
|
||||||
t.Fatalf("%s: %d", path, w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSecurityTLSCookieJarLifecycle(t *testing.T) {
|
|
||||||
a, _ := authFixture(t)
|
|
||||||
a.origin = ""
|
|
||||||
a.loginSlots = nil
|
|
||||||
a.loginSources = nil
|
|
||||||
srv := httptest.NewTLSServer(a.Handler())
|
|
||||||
defer srv.Close()
|
|
||||||
client := srv.Client()
|
|
||||||
client.Jar, _ = cookiejar.New(nil)
|
|
||||||
u, _ := url.Parse(srv.URL + "/api/me")
|
|
||||||
request := func(method, path, body string, status int) []*http.Cookie {
|
|
||||||
t.Helper()
|
|
||||||
r, _ := http.NewRequest(method, srv.URL+path, strings.NewReader(body))
|
|
||||||
r.Header.Set("Origin", srv.URL)
|
|
||||||
r.Header.Set("X-CSRF-Token", cookieValue(client.Jar.Cookies(u), "dsh_csrf"))
|
|
||||||
resp, err := client.Do(r)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
data, _ := io.ReadAll(resp.Body)
|
|
||||||
if resp.StatusCode != status {
|
|
||||||
t.Fatalf("%s: %d %s", path, resp.StatusCode, data)
|
|
||||||
}
|
|
||||||
return resp.Cookies()
|
|
||||||
}
|
|
||||||
check := func(cookies []*http.Cookie, deleting bool) {
|
|
||||||
t.Helper()
|
|
||||||
if len(cookies) != 4 {
|
|
||||||
t.Fatalf("cookies: %v", cookies)
|
|
||||||
}
|
|
||||||
for _, c := range cookies {
|
|
||||||
path := "/"
|
|
||||||
if c.Name == "dsh_refresh" {
|
|
||||||
path = "/api"
|
|
||||||
}
|
|
||||||
if !c.Secure || c.HttpOnly != (c.Name != "dsh_csrf") || c.Path != path || c.SameSite != http.SameSiteLaxMode || (deleting && c.MaxAge != -1) {
|
|
||||||
t.Fatalf("cookie: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
check(request("POST", "/api/login", `{"username":"alice","password":"password-123"}`, 200), false)
|
|
||||||
request("GET", "/api/me", "", 200)
|
|
||||||
old := cookieValue(client.Jar.Cookies(u), "dsh_refresh")
|
|
||||||
check(request("POST", "/api/refresh", "", 200), false)
|
|
||||||
if old == cookieValue(client.Jar.Cookies(u), "dsh_refresh") {
|
|
||||||
t.Fatal("refresh not rotated")
|
|
||||||
}
|
|
||||||
request("GET", "/api/me", "", 200)
|
|
||||||
check(request("POST", "/api/logout", "", 200), true)
|
|
||||||
if len(client.Jar.Cookies(u)) != 0 {
|
|
||||||
t.Fatal("logout left cookies")
|
|
||||||
}
|
|
||||||
request("GET", "/api/me", "", 401)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSecurityLoginBudgetsAndUntrustedHeaders(t *testing.T) {
|
|
||||||
a, _ := authFixture(t)
|
|
||||||
a.origin = ""
|
|
||||||
request := func(peer, origin, body string) int {
|
|
||||||
r := httptest.NewRequest("POST", "http://panel.test/api/login", strings.NewReader(body))
|
|
||||||
r.RemoteAddr = peer + ":1234"
|
|
||||||
r.Header.Set("Origin", origin)
|
|
||||||
r.Header.Set("X-Forwarded-For", fmt.Sprintf("198.51.100.%d", len(a.loginSources)))
|
|
||||||
r.Header.Set("X-Forwarded-Proto", "https")
|
|
||||||
r.Header.Set("X-Forwarded-Host", "evil.test")
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
return w.Code
|
|
||||||
}
|
|
||||||
for i := 0; i < 35; i++ {
|
|
||||||
if got := request("192.0.2.1", "https://evil.test", "{}"); got != 403 {
|
|
||||||
t.Fatal(got)
|
|
||||||
}
|
|
||||||
for _, body := range []string{"{", "{} garbage", "{} {}", strings.Repeat(" ", 4097)} {
|
|
||||||
if got := request("192.0.2.1", "http://panel.test", body); got != 400 {
|
|
||||||
t.Fatal(got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(a.loginSources) != 0 {
|
|
||||||
t.Fatal("invalid requests consumed budget")
|
|
||||||
}
|
|
||||||
for i := 0; i < 30; i++ {
|
|
||||||
if got := request("192.0.2.1", "http://panel.test", "{}"); got != 401 {
|
|
||||||
t.Fatal(i, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got := request("192.0.2.1", "http://panel.test", "{}"); got != 429 {
|
|
||||||
t.Fatal(got)
|
|
||||||
}
|
|
||||||
if got := request("192.0.2.2", "http://panel.test", "{}"); got != 401 {
|
|
||||||
t.Fatal("other source blocked", got)
|
|
||||||
}
|
|
||||||
if len(a.loginSources) != 2 {
|
|
||||||
t.Fatal("forwarded headers changed source")
|
|
||||||
}
|
|
||||||
r := httptest.NewRequest("POST", "http://panel.test", nil)
|
|
||||||
r.Header.Set("X-Forwarded-Proto", "https")
|
|
||||||
if a.secureCookies(r) {
|
|
||||||
t.Fatal("untrusted proto changed cookie policy")
|
|
||||||
}
|
|
||||||
a.origin = "https://panel.test"
|
|
||||||
if !a.secureCookies(r) {
|
|
||||||
t.Fatal("configured HTTPS not secure")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSecurityLoginSlotOverloadRecovery(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
if !a.acquireLoginSlot() {
|
|
||||||
t.Fatal("slot unavailable")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", "{}", "", nil); w.Code != 503 {
|
|
||||||
t.Fatal("overload not rejected", w.Code)
|
|
||||||
}
|
|
||||||
<-a.loginSlots
|
|
||||||
if w := req("POST", "/api/login", "{}", "", nil); w.Code != 401 {
|
|
||||||
t.Fatal("released slot not usable", w.Code)
|
|
||||||
}
|
|
||||||
if len(a.loginSlots) != 7 {
|
|
||||||
t.Fatal("request leaked slot")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSecurityConcurrentSlotsAndBoundedBuckets(t *testing.T) {
|
|
||||||
a := &API{}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
results := make(chan bool, 64)
|
|
||||||
for i := 0; i < 64; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() { defer wg.Done(); results <- a.acquireLoginSlot() }()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(results)
|
|
||||||
count := 0
|
|
||||||
for ok := range results {
|
|
||||||
if ok {
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if count != 8 {
|
|
||||||
t.Fatalf("slots: %d", count)
|
|
||||||
}
|
|
||||||
<-a.loginSlots
|
|
||||||
if !a.acquireLoginSlot() {
|
|
||||||
t.Fatal("released slot unavailable")
|
|
||||||
}
|
|
||||||
for i := 0; i < 1024; i++ {
|
|
||||||
if !a.allowLoginSource(fmt.Sprint(i)) {
|
|
||||||
t.Fatal(i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if a.allowLoginSource("overflow") || len(a.loginSources) != 1024 {
|
|
||||||
t.Fatal("unbounded buckets")
|
|
||||||
}
|
|
||||||
a.loginSources["0"] = loginSource{started: time.Now().Add(-time.Minute), count: 30}
|
|
||||||
if !a.allowLoginSource("overflow") || len(a.loginSources) != 1024 {
|
|
||||||
t.Fatal("expired bucket not reclaimed")
|
|
||||||
}
|
|
||||||
b := &API{}
|
|
||||||
accepted := make(chan bool, 64)
|
|
||||||
for i := 0; i < 64; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() { defer wg.Done(); accepted <- b.allowLoginSource("same") }()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(accepted)
|
|
||||||
count = 0
|
|
||||||
for ok := range accepted {
|
|
||||||
if ok {
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if count != 30 {
|
|
||||||
t.Fatalf("concurrent budget: %d", count)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,93 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestAuthenticationAndAdmin(t *testing.T) {
|
|
||||||
s, e := store.Open(filepath.Join(t.TempDir(), "test.sqlite"))
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
defer s.Close()
|
|
||||||
h, _ := bcrypt.GenerateFromPassword([]byte("test-password-123"), bcrypt.MinCost)
|
|
||||||
if e = s.CreateUser(store.User{ID: "admin", Username: "admin", Admin: true, Quota: 1024}, string(h)); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
a := New(s, t.TempDir())
|
|
||||||
a.origin = "http://panel.test"
|
|
||||||
var access *http.Cookie
|
|
||||||
request := func(method, path, body, csrf string, cookie *http.Cookie) *httptest.ResponseRecorder {
|
|
||||||
r := httptest.NewRequest(method, path, strings.NewReader(body))
|
|
||||||
r.Header.Set("Origin", a.origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
if cookie != nil {
|
|
||||||
r.AddCookie(cookie)
|
|
||||||
if access != nil {
|
|
||||||
r.AddCookie(access)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
if w := request("GET", "/api/me", "", "", nil); w.Code != 401 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
w := request("POST", "/api/login", `{"username":"admin","password":"test-password-123"}`, "", nil)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var login struct {
|
|
||||||
CSRF string `json:"csrf_token"`
|
|
||||||
}
|
|
||||||
json.Unmarshal(w.Body.Bytes(), &login)
|
|
||||||
var cookie *http.Cookie
|
|
||||||
for _, c := range w.Result().Cookies() {
|
|
||||||
if c.Name == "dsh_access" {
|
|
||||||
access = c
|
|
||||||
}
|
|
||||||
if c.Name == "dsh_session" {
|
|
||||||
cookie = c
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if cookie == nil || !cookie.HttpOnly || cookie.Secure {
|
|
||||||
t.Fatal("wrong local cookie settings")
|
|
||||||
}
|
|
||||||
if w = request("GET", "/api/users", "", "", cookie); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w = request("POST", "/api/users", `{"username":"member","password":"member-password-123"}`, "", cookie); w.Code != 403 {
|
|
||||||
t.Fatal("CSRF accepted", w.Code)
|
|
||||||
}
|
|
||||||
if w = request("POST", "/api/users", `{"username":"member","password":"member-password-123"}`, login.CSRF, cookie); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w = request("POST", "/api/logout", "", login.CSRF, cookie); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if w = request("GET", "/api/me", "", "", cookie); w.Code != 401 {
|
|
||||||
t.Fatal("session not revoked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestOriginRequiredWithoutPublicURL(t *testing.T) {
|
|
||||||
a := &API{}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
origin string
|
|
||||||
want bool
|
|
||||||
}{{"", false}, {"http://evil.test", false}, {"http://panel.test", true}} {
|
|
||||||
r := httptest.NewRequest("POST", "http://panel.test/api/login", nil)
|
|
||||||
r.Header.Set("Origin", tc.origin)
|
|
||||||
if a.panelOriginAllowed(r) != tc.want {
|
|
||||||
t.Fatalf("origin %q", tc.origin)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,256 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/hmac"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const accessLifetime = 5 * time.Minute
|
|
||||||
|
|
||||||
var errAuthentication = errors.New("authentication required")
|
|
||||||
|
|
||||||
func (a *API) initAuth() error {
|
|
||||||
if err := os.MkdirAll(a.root, 0700); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
path := filepath.Join(a.root, "auth-signing.key")
|
|
||||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
|
||||||
if err == nil {
|
|
||||||
key := make([]byte, 32)
|
|
||||||
if _, err = rand.Read(key); err == nil {
|
|
||||||
_, err = f.Write(key)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = f.Sync()
|
|
||||||
}
|
|
||||||
closeErr := f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if closeErr != nil {
|
|
||||||
return closeErr
|
|
||||||
}
|
|
||||||
} else if !os.IsExist(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
info, err := os.Lstat(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !info.Mode().IsRegular() || info.Mode().Perm() != 0600 {
|
|
||||||
return errors.New("unsafe signing key permissions")
|
|
||||||
}
|
|
||||||
a.authKey, err = os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if len(a.authKey) != 32 {
|
|
||||||
return errors.New("invalid signing key")
|
|
||||||
}
|
|
||||||
_, err = a.s.DB.Exec(`CREATE TABLE IF NOT EXISTS auth_refresh(sid TEXT PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE, token_hash TEXT NOT NULL);
|
|
||||||
CREATE TABLE IF NOT EXISTS auth_failures(username TEXT PRIMARY KEY, failures INTEGER NOT NULL, locked_until INTEGER NOT NULL);`)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
type accessClaims struct {
|
|
||||||
Subject string `json:"sub"`
|
|
||||||
Session string `json:"sid"`
|
|
||||||
Issuer string `json:"iss"`
|
|
||||||
Audience string `json:"aud"`
|
|
||||||
Expires int64 `json:"exp"`
|
|
||||||
}
|
|
||||||
|
|
||||||
const jwtHeader = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"
|
|
||||||
|
|
||||||
func (a *API) accessJWT(sid, uid string, expires time.Time) string {
|
|
||||||
payload, _ := json.Marshal(accessClaims{uid, sid, "dsh-panel", "dsh-panel", expires.Unix()})
|
|
||||||
body := jwtHeader + "." + base64.RawURLEncoding.EncodeToString(payload)
|
|
||||||
mac := hmac.New(sha256.New, a.authKey)
|
|
||||||
mac.Write([]byte(body))
|
|
||||||
return body + "." + base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
|
|
||||||
}
|
|
||||||
func (a *API) verifyAccess(r *http.Request, sid, uid string) error {
|
|
||||||
if a.authErr != nil || len(a.authKey) != 32 {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
c, err := r.Cookie("dsh_access")
|
|
||||||
if err != nil {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
p := strings.Split(c.Value, ".")
|
|
||||||
if len(p) != 3 || p[0] != jwtHeader {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
sig, err := base64.RawURLEncoding.DecodeString(p[2])
|
|
||||||
if err != nil {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
mac := hmac.New(sha256.New, a.authKey)
|
|
||||||
mac.Write([]byte(p[0] + "." + p[1]))
|
|
||||||
if !hmac.Equal(sig, mac.Sum(nil)) {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
data, err := base64.RawURLEncoding.DecodeString(p[1])
|
|
||||||
if err != nil {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
var claims accessClaims
|
|
||||||
if json.Unmarshal(data, &claims) != nil || claims.Subject != uid || claims.Session != sid || claims.Issuer != "dsh-panel" || claims.Audience != "dsh-panel" || claims.Expires <= time.Now().Unix() || claims.Expires > time.Now().Add(accessLifetime).Unix() {
|
|
||||||
return errAuthentication
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func refreshToken() (string, error) {
|
|
||||||
b := make([]byte, 32)
|
|
||||||
_, err := rand.Read(b)
|
|
||||||
return base64.RawURLEncoding.EncodeToString(b), err
|
|
||||||
}
|
|
||||||
func refreshHash(token string) string {
|
|
||||||
h := sha256.Sum256([]byte(token))
|
|
||||||
return hex.EncodeToString(h[:])
|
|
||||||
}
|
|
||||||
func (a *API) secureCookies(r *http.Request) bool {
|
|
||||||
secure := r.TLS != nil
|
|
||||||
if u, err := url.Parse(a.origin); err == nil && u.Scheme == "https" {
|
|
||||||
secure = true
|
|
||||||
}
|
|
||||||
return secure
|
|
||||||
}
|
|
||||||
func (a *API) authCookies(w http.ResponseWriter, r *http.Request, sid, uid, csrf, refresh string) {
|
|
||||||
for _, c := range []*http.Cookie{
|
|
||||||
{Name: "dsh_session", Value: sid, Path: "/", HttpOnly: true},
|
|
||||||
{Name: "dsh_access", Value: a.accessJWT(sid, uid, time.Now().Add(accessLifetime)), Path: "/", HttpOnly: true, MaxAge: int(accessLifetime.Seconds())},
|
|
||||||
{Name: "dsh_refresh", Value: refresh, Path: "/api", HttpOnly: true},
|
|
||||||
{Name: "dsh_csrf", Value: csrf, Path: "/"},
|
|
||||||
} {
|
|
||||||
c.Secure = a.secureCookies(r)
|
|
||||||
c.SameSite = http.SameSiteLaxMode
|
|
||||||
http.SetCookie(w, c)
|
|
||||||
}
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
}
|
|
||||||
func (a *API) refresh(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if a.authErr != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
sid, err := r.Cookie("dsh_session")
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var uid, csrf string
|
|
||||||
err = a.s.DB.QueryRow("SELECT uid,csrf FROM sessions WHERE id=? AND expires>?", sid.Value, time.Now().Unix()).Scan(&uid, &csrf)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
} else {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
u, err := a.s.UserByID(uid)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
} else {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
old, err := r.Cookie("dsh_refresh")
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
token, err := refreshToken()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
result, err := a.s.DB.Exec("UPDATE auth_refresh SET token_hash=? WHERE sid=? AND token_hash=? AND EXISTS(SELECT 1 FROM sessions WHERE id=? AND expires>?)", refreshHash(token), sid.Value, refreshHash(old.Value), sid.Value, time.Now().Unix())
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
n, err := result.RowsAffected()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if n != 1 {
|
|
||||||
var active bool
|
|
||||||
err = a.s.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sessions JOIN auth_refresh ON auth_refresh.sid=sessions.id WHERE sessions.id=? AND sessions.expires>?)", sid.Value, time.Now().Unix()).Scan(&active)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "session error")
|
|
||||||
} else if active {
|
|
||||||
fail(w, 409, "refresh_conflict")
|
|
||||||
} else {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.authCookies(w, r, sid.Value, u.ID, csrf, token)
|
|
||||||
writeJSON(w, map[string]any{"user": u, "csrf_token": csrf})
|
|
||||||
}
|
|
||||||
func (a *API) accountLocked(username string) (bool, error) {
|
|
||||||
var until int64
|
|
||||||
err := a.s.DB.QueryRow("SELECT locked_until FROM auth_failures WHERE username=?", username).Scan(&until)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return until > time.Now().Unix(), err
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunSessionCleanup removes expired sessions, their refresh tokens and elapsed
|
|
||||||
// login locks every ten minutes; without it the sessions table grows without
|
|
||||||
// bound because SessionUser only filters rows on read.
|
|
||||||
func (a *API) RunSessionCleanup(ctx context.Context) {
|
|
||||||
ticker := time.NewTicker(10 * time.Minute)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
a.cleanupSessions(time.Now().Unix())
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) cleanupSessions(now int64) {
|
|
||||||
for _, q := range []struct{ name, sql string }{
|
|
||||||
{"refresh tokens", "DELETE FROM auth_refresh WHERE sid IN (SELECT id FROM sessions WHERE expires<=?)"},
|
|
||||||
{"sessions", "DELETE FROM sessions WHERE expires<=?"},
|
|
||||||
{"login locks", "DELETE FROM auth_failures WHERE locked_until>0 AND locked_until<=?"},
|
|
||||||
} {
|
|
||||||
if _, err := a.s.DB.Exec(q.sql, now); err != nil {
|
|
||||||
log.Printf("session cleanup (%s): %v", q.name, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (a *API) failedLogin(username string) error {
|
|
||||||
_, err := a.s.DB.Exec(`INSERT INTO auth_failures(username,failures,locked_until) VALUES(?,1,0)
|
|
||||||
ON CONFLICT(username) DO UPDATE SET
|
|
||||||
failures=CASE WHEN locked_until>0 AND locked_until<=? THEN 1 ELSE failures+1 END,
|
|
||||||
locked_until=CASE WHEN locked_until>0 AND locked_until<=? THEN 0 WHEN failures+1>=5 THEN ? ELSE 0 END`, username, time.Now().Unix(), time.Now().Unix(), time.Now().Add(15*time.Minute).Unix())
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,282 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func authFixture(t *testing.T) (*API, func(string, string, string, string, []*http.Cookie) *httptest.ResponseRecorder) {
|
|
||||||
t.Helper()
|
|
||||||
s, e := store.Open(filepath.Join(t.TempDir(), "db"))
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
t.Cleanup(s.Close)
|
|
||||||
h, _ := bcrypt.GenerateFromPassword([]byte("password-123"), bcrypt.MinCost)
|
|
||||||
for _, id := range []string{"alice", "bob"} {
|
|
||||||
if e = s.CreateUser(store.User{ID: id, Username: id}, string(h)); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a := New(s, t.TempDir())
|
|
||||||
a.origin = "https://panel.test"
|
|
||||||
return a, func(method, path, body, csrf string, cookies []*http.Cookie) *httptest.ResponseRecorder {
|
|
||||||
r := httptest.NewRequest(method, path, strings.NewReader(body))
|
|
||||||
r.Header.Set("Origin", a.origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
for _, c := range cookies {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func cookieValue(cs []*http.Cookie, name string) string {
|
|
||||||
for _, c := range cs {
|
|
||||||
if c.Name == name {
|
|
||||||
return c.Value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
func TestRefreshSessionLookupFailureIsServerError(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil)
|
|
||||||
cookies := w.Result().Cookies()
|
|
||||||
a.s.Close()
|
|
||||||
r := httptest.NewRequest("POST", "/api/refresh", nil)
|
|
||||||
for _, c := range cookies {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
a.refresh(w, r)
|
|
||||||
if w.Code != 500 || len(w.Result().Cookies()) != 0 {
|
|
||||||
t.Fatal(w.Code, w.Header(), w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestAccessRefreshLifecycle(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
cs := w.Result().Cookies()
|
|
||||||
sid := cookieValue(cs, "dsh_session")
|
|
||||||
csrf := cookieValue(cs, "dsh_csrf")
|
|
||||||
for _, name := range []string{"dsh_access", "dsh_refresh"} {
|
|
||||||
v := cookieValue(cs, name)
|
|
||||||
if v == "" || strings.Contains(w.Body.String(), v) {
|
|
||||||
t.Fatal("missing/leaked", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, c := range cs {
|
|
||||||
if !c.Secure || (c.Name != "dsh_csrf" && !c.HttpOnly) {
|
|
||||||
t.Fatal("unsafe cookie", c.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w = req("GET", "/api/me", "", "", []*http.Cookie{{Name: "dsh_session", Value: sid}}); w.Code != 401 {
|
|
||||||
t.Fatal("missing access accepted", w.Code)
|
|
||||||
}
|
|
||||||
expired := append([]*http.Cookie{}, cs...)
|
|
||||||
for i, c := range expired {
|
|
||||||
if c.Name == "dsh_access" {
|
|
||||||
expired[i] = &http.Cookie{Name: c.Name, Value: a.accessJWT(sid, "alice", time.Now().Add(-time.Minute))}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w = req("GET", "/api/me", "", "", expired); w.Code != 401 {
|
|
||||||
t.Fatal("expired access accepted")
|
|
||||||
}
|
|
||||||
if w = req("POST", "/api/refresh", "", "", expired); w.Code != 403 {
|
|
||||||
t.Fatal("refresh bypassed csrf", w.Code)
|
|
||||||
}
|
|
||||||
w = req("POST", "/api/refresh", "", csrf, expired)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
fresh := w.Result().Cookies()
|
|
||||||
if cookieValue(fresh, "dsh_refresh") == cookieValue(cs, "dsh_refresh") {
|
|
||||||
t.Fatal("not rotated")
|
|
||||||
}
|
|
||||||
if w = req("POST", "/api/refresh", "", csrf, cs); w.Code != 409 {
|
|
||||||
t.Fatal("replay accepted", w.Code)
|
|
||||||
}
|
|
||||||
if len(w.Result().Cookies()) != 0 || !strings.Contains(w.Body.String(), "refresh_conflict") {
|
|
||||||
t.Fatal("conflict must not issue credentials", w.Header(), w.Body.String())
|
|
||||||
}
|
|
||||||
if w = req("GET", "/api/me", "", "", fresh); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if w = req("POST", "/api/logout", "", csrf, fresh); w.Code != 200 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if w = req("GET", "/api/me", "", "", fresh); w.Code != 401 {
|
|
||||||
t.Fatal("logout access valid")
|
|
||||||
}
|
|
||||||
if w = req("POST", "/api/refresh", "", csrf, fresh); w.Code != 403 {
|
|
||||||
t.Fatal("logout refresh valid")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestAccountLockAndPersistentKey(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
for i := 0; i < 5; i++ {
|
|
||||||
if w := req("POST", "/api/login", `{"username":"alice","password":"wrong"}`, "", nil); w.Code != 401 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil); w.Code != 429 {
|
|
||||||
t.Fatal("not locked", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", `{"username":"bob","password":"password-123"}`, "", nil); w.Code != 200 {
|
|
||||||
t.Fatal("other account locked", w.Code)
|
|
||||||
}
|
|
||||||
b := New(a.s, a.root)
|
|
||||||
if !bytes.Equal(a.authKey, b.authKey) || b.authErr != nil {
|
|
||||||
t.Fatal("key not persistent", b.authErr)
|
|
||||||
}
|
|
||||||
fi, e := os.Stat(filepath.Join(a.root, "auth-signing.key"))
|
|
||||||
if e != nil || fi.Mode().Perm() != 0600 {
|
|
||||||
t.Fatal("unsafe key")
|
|
||||||
}
|
|
||||||
if locked, e := b.accountLocked("alice"); e != nil || !locked {
|
|
||||||
t.Fatal("lock not persistent", e)
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("UPDATE auth_failures SET locked_until=? WHERE username='alice'", time.Now().Add(-time.Second).Unix()); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil); w.Code != 200 {
|
|
||||||
t.Fatal("lock did not expire", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRefreshOriginConcurrentRotationAndJWTBinding(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil)
|
|
||||||
cs := w.Result().Cookies()
|
|
||||||
sid := cookieValue(cs, "dsh_session")
|
|
||||||
csrf := cookieValue(cs, "dsh_csrf")
|
|
||||||
var stored string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT token_hash FROM auth_refresh WHERE sid=?", sid).Scan(&stored); err != nil || stored != refreshHash(cookieValue(cs, "dsh_refresh")) {
|
|
||||||
t.Fatal("refresh not hashed", err)
|
|
||||||
}
|
|
||||||
for _, token := range []string{cookieValue(cs, "dsh_access") + "x", a.accessJWT("other", "alice", time.Now().Add(accessLifetime)), a.accessJWT(sid, "bob", time.Now().Add(accessLifetime)), "eyJhbGciOiJub25lIn0.e30."} {
|
|
||||||
if w = req("GET", "/api/me", "", "", []*http.Cookie{{Name: "dsh_session", Value: sid}, {Name: "dsh_access", Value: token}}); w.Code != 401 {
|
|
||||||
t.Fatal("invalid JWT accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, origin := range []string{"", "https://evil.test"} {
|
|
||||||
r := httptest.NewRequest("POST", "/api/refresh", nil)
|
|
||||||
r.Header.Set("Origin", origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
for _, c := range cs {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
if w.Code != 403 {
|
|
||||||
t.Fatal("origin accepted", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
codes := make(chan int, 2)
|
|
||||||
for i := 0; i < 2; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() { defer wg.Done(); codes <- req("POST", "/api/refresh", "", csrf, cs).Code }()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(codes)
|
|
||||||
counts := map[int]int{}
|
|
||||||
for code := range codes {
|
|
||||||
counts[code]++
|
|
||||||
}
|
|
||||||
if counts[200] != 1 || counts[409] != 1 {
|
|
||||||
t.Fatal(counts)
|
|
||||||
}
|
|
||||||
b := New(a.s, a.root)
|
|
||||||
r := httptest.NewRequest("GET", "/api/me", nil)
|
|
||||||
for _, c := range cs {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
b.Handler().ServeHTTP(w, r)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal("restart rejected access", w.Code)
|
|
||||||
}
|
|
||||||
a.s.DeleteSession(sid)
|
|
||||||
if w = req("GET", "/api/me", "", "", cs); w.Code != 401 {
|
|
||||||
t.Fatal("deleted binding accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnsafeAuthKeyFailsClosed(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if err := os.Chmod(filepath.Join(a.root, "auth-signing.key"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
b := New(a.s, a.root)
|
|
||||||
if b.authErr == nil {
|
|
||||||
t.Fatal("unsafe key accepted")
|
|
||||||
}
|
|
||||||
a.authErr = b.authErr
|
|
||||||
if w := req("POST", "/api/login", `{"username":"alice","password":"password-123"}`, "", nil); w.Code != 503 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSessionCleanupRemovesExpiredRows(t *testing.T) {
|
|
||||||
a, _ := authFixture(t)
|
|
||||||
expired, _, e := a.s.Session("alice")
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
current, _, e := a.s.Session("alice")
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
past := time.Now().Add(-time.Minute).Unix()
|
|
||||||
if _, e = a.s.DB.Exec("UPDATE sessions SET expires=? WHERE id=?", past, expired); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
token, e := refreshToken()
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("INSERT INTO auth_refresh(sid,token_hash) VALUES(?,?)", expired, refreshHash(token)); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
future := time.Now().Add(time.Hour).Unix()
|
|
||||||
if _, e = a.s.DB.Exec("INSERT INTO auth_failures(username,failures,locked_until) VALUES('expired-lock',5,?)", past); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("INSERT INTO auth_failures(username,failures,locked_until) VALUES('active-lock',5,?)", future); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
a.cleanupSessions(time.Now().Unix())
|
|
||||||
var n int
|
|
||||||
for _, q := range []string{
|
|
||||||
"SELECT count(*) FROM sessions WHERE id='" + expired + "'",
|
|
||||||
"SELECT count(*) FROM auth_refresh WHERE sid='" + expired + "'",
|
|
||||||
"SELECT count(*) FROM auth_failures WHERE username='expired-lock'",
|
|
||||||
} {
|
|
||||||
if err := a.s.DB.QueryRow(q).Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatalf("expired row retained: %s (%d, %v)", q, n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, q := range []string{
|
|
||||||
"SELECT count(*) FROM sessions WHERE id='" + current + "'",
|
|
||||||
"SELECT count(*) FROM auth_failures WHERE username='active-lock'",
|
|
||||||
} {
|
|
||||||
if err := a.s.DB.QueryRow(q).Scan(&n); err != nil || n != 1 {
|
|
||||||
t.Fatalf("live row removed: %s (%d, %v)", q, n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,112 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Summary queries never load log bodies; history cannot inflate a page response.
|
|
||||||
const buildSummaryColumns = "id,version,status,error,image_id,created_at,requested_by,COALESCE(recipe_rev,''),COALESCE(platform,''),COALESCE(finished_at,''),experimental"
|
|
||||||
|
|
||||||
func scanBuildSummary(row interface{ Scan(...any) error }, b *buildRecord) error {
|
|
||||||
return row.Scan(&b.ID, &b.Version, &b.Status, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &b.RecipeRev, &b.Platform, &b.Finished, &b.Experimental)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) listBuilds(w http.ResponseWriter, r *http.Request) {
|
|
||||||
limit, offset := 20, 0
|
|
||||||
for key, dest := range map[string]*int{"limit": &limit, "offset": &offset} {
|
|
||||||
if values, ok := r.URL.Query()[key]; ok {
|
|
||||||
if len(values) != 1 {
|
|
||||||
fail(w, 400, "invalid pagination")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
value, err := strconv.Atoi(values[0])
|
|
||||||
if err != nil || value < 0 || (key == "limit" && (value < 1 || value > 100)) {
|
|
||||||
fail(w, 400, "invalid pagination")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
*dest = value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
rows, err := a.s.DB.QueryContext(ctx, "SELECT "+buildSummaryColumns+" FROM image_builds ORDER BY created_at DESC,id DESC LIMIT ? OFFSET ?", limit+1, offset)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
result := []buildRecord{}
|
|
||||||
for rows.Next() {
|
|
||||||
var b buildRecord
|
|
||||||
if err = scanBuildSummary(rows, &b); err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
result = append(result, b)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = rows.Err()
|
|
||||||
}
|
|
||||||
rows.Close()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
hasMore := len(result) > limit
|
|
||||||
if hasMore {
|
|
||||||
result = result[:limit]
|
|
||||||
}
|
|
||||||
if err := a.reconcileBuilds(ctx, result); err != nil {
|
|
||||||
fail(w, 502, "docker image reconciliation failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.startBuildWorker()
|
|
||||||
w.Header().Set("X-Has-More", strconv.FormatBool(hasMore))
|
|
||||||
writeJSON(w, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) buildDetail(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if _, _, err := a.user(r); err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := strings.TrimPrefix(r.URL.Path, "/api/builds/")
|
|
||||||
if id == "" || len(id) > 128 || strings.Contains(id, "/") {
|
|
||||||
fail(w, 404, "build not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
var b buildRecord
|
|
||||||
if err == nil {
|
|
||||||
err = a.s.DB.QueryRowContext(ctx, "SELECT "+buildSummaryColumns+",logs FROM image_builds WHERE id=?", id).Scan(&b.ID, &b.Version, &b.Status, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &b.RecipeRev, &b.Platform, &b.Finished, &b.Experimental, &b.Logs)
|
|
||||||
}
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
fail(w, 404, "build not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
b.Logs = sanitizeBuildLog(b.Logs)
|
|
||||||
writeJSON(w, b)
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildHistoryPaginationAndOnDemandLogs(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Errorf("unexpected Docker request: %s", r.URL) })
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for n := 0; n < 125; n++ {
|
|
||||||
_, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,logs,created_at) VALUES(?,?,?,?,?)", fmt.Sprintf("task-%03d", n), "1.2.3", "failed", "token=secret-value"+string(rune(10))+strings.Repeat("x", buildLogLimit), "2026-01-01")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
request := func(target, session, method string) *httptest.ResponseRecorder {
|
|
||||||
r := httptest.NewRequest(method, target, nil)
|
|
||||||
if session != "" {
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: session})
|
|
||||||
addTestAccess(r, session)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/api/builds/") {
|
|
||||||
a.buildDetail(w, r)
|
|
||||||
} else {
|
|
||||||
a.builds(w, r)
|
|
||||||
}
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
query string
|
|
||||||
count int
|
|
||||||
first, more string
|
|
||||||
}{
|
|
||||||
{"", 20, "task-124", "true"}, {"?limit=100", 100, "task-124", "true"},
|
|
||||||
{"?limit=20&offset=120", 5, "task-004", "false"}, {"?offset=125", 0, "", "false"},
|
|
||||||
{"?limit=20&offset=105", 20, "task-019", "false"}, {"?limit=20&offset=104", 20, "task-020", "true"},
|
|
||||||
} {
|
|
||||||
w := request("/api/builds"+tc.query, sid, "GET")
|
|
||||||
var records []buildRecord
|
|
||||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &records) != nil || len(records) != tc.count {
|
|
||||||
t.Fatalf("%s: %d %s", tc.query, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if tc.count > 0 && records[0].ID != tc.first {
|
|
||||||
t.Fatal(records[0].ID)
|
|
||||||
}
|
|
||||||
if w.Header().Get("X-Has-More") != tc.more || strings.Contains(w.Body.String(), "logs") || strings.Contains(w.Body.String(), "secret-value") {
|
|
||||||
t.Fatal("unbounded or invalid summary response", w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, query := range []string{"?limit=0", "?limit=101", "?limit=-1", "?offset=-1", "?offset=x", "?limit=1&limit=2", "?offset=9999999999999999999999999"} {
|
|
||||||
if w := request("/api/builds"+query, sid, "GET"); w.Code != 400 {
|
|
||||||
t.Fatal(query, w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := request("/api/builds/task-000", sid, "GET")
|
|
||||||
var b buildRecord
|
|
||||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &b) != nil || b.ID != "task-000" || b.Logs == "" || len(b.Logs) > buildLogLimit || strings.Contains(b.Logs, "secret-value") {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for _, path := range []string{"/api/builds/missing", "/api/builds/", "/api/builds/task-000/extra"} {
|
|
||||||
if w := request(path, sid, "GET"); w.Code != 404 {
|
|
||||||
t.Fatal(path, w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := request("/api/builds/task-000", "", "GET"); w.Code != 401 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if w := request("/api/builds", "", "GET"); w.Code != 401 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if w := request("/api/builds/task-000", sid, "POST"); w.Code != 405 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildDetailRouteAndAccess(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker call") })
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('task','1.2.3','failed','now')"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_, csrf, err := a.s.SessionUser(sid)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
path, method string
|
|
||||||
access bool
|
|
||||||
status int
|
|
||||||
}{
|
|
||||||
{"/api/builds/task", "GET", true, 200}, {"/api/builds/task", "GET", false, 401},
|
|
||||||
{"/api/builds/task", "POST", true, 405}, {"/api/builds/", "GET", true, 404},
|
|
||||||
{"/api/builds/task%2Fextra", "GET", true, 404}, {"/api/builds/task/extra", "GET", true, 404},
|
|
||||||
} {
|
|
||||||
r := httptest.NewRequest(tc.method, tc.path, nil)
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
|
|
||||||
if tc.access {
|
|
||||||
addTestAccess(r, sid)
|
|
||||||
}
|
|
||||||
if tc.method != "GET" {
|
|
||||||
r.Header.Set("Origin", "http://example.com")
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
if w.Code != tc.status {
|
|
||||||
t.Fatalf("%s %s: %d %s", tc.method, tc.path, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildPageReconcilesOnlySelectedRecords(t *testing.T) {
|
|
||||||
unavailable := false
|
|
||||||
calls := 0
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
calls++
|
|
||||||
if unavailable {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !strings.HasPrefix(r.URL.Path, "/v1.47/images/") {
|
|
||||||
t.Errorf("unexpected request %s", r.URL)
|
|
||||||
}
|
|
||||||
w.WriteHeader(404)
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "first-image", "1.2.3")
|
|
||||||
seedVerifiedImage(t, a, "second-image", "1.2.4")
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET created_at=CASE version WHEN '1.2.3' THEN 'z' ELSE 'a' END"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
request := func(offset int) *httptest.ResponseRecorder {
|
|
||||||
r := httptest.NewRequest("GET", fmt.Sprintf("/api/builds?limit=1&offset=%d", offset), nil)
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
|
|
||||||
addTestAccess(r, sid)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.builds(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
w := request(0)
|
|
||||||
if w.Code != 200 || calls != 1 || !strings.Contains(w.Body.String(), "missing") {
|
|
||||||
t.Fatal(w.Code, calls, w.Body.String())
|
|
||||||
}
|
|
||||||
var state string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE version='1.2.4'").Scan(&state); err != nil || state != "success" {
|
|
||||||
t.Fatal(state, err)
|
|
||||||
}
|
|
||||||
unavailable = true
|
|
||||||
if w = request(1); w.Code != 502 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,676 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/backup"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var backupIDPattern = regexp.MustCompile("^[a-f0-9]{32}$")
|
|
||||||
var configBodySlots = make(chan struct{}, 4)
|
|
||||||
|
|
||||||
func (a *API) config(w http.ResponseWriter, r *http.Request, i store.Instance, action string) {
|
|
||||||
controller := http.NewResponseController(w)
|
|
||||||
if r.Method != http.MethodGet || action != "backups" || r.URL.Query().Get("download") == "" {
|
|
||||||
var flush func()
|
|
||||||
w, flush = deferJSONResponse(w)
|
|
||||||
defer flush()
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
select {
|
|
||||||
case configBodySlots <- struct{}{}:
|
|
||||||
defer func() {
|
|
||||||
r.Body = http.NoBody
|
|
||||||
<-configBodySlots
|
|
||||||
}()
|
|
||||||
default:
|
|
||||||
fail(w, 503, "configuration uploads busy; retry later")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
limit := int64(4096)
|
|
||||||
multipartUpload := action == "backups" && strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data")
|
|
||||||
if action == "profile" {
|
|
||||||
limit = 2 << 20
|
|
||||||
} else if multipartUpload {
|
|
||||||
limit = min(64<<20, backup.MaxEncryptedBytes+16384)
|
|
||||||
}
|
|
||||||
if !configReadBody(w, r, controller, limit) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if multipartUpload {
|
|
||||||
defer func() {
|
|
||||||
if r.MultipartForm != nil {
|
|
||||||
r.MultipartForm.RemoveAll()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if r.ParseMultipartForm(1<<20) != nil {
|
|
||||||
fail(w, 400, "invalid or oversized backup upload")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
inst := a.instanceLocks.Acquire(i.ID)
|
|
||||||
locked := inst
|
|
||||||
defer func() {
|
|
||||||
if locked != nil {
|
|
||||||
locked.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
a.readersMu.Lock()
|
|
||||||
readers := a.workspaceReaders[i.ID]
|
|
||||||
a.readersMu.Unlock()
|
|
||||||
if readers > 0 {
|
|
||||||
fail(w, 409, "instance files busy; retry later")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
user := a.userLocks.Acquire(i.UID)
|
|
||||||
defer user.Release()
|
|
||||||
}
|
|
||||||
current, e := a.s.GetInstance(i.ID)
|
|
||||||
if e != nil || current.UID != i.UID {
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
i = current
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
admissionMu.RLock()
|
|
||||||
defer admissionMu.RUnlock()
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var blocked int
|
|
||||||
if err = a.s.DB.QueryRowContext(r.Context(), "SELECT count(*) FROM image_admissions WHERE image_id=? AND state='blocked'", i.ImageID).Scan(&blocked); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if blocked > 0 {
|
|
||||||
fail(w, 409, "image is blocked; configuration mutations and automatic restarts denied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if i.Status == "error" || a.upgradeRecoveryRequired(i) {
|
|
||||||
fail(w, 409, "instance recovery required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
home := filepath.Join(a.root, "workspaces", i.UID, "instances", i.ID, "home")
|
|
||||||
dir := filepath.Join(a.root, "backups", i.UID, i.ID)
|
|
||||||
storage, e := os.OpenRoot(a.root)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "storage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer storage.Close()
|
|
||||||
if configCheckPath(storage, filepath.ToSlash(filepath.Join("backups", i.UID, i.ID, "archive"))) != nil || configCheckPath(storage, filepath.ToSlash(filepath.Join("workspaces", i.UID, "instances", i.ID, "home", "profile"))) != nil {
|
|
||||||
fail(w, 400, "unsafe storage path")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e = os.MkdirAll(dir, 0700); e != nil {
|
|
||||||
fail(w, 500, "backup storage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
wasRunning := false
|
|
||||||
if r.Method != "GET" {
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
c, e := a.docker.Inspect(ctx, i.ContainerID)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 502, "container inspection failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
wasRunning = c.State.Running
|
|
||||||
if c.State.Paused || c.State.Restarting {
|
|
||||||
fail(w, 409, "unsupported container state")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.State.Running && action == "profile" {
|
|
||||||
fail(w, 409, "stop instance before changing configuration or backing up")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
switch action {
|
|
||||||
case "profile":
|
|
||||||
parent, e := upgradeRoot(a.root, i)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "unsafe home")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer parent.Close()
|
|
||||||
if e = configCheckPath(parent, "home/profiles/web/cordis.patch.yml"); e != nil {
|
|
||||||
fail(w, 400, "unsafe profile")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
root, e := configProfileRoot(parent, r.Method == "PUT")
|
|
||||||
if e != nil {
|
|
||||||
if r.Method == "GET" && errors.Is(e, os.ErrNotExist) {
|
|
||||||
writeJSON(w, map[string]string{"content": ""})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fail(w, 404, "home unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
const path = "cordis.patch.yml"
|
|
||||||
if r.Method == "GET" {
|
|
||||||
f, e := root.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
||||||
if errors.Is(e, os.ErrNotExist) {
|
|
||||||
writeJSON(w, map[string]string{"content": ""})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "profile unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
st, e := f.Stat()
|
|
||||||
if e != nil || !st.Mode().IsRegular() || st.Size() > 1<<20 {
|
|
||||||
fail(w, 400, "invalid profile")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
b, e := io.ReadAll(io.LimitReader(f, 1<<20))
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "read failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]string{"content": string(b)})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != "PUT" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct {
|
|
||||||
Content string `json:"content"`
|
|
||||||
Restart bool `json:"restart"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 2<<20)).Decode(&in) != nil || len(in.Content) > 1<<20 {
|
|
||||||
fail(w, 400, "invalid profile")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var document any
|
|
||||||
if yaml.Unmarshal([]byte(in.Content), &document) != nil {
|
|
||||||
fail(w, 400, "invalid YAML")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var previous int64
|
|
||||||
if st, err := root.Lstat(path); err == nil {
|
|
||||||
if !st.Mode().IsRegular() {
|
|
||||||
fail(w, 400, "invalid profile")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
previous = st.Size()
|
|
||||||
} else if !errors.Is(err, os.ErrNotExist) {
|
|
||||||
fail(w, 400, "profile unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e = a.quotaGrowth(i.UID, int64(len(in.Content))-previous); e != nil {
|
|
||||||
fail(w, 413, "quota exceeded or usage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tmp := ".panel-" + store.ID()
|
|
||||||
f, e := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "profile write failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer root.Remove(tmp)
|
|
||||||
_, e = f.WriteString(in.Content)
|
|
||||||
if e == nil {
|
|
||||||
e = f.Chown(1000, 1000)
|
|
||||||
}
|
|
||||||
if e == nil {
|
|
||||||
e = f.Sync()
|
|
||||||
}
|
|
||||||
f.Close()
|
|
||||||
if e == nil {
|
|
||||||
e = root.Rename(tmp, path)
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "profile write failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.s.AddAudit(i.UID, "profile.save", i.ID, "")
|
|
||||||
if in.Restart {
|
|
||||||
if a.configRestart(i) != nil {
|
|
||||||
fail(w, 502, "profile saved but restart failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 45*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if a.upgradeReady(ctx, i.ContainerID, i.ID, a.runtimeHost(r)) != nil {
|
|
||||||
a.s.SetInstance(i.ID, "error")
|
|
||||||
fail(w, 502, "profile saved but runtime readiness failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.s.SetInstance(i.ID, "running")
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
case "backups":
|
|
||||||
if r.Method == "GET" {
|
|
||||||
if id := r.URL.Query().Get("download"); id != "" {
|
|
||||||
if !backupIDPattern.MatchString(id) {
|
|
||||||
fail(w, 400, "invalid backup ID")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
w.Header().Set("Content-Disposition", "attachment; filename="+id+".dshbak")
|
|
||||||
root, e := os.OpenRoot(dir)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 404, "backup unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
f, e := root.OpenFile(id+".dshbak", os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 404, "backup unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
st, e := f.Stat()
|
|
||||||
if e != nil || !st.Mode().IsRegular() {
|
|
||||||
fail(w, 400, "invalid backup")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
locked.Release()
|
|
||||||
locked = nil
|
|
||||||
http.ServeContent(w, r, id+".dshbak", st.ModTime(), f)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
entries, e := os.ReadDir(dir)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "backup listing failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
out := []map[string]any{}
|
|
||||||
for _, entry := range entries {
|
|
||||||
if entry.IsDir() || filepath.Ext(entry.Name()) != ".dshbak" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
st, e := entry.Info()
|
|
||||||
if e != nil || !st.Mode().IsRegular() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
id := entry.Name()[:len(entry.Name())-7]
|
|
||||||
kind, schema := "legacy", 0
|
|
||||||
_ = a.s.DB.QueryRow("SELECT kind,schema FROM backups WHERE id=? AND uid=? AND instance_id=?", id, i.UID, i.ID).Scan(&kind, &schema)
|
|
||||||
out = append(out, map[string]any{"id": id, "size": st.Size(), "created_at": st.ModTime(), "kind": kind, "schema": schema})
|
|
||||||
}
|
|
||||||
writeJSON(w, out)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != "POST" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
entries, _ := os.ReadDir(dir)
|
|
||||||
if len(entries) >= 20 {
|
|
||||||
fail(w, 409, "backup retention limit reached (20); administrator cleanup required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := store.ID()
|
|
||||||
path := filepath.Join(dir, id+".dshbak")
|
|
||||||
kind, schema := "manual", 2
|
|
||||||
if strings.HasPrefix(r.Header.Get("Content-Type"), "multipart/form-data") {
|
|
||||||
kind, schema = "imported", 0
|
|
||||||
source, _, err := r.FormFile("file")
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 400, "backup file required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer source.Close()
|
|
||||||
dest, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "backup storage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
n, err := io.Copy(dest, io.LimitReader(source, backup.MaxEncryptedBytes+1))
|
|
||||||
closeErr := dest.Close()
|
|
||||||
if err != nil || closeErr != nil || n > backup.MaxEncryptedBytes {
|
|
||||||
os.Remove(path)
|
|
||||||
fail(w, 400, "invalid or oversized backup upload")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
key, _ := a.configKey()
|
|
||||||
_, cleanup, err := backup.Prepare(path, r.FormValue("password"), key)
|
|
||||||
clear(key)
|
|
||||||
if err != nil {
|
|
||||||
os.Remove(path)
|
|
||||||
fail(w, 400, "invalid backup or original backup password")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
cleanup()
|
|
||||||
} else {
|
|
||||||
var in struct {
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&in) != nil || len(in.Password) == 0 || len(in.Password) > 72 {
|
|
||||||
fail(w, 400, "login password required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !a.reauthenticate(w, i.UID, in.Password) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if wasRunning {
|
|
||||||
if e = a.configStop(r.Context(), i.ContainerID); e != nil {
|
|
||||||
a.s.SetInstance(i.ID, "error")
|
|
||||||
fail(w, 502, "runtime stop failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e = backup.CreateWithPassword(home, path, in.Password, backup.Metadata{DSHVersion: i.Version, CreatedAt: time.Now().UTC()}); e != nil {
|
|
||||||
if wasRunning {
|
|
||||||
a.configRestart(i)
|
|
||||||
}
|
|
||||||
fail(w, 400, "backup failed: unsafe or oversized configuration")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
st, _ := os.Stat(path)
|
|
||||||
if e = a.indexBackup(i, id, kind, schema, st.Size()); e != nil {
|
|
||||||
os.Remove(path)
|
|
||||||
if wasRunning && kind == "manual" {
|
|
||||||
a.configRestart(i)
|
|
||||||
}
|
|
||||||
fail(w, 500, "backup index unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if wasRunning && kind == "manual" && a.configRestart(i) != nil {
|
|
||||||
fail(w, 502, "backup saved; runtime restart requires recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.s.AddAudit(i.UID, "backup.create", i.ID, id)
|
|
||||||
writeJSON(w, map[string]any{"id": id, "size": st.Size(), "created_at": st.ModTime(), "kind": kind, "schema": schema})
|
|
||||||
case "restore":
|
|
||||||
if r.Method != "POST" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct {
|
|
||||||
ID string `json:"backup_id"`
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096)).Decode(&in) != nil || !backupIDPattern.MatchString(in.ID) {
|
|
||||||
fail(w, 400, "invalid backup ID")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
key, _ := a.configKey()
|
|
||||||
restore, cleanup, e := backup.Prepare(filepath.Join(dir, in.ID+".dshbak"), in.Password, key)
|
|
||||||
clear(key)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "invalid backup or original backup password")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer cleanup()
|
|
||||||
staging, e := os.MkdirTemp("", "dsh-restore-quota-")
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "restore staging unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer os.RemoveAll(staging)
|
|
||||||
if e = restore(staging); e != nil {
|
|
||||||
fail(w, 400, "invalid restore contents")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
staged, e := os.OpenRoot(staging)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "restore staging unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
restoredBytes, e := quotaTreeBytes(staged, ".")
|
|
||||||
staged.Close()
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 413, "restore size unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 3*time.Minute)
|
|
||||||
defer cancel()
|
|
||||||
parent, e := upgradeRoot(a.root, i)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "storage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer parent.Close()
|
|
||||||
f, e := parent.Open(".")
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "snapshot storage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
entries, e := f.ReadDir(-1)
|
|
||||||
f.Close()
|
|
||||||
count := 0
|
|
||||||
for _, entry := range entries {
|
|
||||||
if strings.HasPrefix(entry.Name(), "pre-restore-") || strings.HasPrefix(entry.Name(), "failed-restore-") {
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e != nil || count >= 3 {
|
|
||||||
fail(w, 409, "restore snapshot limit reached; administrator cleanup required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
snapshot := "pre-restore-" + store.ID()
|
|
||||||
if _, e = a.quotaSnapshot(i.UID, parent, restoredBytes); e != nil {
|
|
||||||
fail(w, 413, "quota exceeded or snapshot size unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if wasRunning {
|
|
||||||
if e = a.configStop(ctx, i.ContainerID); e != nil {
|
|
||||||
a.s.SetInstance(i.ID, "error")
|
|
||||||
fail(w, 502, "runtime stop failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
snapshotBytes, quotaErr := a.quotaSnapshot(i.UID, parent, restoredBytes)
|
|
||||||
if quotaErr != nil {
|
|
||||||
if wasRunning {
|
|
||||||
a.configRestart(i)
|
|
||||||
}
|
|
||||||
fail(w, 413, "quota exceeded or snapshot size unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e = upgradeCopy(r.Context(), parent, "home", snapshot, snapshotBytes); e != nil {
|
|
||||||
cleanupErr := parent.RemoveAll(snapshot)
|
|
||||||
if wasRunning {
|
|
||||||
a.configRestart(i)
|
|
||||||
}
|
|
||||||
if cleanupErr != nil {
|
|
||||||
a.s.AddAudit(i.UID, "backup.snapshot.cleanup_failed", i.ID, snapshot)
|
|
||||||
fail(w, 500, "restore snapshot cleanup requires administrator recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fail(w, 400, "cannot snapshot home for restore")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("UPDATE instances SET status='error' WHERE id=?", i.ID); e != nil {
|
|
||||||
if wasRunning {
|
|
||||||
a.configRestart(i)
|
|
||||||
}
|
|
||||||
fail(w, 500, "cannot record restore state")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
restoreErr := restore(home)
|
|
||||||
if restoreErr == nil {
|
|
||||||
var root *os.Root
|
|
||||||
root, restoreErr = os.OpenRoot(home)
|
|
||||||
if restoreErr == nil {
|
|
||||||
for _, p := range []string{"profiles", ".credentials.yaml", "cordis.patch.yml", "dsh-ssh.json", "skins", "pets", "agent-presets", ".agent-presets"} {
|
|
||||||
if restoreErr = configChownTree(root, p); restoreErr != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
root.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if restoreErr == nil {
|
|
||||||
restoreErr = a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(i.ContainerID)+"/start", nil, nil)
|
|
||||||
}
|
|
||||||
if restoreErr == nil {
|
|
||||||
restoreErr = a.upgradeReady(ctx, i.ContainerID, i.ID, a.runtimeHost(r))
|
|
||||||
}
|
|
||||||
if restoreErr == nil && !wasRunning {
|
|
||||||
restoreErr = a.configStop(ctx, i.ContainerID)
|
|
||||||
}
|
|
||||||
if restoreErr != nil {
|
|
||||||
rollbackCtx, done := context.WithTimeout(context.Background(), 45*time.Second)
|
|
||||||
defer done()
|
|
||||||
if a.configStop(rollbackCtx, i.ContainerID) != nil {
|
|
||||||
fail(w, 500, "restore rollback requires runtime stop")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
failed := "failed-restore-" + store.ID()
|
|
||||||
if parent.Rename("home", failed) != nil || parent.Rename(snapshot, "home") != nil || upgradeSync(parent) != nil {
|
|
||||||
a.s.SetInstance(i.ID, "error")
|
|
||||||
fail(w, 500, "restore rollback requires administrator recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if wasRunning && a.configRestart(i) != nil {
|
|
||||||
fail(w, 500, "restore rollback restart requires recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
status := "exited"
|
|
||||||
if wasRunning {
|
|
||||||
status = "running"
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("UPDATE instances SET status=? WHERE id=?", status, i.ID); e != nil {
|
|
||||||
fail(w, 500, "restore state requires administrator recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fail(w, 400, "restore rejected; previous home restored")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
status := "exited"
|
|
||||||
if wasRunning {
|
|
||||||
status = "running"
|
|
||||||
}
|
|
||||||
if _, e = a.s.DB.Exec("UPDATE instances SET status=? WHERE id=?", status, i.ID); e != nil {
|
|
||||||
fail(w, 500, "restore state requires administrator recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
runtimeAuthCache.Delete(runtimeAuthKey{a, i.ID})
|
|
||||||
a.s.AddAudit(i.UID, "backup.restore", i.ID, in.ID)
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
default:
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func configReadBody(w http.ResponseWriter, r *http.Request, controller *http.ResponseController, limit int64) bool {
|
|
||||||
if r.ContentLength > limit {
|
|
||||||
fail(w, 413, "configuration request too large")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if err := controller.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil && !errors.Is(err, http.ErrNotSupported) {
|
|
||||||
fail(w, 500, "cannot set configuration read deadline")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
defer controller.SetReadDeadline(time.Time{})
|
|
||||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, limit))
|
|
||||||
if err != nil {
|
|
||||||
var oversized *http.MaxBytesError
|
|
||||||
switch {
|
|
||||||
case errors.As(err, &oversized):
|
|
||||||
fail(w, 413, "configuration request too large")
|
|
||||||
case errors.Is(err, os.ErrDeadlineExceeded):
|
|
||||||
fail(w, 408, "configuration upload timed out")
|
|
||||||
default:
|
|
||||||
fail(w, 400, "cannot read configuration request")
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
r.Body = io.NopCloser(bytes.NewReader(body))
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) configKey() ([]byte, error) {
|
|
||||||
path := filepath.Join(a.root, "backup.key")
|
|
||||||
f, e := os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
||||||
if e == nil {
|
|
||||||
defer f.Close()
|
|
||||||
st, err := f.Stat()
|
|
||||||
if err != nil || !st.Mode().IsRegular() {
|
|
||||||
return nil, errors.New("invalid key")
|
|
||||||
}
|
|
||||||
b, e := io.ReadAll(io.LimitReader(f, 33))
|
|
||||||
if e != nil || len(b) != 32 {
|
|
||||||
return nil, errors.New("invalid key")
|
|
||||||
}
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
|
|
||||||
func configCheckPath(root *os.Root, name string) error {
|
|
||||||
parts := strings.Split(name, "/")
|
|
||||||
for n := range parts {
|
|
||||||
st, e := root.Lstat(strings.Join(parts[:n+1], "/"))
|
|
||||||
if errors.Is(e, os.ErrNotExist) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
if st.Mode()&os.ModeSymlink != 0 || (n < len(parts)-1 && !st.IsDir()) {
|
|
||||||
return errors.New("unsafe configuration path")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func configProfileRoot(parent *os.Root, create bool) (*os.Root, error) {
|
|
||||||
root, err := parent.OpenRoot(".")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
for _, name := range []string{"home", "profiles", "web"} {
|
|
||||||
created := false
|
|
||||||
st, e := root.Lstat(name)
|
|
||||||
if errors.Is(e, os.ErrNotExist) && create && name != "home" {
|
|
||||||
e = root.Mkdir(name, 0750)
|
|
||||||
if e == nil {
|
|
||||||
created = true
|
|
||||||
st, e = root.Lstat(name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
root.Close()
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
if !st.IsDir() {
|
|
||||||
root.Close()
|
|
||||||
return nil, errors.New("unsafe profile directory")
|
|
||||||
}
|
|
||||||
next, e := root.OpenRoot(name)
|
|
||||||
root.Close()
|
|
||||||
if e != nil {
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
actual, e := next.Stat(".")
|
|
||||||
if e != nil || !os.SameFile(st, actual) {
|
|
||||||
next.Close()
|
|
||||||
return nil, errors.New("profile directory changed")
|
|
||||||
}
|
|
||||||
if created {
|
|
||||||
if e = next.Chown(".", 1000, 1000); e != nil {
|
|
||||||
next.Close()
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
root = next
|
|
||||||
}
|
|
||||||
return root, nil
|
|
||||||
}
|
|
||||||
@@ -1,200 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"mime/multipart"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
type configPausedBody struct {
|
|
||||||
io.Reader
|
|
||||||
started chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
once sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *configPausedBody) Read(p []byte) (int, error) {
|
|
||||||
b.once.Do(func() { close(b.started) })
|
|
||||||
<-b.release
|
|
||||||
return b.Reader.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigSlowBodyRevalidatesWithoutLocks(t *testing.T) {
|
|
||||||
for _, action := range []string{"profile", "backups", "restore", "multipart"} {
|
|
||||||
for _, change := range []string{"lease", "deleted"} {
|
|
||||||
t.Run(action+"/"+change, func(t *testing.T) {
|
|
||||||
a, i, _ := configTestAPI(t, false)
|
|
||||||
a.workspaceReaders = make(map[string]int)
|
|
||||||
body := &configPausedBody{Reader: strings.NewReader(`{}`), started: make(chan struct{}), release: make(chan struct{})}
|
|
||||||
r := httptest.NewRequest("POST", "/", body)
|
|
||||||
if action == "profile" {
|
|
||||||
r.Method = "PUT"
|
|
||||||
}
|
|
||||||
target := action
|
|
||||||
if action == "multipart" {
|
|
||||||
target = "backups"
|
|
||||||
var buf bytes.Buffer
|
|
||||||
form := multipart.NewWriter(&buf)
|
|
||||||
part, _ := form.CreateFormFile("file", "test.dshbak")
|
|
||||||
part.Write(bytes.Repeat([]byte("x"), 2<<20))
|
|
||||||
form.Close()
|
|
||||||
body.Reader = &buf
|
|
||||||
r.Header.Set("Content-Type", form.FormDataContentType())
|
|
||||||
t.Setenv("TMPDIR", t.TempDir())
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() { defer close(done); a.config(w, r, i, target) }()
|
|
||||||
defer func() {
|
|
||||||
select {
|
|
||||||
case <-body.release:
|
|
||||||
default:
|
|
||||||
close(body.release)
|
|
||||||
}
|
|
||||||
<-done
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-body.started:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("body read did not start")
|
|
||||||
}
|
|
||||||
if a.instanceLocks.held() || a.userLocks.held() {
|
|
||||||
t.Fatal("slow body holds API locks")
|
|
||||||
}
|
|
||||||
if change == "lease" {
|
|
||||||
a.readersMu.Lock()
|
|
||||||
a.workspaceReaders[i.ID]++
|
|
||||||
a.readersMu.Unlock()
|
|
||||||
}
|
|
||||||
want := 409
|
|
||||||
if change == "deleted" {
|
|
||||||
if _, err := a.s.DB.Exec("DELETE FROM instances WHERE id=?", i.ID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want = 404
|
|
||||||
}
|
|
||||||
close(body.release)
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("config did not finish")
|
|
||||||
}
|
|
||||||
if w.Code != want {
|
|
||||||
t.Fatalf("status %d: %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if action == "multipart" {
|
|
||||||
entries, err := os.ReadDir(os.Getenv("TMPDIR"))
|
|
||||||
if err != nil || len(entries) != 0 {
|
|
||||||
t.Fatalf("multipart temp files remain: %v %v", entries, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type configDeadlineWriter struct {
|
|
||||||
*httptest.ResponseRecorder
|
|
||||||
deadlines []time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *configDeadlineWriter) SetReadDeadline(deadline time.Time) error {
|
|
||||||
w.deadlines = append(w.deadlines, deadline)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type configErrorBody struct{}
|
|
||||||
|
|
||||||
func (configErrorBody) Read([]byte) (int, error) { return 0, os.ErrDeadlineExceeded }
|
|
||||||
|
|
||||||
func TestConfigBodyErrorsReturnWithoutLocks(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
body io.Reader
|
|
||||||
want int
|
|
||||||
}{
|
|
||||||
{"oversized", strings.NewReader(strings.Repeat("x", 4097)), 413},
|
|
||||||
{"timeout", configErrorBody{}, 408},
|
|
||||||
{"multipart", strings.NewReader("bad multipart"), 400},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
a := &API{}
|
|
||||||
r := httptest.NewRequest("POST", "/", tc.body)
|
|
||||||
r.ContentLength = -1
|
|
||||||
if tc.name == "multipart" {
|
|
||||||
r.Header.Set("Content-Type", "multipart/form-data; boundary=test")
|
|
||||||
}
|
|
||||||
w := &configDeadlineWriter{ResponseRecorder: httptest.NewRecorder()}
|
|
||||||
inst := a.instanceLocks.Acquire("")
|
|
||||||
user := a.userLocks.Acquire("")
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() { defer close(done); a.config(w, r, store.Instance{}, "backups") }()
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
user.Release()
|
|
||||||
inst.Release()
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
user.Release()
|
|
||||||
inst.Release()
|
|
||||||
t.Fatal("body error waited for locks")
|
|
||||||
}
|
|
||||||
if w.Code != tc.want {
|
|
||||||
t.Fatalf("status %d: %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if len(w.deadlines) != 2 || w.deadlines[0].IsZero() || !w.deadlines[1].IsZero() {
|
|
||||||
t.Fatalf("deadlines: %v", w.deadlines)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigBodySlotsRejectWithoutReading(t *testing.T) {
|
|
||||||
for n := 0; n < cap(configBodySlots); n++ {
|
|
||||||
configBodySlots <- struct{}{}
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
for n := 0; n < cap(configBodySlots); n++ {
|
|
||||||
<-configBodySlots
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
(&API{}).config(w, httptest.NewRequest("POST", "/", configErrorBody{}), store.Instance{}, "backups")
|
|
||||||
if w.Code != 503 {
|
|
||||||
t.Fatalf("status %d", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigSlowResponseReleasesLocks(t *testing.T) {
|
|
||||||
for _, tc := range []struct{ method, action string }{{"POST", "restore"}, {"GET", "profile"}, {"GET", "backups"}} {
|
|
||||||
t.Run(tc.method+"/"+tc.action, func(t *testing.T) {
|
|
||||||
a, i, _ := configTestAPI(t, false)
|
|
||||||
w := &pausedDownload{ResponseRecorder: httptest.NewRecorder(), started: make(chan struct{}), release: make(chan struct{})}
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(done)
|
|
||||||
a.config(w, httptest.NewRequest(tc.method, "/", strings.NewReader(`{}`)), i, tc.action)
|
|
||||||
}()
|
|
||||||
defer func() { close(w.release); <-done }()
|
|
||||||
select {
|
|
||||||
case <-w.started:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("response write did not start")
|
|
||||||
}
|
|
||||||
if a.instanceLocks.held() {
|
|
||||||
t.Fatal("slow response holds instance lock")
|
|
||||||
}
|
|
||||||
if a.userLocks.held() {
|
|
||||||
t.Fatal("slow response holds user lock")
|
|
||||||
}
|
|
||||||
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,80 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *API) indexBackup(i store.Instance, id, kind string, schema int, size int64) error {
|
|
||||||
_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS backups (id TEXT PRIMARY KEY, uid TEXT NOT NULL, instance_id TEXT NOT NULL, kind TEXT NOT NULL, schema INTEGER NOT NULL, size INTEGER NOT NULL, created_at INTEGER NOT NULL)")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = a.s.DB.Exec("INSERT INTO backups(id,uid,instance_id,kind,schema,size,created_at) VALUES(?,?,?,?,?,?,?)", id, i.UID, i.ID, kind, schema, size, time.Now().Unix())
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func configChownTree(root *os.Root, name string) error {
|
|
||||||
st, err := root.Lstat(name)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if st.Mode()&os.ModeSymlink != 0 {
|
|
||||||
return root.Lchown(name, 1000, 1000)
|
|
||||||
}
|
|
||||||
if err = root.Chown(name, 1000, 1000); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !st.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
f, err := root.Open(name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
entries, err := f.ReadDir(-1)
|
|
||||||
f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, entry := range entries {
|
|
||||||
if err = configChownTree(root, filepath.Join(name, entry.Name())); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) configStop(ctx context.Context, id string) error {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(id)+"/stop?t=10", nil, nil); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
c, err := a.docker.Inspect(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if c.State.Running || c.State.Paused || c.State.Restarting {
|
|
||||||
return errors.New("runtime did not stop")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) configRestart(i store.Instance) error {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
err := a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(i.ContainerID)+"/start", nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
a.s.SetInstance(i.ID, "error")
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,404 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"mime/multipart"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestConfigPasswordBackupImport(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
hash, _ := bcrypt.GenerateFromPassword([]byte("original-password"), bcrypt.MinCost)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET pass_hash=? WHERE id=?", string(hash), i.UID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "secret: original")
|
|
||||||
for _, body := range []string{`{}`, `{"password":"wrong"}`} {
|
|
||||||
if w := configCall(a, i, "POST", "backups", body); w.Code < 400 {
|
|
||||||
t.Fatal("accepted wrong password")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var out struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
}
|
|
||||||
json.Unmarshal(w.Body.Bytes(), &out)
|
|
||||||
if _, err := os.Stat(filepath.Join(a.root, "backup.key")); !os.IsNotExist(err) {
|
|
||||||
t.Fatal("new backup persisted key")
|
|
||||||
}
|
|
||||||
encrypted, err := os.ReadFile(filepath.Join(a.root, "backups", i.UID, i.ID, out.ID+".dshbak"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if bytes.Contains(encrypted, []byte("secret: original")) {
|
|
||||||
t.Fatal("plaintext archive")
|
|
||||||
}
|
|
||||||
hash, _ = bcrypt.GenerateFromPassword([]byte("changed-password"), bcrypt.MinCost)
|
|
||||||
a.s.DB.Exec("UPDATE users SET pass_hash=? WHERE id=?", string(hash), i.UID)
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "secret: changed")
|
|
||||||
w = configCall(a, i, "POST", "restore", `{"backup_id":"`+out.ID+`","password":"wrong"}`)
|
|
||||||
if w.Code != 400 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
entries, _ := os.ReadDir(filepath.Dir(home))
|
|
||||||
for _, entry := range entries {
|
|
||||||
if strings.HasPrefix(entry.Name(), "pre-restore-") {
|
|
||||||
t.Fatal("bad password created snapshot")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var body bytes.Buffer
|
|
||||||
form := multipart.NewWriter(&body)
|
|
||||||
part, _ := form.CreateFormFile("file", "external.dshbak")
|
|
||||||
part.Write(encrypted)
|
|
||||||
form.WriteField("password", "original-password")
|
|
||||||
form.Close()
|
|
||||||
req := httptest.NewRequest("POST", "/api/instances/one/backups", &body)
|
|
||||||
req.Header.Set("Content-Type", form.FormDataContentType())
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
a.config(w, req, i, "backups")
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
w = configCall(a, i, "POST", "restore", `{"backup_id":"`+out.ID+`","password":"original-password"}`)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
restored, _ := os.ReadFile(filepath.Join(home, ".credentials.yaml"))
|
|
||||||
if string(restored) != "secret: original" {
|
|
||||||
t.Fatal(string(restored))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func configTestAPI(t *testing.T, running bool, failures ...string) (*API, store.Instance, string) {
|
|
||||||
t.Helper()
|
|
||||||
var mu sync.Mutex
|
|
||||||
httpFailure := false
|
|
||||||
backupRestart := running
|
|
||||||
failure := ""
|
|
||||||
if len(failures) != 0 {
|
|
||||||
failure = failures[0]
|
|
||||||
}
|
|
||||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if httpFailure {
|
|
||||||
httpFailure = false
|
|
||||||
running = false
|
|
||||||
w.WriteHeader(503)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.URL.Query().Get("token") == "test-token" {
|
|
||||||
http.SetCookie(w, &http.Cookie{Name: "dsh_auth", Value: "ready"})
|
|
||||||
w.WriteHeader(303)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(200)
|
|
||||||
}))
|
|
||||||
t.Cleanup(upstream.Close)
|
|
||||||
transport := runtimeTransport
|
|
||||||
runtimeTransport = transport.Clone()
|
|
||||||
runtimeTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
|
|
||||||
return (&net.Dialer{}).DialContext(ctx, network, strings.TrimPrefix(upstream.URL, "http://"))
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { runtimeTransport.CloseIdleConnections(); runtimeTransport = transport })
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
switch {
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/stop"):
|
|
||||||
running = false
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/start"):
|
|
||||||
if backupRestart {
|
|
||||||
backupRestart = false
|
|
||||||
running = true
|
|
||||||
w.WriteHeader(204)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
running = failure != "reconcile"
|
|
||||||
httpFailure = failure == "readiness"
|
|
||||||
failure = ""
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case strings.Contains(r.URL.Path, "/exec"), strings.HasSuffix(r.URL.Path, "/restart"):
|
|
||||||
t.Error("restore must use entrypoint prestart, not exec/restart")
|
|
||||||
w.WriteHeader(500)
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/logs"):
|
|
||||||
fmt.Fprint(w, "http://runtime/?token=test-token")
|
|
||||||
default:
|
|
||||||
writeJSON(w, map[string]any{"State": map[string]any{"Running": running}, "NetworkSettings": map[string]any{"Networks": map[string]any{"dsh-inst-one": map[string]string{"IPAddress": "127.0.0.1"}}}})
|
|
||||||
}
|
|
||||||
})
|
|
||||||
hash, _ := bcrypt.GenerateFromPassword([]byte("original-password"), bcrypt.MinCost)
|
|
||||||
a.s.DB.Exec("UPDATE users SET pass_hash=?,quota=10737418240 WHERE id='owner'", string(hash))
|
|
||||||
i := store.Instance{ID: "one", UID: "owner", Name: "one", ContainerID: "old", Status: "exited", Created: time.Now()}
|
|
||||||
if e := a.s.AddInstance(i); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
home := filepath.Join(a.root, "workspaces", "owner", "instances", "one", "home")
|
|
||||||
if e := os.MkdirAll(home, 0750); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
return a, i, home
|
|
||||||
}
|
|
||||||
func configCall(a *API, i store.Instance, method, action, body string) *httptest.ResponseRecorder {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest(method, "/api/instances/one/"+action, strings.NewReader(body))
|
|
||||||
a.config(w, r, i, strings.Split(action, "?")[0])
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
func configWrite(t *testing.T, path, content string) {
|
|
||||||
t.Helper()
|
|
||||||
if e := os.WriteFile(path, []byte(content), 0600); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestConfigProfile(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
for _, tc := range []struct {
|
|
||||||
method, body string
|
|
||||||
code int
|
|
||||||
}{
|
|
||||||
{"GET", "", 200}, {"PUT", `{"content":"foo: ["}`, 400}, {"PUT", `{"content":"foo: bar\n"}`, 200}, {"GET", "", 200},
|
|
||||||
} {
|
|
||||||
w := configCall(a, i, tc.method, "profile", tc.body)
|
|
||||||
if w.Code != tc.code {
|
|
||||||
t.Fatalf("%s: %d %s", tc.method, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, path := range []string{"profiles", "profiles/web", "profiles/web/cordis.patch.yml"} {
|
|
||||||
st, e := os.Stat(filepath.Join(home, path))
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
stat := st.Sys().(*syscall.Stat_t)
|
|
||||||
if stat.Uid != 1000 || stat.Gid != 1000 {
|
|
||||||
t.Fatalf("ownership of %s: %+v", path, stat)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := configCall(a, i, "GET", "profile", "")
|
|
||||||
if !strings.Contains(w.Body.String(), "foo: bar") {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestConfigProfileSymlinks(t *testing.T) {
|
|
||||||
for _, level := range []string{"profiles", "web", "leaf", "home"} {
|
|
||||||
t.Run(level, func(t *testing.T) {
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
secret := "credential-secret-never-return"
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), secret)
|
|
||||||
if e := os.MkdirAll(filepath.Join(home, "profiles", "web"), 0750); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
var link, target string
|
|
||||||
switch level {
|
|
||||||
case "profiles":
|
|
||||||
link = filepath.Join(home, "profiles")
|
|
||||||
target = "private"
|
|
||||||
if e := os.MkdirAll(filepath.Join(home, "private", "web"), 0750); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, "private", "web", "cordis.patch.yml"), secret)
|
|
||||||
case "web":
|
|
||||||
link = filepath.Join(home, "profiles", "web")
|
|
||||||
target = "../private"
|
|
||||||
if e := os.Mkdir(filepath.Join(home, "private"), 0750); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, "private", "cordis.patch.yml"), secret)
|
|
||||||
case "leaf":
|
|
||||||
link = filepath.Join(home, "profiles", "web", "cordis.patch.yml")
|
|
||||||
target = "../../.credentials.yaml"
|
|
||||||
case "home":
|
|
||||||
link = home
|
|
||||||
target = home + "-other"
|
|
||||||
if e := os.Rename(home, target); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if level != "home" {
|
|
||||||
if e := os.RemoveAll(link); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e := os.Symlink(target, link); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
for _, method := range []string{"GET", "PUT"} {
|
|
||||||
w := configCall(a, i, method, "profile", `{"content":"foo: bar"}`)
|
|
||||||
if w.Code != 400 || strings.Contains(w.Body.String(), secret) {
|
|
||||||
t.Fatalf("%s: %d %s", method, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestConfigRunningRejected(t *testing.T) {
|
|
||||||
a, i, _ := configTestAPI(t, true)
|
|
||||||
for _, tc := range []struct{ method, action string }{{"PUT", "profile"}} {
|
|
||||||
w := configCall(a, i, tc.method, tc.action, `{}`)
|
|
||||||
if w.Code != 409 {
|
|
||||||
t.Fatalf("%s: %d", tc.action, w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestConfigBackupRestore(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
const secret = "private-credential-token"
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), secret)
|
|
||||||
if w := configCall(a, i, "PUT", "profile", `{"content":"foo: old"}`); w.Code != 200 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
|
|
||||||
var result struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
}
|
|
||||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || !backupIDPattern.MatchString(result.ID) {
|
|
||||||
t.Fatalf("%d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
w = configCall(a, i, "GET", "backups?download="+result.ID, "")
|
|
||||||
if w.Code != 200 || !bytes.HasPrefix(w.Body.Bytes(), []byte("DSHPASS")) || bytes.Contains(w.Body.Bytes(), []byte(secret)) {
|
|
||||||
t.Fatalf("unsafe download: %d", w.Code)
|
|
||||||
}
|
|
||||||
if e := os.RemoveAll(filepath.Join(home, "profiles")); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "changed")
|
|
||||||
body := fmt.Sprintf(`{"backup_id":%q,"password":"original-password"}`, result.ID)
|
|
||||||
for n := 0; n < 3; n++ {
|
|
||||||
w = configCall(a, i, "POST", "restore", body)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatalf("%d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w = configCall(a, i, "POST", "restore", body)
|
|
||||||
if w.Code != 409 {
|
|
||||||
t.Fatalf("snapshot retention: %d", w.Code)
|
|
||||||
}
|
|
||||||
got, e := os.ReadFile(filepath.Join(home, ".credentials.yaml"))
|
|
||||||
if e != nil || string(got) != secret {
|
|
||||||
t.Fatalf("restore: %q %v", got, e)
|
|
||||||
}
|
|
||||||
for _, p := range []string{"profiles", "profiles/web", "profiles/web/cordis.patch.yml"} {
|
|
||||||
st, e := os.Stat(filepath.Join(home, p))
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
if st.Sys().(*syscall.Stat_t).Uid != 1000 {
|
|
||||||
t.Fatalf("not owned by runtime: %s", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
state, e := a.s.GetInstance(i.ID)
|
|
||||||
if e != nil || state.Status != "exited" {
|
|
||||||
t.Fatalf("state: %+v %v", state, e)
|
|
||||||
}
|
|
||||||
id := store.ID()
|
|
||||||
if e := os.Symlink(filepath.Join(a.root, "backup.key"), filepath.Join(a.root, "backups", i.UID, i.ID, id+".dshbak")); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
w = configCall(a, i, "GET", "backups?download="+id, "")
|
|
||||||
if w.Code == 200 {
|
|
||||||
t.Fatal("symlink download exposed key")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestConfigRestoreInvalidArchive(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
configWrite(t, filepath.Join(home, "cordis.patch.yml"), "original")
|
|
||||||
id := store.ID()
|
|
||||||
dir := filepath.Join(a.root, "backups", i.UID, i.ID)
|
|
||||||
if e := os.MkdirAll(dir, 0700); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(dir, id+".dshbak"), "corrupt")
|
|
||||||
w := configCall(a, i, "POST", "restore", fmt.Sprintf(`{"backup_id":%q}`, id))
|
|
||||||
if w.Code != 400 || !strings.Contains(w.Body.String(), "invalid backup") {
|
|
||||||
t.Fatalf("%d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
got, e := os.ReadFile(filepath.Join(home, "cordis.patch.yml"))
|
|
||||||
if e != nil || string(got) != "original" {
|
|
||||||
t.Fatalf("rollback: %q %v", got, e)
|
|
||||||
}
|
|
||||||
a.s.SetInstance(i.ID, "error")
|
|
||||||
w = configCall(a, i, "POST", "restore", fmt.Sprintf(`{"backup_id":%q}`, id))
|
|
||||||
if w.Code != 409 {
|
|
||||||
t.Fatalf("recovery guard: %d", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConfigRestoreOrchestration(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
for _, running := range []bool{false, true} {
|
|
||||||
for _, failure := range []string{"", "reconcile", "readiness"} {
|
|
||||||
t.Run(fmt.Sprintf("running=%v/%s", running, failure), func(t *testing.T) {
|
|
||||||
a, i, home := configTestAPI(t, running, failure)
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "archive")
|
|
||||||
if err := os.MkdirAll(filepath.Join(home, "pets", "nested"), 0750); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, "pets", "nested", "pet.json"), "{}")
|
|
||||||
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var out struct{ ID string }
|
|
||||||
json.Unmarshal(w.Body.Bytes(), &out)
|
|
||||||
var kind string
|
|
||||||
var schema int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT kind,schema FROM backups WHERE id=?", out.ID).Scan(&kind, &schema); err != nil || kind != "manual" || schema != 2 {
|
|
||||||
t.Fatalf("index: %s %d %v", kind, schema, err)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "before-restore")
|
|
||||||
w = configCall(a, i, "POST", "restore", fmt.Sprintf(`{"backup_id":%q,"password":"original-password"}`, out.ID))
|
|
||||||
wantCode, wantContent := 200, "archive"
|
|
||||||
if failure != "" {
|
|
||||||
wantCode, wantContent = 400, "before-restore"
|
|
||||||
}
|
|
||||||
if w.Code != wantCode {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
b, _ := os.ReadFile(filepath.Join(home, ".credentials.yaml"))
|
|
||||||
if string(b) != wantContent {
|
|
||||||
t.Fatalf("home: %s", b)
|
|
||||||
}
|
|
||||||
c, err := a.docker.Inspect(context.Background(), i.ContainerID)
|
|
||||||
if err != nil || c.State.Running != running {
|
|
||||||
t.Fatalf("runtime state: %+v %v", c, err)
|
|
||||||
}
|
|
||||||
st, err := os.Stat(filepath.Join(home, "pets", "nested", "pet.json"))
|
|
||||||
if err != nil || st.Sys().(*syscall.Stat_t).Uid != 1000 {
|
|
||||||
t.Fatalf("resource ownership: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,97 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRuntimePrestart(t *testing.T) {
|
|
||||||
for _, status := range []string{"0", "23", "124", "137", "empty"} {
|
|
||||||
t.Run(status, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
if status != "empty" {
|
|
||||||
if err := os.MkdirAll(filepath.Join(dir, "profiles/web"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "profiles/web/package.json"), []byte("{}"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
scripts := map[string]string{
|
|
||||||
"node": "#!/bin/bash\nexit 0\n",
|
|
||||||
"timeout": `#!/bin/bash
|
|
||||||
printf "timeout %s\n" "$*" >> "$TRACE"
|
|
||||||
test "$1 $2 $3" = "--signal=TERM --kill-after=5s 120s" || exit 90
|
|
||||||
shift 3
|
|
||||||
exec "$@"
|
|
||||||
`,
|
|
||||||
"dsh": `#!/bin/bash
|
|
||||||
printf "dsh %s\n" "$*" >> "$TRACE"
|
|
||||||
test "$npm_config_ignore_scripts:$npm_config_ignore_pnpmfile" = true:true || exit 91
|
|
||||||
if [ "$1" = plugin ]; then
|
|
||||||
test "$*" = "plugin --profile web install --ignore-scripts" || exit 92
|
|
||||||
exit "$PLUGIN_STATUS"
|
|
||||||
fi
|
|
||||||
exit 0
|
|
||||||
`,
|
|
||||||
"caddy": "#!/bin/bash\necho caddy >> \"$TRACE\"\nsleep 0.1\n",
|
|
||||||
}
|
|
||||||
for name, script := range scripts {
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(script), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
cmd := exec.CommandContext(ctx, "bash", "../../image/entrypoint.sh", "web", "--no-open", "--port", "3080")
|
|
||||||
cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), "DSH_HOME="+dir, "TRACE="+filepath.Join(dir, "trace"), "PLUGIN_STATUS="+status)
|
|
||||||
output, err := cmd.CombinedOutput()
|
|
||||||
data, readErr := os.ReadFile(filepath.Join(dir, "trace"))
|
|
||||||
if readErr != nil {
|
|
||||||
t.Fatal(readErr)
|
|
||||||
}
|
|
||||||
trace := string(data)
|
|
||||||
if status == "empty" {
|
|
||||||
if err != nil || strings.Contains(trace, "plugin") || !strings.Contains(trace, "dsh web") {
|
|
||||||
t.Fatalf("%v: %s: %s", err, trace, output)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !strings.Contains(trace, "dsh plugin --profile web install --ignore-scripts") {
|
|
||||||
t.Fatal(trace)
|
|
||||||
}
|
|
||||||
if status == "0" {
|
|
||||||
if err != nil || !strings.Contains(string(output), "reconciliation complete") {
|
|
||||||
t.Fatalf("%v: %s", err, output)
|
|
||||||
}
|
|
||||||
if !strings.Contains(trace, "dsh web") {
|
|
||||||
t.Fatal(trace)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err == nil || !strings.Contains(string(output), "failed (exit "+status+")") {
|
|
||||||
t.Fatalf("%v: %s", err, output)
|
|
||||||
}
|
|
||||||
if strings.Contains(trace, "caddy") || strings.Contains(trace, "dsh web") {
|
|
||||||
t.Fatalf("application started after failure: %s", trace)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRuntimePNPMPinned(t *testing.T) {
|
|
||||||
data, err := os.ReadFile("../../image/Dockerfile")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, required := range []string{"[email protected]", `test "$(pnpm --version)" = "10.11.0"`, "USER node", `"/usr/bin/tini", "-g"`} {
|
|
||||||
if !strings.Contains(string(data), required) {
|
|
||||||
t.Errorf("missing %s", required)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,825 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/zip"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"io"
|
|
||||||
"mime"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
const maxUpload = 32 << 20
|
|
||||||
|
|
||||||
var uploadSlots = make(chan struct{}, 4)
|
|
||||||
|
|
||||||
const (
|
|
||||||
maxZipEntries = 10000
|
|
||||||
maxZipDepth = 64
|
|
||||||
maxZipBytes int64 = 10 << 30
|
|
||||||
zipTimeout = 2 * time.Minute
|
|
||||||
)
|
|
||||||
|
|
||||||
var zipSlots = make(chan struct{}, 4)
|
|
||||||
var errZipLimit = errors.New("archive limit exceeded")
|
|
||||||
|
|
||||||
func renameNoReplace(root *os.Root, old, next string) error {
|
|
||||||
anchor, err := root.Open(".")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer anchor.Close()
|
|
||||||
openParent := func(p string) (int, error) {
|
|
||||||
return unix.Openat2(int(anchor.Fd()), path.Dir(p), &unix.OpenHow{
|
|
||||||
Flags: unix.O_RDONLY | unix.O_DIRECTORY | unix.O_CLOEXEC,
|
|
||||||
Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
src, err := openParent(old)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer unix.Close(src)
|
|
||||||
dst, err := openParent(next)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer unix.Close(dst)
|
|
||||||
var st unix.Stat_t
|
|
||||||
if err = unix.Fstatat(src, path.Base(old), &st, unix.AT_SYMLINK_NOFOLLOW); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if st.Mode&unix.S_IFMT == unix.S_IFLNK {
|
|
||||||
return errors.New("unsafe path")
|
|
||||||
}
|
|
||||||
return unix.Renameat2(src, path.Base(old), dst, path.Base(next), unix.RENAME_NOREPLACE)
|
|
||||||
}
|
|
||||||
|
|
||||||
func safeRel(p string) (string, error) {
|
|
||||||
p = strings.TrimSpace(strings.ReplaceAll(p, `\`, `/`))
|
|
||||||
if p == "" {
|
|
||||||
return ".", nil
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(p, "/") {
|
|
||||||
return "", errors.New("invalid path")
|
|
||||||
}
|
|
||||||
c := path.Clean(p)
|
|
||||||
if c == ".." || strings.HasPrefix(c, "../") {
|
|
||||||
return "", errors.New("invalid path")
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func workspacePath(uid string) string { return path.Join("workspaces", uid, "files") }
|
|
||||||
|
|
||||||
func (a *API) openWorkspace(u store.User, instanceIDs ...string) (*os.Root, error) {
|
|
||||||
instanceID := ""
|
|
||||||
if len(instanceIDs) > 0 {
|
|
||||||
instanceID = instanceIDs[0]
|
|
||||||
}
|
|
||||||
base, err := os.OpenRoot(a.root)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer base.Close()
|
|
||||||
parts := []string{"workspaces", path.Join("workspaces", u.ID), workspacePath(u.ID)}
|
|
||||||
if instanceID != "" {
|
|
||||||
rel := path.Join("workspaces", u.ID, "instances", instanceID, "home")
|
|
||||||
if err := rootSafe(base, rel, false); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return base.OpenRoot(rel)
|
|
||||||
}
|
|
||||||
for _, p := range parts {
|
|
||||||
if err = rootSafe(base, p, true); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if err = base.Mkdir(p, 0750); err != nil && !errors.Is(err, os.ErrExist) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err = base.Chown(workspacePath(u.ID), 1000, 1000); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return base.OpenRoot(workspacePath(u.ID))
|
|
||||||
}
|
|
||||||
|
|
||||||
func rootSafe(root *os.Root, rel string, finalMayMissing bool) error {
|
|
||||||
if rel == "." {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
parts := strings.Split(rel, "/")
|
|
||||||
for i := range parts {
|
|
||||||
q := strings.Join(parts[:i+1], "/")
|
|
||||||
st, err := root.Lstat(q)
|
|
||||||
if err != nil {
|
|
||||||
if finalMayMissing && i == len(parts)-1 && errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if st.Mode()&os.ModeSymlink != 0 || (i < len(parts)-1 && !st.IsDir()) {
|
|
||||||
return errors.New("unsafe path")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) workspace(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/api/ws/download" {
|
|
||||||
var flush func()
|
|
||||||
w, flush = deferJSONResponse(w)
|
|
||||||
defer flush()
|
|
||||||
}
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
instanceID := strings.TrimSpace(r.URL.Query().Get("instance"))
|
|
||||||
if instanceID == "" && r.URL.Path != "/api/ws/usage" {
|
|
||||||
fail(w, http.StatusGone, "shared files removed; instance required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if instanceID != "" {
|
|
||||||
// Pin this instance's home without holding the instance lifecycle lock
|
|
||||||
// across client reads/writes. Lifecycle mutations reject active leases.
|
|
||||||
inst := a.instanceLocks.Acquire(instanceID)
|
|
||||||
i, e := a.s.GetInstance(instanceID)
|
|
||||||
if e != nil || i.UID != u.ID {
|
|
||||||
inst.Release()
|
|
||||||
fail(w, 404, "instance not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet && (i.Status == "error" || a.upgradeRecoveryRequired(i)) {
|
|
||||||
inst.Release()
|
|
||||||
fail(w, 409, "instance requires recovery")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.readersMu.Lock()
|
|
||||||
if a.workspaceReaders == nil {
|
|
||||||
a.workspaceReaders = make(map[string]int)
|
|
||||||
}
|
|
||||||
a.workspaceReaders[instanceID]++
|
|
||||||
a.readersMu.Unlock()
|
|
||||||
inst.Release()
|
|
||||||
defer func() {
|
|
||||||
a.readersMu.Lock()
|
|
||||||
if a.workspaceReaders != nil {
|
|
||||||
a.workspaceReaders[instanceID]--
|
|
||||||
if a.workspaceReaders[instanceID] == 0 {
|
|
||||||
delete(a.workspaceReaders, instanceID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.readersMu.Unlock()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
root, err := a.openWorkspace(u, instanceID)
|
|
||||||
user.Release()
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "storage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
switch strings.TrimPrefix(r.URL.Path, "/api/ws/") {
|
|
||||||
case "usage":
|
|
||||||
a.wsUsage(w, r, root, u)
|
|
||||||
case "tree":
|
|
||||||
a.wsTree(w, r, root, u)
|
|
||||||
case "mkdir":
|
|
||||||
a.wsMkdir(w, r, root, u)
|
|
||||||
case "rename":
|
|
||||||
a.wsRename(w, r, root, u)
|
|
||||||
case "delete":
|
|
||||||
a.wsDelete(w, r, root, u)
|
|
||||||
case "upload":
|
|
||||||
a.wsUpload(w, r, root, u)
|
|
||||||
case "download":
|
|
||||||
a.wsDownload(w, r, root)
|
|
||||||
default:
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func entry(root *os.Root, rel string) (os.FileInfo, error) {
|
|
||||||
if err := rootSafe(root, rel, false); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return root.Lstat(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsUsage(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
|
|
||||||
if r.Method != "GET" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
n, e := a.userUsage(u)
|
|
||||||
user.Release()
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 500, "usage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]int64{"used_bytes": n, "quota_bytes": u.Quota})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsTree(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
locked := user
|
|
||||||
defer func() {
|
|
||||||
if locked != nil {
|
|
||||||
locked.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
rel, err := safeRel(r.URL.Query().Get("path"))
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 400, "invalid path")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
st, err := entry(root, rel)
|
|
||||||
if err != nil || !st.IsDir() {
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
es, err := readRootDir(root, rel)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, errDirectoryLimit) {
|
|
||||||
fail(w, 413, "directory entry limit exceeded")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
out := make([]map[string]any, 0, len(es))
|
|
||||||
for _, de := range es {
|
|
||||||
p := de.Name()
|
|
||||||
if rel != "." {
|
|
||||||
p = rel + "/" + p
|
|
||||||
}
|
|
||||||
st, err := entry(root, p)
|
|
||||||
if err != nil || (!st.IsDir() && !st.Mode().IsRegular()) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
sz := int64(0)
|
|
||||||
if !st.IsDir() {
|
|
||||||
sz = st.Size()
|
|
||||||
}
|
|
||||||
out = append(out, map[string]any{"name": de.Name(), "path": p, "is_dir": st.IsDir(), "size": sz, "mod_time": st.ModTime()})
|
|
||||||
}
|
|
||||||
sort.Slice(out, func(i, j int) bool { return out[i]["name"].(string) < out[j]["name"].(string) })
|
|
||||||
locked.Release()
|
|
||||||
locked = nil
|
|
||||||
writeJSON(w, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsMkdir(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var x struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(r.Body).Decode(&x) != nil {
|
|
||||||
fail(w, 400, "invalid json")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p, e := safeRel(x.Path)
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
locked := user
|
|
||||||
defer func() {
|
|
||||||
if locked != nil {
|
|
||||||
locked.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if e != nil || p == "." || rootSafe(root, path.Dir(p), false) != nil {
|
|
||||||
fail(w, 400, "cannot create directory")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e = root.Mkdir(p, 0750); e != nil {
|
|
||||||
fail(w, 400, "cannot create directory")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if f, e := root.OpenFile(p, os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0); e == nil {
|
|
||||||
_ = f.Chown(1000, 1000)
|
|
||||||
f.Close()
|
|
||||||
}
|
|
||||||
locked.Release()
|
|
||||||
locked = nil
|
|
||||||
a.s.AddAudit(u.ID, "workspace.mkdir", p, "")
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsRename(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var x struct {
|
|
||||||
Path string `json:"path"`
|
|
||||||
NewPath string `json:"new_path"`
|
|
||||||
}
|
|
||||||
if json.NewDecoder(r.Body).Decode(&x) != nil {
|
|
||||||
fail(w, 400, "invalid json")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p, e := safeRel(x.Path)
|
|
||||||
q, e2 := safeRel(x.NewPath)
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
locked := user
|
|
||||||
defer func() {
|
|
||||||
if locked != nil {
|
|
||||||
locked.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if e != nil || e2 != nil || p == "." || q == "." || rootSafe(root, p, false) != nil || rootSafe(root, path.Dir(q), false) != nil {
|
|
||||||
fail(w, 400, "rename failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
st, e := root.Lstat(p)
|
|
||||||
if e != nil || st.Mode()&os.ModeSymlink != 0 {
|
|
||||||
fail(w, 400, "rename failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, e = root.Lstat(q); e == nil {
|
|
||||||
fail(w, 400, "rename failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e = renameNoReplace(root, p, q); e != nil {
|
|
||||||
fail(w, 400, "rename failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
locked.Release()
|
|
||||||
locked = nil
|
|
||||||
a.s.AddAudit(u.ID, "workspace.rename", p, q)
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsDelete(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
locked := user
|
|
||||||
defer func() {
|
|
||||||
if locked != nil {
|
|
||||||
locked.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
if r.Method != http.MethodDelete {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p, e := safeRel(r.URL.Query().Get("path"))
|
|
||||||
if e != nil || p == "." {
|
|
||||||
fail(w, 400, "delete failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
st, e := entry(root, p)
|
|
||||||
if e != nil || st.Mode()&os.ModeSymlink != 0 {
|
|
||||||
fail(w, 400, "delete failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if st.IsDir() {
|
|
||||||
es, e := readRootDir(root, p)
|
|
||||||
if e != nil || len(es) != 0 {
|
|
||||||
fail(w, 400, "delete failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if e = root.Remove(p); e != nil {
|
|
||||||
fail(w, 400, "delete failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
locked.Release()
|
|
||||||
locked = nil
|
|
||||||
a.s.AddAudit(u.ID, "workspace.delete", p, "")
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsUpload(w http.ResponseWriter, r *http.Request, root *os.Root, u store.User) {
|
|
||||||
if r.Method != "POST" {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case uploadSlots <- struct{}{}:
|
|
||||||
defer func() { <-uploadSlots }()
|
|
||||||
default:
|
|
||||||
fail(w, 429, "too many concurrent uploads")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
usedBefore, usageErr := a.userUsage(u)
|
|
||||||
if usageErr != nil || usedBefore >= u.Quota {
|
|
||||||
fail(w, 413, "quota exceeded or usage unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
r.Body = http.MaxBytesReader(w, r.Body, maxUpload+(1<<20))
|
|
||||||
controller := http.NewResponseController(w)
|
|
||||||
if err := controller.SetReadDeadline(time.Now().Add(30 * time.Second)); err != nil && !errors.Is(err, http.ErrNotSupported) {
|
|
||||||
fail(w, 400, "upload unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer controller.SetReadDeadline(time.Time{})
|
|
||||||
if r.ParseMultipartForm(1<<20) != nil {
|
|
||||||
fail(w, 400, "invalid upload")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer r.MultipartForm.RemoveAll()
|
|
||||||
user := a.userLocks.Acquire(u.ID)
|
|
||||||
locked := user
|
|
||||||
defer func() {
|
|
||||||
if locked != nil {
|
|
||||||
locked.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
dir, e := safeRel(r.FormValue("path"))
|
|
||||||
if e != nil || rootSafe(root, dir, false) != nil {
|
|
||||||
fail(w, 400, "invalid directory")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
input, h, e := r.FormFile("file")
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "file required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer input.Close()
|
|
||||||
name, e := safeRel(h.Filename)
|
|
||||||
if e != nil || name == "." || path.Base(name) != name {
|
|
||||||
fail(w, 400, "invalid filename")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !uploadExtensionAllowed(name, os.Getenv("UPLOAD_ALLOWED_EXTENSIONS")) {
|
|
||||||
fail(w, 415, "upload file type denied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
target := path.Join(dir, name)
|
|
||||||
if _, e = root.Lstat(target); !errors.Is(e, os.ErrNotExist) {
|
|
||||||
fail(w, 409, "target exists")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRow("SELECT quota FROM users WHERE id=?", u.ID).Scan(&u.Quota); err != nil {
|
|
||||||
fail(w, 413, "quota unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
used, e := a.userUsage(u)
|
|
||||||
if e != nil || h.Size > maxUpload || used+h.Size > u.Quota {
|
|
||||||
fail(w, 413, "quota or upload limit exceeded")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tmp := path.Join(dir, ".upload-"+store.ID())
|
|
||||||
f, e := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "upload directory unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer root.Remove(tmp)
|
|
||||||
n, e := io.Copy(f, io.LimitReader(input, maxUpload+1))
|
|
||||||
if e == nil && (n > maxUpload || used+n > u.Quota) {
|
|
||||||
e = errors.New("quota exceeded")
|
|
||||||
}
|
|
||||||
if e == nil {
|
|
||||||
e = f.Chown(1000, 1000)
|
|
||||||
}
|
|
||||||
if e == nil {
|
|
||||||
e = f.Sync()
|
|
||||||
}
|
|
||||||
f.Close()
|
|
||||||
if e == nil {
|
|
||||||
e = renameNoReplace(root, tmp, target)
|
|
||||||
}
|
|
||||||
if errors.Is(e, os.ErrExist) {
|
|
||||||
fail(w, 409, "target exists")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "upload failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
locked.Release()
|
|
||||||
locked = nil
|
|
||||||
a.s.AddAudit(u.ID, "workspace.upload", target, "")
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Empty policy permits arbitrary workspace files; configured entries are exact
|
|
||||||
// case-insensitive extensions, not MIME claims supplied by the client.
|
|
||||||
func uploadExtensionAllowed(name, policy string) bool {
|
|
||||||
if strings.TrimSpace(policy) == "" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
ext := strings.ToLower(path.Ext(name))
|
|
||||||
for _, entry := range strings.Split(policy, ",") {
|
|
||||||
entry = strings.ToLower(strings.TrimSpace(entry))
|
|
||||||
if strings.HasPrefix(entry, ".") && len(entry) > 1 && ext == entry {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) userUsage(u store.User) (int64, error) {
|
|
||||||
base, e := os.OpenRoot(a.root)
|
|
||||||
if e != nil {
|
|
||||||
return 0, e
|
|
||||||
}
|
|
||||||
defer base.Close()
|
|
||||||
root, e := base.OpenRoot(path.Join("workspaces", u.ID))
|
|
||||||
if e != nil {
|
|
||||||
return 0, e
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
var total int64
|
|
||||||
count := 0
|
|
||||||
var walk func(string, int) error
|
|
||||||
walk = func(p string, depth int) error {
|
|
||||||
count++
|
|
||||||
if count > 100000 || depth > 128 {
|
|
||||||
return errors.New("tree limit")
|
|
||||||
}
|
|
||||||
st, e := root.Lstat(p)
|
|
||||||
if e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
if st.Mode().IsRegular() {
|
|
||||||
total += st.Size()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !st.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
f, e := root.OpenFile(p, os.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0)
|
|
||||||
if e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
for {
|
|
||||||
ds, readErr := f.ReadDir(128)
|
|
||||||
for _, d := range ds {
|
|
||||||
if e = walk(path.Join(p, d.Name()), depth+1); e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if readErr == io.EOF {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if readErr != nil {
|
|
||||||
return readErr
|
|
||||||
}
|
|
||||||
if count >= 100000 {
|
|
||||||
return errors.New("tree limit")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
e = walk(".", 0)
|
|
||||||
return total, e
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsDownload(w http.ResponseWriter, r *http.Request, root *os.Root) {
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
p, e := safeRel(r.URL.Query().Get("path"))
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 400, "invalid path")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
anchor, e := root.Open(".")
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer anchor.Close()
|
|
||||||
f, e := openDownload(anchor, p)
|
|
||||||
if e != nil {
|
|
||||||
fail(w, 404, "not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
st, e := f.Stat()
|
|
||||||
if e == nil && st.IsDir() {
|
|
||||||
a.wsDownloadZip(w, r, f, p)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if e != nil || !st.Mode().IsRegular() {
|
|
||||||
fail(w, 400, "invalid file")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Disposition", "attachment")
|
|
||||||
http.ServeContent(w, r, path.Base(p), st.ModTime(), f)
|
|
||||||
}
|
|
||||||
|
|
||||||
func openDownload(anchor *os.File, p string) (*os.File, error) {
|
|
||||||
fd, err := unix.Openat2(int(anchor.Fd()), p, &unix.OpenHow{
|
|
||||||
Flags: unix.O_RDONLY | unix.O_NONBLOCK | unix.O_CLOEXEC,
|
|
||||||
Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_NO_XDEV,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
f := os.NewFile(uintptr(fd), p)
|
|
||||||
st, err := f.Stat()
|
|
||||||
if err == nil && (!st.IsDir() && !st.Mode().IsRegular() || st.Mode().IsRegular() && st.Sys().(*syscall.Stat_t).Nlink != 1) {
|
|
||||||
err = errors.New("unsafe file")
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
f.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return f, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type zipEntry struct {
|
|
||||||
name string
|
|
||||||
info os.FileInfo
|
|
||||||
}
|
|
||||||
|
|
||||||
func zipManifest(ctx context.Context, anchor *os.File) ([]zipEntry, error) {
|
|
||||||
var entries []zipEntry
|
|
||||||
var total int64
|
|
||||||
var walk func(string, int) error
|
|
||||||
walk = func(p string, depth int) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if depth > maxZipDepth || len(entries) >= maxZipEntries {
|
|
||||||
return errZipLimit
|
|
||||||
}
|
|
||||||
f, err := openDownload(anchor, p)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
st, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if p != "." {
|
|
||||||
if st.Size() < 0 || st.Mode().IsRegular() && st.Size() > maxZipBytes-total {
|
|
||||||
return errZipLimit
|
|
||||||
}
|
|
||||||
if st.Mode().IsRegular() {
|
|
||||||
total += st.Size()
|
|
||||||
}
|
|
||||||
entries = append(entries, zipEntry{p, st})
|
|
||||||
}
|
|
||||||
if !st.IsDir() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
ds, err := f.ReadDir(128)
|
|
||||||
if err != nil && err != io.EOF {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, d := range ds {
|
|
||||||
if strings.ContainsAny(d.Name(), "\\:") {
|
|
||||||
return errors.New("unsafe archive name")
|
|
||||||
}
|
|
||||||
if e := walk(path.Join(p, d.Name()), depth+1); e != nil {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err == io.EOF {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
err := walk(".", 0)
|
|
||||||
return entries, err
|
|
||||||
}
|
|
||||||
|
|
||||||
type zipContextWriter struct {
|
|
||||||
ctx context.Context
|
|
||||||
w io.Writer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w zipContextWriter) Write(p []byte) (int, error) {
|
|
||||||
if err := w.ctx.Err(); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return w.w.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func streamZip(ctx context.Context, w io.Writer, anchor *os.File, entries []zipEntry) error {
|
|
||||||
z := zip.NewWriter(zipContextWriter{ctx, w})
|
|
||||||
buf := make([]byte, 32<<10)
|
|
||||||
for _, item := range entries {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
f, err := openDownload(anchor, item.name)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err = func() error {
|
|
||||||
defer f.Close()
|
|
||||||
st, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !os.SameFile(st, item.info) || st.Mode() != item.info.Mode() || st.Size() != item.info.Size() || !st.ModTime().Equal(item.info.ModTime()) {
|
|
||||||
return errors.New("file changed")
|
|
||||||
}
|
|
||||||
h, err := zip.FileInfoHeader(st)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
h.Name = item.name
|
|
||||||
if st.IsDir() {
|
|
||||||
h.Name += "/"
|
|
||||||
}
|
|
||||||
dst, err := z.CreateHeader(h)
|
|
||||||
if err != nil || st.IsDir() {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
n, err := io.CopyBuffer(dst, io.LimitReader(f, item.info.Size()), buf)
|
|
||||||
if err == nil && n != item.info.Size() {
|
|
||||||
err = io.ErrUnexpectedEOF
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
after, statErr := f.Stat()
|
|
||||||
if statErr != nil {
|
|
||||||
return statErr
|
|
||||||
}
|
|
||||||
if after.Size() != st.Size() || !after.ModTime().Equal(st.ModTime()) {
|
|
||||||
return errors.New("file changed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return z.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) wsDownloadZip(w http.ResponseWriter, r *http.Request, anchor *os.File, p string) {
|
|
||||||
select {
|
|
||||||
case zipSlots <- struct{}{}:
|
|
||||||
defer func() { <-zipSlots }()
|
|
||||||
default:
|
|
||||||
fail(w, 429, "too many archive downloads")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), zipTimeout)
|
|
||||||
defer cancel()
|
|
||||||
controller := http.NewResponseController(w)
|
|
||||||
if err := controller.SetWriteDeadline(time.Now().Add(zipTimeout)); err != nil && !errors.Is(err, http.ErrNotSupported) {
|
|
||||||
fail(w, 500, "download unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer controller.SetWriteDeadline(time.Time{})
|
|
||||||
entries, err := zipManifest(ctx, anchor)
|
|
||||||
if err != nil {
|
|
||||||
code := 400
|
|
||||||
if errors.Is(err, errZipLimit) {
|
|
||||||
code = 413
|
|
||||||
}
|
|
||||||
fail(w, code, "cannot archive directory")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
name := path.Base(p)
|
|
||||||
if p == "." {
|
|
||||||
name = "workspace"
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/zip")
|
|
||||||
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": name + ".zip"}))
|
|
||||||
if err := streamZip(ctx, w, anchor, entries); err != nil {
|
|
||||||
// A failed stream must not become a successful, silently incomplete archive.
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const maxDirectoryEntries = 10000
|
|
||||||
|
|
||||||
var errDirectoryLimit = errors.New("directory entry limit exceeded")
|
|
||||||
|
|
||||||
func readRootDir(root *os.Root, p string) ([]os.DirEntry, error) {
|
|
||||||
f, e := root.OpenFile(p, os.O_RDONLY|syscall.O_DIRECTORY|syscall.O_NOFOLLOW, 0)
|
|
||||||
if e != nil {
|
|
||||||
return nil, e
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
entries, err := f.ReadDir(maxDirectoryEntries + 1)
|
|
||||||
if len(entries) > maxDirectoryEntries {
|
|
||||||
return nil, errDirectoryLimit
|
|
||||||
}
|
|
||||||
if err == io.EOF {
|
|
||||||
err = nil
|
|
||||||
}
|
|
||||||
return entries, err
|
|
||||||
}
|
|
||||||
@@ -1,543 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/zip"
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"io"
|
|
||||||
"mime/multipart"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestUploadExtensionPolicy(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, policy string
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
{"file.exe", "", true}, {"file.TXT", ".txt, .md", true},
|
|
||||||
{"file.txt.exe", ".txt", false}, {"file", ".txt", false},
|
|
||||||
{"file.exe", "*,txt", false}, {"file.json", ".txt,.json", true},
|
|
||||||
} {
|
|
||||||
if got := uploadExtensionAllowed(tc.name, tc.policy); got != tc.want {
|
|
||||||
t.Errorf("%q %q: %v", tc.name, tc.policy, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func downloadTestRoot(t *testing.T) (string, *os.Root) {
|
|
||||||
t.Helper()
|
|
||||||
dir := t.TempDir()
|
|
||||||
root, err := os.OpenRoot(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { root.Close() })
|
|
||||||
return dir, root
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDirectoryDownloadZip(t *testing.T) {
|
|
||||||
dir, root := downloadTestRoot(t)
|
|
||||||
if err := os.MkdirAll(filepath.Join(dir, "docs/empty"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "docs/你好.txt"), []byte("hello zip"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, p := range []string{"docs", ".", "", "docs/empty"} {
|
|
||||||
t.Run(p, func(t *testing.T) {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
(&API{}).wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path="+url.QueryEscape(p), nil), root)
|
|
||||||
if w.Code != 200 || w.Header().Get("Content-Type") != "application/zip" || !strings.Contains(w.Header().Get("Content-Disposition"), ".zip") {
|
|
||||||
t.Fatal(w.Code, w.Header(), w.Body.String())
|
|
||||||
}
|
|
||||||
z, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
want := map[string]string{"empty/": "", "你好.txt": "hello zip"}
|
|
||||||
if p == "." || p == "" {
|
|
||||||
want = map[string]string{"docs/": "", "docs/empty/": "", "docs/你好.txt": "hello zip"}
|
|
||||||
}
|
|
||||||
if p == "docs/empty" {
|
|
||||||
want = map[string]string{}
|
|
||||||
}
|
|
||||||
if len(z.File) != len(want) {
|
|
||||||
t.Fatalf("entries %d want %d", len(z.File), len(want))
|
|
||||||
}
|
|
||||||
for _, f := range z.File {
|
|
||||||
expect, ok := want[f.Name]
|
|
||||||
if !ok {
|
|
||||||
t.Fatal("unexpected entry", f.Name)
|
|
||||||
}
|
|
||||||
r, err := f.Open()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
b, err := io.ReadAll(r)
|
|
||||||
r.Close()
|
|
||||||
if err != nil || string(b) != expect {
|
|
||||||
t.Fatal(f.Name, string(b), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
r := httptest.NewRequest("GET", "/api/ws/download?path=docs/"+url.QueryEscape("你好.txt"), nil)
|
|
||||||
r.Header.Set("Range", "bytes=1-3")
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
(&API{}).wsDownload(w, r, root)
|
|
||||||
if w.Code != 206 || w.Body.String() != "ell" {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDirectoryDownloadRejectsUnsafeEntries(t *testing.T) {
|
|
||||||
for _, kind := range []string{"symlink", "internal-link", "hardlink", "fifo", "backslash", "drive"} {
|
|
||||||
t.Run(kind, func(t *testing.T) {
|
|
||||||
dir, root := downloadTestRoot(t)
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "docs"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
outside := filepath.Join(t.TempDir(), "secret")
|
|
||||||
if err := os.WriteFile(outside, []byte("secret"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
target := filepath.Join(dir, "docs/unsafe")
|
|
||||||
var err error
|
|
||||||
switch kind {
|
|
||||||
case "symlink":
|
|
||||||
err = os.Symlink(outside, target)
|
|
||||||
case "internal-link":
|
|
||||||
err = os.Symlink(".", target)
|
|
||||||
case "hardlink":
|
|
||||||
err = os.Link(outside, target)
|
|
||||||
case "fifo":
|
|
||||||
err = syscall.Mkfifo(target, 0600)
|
|
||||||
case "backslash":
|
|
||||||
err = os.WriteFile(filepath.Join(dir, `docs/..\secret`), nil, 0600)
|
|
||||||
case "drive":
|
|
||||||
err = os.WriteFile(filepath.Join(dir, "docs/C:secret"), nil, 0600)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
(&API{}).wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path=docs", nil), root)
|
|
||||||
if w.Code != 400 || strings.Contains(w.Body.String(), "secret") {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
dir, root := downloadTestRoot(t)
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "real"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "real/file"), []byte("secret"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.Symlink("real", filepath.Join(dir, "link")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, p := range []string{"../secret", "/etc/passwd", `..\secret`, "link", "link/file"} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
(&API{}).wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path="+url.QueryEscape(p), nil), root)
|
|
||||||
if w.Code < 400 {
|
|
||||||
t.Fatal("unsafe path accepted", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDirectoryDownloadLimits(t *testing.T) {
|
|
||||||
for _, limit := range []string{"size", "depth", "count"} {
|
|
||||||
t.Run(limit, func(t *testing.T) {
|
|
||||||
dir, root := downloadTestRoot(t)
|
|
||||||
switch limit {
|
|
||||||
case "size":
|
|
||||||
f, err := os.Create(filepath.Join(dir, "large"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
err = f.Truncate(maxZipBytes + 1)
|
|
||||||
f.Close()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case "depth":
|
|
||||||
if err := os.MkdirAll(filepath.Join(dir, strings.Repeat("d/", maxZipDepth+1)), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case "count":
|
|
||||||
for i := 0; i <= maxZipEntries; i++ {
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, fmt.Sprint(i)), nil, 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
(&API{}).wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path=.", nil), root)
|
|
||||||
if w.Code != 413 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestZipManifestRechecksAndCancellation(t *testing.T) {
|
|
||||||
for _, change := range []string{"symlink", "grow", "cancel"} {
|
|
||||||
t.Run(change, func(t *testing.T) {
|
|
||||||
dir, root := downloadTestRoot(t)
|
|
||||||
p := filepath.Join(dir, "file")
|
|
||||||
if err := os.WriteFile(p, []byte("data"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f, err := root.Open(".")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
es, err := zipManifest(ctx, f)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
switch change {
|
|
||||||
case "symlink":
|
|
||||||
if err := os.Remove(p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.Symlink("/etc/passwd", p); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case "grow":
|
|
||||||
if err := os.WriteFile(p, []byte("more data"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
case "cancel":
|
|
||||||
cancel()
|
|
||||||
}
|
|
||||||
if err := streamZip(ctx, io.Discard, f, es); err == nil {
|
|
||||||
t.Fatal("unsafe stream accepted")
|
|
||||||
}
|
|
||||||
if change == "cancel" {
|
|
||||||
if _, err := zipManifest(ctx, f); !errors.Is(err, context.Canceled) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type failingZipWriter struct{}
|
|
||||||
|
|
||||||
func (failingZipWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe }
|
|
||||||
|
|
||||||
func TestZipWriteFailureAndSizeBound(t *testing.T) {
|
|
||||||
dir, root := downloadTestRoot(t)
|
|
||||||
p := filepath.Join(dir, "file")
|
|
||||||
if err := os.WriteFile(p, bytes.Repeat([]byte("x"), 65536), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
f, err := root.Open(".")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
es, err := zipManifest(context.Background(), f)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := streamZip(context.Background(), failingZipWriter{}, f, es); !errors.Is(err, io.ErrClosedPipe) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := &growingZipWriter{path: p}
|
|
||||||
if err := streamZip(context.Background(), w, f, es); err == nil {
|
|
||||||
t.Fatal("concurrent growth accepted")
|
|
||||||
}
|
|
||||||
if w.Len() > 65536+1024 {
|
|
||||||
t.Fatal("stream exceeded manifest bound", w.Len())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type growingZipWriter struct {
|
|
||||||
bytes.Buffer
|
|
||||||
path string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *growingZipWriter) Write(p []byte) (int, error) {
|
|
||||||
if w.path != "" {
|
|
||||||
f, err := os.OpenFile(w.path, os.O_WRONLY|os.O_APPEND, 0)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
_, err = f.Write([]byte("growth"))
|
|
||||||
f.Close()
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
w.path = ""
|
|
||||||
}
|
|
||||||
return w.Buffer.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
type blockedZipWriter struct {
|
|
||||||
*httptest.ResponseRecorder
|
|
||||||
started chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
deadlines []time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *blockedZipWriter) Write(p []byte) (int, error) {
|
|
||||||
if w.started != nil {
|
|
||||||
close(w.started)
|
|
||||||
w.started = nil
|
|
||||||
<-w.release
|
|
||||||
}
|
|
||||||
return w.ResponseRecorder.Write(p)
|
|
||||||
}
|
|
||||||
func (w *blockedZipWriter) SetWriteDeadline(d time.Time) error {
|
|
||||||
w.deadlines = append(w.deadlines, d)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestZipTransferNoLocksAndConcurrencyLimit(t *testing.T) {
|
|
||||||
_, root := downloadTestRoot(t)
|
|
||||||
a := &API{}
|
|
||||||
started := make(chan struct{})
|
|
||||||
w := &blockedZipWriter{ResponseRecorder: httptest.NewRecorder(), started: started, release: make(chan struct{})}
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(done)
|
|
||||||
a.wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path=.", nil), root)
|
|
||||||
}()
|
|
||||||
defer func() { close(w.release); <-done }()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("download not started")
|
|
||||||
}
|
|
||||||
if a.userLocks.held() {
|
|
||||||
t.Fatal("user write lock held across write")
|
|
||||||
}
|
|
||||||
if a.instanceLocks.held() {
|
|
||||||
t.Fatal("instance lock held across write")
|
|
||||||
}
|
|
||||||
if len(w.deadlines) != 1 || w.deadlines[0].IsZero() {
|
|
||||||
t.Fatal("missing deadline")
|
|
||||||
}
|
|
||||||
for i := 1; i < cap(zipSlots); i++ {
|
|
||||||
zipSlots <- struct{}{}
|
|
||||||
}
|
|
||||||
defer func() {
|
|
||||||
for i := 1; i < cap(zipSlots); i++ {
|
|
||||||
<-zipSlots
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
busy := httptest.NewRecorder()
|
|
||||||
a.wsDownload(busy, httptest.NewRequest("GET", "/api/ws/download?path=.", nil), root)
|
|
||||||
if busy.Code != 429 {
|
|
||||||
t.Fatal(busy.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRenameNoReplace(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
root, err := os.OpenRoot(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
for _, source := range []string{"source", ".upload-test"} {
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, source), []byte("new"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := root.Lstat("target"); !errors.Is(err, os.ErrNotExist) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "target"), []byte("runtime"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := renameNoReplace(root, source, "target"); !errors.Is(err, os.ErrExist) {
|
|
||||||
t.Fatalf("overwrite: %v", err)
|
|
||||||
}
|
|
||||||
b, err := root.ReadFile("target")
|
|
||||||
if err != nil || string(b) != "runtime" {
|
|
||||||
t.Fatalf("target changed: %q %v", b, err)
|
|
||||||
}
|
|
||||||
if err := root.Remove("target"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := renameNoReplace(root, source, "target"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := root.Remove("target"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := os.Mkdir(filepath.Join(dir, "real"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.Symlink("real", filepath.Join(dir, "link")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "source"), []byte("new"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, target := range []string{"link/target", "../target"} {
|
|
||||||
if err := renameNoReplace(root, "source", target); err == nil {
|
|
||||||
t.Fatalf("unsafe parent accepted: %s", target)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := renameNoReplace(root, "link", "target"); err == nil {
|
|
||||||
t.Fatal("symlink source accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type stalledUpload struct {
|
|
||||||
started chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *stalledUpload) Read([]byte) (int, error) {
|
|
||||||
close(b.started)
|
|
||||||
<-b.release
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
func (b *stalledUpload) Close() error { return nil }
|
|
||||||
|
|
||||||
type deadlineRecorder struct {
|
|
||||||
*httptest.ResponseRecorder
|
|
||||||
deadlines []time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *deadlineRecorder) SetReadDeadline(d time.Time) error {
|
|
||||||
w.deadlines = append(w.deadlines, d)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLegacyWorkspaceRemoved(t *testing.T) {
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Fatal("unexpected docker call") })
|
|
||||||
for _, method := range []string{"GET", "POST", "DELETE"} {
|
|
||||||
for _, endpoint := range []string{"tree", "download", "upload", "mkdir", "rename", "delete"} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.workspace(w, runtimeRequest(method, "/api/ws/"+endpoint, "", sid))
|
|
||||||
if w.Code != http.StatusGone {
|
|
||||||
t.Fatalf("%s %s: %d", method, endpoint, w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWorkspaceSafetyAndUpload(t *testing.T) {
|
|
||||||
s, err := store.Open(filepath.Join(t.TempDir(), "test.sqlite"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer s.Close()
|
|
||||||
a := &API{s: s, root: t.TempDir()}
|
|
||||||
u := store.User{ID: "user", Quota: 8}
|
|
||||||
if err := s.CreateUser(u, "unused"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
root, e := a.openWorkspace(u)
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
for _, p := range []string{"", ".", "a/..", "../other", "/tmp"} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.wsDelete(w, httptest.NewRequest("DELETE", "/api/ws/delete?path="+p, nil), root, u)
|
|
||||||
if w.Code < 400 {
|
|
||||||
t.Fatalf("deleted %q", p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
outside := t.TempDir()
|
|
||||||
os.WriteFile(filepath.Join(outside, "secret"), []byte("secret"), 0600)
|
|
||||||
if e = os.Symlink(outside, filepath.Join(a.root, "workspaces/user/files/link")); e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path=link/secret", nil), root)
|
|
||||||
if w.Code < 400 {
|
|
||||||
t.Fatal("escaped root")
|
|
||||||
}
|
|
||||||
upload := func(name, content string) int {
|
|
||||||
var b bytes.Buffer
|
|
||||||
m := multipart.NewWriter(&b)
|
|
||||||
m.WriteField("path", ".")
|
|
||||||
f, _ := m.CreateFormFile("file", name)
|
|
||||||
f.Write([]byte(content))
|
|
||||||
m.Close()
|
|
||||||
r := httptest.NewRequest("POST", "/api/ws/upload", &b)
|
|
||||||
r.Header.Set("Content-Type", m.FormDataContentType())
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.wsUpload(w, r, root, u)
|
|
||||||
return w.Code
|
|
||||||
}
|
|
||||||
if code := upload("file.txt", "hello"); code != 200 {
|
|
||||||
t.Fatal(code)
|
|
||||||
}
|
|
||||||
if code := upload("big.txt", "12345"); code != 413 {
|
|
||||||
t.Fatal("quota", code)
|
|
||||||
}
|
|
||||||
if code := upload("file.txt", "x"); code != 409 {
|
|
||||||
t.Fatal("overwrite", code)
|
|
||||||
}
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
a.wsDownload(w, httptest.NewRequest("GET", "/api/ws/download?path=file.txt", nil), root)
|
|
||||||
if w.Code != 200 || w.Body.String() != "hello" {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for _, op := range []struct{ action, body string }{
|
|
||||||
{"mkdir", `{"path":"folder"}`},
|
|
||||||
{"rename", `{"path":"folder","new_path":"renamed"}`},
|
|
||||||
{"delete", ""},
|
|
||||||
} {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := httptest.NewRequest("POST", "/api/ws/"+op.action, strings.NewReader(op.body))
|
|
||||||
switch op.action {
|
|
||||||
case "mkdir":
|
|
||||||
a.wsMkdir(w, r, root, u)
|
|
||||||
case "rename":
|
|
||||||
a.wsRename(w, r, root, u)
|
|
||||||
case "delete":
|
|
||||||
a.wsDelete(w, httptest.NewRequest("DELETE", "/api/ws/delete?path=renamed", nil), root, u)
|
|
||||||
}
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(op.action, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rows, err := s.DB.Query("SELECT uid,action,target,detail FROM audit ORDER BY id")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
want := [][3]string{{"workspace.upload", "file.txt", ""}, {"workspace.mkdir", "folder", ""}, {"workspace.rename", "folder", "renamed"}, {"workspace.delete", "renamed", ""}}
|
|
||||||
i := 0
|
|
||||||
for rows.Next() {
|
|
||||||
var uid, action, target, detail string
|
|
||||||
if err := rows.Scan(&uid, &action, &target, &detail); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if i >= len(want) || uid != u.ID || [3]string{action, target, detail} != want[i] {
|
|
||||||
t.Fatal("unexpected audit", uid, action, target, detail)
|
|
||||||
}
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
if err := rows.Err(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if i != len(want) {
|
|
||||||
t.Fatal("missing audit entries", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/zip"
|
|
||||||
"bytes"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Full-home access is an explicit owner capability, not a credential write-only
|
|
||||||
// boundary. Keep ownership, authentication and traversal checks independent.
|
|
||||||
func TestFullHomeOwnerPolicy(t *testing.T) {
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker operation") })
|
|
||||||
for _, i := range []store.Instance{{ID: "owned", UID: "owner", Status: "running"}, {ID: "foreign", UID: "other", Status: "running"}} {
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
home := filepath.Join(a.root, "workspaces", i.UID, "instances", i.ID, "home")
|
|
||||||
if err := os.MkdirAll(filepath.Join(home, "profiles", "web"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for path, contents := range map[string]string{".credentials.yaml": "synthetic-test-credential", "dsh-ssh.json": "synthetic-test-ssh", "profiles/web/cordis.patch.yml": "old: value"} {
|
|
||||||
if err := os.WriteFile(filepath.Join(home, path), []byte(contents), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
request := func(method, path, body, session string) *httptest.ResponseRecorder {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.workspace(w, runtimeRequest(method, path, body, session))
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
for _, name := range []string{".credentials.yaml", "dsh-ssh.json"} {
|
|
||||||
w := request("GET", "/api/ws/download?instance=owned&path="+name, "", sid)
|
|
||||||
if w.Code != 200 || w.Body.Len() == 0 {
|
|
||||||
t.Fatal(name, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w = request("GET", "/api/ws/download?instance=foreign&path="+name, "", sid); w.Code != 404 {
|
|
||||||
t.Fatal("cross-owner read", w.Code)
|
|
||||||
}
|
|
||||||
if w = request("GET", "/api/ws/download?instance=owned&path="+name, "", ""); w.Code != 401 {
|
|
||||||
t.Fatal("anonymous read", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := request("GET", "/api/ws/download?instance=owned&path=.", "", sid)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
archive, err := zip.NewReader(bytes.NewReader(w.Body.Bytes()), int64(w.Body.Len()))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found := false
|
|
||||||
for _, f := range archive.File {
|
|
||||||
if f.Name == ".credentials.yaml" {
|
|
||||||
r, e := f.Open()
|
|
||||||
if e != nil {
|
|
||||||
t.Fatal(e)
|
|
||||||
}
|
|
||||||
data, e := io.ReadAll(r)
|
|
||||||
r.Close()
|
|
||||||
if e != nil || string(data) != "synthetic-test-credential" {
|
|
||||||
t.Fatal("wrong credential archive entry", e)
|
|
||||||
}
|
|
||||||
found = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
t.Fatal("owner archive unexpectedly filters credentials")
|
|
||||||
}
|
|
||||||
// Running state intentionally does not force the controlled profile workflow.
|
|
||||||
w = request("POST", "/api/ws/rename?instance=owned", `{"path":"profiles/web/cordis.patch.yml","new_path":"profiles/web/saved.yml"}`, sid)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal("owner profile rename blocked", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
w = request("GET", "/api/ws/download?instance=owned&path=../../other/instances/foreign/home/.credentials.yaml", "", sid)
|
|
||||||
if w.Code != 400 && w.Code != 404 {
|
|
||||||
t.Fatal("traversal allowed", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,735 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/tar"
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
imageassets "git.misaka.ren/m1saka/dsh-web/image"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Caller holds buildQueue; interrupted jobs cannot be reported as successful.
|
|
||||||
func (a *API) ensureBuilds() error {
|
|
||||||
if buildQueue.initialized[a.s.DB] {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
_, err := a.s.DB.Exec(`CREATE TABLE IF NOT EXISTS image_builds(id TEXT PRIMARY KEY,version TEXT NOT NULL,status TEXT NOT NULL,logs TEXT NOT NULL DEFAULT '',error TEXT NOT NULL DEFAULT '',image_id TEXT NOT NULL DEFAULT '',created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS image_build_requests(user_id TEXT NOT NULL,build_id TEXT PRIMARY KEY,created_at TEXT NOT NULL); CREATE TABLE IF NOT EXISTS images(image_id TEXT PRIMARY KEY,version TEXT NOT NULL,recipe_rev TEXT,platform TEXT,tag TEXT NOT NULL DEFAULT '',status TEXT NOT NULL,built_at TEXT NOT NULL,usage_count INTEGER NOT NULL DEFAULT 0);`)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, q := range []string{
|
|
||||||
"CREATE INDEX IF NOT EXISTS image_builds_history ON image_builds(created_at DESC,id DESC)",
|
|
||||||
"ALTER TABLE image_builds ADD COLUMN requested_by TEXT NOT NULL DEFAULT ''",
|
|
||||||
"ALTER TABLE image_builds ADD COLUMN recipe_rev TEXT",
|
|
||||||
"ALTER TABLE image_builds ADD COLUMN platform TEXT",
|
|
||||||
"ALTER TABLE image_builds ADD COLUMN finished_at TEXT",
|
|
||||||
"ALTER TABLE image_builds ADD COLUMN experimental INTEGER NOT NULL DEFAULT 0",
|
|
||||||
"CREATE TABLE IF NOT EXISTS image_admissions(image_id TEXT NOT NULL,version TEXT NOT NULL,recipe_rev TEXT NOT NULL,platform TEXT NOT NULL,state TEXT NOT NULL,evidence TEXT NOT NULL,admitted_by TEXT NOT NULL,admitted_at TEXT NOT NULL,PRIMARY KEY(image_id,version,recipe_rev,platform))",
|
|
||||||
"ALTER TABLE images ADD COLUMN recipe_rev TEXT",
|
|
||||||
"ALTER TABLE images ADD COLUMN platform TEXT",
|
|
||||||
"ALTER TABLE images ADD COLUMN usage_count INTEGER NOT NULL DEFAULT 0",
|
|
||||||
} {
|
|
||||||
if _, e := a.s.DB.Exec(q); e != nil && !strings.Contains(strings.ToLower(e.Error()), "duplicate column") {
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, err = a.s.DB.Exec("DROP INDEX IF EXISTS image_builds_version; CREATE UNIQUE INDEX IF NOT EXISTS image_builds_identity ON image_builds(version,recipe_rev,platform) WHERE recipe_rev IS NOT NULL AND platform IS NOT NULL AND status IN ('queued','running','validating','success')")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='version requires administrator experimental build' WHERE status='queued' AND experimental=0 AND version NOT IN ('0.2.0-rc.1','0.2.0-rc.2')"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !buildQueue.initialized[a.s.DB] {
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='image template identity is unknown' WHERE status='queued' AND (recipe_rev IS NULL OR recipe_rev<>? OR platform IS NULL OR platform<>?)", imageRecipeRev, imagePlatform); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE image_builds SET status='failed',error='build interrupted by process restart' WHERE status IN ('running','validating')"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
buildQueue.initialized[a.s.DB] = true
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (a *API) builds(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, authErr := a.user(r)
|
|
||||||
if authErr != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet && r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct {
|
|
||||||
Version string `json:"version"`
|
|
||||||
Experimental bool `json:"experimental"`
|
|
||||||
}
|
|
||||||
if r.Method == http.MethodPost {
|
|
||||||
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
|
|
||||||
dec.DisallowUnknownFields()
|
|
||||||
if dec.Decode(&in) != nil || dec.Decode(new(any)) != io.EOF || !validBuildVersion(in.Version) {
|
|
||||||
fail(w, 400, "a fixed version is required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if r.Method == http.MethodGet {
|
|
||||||
a.listBuilds(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if in.Experimental && !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
b, err := a.enqueueBuild(r.Context(), u.ID, in.Version, in.Experimental)
|
|
||||||
if err != nil {
|
|
||||||
status := 502
|
|
||||||
if errors.Is(err, errBuildUnsupported) {
|
|
||||||
status = 400
|
|
||||||
}
|
|
||||||
if errors.Is(err, errBuildRateLimited) {
|
|
||||||
status = 429
|
|
||||||
w.Header().Set("Retry-After", "3600")
|
|
||||||
}
|
|
||||||
fail(w, status, sanitizeBuildLog(err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusAccepted)
|
|
||||||
writeJSON(w, b)
|
|
||||||
}
|
|
||||||
|
|
||||||
var errBuildRateLimited = errors.New("build limit: three new tasks per user per hour")
|
|
||||||
|
|
||||||
// lookupActiveBuild returns the active or successful record for a version; it
|
|
||||||
// performs no I/O and must be called with the build queue held.
|
|
||||||
func (a *API) lookupActiveBuild(version string) (buildRecord, bool, error) {
|
|
||||||
var b buildRecord
|
|
||||||
var finished, recipe, platform sql.NullString
|
|
||||||
err := a.s.DB.QueryRow("SELECT id,version,status,logs,error,image_id,created_at,requested_by,recipe_rev,platform,finished_at,experimental FROM image_builds WHERE version=? AND recipe_rev=? AND platform=? AND status IN ('queued','running','validating','success')", version, imageRecipeRev, imagePlatform).Scan(&b.ID, &b.Version, &b.Status, &b.Logs, &b.Error, &b.ImageID, &b.Created, &b.RequestedBy, &recipe, &platform, &finished, &b.Experimental)
|
|
||||||
b.RecipeRev, b.Platform, b.Finished = recipe.String, platform.String, finished.String
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return b, false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return b, false, err
|
|
||||||
}
|
|
||||||
return b, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) enqueueBuild(ctx context.Context, uid, version string, experiment ...bool) (buildRecord, error) {
|
|
||||||
experimental := len(experiment) > 0 && experiment[0]
|
|
||||||
if experimental {
|
|
||||||
u, err := a.s.UserByID(uid)
|
|
||||||
if err != nil || !u.Admin {
|
|
||||||
return buildRecord{}, errors.New("administrator required")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !experimental && !normalBuildAllowed(version) {
|
|
||||||
return buildRecord{}, errBuildUnsupported
|
|
||||||
}
|
|
||||||
if !validBuildVersion(version) {
|
|
||||||
return buildRecord{}, errors.New("a fixed version is required")
|
|
||||||
}
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
var b buildRecord
|
|
||||||
var found bool
|
|
||||||
if err == nil {
|
|
||||||
b, found, err = a.lookupActiveBuild(version)
|
|
||||||
}
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
if found && !experimental && b.Experimental && b.Status != "success" {
|
|
||||||
return b, errBuildUnsupported
|
|
||||||
}
|
|
||||||
if found && b.Status != "success" {
|
|
||||||
a.startBuildWorker()
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
// Verifying a previously built image talks to Docker; keep that I/O
|
|
||||||
// out of the queue lock shared with instance creation and upgrades.
|
|
||||||
verify, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
||||||
id, lookupErr := a.localBuildImage(verify, b.Version, b.ImageID)
|
|
||||||
cancel()
|
|
||||||
if lookupErr != nil {
|
|
||||||
return b, lookupErr
|
|
||||||
}
|
|
||||||
if id != "" && id == b.ImageID {
|
|
||||||
if !experimental {
|
|
||||||
im, e := a.evaluateImage(ctx, id)
|
|
||||||
if e != nil {
|
|
||||||
return b, e
|
|
||||||
}
|
|
||||||
if !im.Selectable {
|
|
||||||
return b, errBuildUnsupported
|
|
||||||
}
|
|
||||||
}
|
|
||||||
a.startBuildWorker()
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
buildQueue.Lock()
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE image_builds SET status='missing',error='image no longer available' WHERE id=? AND status='success'", b.ID); err == nil {
|
|
||||||
// Another request may have queued a replacement while the image
|
|
||||||
// was being verified; honor it instead of inserting a duplicate.
|
|
||||||
b, found, err = a.lookupActiveBuild(version)
|
|
||||||
}
|
|
||||||
if err == nil && !found {
|
|
||||||
b, err = a.insertBuild(uid, version, experimental)
|
|
||||||
}
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
if !experimental && found && b.Experimental {
|
|
||||||
return b, errBuildUnsupported
|
|
||||||
}
|
|
||||||
a.startBuildWorker()
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
// Re-check after the unlocked phases: another request may have inserted.
|
|
||||||
b, found, err = a.lookupActiveBuild(version)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
if !experimental && b.Experimental {
|
|
||||||
return b, errBuildUnsupported
|
|
||||||
}
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
b, err = a.insertBuild(uid, version, experimental)
|
|
||||||
if err != nil {
|
|
||||||
return b, err
|
|
||||||
}
|
|
||||||
a.startBuildWorkerLocked()
|
|
||||||
return b, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// insertBuild applies the per-user rate limit and inserts a queued record;
|
|
||||||
// call with the build queue held.
|
|
||||||
func (a *API) insertBuild(uid, version string, experiment ...bool) (buildRecord, error) {
|
|
||||||
var count int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests WHERE user_id=? AND created_at>?", uid, time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)).Scan(&count); err != nil {
|
|
||||||
return buildRecord{}, err
|
|
||||||
}
|
|
||||||
if count >= 3 {
|
|
||||||
return buildRecord{}, errBuildRateLimited
|
|
||||||
}
|
|
||||||
b := buildRecord{ID: store.ID(), Version: version, Status: "queued", Created: time.Now().UTC().Format(time.RFC3339Nano), RequestedBy: uid, RecipeRev: imageRecipeRev, Platform: imagePlatform}
|
|
||||||
b.Experimental = len(experiment) > 0 && experiment[0]
|
|
||||||
return b, a.insertBuildRequest(b, uid)
|
|
||||||
}
|
|
||||||
func (a *API) startBuildWorker() {
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
a.startBuildWorkerLocked()
|
|
||||||
}
|
|
||||||
|
|
||||||
// startBuildWorkerLocked requires the caller to hold buildQueue: the active map
|
|
||||||
// is only safe under that lock.
|
|
||||||
func (a *API) startBuildWorkerLocked() {
|
|
||||||
if buildQueue.active[a.s.DB] {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
buildQueue.active[a.s.DB] = true
|
|
||||||
go func() {
|
|
||||||
var lastCleanupAudit time.Time
|
|
||||||
var nextCleanup time.Time
|
|
||||||
var recoveryErr error
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-a.s.Done():
|
|
||||||
buildQueue.Lock()
|
|
||||||
delete(buildQueue.active, a.s.DB)
|
|
||||||
buildQueue.Unlock()
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if time.Now().After(nextCleanup) {
|
|
||||||
buildWorker.Lock()
|
|
||||||
recovery, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
||||||
recoveryErr = a.recoverBuildChecks(recovery)
|
|
||||||
cancel()
|
|
||||||
buildWorker.Unlock()
|
|
||||||
if errors.Is(recoveryErr, errBuildCheckUncertain) {
|
|
||||||
nextCleanup = time.Now().Add(time.Minute)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if recoveryErr != nil {
|
|
||||||
if time.Since(lastCleanupAudit) >= time.Minute {
|
|
||||||
a.s.AddAudit("", "build.cleanup_error", "", sanitizeBuildLog(recoveryErr.Error()))
|
|
||||||
lastCleanupAudit = time.Now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildQueue.Lock()
|
|
||||||
var b buildRecord
|
|
||||||
err := a.s.DB.QueryRow("SELECT id,version,experimental,requested_by FROM image_builds WHERE status='queued' ORDER BY created_at,id LIMIT 1").Scan(&b.ID, &b.Version, &b.Experimental, &b.RequestedBy)
|
|
||||||
if err == nil {
|
|
||||||
_, err = a.s.DB.Exec("UPDATE image_builds SET status='running' WHERE id=?", b.ID)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
if !errors.Is(err, sql.ErrNoRows) || recoveryErr != nil {
|
|
||||||
buildQueue.Unlock()
|
|
||||||
select {
|
|
||||||
case <-a.s.Done():
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
delete(buildQueue.active, a.s.DB)
|
|
||||||
buildQueue.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
buildQueue.Unlock()
|
|
||||||
a.runImageBuild(b)
|
|
||||||
nextCleanup = time.Time{}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
func imageBuildContext() (io.Reader, error) {
|
|
||||||
var buf bytes.Buffer
|
|
||||||
tw := tar.NewWriter(&buf)
|
|
||||||
for _, name := range []string{"Dockerfile", "Caddyfile", "entrypoint.sh", "install-mobile.mjs", "install-free-search.mjs", "install-workspace.mjs", "init-profile.mjs"} {
|
|
||||||
dataBytes, err := imageassets.Files.ReadFile(name)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
data := string(dataBytes)
|
|
||||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0755, Size: int64(len(data))}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if _, err := tw.Write([]byte(data)); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := tw.Close(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &buf, nil
|
|
||||||
}
|
|
||||||
func (a *API) runImageBuild(b buildRecord) {
|
|
||||||
buildWorker.Lock()
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
||||||
defer cancel()
|
|
||||||
logs := ""
|
|
||||||
rawLogs := ""
|
|
||||||
var logErr error
|
|
||||||
appendLog := func(s string) {
|
|
||||||
if len(rawLogs) < buildLogLimit {
|
|
||||||
remaining := buildLogLimit - len(rawLogs)
|
|
||||||
if len(s) > remaining {
|
|
||||||
s = s[:remaining]
|
|
||||||
}
|
|
||||||
rawLogs += s
|
|
||||||
}
|
|
||||||
logs = sanitizeBuildLog(rawLogs)
|
|
||||||
if logErr == nil {
|
|
||||||
_, logErr = a.s.DB.Exec("UPDATE image_builds SET logs=? WHERE id=?", logs, b.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
tarball, err := imageBuildContext()
|
|
||||||
if !b.Experimental && !normalBuildAllowed(b.Version) {
|
|
||||||
err = errBuildUnsupported
|
|
||||||
}
|
|
||||||
if b.Experimental {
|
|
||||||
u, e := a.s.UserByID(b.RequestedBy)
|
|
||||||
if e != nil || !u.Admin {
|
|
||||||
err = errors.New("experimental build requester is not an administrator")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var id string
|
|
||||||
if err == nil {
|
|
||||||
id, err = a.localBuildImage(ctx, b.Version, "")
|
|
||||||
}
|
|
||||||
if err == nil && id == "" {
|
|
||||||
_, err = fetchNPMVersion(ctx, npmBuildClient, b.Version)
|
|
||||||
if err == nil {
|
|
||||||
err = a.checkBuildDisk(ctx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err == nil && id == "" {
|
|
||||||
id, err = a.docker.Build(ctx, tarball, b.Version, "dsh-web:"+b.Version+"-"+imageRecipeRev, appendLog)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
var matches bool
|
|
||||||
matches, err = a.imageMatches(ctx, id, b.Version)
|
|
||||||
if err == nil && !matches {
|
|
||||||
err = errors.New("image template or platform mismatch")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = a.s.DB.Exec("UPDATE image_builds SET status='validating',image_id=? WHERE id=?", id, b.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = a.verifyBuiltImage(ctx, id, b.Version, b.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = logErr
|
|
||||||
}
|
|
||||||
status, detail := "success", ""
|
|
||||||
if err != nil {
|
|
||||||
status = "failed"
|
|
||||||
detail = sanitizeBuildLog(err.Error())
|
|
||||||
appendLog(string(rune(10)) + detail)
|
|
||||||
}
|
|
||||||
finished := time.Now().UTC().Format(time.RFC3339Nano)
|
|
||||||
buildWorker.Unlock()
|
|
||||||
if err := a.persistBuildResult(b, status, logs, detail, id, finished); err != nil {
|
|
||||||
go a.finishBuild(b, status, logs, detail, id, finished)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (a *API) verifyBuiltImage(parent context.Context, image, version string, buildIDs ...string) (result error) {
|
|
||||||
ctx, cancel := context.WithTimeout(parent, 90*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
id := store.ID()
|
|
||||||
if len(buildIDs) != 0 {
|
|
||||||
id = buildIDs[0]
|
|
||||||
}
|
|
||||||
if err := a.ensureBuildChecks(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES(?,?,?)", id, image, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var created struct {
|
|
||||||
ID string `json:"Id"`
|
|
||||||
}
|
|
||||||
rejected := false
|
|
||||||
defer func() {
|
|
||||||
cleanup, c := context.WithTimeout(context.Background(), 10*time.Second)
|
|
||||||
defer c()
|
|
||||||
var err error
|
|
||||||
if rejected {
|
|
||||||
_, err = a.s.DB.Exec("DELETE FROM image_build_checks WHERE build_id=?", id)
|
|
||||||
} else if created.ID != "" {
|
|
||||||
err = a.deleteBuildCheck(cleanup, id, created.ID)
|
|
||||||
} else {
|
|
||||||
err = a.cleanupBuildCheck(cleanup, id, image)
|
|
||||||
}
|
|
||||||
result = errors.Join(result, err)
|
|
||||||
}()
|
|
||||||
config := map[string]any{"Image": image, "User": "1000:1000", "Labels": map[string]string{"dsh-mgr.role": "build-check", "dsh-mgr.build": id}, "HostConfig": map[string]any{"NetworkMode": "none", "ReadonlyRootfs": true, "CapDrop": []string{"ALL"}, "SecurityOpt": []string{"no-new-privileges:true"}, "Memory": int64(1 << 30), "NanoCpus": int64(1e9), "PidsLimit": 256, "Tmpfs": map[string]string{"/tmp": "rw,exec,nosuid,nodev,size=256m,mode=1777", "/workspace": "rw,nosuid,nodev,size=256m,uid=1000,gid=1000"}}}
|
|
||||||
if err := a.docker.Do(ctx, http.MethodPost, "/containers/create?name="+url.QueryEscape(buildCheckName(id)), config, &created); err != nil {
|
|
||||||
rejected = dockerCreateRejected(err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if created.ID == "" {
|
|
||||||
return errors.New("verification container ID missing")
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE image_build_checks SET container_id=? WHERE build_id=?", created.ID, id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
path := "/containers/" + url.PathEscape(created.ID)
|
|
||||||
if err := a.docker.Do(ctx, http.MethodPost, path+"/start", nil, nil); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
script := `const end=Date.now()+60000;async function probe(){try{const r=await fetch('http://127.0.0.1:8080/',{redirect:'manual',signal:AbortSignal.timeout(2000)});if(r.status>=200&&r.status<500)process.exit(0);}catch{}if(Date.now()>end)process.exit(1);setTimeout(probe,500);}probe();`
|
|
||||||
{
|
|
||||||
expected, _ := json.Marshal(version)
|
|
||||||
script = `const cp=require('node:child_process');const actual=cp.execFileSync('dsh',['--version'],{encoding:'utf8',timeout:20000}).trim();if(actual!==` + string(expected) + `)process.exit(2);` + script
|
|
||||||
}
|
|
||||||
script = `const fs=require('node:fs');for(const [k,v] of Object.entries({DSH_HOME:'/workspace',HOME:'/workspace',XDG_DATA_HOME:'/workspace/.local/share',XDG_CONFIG_HOME:'/workspace/.config'})){if(process.env[k]!==v)process.exit(3);}if(!fs.lstatSync('/data/dsh-home').isSymbolicLink()||fs.realpathSync('/data/dsh-home')!=='/workspace')process.exit(4);fs.writeFileSync('/workspace/.home-check','isolated');if(fs.readFileSync('/data/dsh-home/.home-check','utf8')!=='isolated')process.exit(5);fs.unlinkSync('/workspace/.home-check');` + script
|
|
||||||
var exec struct {
|
|
||||||
ID string `json:"Id"`
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, http.MethodPost, path+"/exec", map[string]any{"Cmd": []string{"node", "-e", script}, "User": "1000:1000"}, &exec); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if exec.ID == "" {
|
|
||||||
return errors.New("verification exec ID missing")
|
|
||||||
}
|
|
||||||
ep := "/exec/" + url.PathEscape(exec.ID)
|
|
||||||
if err := a.docker.Do(ctx, http.MethodPost, ep+"/start", map[string]bool{"Detach": true, "Tty": false}, nil); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
ticker := time.NewTicker(250 * time.Millisecond)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
var state struct {
|
|
||||||
Running bool
|
|
||||||
ExitCode int
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, http.MethodGet, ep+"/json", nil, &state); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !state.Running {
|
|
||||||
if state.ExitCode != 0 {
|
|
||||||
return fmt.Errorf("image startup verification failed (exit %d)", state.ExitCode)
|
|
||||||
}
|
|
||||||
container, err := a.docker.Inspect(ctx, created.ID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !container.State.Running {
|
|
||||||
return errors.New("verification container exited")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return ctx.Err()
|
|
||||||
case <-ticker.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const buildLogLimit = 64 << 10
|
|
||||||
|
|
||||||
var fixedVersion = regexp.MustCompile(`^(0|[1-9][0-9]*)[.](0|[1-9][0-9]*)[.](0|[1-9][0-9]*)(-[0-9A-Za-z-]+([.][0-9A-Za-z-]+)*)?$`)
|
|
||||||
var buildSecrets = regexp.MustCompile(`(?i)(authorization["']?[[:space:]]*[:=][[:space:]]*["']?(?:bearer[[:space:]]+|basic[[:space:]]+)?|(?:token|password|passwd|secret|api[_-]?key|cookie)["']?[[:space:]]*[:=][[:space:]]*["']?)[^[:space:]"'&,;]+`)
|
|
||||||
var buildURLCredentials = regexp.MustCompile(`(https?://)[^/@[:space:]]+:[^/@[:space:]]+@`)
|
|
||||||
var buildQueue = struct {
|
|
||||||
sync.Mutex
|
|
||||||
initialized map[*sql.DB]bool
|
|
||||||
active map[*sql.DB]bool
|
|
||||||
}{initialized: map[*sql.DB]bool{}, active: map[*sql.DB]bool{}}
|
|
||||||
var buildWorker sync.Mutex
|
|
||||||
|
|
||||||
type imageRecord struct {
|
|
||||||
Admission string `json:"admission"`
|
|
||||||
AdmissionReason string `json:"admission_reason"`
|
|
||||||
Selectable bool `json:"selectable"`
|
|
||||||
ID string `json:"id"`
|
|
||||||
ImageID string `json:"image_id"`
|
|
||||||
Version string `json:"version"`
|
|
||||||
Tag string `json:"tag,omitempty"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
Created string `json:"created_at"`
|
|
||||||
RecipeRev string `json:"recipe_rev,omitempty"`
|
|
||||||
Platform string `json:"platform,omitempty"`
|
|
||||||
UsageCount int `json:"usage_count"`
|
|
||||||
}
|
|
||||||
type buildRecord struct {
|
|
||||||
Experimental bool `json:"experimental"`
|
|
||||||
ImageID string `json:"image_id,omitempty"`
|
|
||||||
ID string `json:"id"`
|
|
||||||
Version string `json:"version"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
Logs string `json:"logs,omitempty"`
|
|
||||||
Error string `json:"error,omitempty"`
|
|
||||||
Created string `json:"created_at"`
|
|
||||||
RequestedBy string `json:"requested_by,omitempty"`
|
|
||||||
RecipeRev string `json:"recipe_rev,omitempty"`
|
|
||||||
Platform string `json:"platform,omitempty"`
|
|
||||||
Finished string `json:"finished_at,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) images(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == http.MethodDelete {
|
|
||||||
a.deleteImage(w, r)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, _, err := a.user(r); err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
result, err := a.imageIndex(ctx)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 502, "image index reconciliation failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
// verifiedImagesTTL bounds how long a cached admission evaluation is reused
|
|
||||||
// by the one-second intent poller.
|
|
||||||
var verifiedImagesTTL = 5 * time.Second
|
|
||||||
|
|
||||||
type evalCacheEntry struct {
|
|
||||||
im imageRecord
|
|
||||||
at time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// evaluateImageCached reuses recent per-image evaluations for the intent
|
|
||||||
// poller so repeated ticks do not hammer Docker. Deletions and admission
|
|
||||||
// writes bump imageDeleteGeneration and invalidate every entry.
|
|
||||||
func (a *API) evaluateImageCached(ctx context.Context, gen uint64, id string) (imageRecord, error) {
|
|
||||||
a.evalCacheMu.Lock()
|
|
||||||
if entry, ok := a.evalCache[id]; ok && a.evalCacheGen == gen && time.Since(entry.at) < verifiedImagesTTL {
|
|
||||||
a.evalCacheMu.Unlock()
|
|
||||||
return entry.im, nil
|
|
||||||
}
|
|
||||||
a.evalCacheMu.Unlock()
|
|
||||||
im, err := a.evaluateImage(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return im, err
|
|
||||||
}
|
|
||||||
a.evalCacheMu.Lock()
|
|
||||||
if a.evalCache == nil || a.evalCacheGen != gen {
|
|
||||||
a.evalCache, a.evalCacheGen = make(map[string]evalCacheEntry), gen
|
|
||||||
}
|
|
||||||
a.evalCache[id] = evalCacheEntry{im: im, at: time.Now()}
|
|
||||||
a.evalCacheMu.Unlock()
|
|
||||||
return im, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) successfulImageIDs(ctx context.Context) ([]string, error) {
|
|
||||||
rows, err := a.s.DB.QueryContext(ctx, "SELECT DISTINCT image_id FROM image_builds WHERE status='success' AND image_id<>''")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ids := []string{}
|
|
||||||
for rows.Next() {
|
|
||||||
var id string
|
|
||||||
if err = rows.Scan(&id); err != nil {
|
|
||||||
rows.Close()
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ids = append(ids, id)
|
|
||||||
}
|
|
||||||
err = rows.Err()
|
|
||||||
rows.Close()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return ids, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) verifiedImages(ctx context.Context) (map[string]string, error) {
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ids, err := a.successfulImageIDs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result := map[string]string{}
|
|
||||||
for _, id := range ids {
|
|
||||||
im, err := a.evaluateImage(ctx, id)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if im.Selectable {
|
|
||||||
result[id] = im.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// verifiedImagesPoll is the intent poller's view: build rows are reread every
|
|
||||||
// tick, but per-image Docker evaluation is cached briefly so a stuck waiting
|
|
||||||
// intent cannot turn the one-second loop into Docker hammering.
|
|
||||||
func (a *API) verifiedImagesPoll(ctx context.Context) (map[string]string, error) {
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
gen := a.imageDeleteGeneration
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
ids, err := a.successfulImageIDs(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result := map[string]string{}
|
|
||||||
for _, id := range ids {
|
|
||||||
im, err := a.evaluateImageCached(ctx, gen, id)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if im.Selectable {
|
|
||||||
result[id] = im.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
func (a *API) latest(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if _, _, err := a.user(r); err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
v, err := fetchLatest(r.Context(), npmBuildClient)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 502, "npm version lookup failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]any{"version": v, "checked_at": time.Now().UTC().Format(time.RFC3339), "supported": normalBuildAllowed(v), "build_allowed": normalBuildAllowed(v), "admission": defaultAdmission(v, imageRecipeRev, imagePlatform), "admission_reason": "normal builds require a built-in tested tuple; unknown versions require administrator experimental validation and admission"})
|
|
||||||
}
|
|
||||||
func fetchLatest(ctx context.Context, client *http.Client) (string, error) {
|
|
||||||
return fetchNPMVersion(ctx, client, "latest")
|
|
||||||
}
|
|
||||||
func fetchNPMVersion(ctx context.Context, client *http.Client, version string) (string, error) {
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://registry.npmjs.org/@deepseek-ai%2Fdsh/"+url.PathEscape(version), nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != 200 {
|
|
||||||
return "", fmt.Errorf("npm status %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
var v struct{ Version string }
|
|
||||||
if err = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&v); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if !validBuildVersion(v.Version) || (version != "latest" && v.Version != version) {
|
|
||||||
return "", errors.New("invalid npm version")
|
|
||||||
}
|
|
||||||
return v.Version, nil
|
|
||||||
}
|
|
||||||
func validBuildVersion(v string) bool { return len(v) <= 100 && fixedVersion.MatchString(v) }
|
|
||||||
func sanitizeBuildLog(s string) string {
|
|
||||||
s = buildURLCredentials.ReplaceAllString(s, "$1[REDACTED]@")
|
|
||||||
s = buildSecrets.ReplaceAllString(s, "$1[REDACTED]")
|
|
||||||
s = strings.ToValidUTF8(s, "")
|
|
||||||
if len(s) > buildLogLimit {
|
|
||||||
s = strings.ToValidUTF8(s[:buildLogLimit], "")
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) localBuildImage(ctx context.Context, version, preferred string) (string, error) {
|
|
||||||
filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime", "dsh-mgr.version=" + version}})
|
|
||||||
var images []struct {
|
|
||||||
ID string
|
|
||||||
Labels map[string]string
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &images); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
for _, im := range images {
|
|
||||||
if im.ID != "" && im.Labels["dsh-mgr.role"] == "runtime" && im.Labels["dsh-mgr.version"] == version && im.Labels["dsh-mgr.recipe"] == imageRecipeRev && (preferred == "" || im.ID == preferred) {
|
|
||||||
ok, err := a.imageMatches(ctx, im.ID, version)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if ok {
|
|
||||||
return im.ID, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
@@ -1,147 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *API) deleteImage(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := r.URL.Query().Get("image_id")
|
|
||||||
if !runtimeImageID.MatchString(id) {
|
|
||||||
fail(w, 400, "invalid image_id")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Hold the build queue through the checks and the Docker deletion so a
|
|
||||||
// record cannot flip to missing mid-flight. Upgrades write their journal
|
|
||||||
// entry before creating containers; the transaction below re-checks that
|
|
||||||
// journal, and an upgrade racing an administrator deletion fails safely
|
|
||||||
// with its own rollback path.
|
|
||||||
admissionMu.Lock()
|
|
||||||
defer admissionMu.Unlock()
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
if buildQueue.active[a.s.DB] {
|
|
||||||
fail(w, 409, "image build worker is active")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
tx, err := a.s.DB.BeginTx(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
// Take SQLite's writer lock before checking intents: reservation does not use the build queue.
|
|
||||||
if _, err = tx.ExecContext(ctx, "UPDATE images SET status=status WHERE image_id=?", id); err != nil {
|
|
||||||
fail(w, 500, "image index unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var version string
|
|
||||||
err = tx.QueryRowContext(ctx, "SELECT version FROM images WHERE image_id=?", id).Scan(&version)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
fail(w, 404, "managed image not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "image index unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, check := range []struct{ table, query string }{
|
|
||||||
{"image_builds", "SELECT count(*) FROM image_builds WHERE status IN ('queued','running','validating')"},
|
|
||||||
{"creation_intents", "SELECT count(*) FROM creation_intents WHERE status IN ('waiting','creating')"},
|
|
||||||
{"upgrade_journal", "SELECT count(*) FROM upgrade_journal WHERE phase NOT IN ('complete','rolled_back')"},
|
|
||||||
} {
|
|
||||||
var exists, count int
|
|
||||||
if err = tx.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name=?", check.table).Scan(&exists); err == nil && exists > 0 {
|
|
||||||
err = tx.QueryRowContext(ctx, check.query).Scan(&count)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "active task lookup failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if count > 0 {
|
|
||||||
fail(w, 409, "active task prevents image deletion")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var references int
|
|
||||||
if err = tx.QueryRowContext(ctx, "SELECT count(*) FROM instances WHERE image_id=?", id).Scan(&references); err != nil {
|
|
||||||
fail(w, 500, "instance reference lookup failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if references > 0 {
|
|
||||||
fail(w, 409, "image is referenced by an instance")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var image struct {
|
|
||||||
ID string `json:"Id"`
|
|
||||||
Config struct{ Labels map[string]string }
|
|
||||||
}
|
|
||||||
path := "/images/" + url.PathEscape(id)
|
|
||||||
if err = a.docker.Do(ctx, http.MethodGet, path+"/json", nil, &image); err != nil {
|
|
||||||
fail(w, 502, "image inspection failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if image.ID != id || image.Config.Labels["dsh-mgr.role"] != "runtime" || version == "" || image.Config.Labels["dsh-mgr.version"] != version {
|
|
||||||
fail(w, 409, "managed runtime image required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var containers []struct{ ImageID string }
|
|
||||||
if err = a.docker.Do(ctx, http.MethodGet, "/containers/json?all=1", nil, &containers); err != nil || containers == nil {
|
|
||||||
fail(w, 502, "container reference lookup failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, container := range containers {
|
|
||||||
if container.ImageID == "" {
|
|
||||||
fail(w, 502, "container image identity unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if container.ImageID == id {
|
|
||||||
fail(w, 409, "image is referenced by a container")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err = tx.ExecContext(ctx, "UPDATE images SET status='missing',usage_count=0 WHERE image_id=?", id); err == nil {
|
|
||||||
_, err = tx.ExecContext(ctx, "UPDATE image_builds SET status='missing',error='image deleted by administrator' WHERE image_id=? AND status='success'", id)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(ctx, "INSERT INTO audit(uid,action,target,detail,at) VALUES(?,'image.delete',?,'',?)", u.ID, id, time.Now().Unix())
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "image deletion recording failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = a.docker.Do(ctx, http.MethodDelete, path+"?force=false", nil, nil); err != nil {
|
|
||||||
status := http.StatusBadGateway
|
|
||||||
var apiErr *docker.APIError
|
|
||||||
if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusConflict {
|
|
||||||
status = http.StatusConflict
|
|
||||||
}
|
|
||||||
fail(w, status, "docker image deletion failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Docker deletion is already authoritative, even if the DB commit fails.
|
|
||||||
// Any inventory observation taken before it must be discarded.
|
|
||||||
a.imageDeleteGeneration++
|
|
||||||
if err = tx.Commit(); err != nil {
|
|
||||||
fail(w, 500, "image deleted but recording failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]string{"image_id": id, "status": "missing"})
|
|
||||||
}
|
|
||||||
@@ -1,173 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestImageDelete(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
for _, mode := range []string{"success", "anonymous", "non-admin", "invalid", "unmanaged", "role", "instance", "container-running", "container-exited", "containers-failure", "containers-null", "containers-unknown", "inspect-failure", "delete-failure", "delete-conflict", "queued", "running", "validating", "worker", "waiting", "creating", "journal", "complete", "rolled_back", "db-failure", "task-query-failure", "audit-failure"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
var deletes atomic.Int32
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case r.Method == http.MethodDelete:
|
|
||||||
deletes.Add(1)
|
|
||||||
if r.URL.Path != "/v1.47/images/"+id || r.URL.Query().Get("force") != "false" {
|
|
||||||
t.Errorf("unsafe delete: %s", r.URL)
|
|
||||||
}
|
|
||||||
if mode == "delete-failure" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "delete-conflict" {
|
|
||||||
w.WriteHeader(409)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, []any{})
|
|
||||||
case r.URL.Path == "/v1.47/images/"+id+"/json":
|
|
||||||
if mode == "inspect-failure" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "role" {
|
|
||||||
writeJSON(w, map[string]any{"Id": id})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeImageInspect(w, id, "1.2.3")
|
|
||||||
case r.URL.Path == "/v1.47/containers/json":
|
|
||||||
if r.URL.RawQuery != "all=1" {
|
|
||||||
t.Errorf("must inspect all containers: %s", r.URL)
|
|
||||||
}
|
|
||||||
if mode == "containers-failure" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "containers-null" {
|
|
||||||
writeJSON(w, nil)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "containers-unknown" {
|
|
||||||
writeJSON(w, []any{map[string]string{"Image": "tag"}})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(mode, "container-") {
|
|
||||||
writeJSON(w, []any{map[string]string{"ImageID": id, "State": strings.TrimPrefix(mode, "container-")}})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, []any{})
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected Docker request: %s %s", r.Method, r.URL)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
exec := func(q string, args ...any) {
|
|
||||||
t.Helper()
|
|
||||||
if _, err := a.s.DB.Exec(q, args...); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
seedVerifiedImage(t, a, id, "1.2.3")
|
|
||||||
exec("INSERT INTO images(image_id,version,status,built_at) VALUES(?,'1.2.3','available','now')", id)
|
|
||||||
if mode != "non-admin" {
|
|
||||||
exec("UPDATE users SET admin=1 WHERE id='u'")
|
|
||||||
}
|
|
||||||
want := 409
|
|
||||||
switch mode {
|
|
||||||
case "success", "complete", "rolled_back":
|
|
||||||
want = 200
|
|
||||||
case "anonymous":
|
|
||||||
sid = ""
|
|
||||||
want = 401
|
|
||||||
case "non-admin":
|
|
||||||
want = 403
|
|
||||||
case "invalid":
|
|
||||||
want = 400
|
|
||||||
case "unmanaged":
|
|
||||||
exec("DELETE FROM images")
|
|
||||||
want = 404
|
|
||||||
case "inspect-failure", "containers-failure", "containers-null", "containers-unknown", "delete-failure":
|
|
||||||
want = 502
|
|
||||||
case "instance":
|
|
||||||
exec("INSERT INTO instances(id,uid,name,image_id,created) VALUES('i','other','i',?,0)", id)
|
|
||||||
case "queued", "running", "validating":
|
|
||||||
exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('active','2.0.0',?,'now')", mode)
|
|
||||||
case "worker":
|
|
||||||
buildQueue.Lock()
|
|
||||||
buildQueue.active[a.s.DB] = true
|
|
||||||
buildQueue.Unlock()
|
|
||||||
defer func() { buildQueue.Lock(); delete(buildQueue.active, a.s.DB); buildQueue.Unlock() }()
|
|
||||||
case "waiting", "creating":
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
exec("INSERT INTO creation_intents(id,uid,name,version,status,host,created) VALUES('intent','other','i','2.0.0',?,'host',0)", mode)
|
|
||||||
case "db-failure":
|
|
||||||
exec("ALTER TABLE instances RENAME COLUMN image_id TO broken")
|
|
||||||
want = 500
|
|
||||||
case "task-query-failure":
|
|
||||||
exec("CREATE TABLE upgrade_journal(broken TEXT)")
|
|
||||||
want = 500
|
|
||||||
case "audit-failure":
|
|
||||||
exec("CREATE TRIGGER deny_audit BEFORE INSERT ON audit BEGIN SELECT RAISE(FAIL,'unavailable'); END")
|
|
||||||
want = 500
|
|
||||||
}
|
|
||||||
if mode == "journal" || mode == "complete" || mode == "rolled_back" {
|
|
||||||
exec("CREATE TABLE upgrade_journal(phase TEXT)")
|
|
||||||
exec("INSERT INTO upgrade_journal VALUES(?)", mode)
|
|
||||||
}
|
|
||||||
queryID := id
|
|
||||||
if mode == "invalid" {
|
|
||||||
queryID = "runtime:latest"
|
|
||||||
}
|
|
||||||
r := httptest.NewRequest(http.MethodDelete, "/api/images?image_id="+queryID, nil)
|
|
||||||
if sid != "" {
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
|
|
||||||
addTestAccess(r, sid)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.images(w, r)
|
|
||||||
if w.Code != want {
|
|
||||||
t.Fatalf("status=%d want=%d body=%s", w.Code, want, w.Body.String())
|
|
||||||
}
|
|
||||||
wantDeletes := int32(0)
|
|
||||||
if want == 200 || mode == "delete-failure" || mode == "delete-conflict" {
|
|
||||||
wantDeletes = 1
|
|
||||||
}
|
|
||||||
if deletes.Load() != wantDeletes {
|
|
||||||
t.Fatalf("deletes=%d want=%d", deletes.Load(), wantDeletes)
|
|
||||||
}
|
|
||||||
if mode != "unmanaged" {
|
|
||||||
var status string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM images WHERE image_id=?", id).Scan(&status); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
wantStatus := "available"
|
|
||||||
if want == 200 {
|
|
||||||
wantStatus = "missing"
|
|
||||||
}
|
|
||||||
if status != wantStatus {
|
|
||||||
t.Fatalf("image status=%s want=%s", status, wantStatus)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var audits int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='image.delete' AND uid='u' AND target=?", id).Scan(&audits); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if (audits == 1) != (want == 200) {
|
|
||||||
t.Fatalf("audits=%d", audits)
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE image_id=?", id).Scan(&status); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if (status == "missing") != (want == 200) {
|
|
||||||
t.Fatalf("build status=%s", status)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,128 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestEnqueueBuildConcurrentMissingImageReturnsReplacement(t *testing.T) {
|
|
||||||
inspections := make(chan chan struct{}, 2)
|
|
||||||
stop := make(chan struct{})
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
t.Errorf("unexpected Docker method: %s", r.Method)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/v1.47/images/json":
|
|
||||||
writeJSON(w, []map[string]any{{"Id": "old-image", "Labels": map[string]string{
|
|
||||||
"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev,
|
|
||||||
}}})
|
|
||||||
case "/v1.47/images/old-image/json":
|
|
||||||
release := make(chan struct{})
|
|
||||||
select {
|
|
||||||
case inspections <- release:
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-release:
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
case <-stop:
|
|
||||||
case <-r.Context().Done():
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected Docker path: %s", r.URL.Path)
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "old-image", "0.2.0-rc.1")
|
|
||||||
var oldID string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT id FROM image_builds WHERE image_id='old-image'").Scan(&oldID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// Keep the replacement queued without starting a background worker.
|
|
||||||
buildQueue.Lock()
|
|
||||||
buildQueue.active[a.s.DB] = true
|
|
||||||
buildQueue.Unlock()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
var submissions sync.WaitGroup
|
|
||||||
t.Cleanup(func() {
|
|
||||||
close(stop)
|
|
||||||
cancel()
|
|
||||||
submissions.Wait()
|
|
||||||
buildQueue.Lock()
|
|
||||||
delete(buildQueue.active, a.s.DB)
|
|
||||||
delete(buildQueue.initialized, a.s.DB)
|
|
||||||
buildQueue.Unlock()
|
|
||||||
})
|
|
||||||
type result struct {
|
|
||||||
build buildRecord
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
submit := func() <-chan result {
|
|
||||||
done := make(chan result, 1)
|
|
||||||
submissions.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer submissions.Done()
|
|
||||||
b, err := a.enqueueBuild(ctx, "u", "0.2.0-rc.1")
|
|
||||||
done <- result{b, err}
|
|
||||||
}()
|
|
||||||
return done
|
|
||||||
}
|
|
||||||
waitInspection := func() chan struct{} {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case release := <-inspections:
|
|
||||||
return release
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("submission did not inspect the old image")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
waitResult := func(done <-chan result) buildRecord {
|
|
||||||
t.Helper()
|
|
||||||
select {
|
|
||||||
case got := <-done:
|
|
||||||
if got.err != nil {
|
|
||||||
t.Fatal(got.err)
|
|
||||||
}
|
|
||||||
return got.build
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("submission did not finish")
|
|
||||||
return buildRecord{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
firstDone := submit()
|
|
||||||
firstRelease := waitInspection()
|
|
||||||
secondDone := submit()
|
|
||||||
secondRelease := waitInspection()
|
|
||||||
close(firstRelease)
|
|
||||||
first := waitResult(firstDone)
|
|
||||||
if first.ID == "" || first.ID == oldID || first.Status != "queued" {
|
|
||||||
t.Fatalf("first submission did not queue a replacement: %+v", first)
|
|
||||||
}
|
|
||||||
close(secondRelease)
|
|
||||||
second := waitResult(secondDone)
|
|
||||||
if second.ID != first.ID || second.Status != "queued" {
|
|
||||||
t.Errorf("second submission returned id=%q status=%q; want replacement id=%q status=queued", second.ID, second.Status, first.ID)
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id=?", oldID).Scan(&status); err != nil || status != "missing" {
|
|
||||||
t.Errorf("old build status=%q, err=%v; want missing", status, err)
|
|
||||||
}
|
|
||||||
for query, want := range map[string]int{
|
|
||||||
"SELECT count(*) FROM image_builds": 2,
|
|
||||||
"SELECT count(*) FROM image_builds WHERE status='queued'": 1,
|
|
||||||
"SELECT count(*) FROM image_build_requests": 1,
|
|
||||||
} {
|
|
||||||
var count int
|
|
||||||
if err := a.s.DB.QueryRow(query).Scan(&count); err != nil || count != want {
|
|
||||||
t.Errorf("%s: count=%d, err=%v; want %d", query, count, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,140 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestImageIndexDoesNotUndoConcurrentDelete(t *testing.T) {
|
|
||||||
for _, initial := range []string{"available", "missing"} {
|
|
||||||
t.Run(initial, func(t *testing.T) {
|
|
||||||
id := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
blocker := "sha256:" + strings.Repeat("b", 64)
|
|
||||||
reached, release := make(chan struct{}), make(chan struct{})
|
|
||||||
var once sync.Once
|
|
||||||
var reachedOnce sync.Once
|
|
||||||
var deleted atomic.Bool
|
|
||||||
unblock := func() { once.Do(func() { close(release) }) }
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case r.Method == http.MethodDelete:
|
|
||||||
deleted.Store(true)
|
|
||||||
writeJSON(w, []any{})
|
|
||||||
case r.URL.Path == "/v1.47/images/"+id+"/json":
|
|
||||||
if deleted.Load() {
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeImageInspect(w, id, "1.2.3")
|
|
||||||
case r.URL.Path == "/v1.47/images/"+blocker+"/json":
|
|
||||||
reachedOnce.Do(func() { close(reached) })
|
|
||||||
select {
|
|
||||||
case <-release:
|
|
||||||
case <-r.Context().Done():
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(http.StatusNotFound)
|
|
||||||
case r.URL.Path == "/v1.47/images/json", r.URL.Path == "/v1.47/containers/json":
|
|
||||||
writeJSON(w, []any{})
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected Docker request: %s %s", r.Method, r.URL)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
defer unblock()
|
|
||||||
seedVerifiedImage(t, a, id, "1.2.3")
|
|
||||||
for _, q := range []struct {
|
|
||||||
sql string
|
|
||||||
args []any
|
|
||||||
}{
|
|
||||||
{"UPDATE users SET admin=1 WHERE id='u'", nil},
|
|
||||||
{"INSERT INTO images(image_id,version,status,built_at) VALUES(?,'1.2.3',?,'2'),(?,'2.0.0','missing','1')", []any{id, initial, blocker}},
|
|
||||||
} {
|
|
||||||
if _, err := a.s.DB.Exec(q.sql, q.args...); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
type outcome struct {
|
|
||||||
images []imageRecord
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
done := make(chan outcome, 1)
|
|
||||||
go func() { images, err := a.imageIndex(ctx); done <- outcome{images, err} }()
|
|
||||||
select {
|
|
||||||
case <-reached:
|
|
||||||
case <-ctx.Done():
|
|
||||||
t.Fatal("index did not reach second image")
|
|
||||||
}
|
|
||||||
r := httptest.NewRequest(http.MethodDelete, "/api/images?image_id="+id, nil).WithContext(ctx)
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
|
|
||||||
addTestAccess(r, sid)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.images(w, r)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatalf("delete: %d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
unblock()
|
|
||||||
select {
|
|
||||||
case out := <-done:
|
|
||||||
if out.err != nil {
|
|
||||||
t.Fatal(out.err)
|
|
||||||
}
|
|
||||||
matches := 0
|
|
||||||
for _, im := range out.images {
|
|
||||||
if im.ImageID == id {
|
|
||||||
matches++
|
|
||||||
}
|
|
||||||
if im.ImageID == id && im.Status != "missing" {
|
|
||||||
t.Errorf("returned deleted image as %s", im.Status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if matches != 1 {
|
|
||||||
t.Errorf("returned %d records for deleted image, want 1", matches)
|
|
||||||
}
|
|
||||||
case <-ctx.Done():
|
|
||||||
t.Fatal("index did not finish")
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM images WHERE image_id=?", id).Scan(&status); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if status != "missing" {
|
|
||||||
t.Errorf("persisted deleted image as %s", status)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageIndexBoundsInvalidatedSnapshots(t *testing.T) {
|
|
||||||
var a *API
|
|
||||||
var calls atomic.Int32
|
|
||||||
a, _ = imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/v1.47/images/json" {
|
|
||||||
t.Errorf("unexpected request: %s", r.URL)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Model a completed deletion during every unlocked observation phase.
|
|
||||||
buildQueue.Lock()
|
|
||||||
a.imageDeleteGeneration++
|
|
||||||
buildQueue.Unlock()
|
|
||||||
calls.Add(1)
|
|
||||||
writeJSON(w, []any{})
|
|
||||||
})
|
|
||||||
images, err := a.imageIndex(context.Background())
|
|
||||||
if !errors.Is(err, errImageIndexChanged) || images != nil {
|
|
||||||
t.Fatalf("images=%v err=%v; want no inventory and invalidation error", images, err)
|
|
||||||
}
|
|
||||||
if calls.Load() != 3 {
|
|
||||||
t.Fatalf("attempts=%d, want 3", calls.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,272 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var npmBuildClient = &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
|
||||||
|
|
||||||
const imageRecipeRev = "7"
|
|
||||||
|
|
||||||
var imagePlatform = runtime.GOOS + "/" + runtime.GOARCH
|
|
||||||
|
|
||||||
const minimumBuildDiskBytes = uint64(2 << 30)
|
|
||||||
|
|
||||||
var buildDiskAvailable = func(path string) (uint64, error) {
|
|
||||||
var stat syscall.Statfs_t
|
|
||||||
if err := syscall.Statfs(path, &stat); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return uint64(stat.Bavail) * uint64(stat.Bsize), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type inspectedImage struct {
|
|
||||||
ID string `json:"Id"`
|
|
||||||
Os, Architecture, Variant, Created string
|
|
||||||
RepoTags []string
|
|
||||||
Config struct{ Labels map[string]string }
|
|
||||||
}
|
|
||||||
|
|
||||||
func (im inspectedImage) platform() string {
|
|
||||||
if im.Os == "" || im.Architecture == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
p := im.Os + "/" + im.Architecture
|
|
||||||
if im.Variant != "" {
|
|
||||||
p += "/" + im.Variant
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) inspectLocalImage(ctx context.Context, id string) (*inspectedImage, error) {
|
|
||||||
if id == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
var im inspectedImage
|
|
||||||
if err := a.docker.Do(ctx, http.MethodGet, "/images/"+url.PathEscape(id)+"/json", nil, &im); err != nil {
|
|
||||||
var apiErr *docker.APIError
|
|
||||||
if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &im, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) imageMatches(ctx context.Context, id, version string) (bool, error) {
|
|
||||||
im, err := a.inspectLocalImage(ctx, id)
|
|
||||||
if err != nil || im == nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
return im.ID == id && im.platform() == imagePlatform && im.Config.Labels["dsh-mgr.role"] == "runtime" && im.Config.Labels["dsh-mgr.version"] == version && im.Config.Labels["dsh-mgr.recipe"] == imageRecipeRev, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var errImageIndexChanged = errors.New("image inventory changed during reconciliation")
|
|
||||||
|
|
||||||
func (a *API) imageIndex(ctx context.Context) ([]imageRecord, error) {
|
|
||||||
// Retry stale observations without holding the queue across Docker I/O.
|
|
||||||
// Bound retries so sustained deletions cannot keep a request alive forever.
|
|
||||||
for attempt := 0; attempt < 3; attempt++ {
|
|
||||||
result, err := a.imageIndexSnapshot(ctx)
|
|
||||||
if !errors.Is(err, errImageIndexChanged) {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil, errImageIndexChanged
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) imageIndexSnapshot(ctx context.Context) ([]imageRecord, error) {
|
|
||||||
// Phase 1: read the managed index under the queue lock (DB only).
|
|
||||||
buildQueue.Lock()
|
|
||||||
generation := a.imageDeleteGeneration
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
if err == nil {
|
|
||||||
_, err = a.s.DB.ExecContext(ctx, `INSERT OR IGNORE INTO images(image_id,version,recipe_rev,platform,tag,status,built_at)
|
|
||||||
SELECT image_id,version,recipe_rev,platform,'','missing',COALESCE(finished_at,created_at) FROM image_builds WHERE status IN ('success','missing') AND image_id<>'' ORDER BY created_at DESC,id DESC`)
|
|
||||||
}
|
|
||||||
var rows *sql.Rows
|
|
||||||
if err == nil {
|
|
||||||
rows, err = a.s.DB.QueryContext(ctx, `SELECT image_id,version,COALESCE(recipe_rev,''),COALESCE(platform,''),tag,status,built_at,
|
|
||||||
(SELECT count(*) FROM instances WHERE instances.image_id=images.image_id) FROM images ORDER BY built_at DESC,image_id`)
|
|
||||||
}
|
|
||||||
result := []imageRecord{}
|
|
||||||
if err == nil {
|
|
||||||
for rows.Next() {
|
|
||||||
var im imageRecord
|
|
||||||
if err = rows.Scan(&im.ImageID, &im.Version, &im.RecipeRev, &im.Platform, &im.Tag, &im.Status, &im.Created, &im.UsageCount); err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
im.ID = im.ImageID
|
|
||||||
result = append(result, im)
|
|
||||||
}
|
|
||||||
scanErr := rows.Err()
|
|
||||||
rows.Close()
|
|
||||||
if err == nil {
|
|
||||||
err = scanErr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Phase 2: inspect Docker without the queue lock; per-image state comes
|
|
||||||
// from real daemon observations, and instance creation must not wait here.
|
|
||||||
known := make(map[string]bool, len(result))
|
|
||||||
for _, im := range result {
|
|
||||||
known[im.ImageID] = true
|
|
||||||
}
|
|
||||||
var local []struct {
|
|
||||||
ID string
|
|
||||||
Labels map[string]string
|
|
||||||
}
|
|
||||||
{
|
|
||||||
filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=runtime"}})
|
|
||||||
if err := a.docker.Do(ctx, http.MethodGet, "/images/json?filters="+url.QueryEscape(string(filters)), nil, &local); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, im := range local {
|
|
||||||
if im.ID != "" && !known[im.ID] && im.Labels["dsh-mgr.role"] == "runtime" {
|
|
||||||
known[im.ID] = true
|
|
||||||
result = append(result, imageRecord{ID: im.ID, ImageID: im.ID})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := range result {
|
|
||||||
im := &result[i]
|
|
||||||
im.Status = "missing"
|
|
||||||
im.Admission = "existing_only"
|
|
||||||
im.AdmissionReason = "local image missing"
|
|
||||||
actual, err := a.inspectLocalImage(ctx, im.ImageID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if actual == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
im.Status = "unverified"
|
|
||||||
im.Platform = actual.platform()
|
|
||||||
im.Version = actual.Config.Labels["dsh-mgr.version"]
|
|
||||||
im.RecipeRev = actual.Config.Labels["dsh-mgr.recipe"]
|
|
||||||
im.Tag = ""
|
|
||||||
if len(actual.RepoTags) > 0 {
|
|
||||||
im.Tag = actual.RepoTags[0]
|
|
||||||
}
|
|
||||||
if actual.Created != "" {
|
|
||||||
im.Created = actual.Created
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM instances WHERE image_id=?", im.ImageID).Scan(&im.UsageCount); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
policy, err := a.evaluateImage(ctx, im.ImageID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
im.Status, im.Admission, im.AdmissionReason, im.Selectable = policy.Status, policy.Admission, policy.AdmissionReason, policy.Selectable
|
|
||||||
}
|
|
||||||
// Phase 3: write observed state back under the queue lock (DB only).
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
if generation != a.imageDeleteGeneration {
|
|
||||||
return nil, errImageIndexChanged
|
|
||||||
}
|
|
||||||
tx, err := a.s.DB.BeginTx(ctx, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
for _, im := range result {
|
|
||||||
if _, err = tx.ExecContext(ctx, `INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at,usage_count) VALUES(?,?,?,?,?,?,?,?)
|
|
||||||
ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at,usage_count=excluded.usage_count`, im.ImageID, im.Version, im.RecipeRev, im.Platform, im.Tag, im.Status, im.Created, im.UsageCount); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if im.Status == "missing" {
|
|
||||||
if _, err = tx.ExecContext(ctx, "UPDATE image_builds SET status='missing',error='image no longer available or template identity is unknown' WHERE image_id=? AND status='success'", im.ImageID); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return result, tx.Commit()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) StartImageBuilds() error {
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
if err := a.ensureBuilds(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
a.startBuildWorkerLocked()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) checkBuildDisk(ctx context.Context) error {
|
|
||||||
var info struct{ DockerRootDir string }
|
|
||||||
if err := a.docker.Do(ctx, http.MethodGet, "/info", nil, &info); err != nil {
|
|
||||||
return fmt.Errorf("build disk check: %w", err)
|
|
||||||
}
|
|
||||||
// A matching pathname alone cannot establish daemon filesystem identity.
|
|
||||||
confirmed := os.Getenv("DSH_DOCKER_ROOT_VISIBLE")
|
|
||||||
if !filepath.IsAbs(info.DockerRootDir) || confirmed != info.DockerRootDir {
|
|
||||||
return errors.New("Docker disk capacity unknown: mount the daemon DockerRootDir at the identical path and set DSH_DOCKER_ROOT_VISIBLE to that path; new builds are blocked")
|
|
||||||
}
|
|
||||||
available, err := buildDiskAvailable(confirmed)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("Docker disk capacity unavailable: %w", err)
|
|
||||||
}
|
|
||||||
return requireBuildDisk(available)
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireBuildDisk(available uint64) error {
|
|
||||||
if available < minimumBuildDiskBytes {
|
|
||||||
return fmt.Errorf("insufficient Docker disk space: %d available bytes, require %d", available, minimumBuildDiskBytes)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) insertBuildRequest(b buildRecord, user string) error {
|
|
||||||
tx, err := a.s.DB.Begin()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
if _, err = tx.Exec("INSERT INTO image_builds(id,version,status,created_at,requested_by,recipe_rev,platform,experimental) VALUES(?,?,?,?,?,?,?,?)", b.ID, b.Version, b.Status, b.Created, b.RequestedBy, b.RecipeRev, b.Platform, b.Experimental); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if _, err = tx.Exec("INSERT INTO image_build_requests(user_id,build_id,created_at) VALUES(?,?,?)", user, b.ID, b.Created); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return tx.Commit()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) reconcileBuilds(parent context.Context, records []buildRecord) error {
|
|
||||||
ctx, cancel := context.WithTimeout(parent, 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
for i := range records {
|
|
||||||
b := &records[i]
|
|
||||||
if b.Status != "success" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
actual, err := a.inspectLocalImage(ctx, b.ImageID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if actual == nil || actual.ID != b.ImageID || actual.Config.Labels["dsh-mgr.role"] != "runtime" || actual.Config.Labels["dsh-mgr.version"] != b.Version || actual.Config.Labels["dsh-mgr.recipe"] != b.RecipeRev || actual.platform() != b.Platform {
|
|
||||||
b.Status, b.Error = "missing", "image no longer available"
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET status=?,error=? WHERE id=? AND status='success'", b.Status, b.Error, b.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,422 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestImageLegacyMigrationNeverCertifies(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/json" {
|
|
||||||
io.WriteString(w, "[]")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
|
|
||||||
})
|
|
||||||
_, err := a.s.DB.Exec(`CREATE TABLE image_builds(id TEXT PRIMARY KEY,version TEXT NOT NULL,status TEXT NOT NULL,logs TEXT NOT NULL DEFAULT '',error TEXT NOT NULL DEFAULT '',image_id TEXT NOT NULL DEFAULT '',created_at TEXT NOT NULL);
|
|
||||||
CREATE UNIQUE INDEX image_builds_version ON image_builds(version) WHERE status IN ('queued','running','validating','success');
|
|
||||||
INSERT INTO image_builds(id,version,status,image_id,created_at) VALUES('legacy','0.2.0-rc.1','success','legacy-image','now'),('queued','0.2.0-rc.2','queued','','now');
|
|
||||||
CREATE TABLE images(image_id TEXT PRIMARY KEY,version TEXT NOT NULL,tag TEXT NOT NULL DEFAULT '',status TEXT NOT NULL,built_at TEXT NOT NULL);
|
|
||||||
INSERT INTO images VALUES('legacy-image','0.2.0-rc.1','','available','now');`)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
verified, err := a.verifiedImages(context.Background())
|
|
||||||
if err != nil || len(verified) != 0 {
|
|
||||||
t.Fatalf("%v %v", verified, err)
|
|
||||||
}
|
|
||||||
var recipe, platform sql.NullString
|
|
||||||
if err = a.s.DB.QueryRow("SELECT recipe_rev,platform FROM image_builds WHERE id='legacy'").Scan(&recipe, &platform); err != nil || recipe.Valid || platform.Valid {
|
|
||||||
t.Fatalf("legacy metadata inferred: %v %v %v", recipe, platform, err)
|
|
||||||
}
|
|
||||||
w := imageRequest(a.images, sid, "GET", "")
|
|
||||||
var images []imageRecord
|
|
||||||
if err = json.Unmarshal(w.Body.Bytes(), &images); err != nil || len(images) != 1 || images[0].Status != "unverified" || images[0].RecipeRev != imageRecipeRev || images[0].Platform != imagePlatform {
|
|
||||||
t.Fatal(w.Body.String(), err)
|
|
||||||
}
|
|
||||||
if err = a.s.DB.QueryRow("SELECT recipe_rev,platform FROM image_builds WHERE id='legacy'").Scan(&recipe, &platform); err != nil || recipe.Valid || platform.Valid {
|
|
||||||
t.Fatalf("GET certified legacy metadata: %v %v %v", recipe, platform, err)
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='queued'").Scan(&status); err != nil || status != "failed" {
|
|
||||||
t.Fatal(status, err)
|
|
||||||
}
|
|
||||||
seedVerifiedImage(t, a, "new-image", "0.2.0-rc.1")
|
|
||||||
verified, err = a.verifiedImages(context.Background())
|
|
||||||
if err != nil || len(verified) != 1 || verified["new-image"] != "0.2.0-rc.1" {
|
|
||||||
t.Fatal(verified, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageInspectIdentity(t *testing.T) {
|
|
||||||
for _, mode := range []string{"valid", "recipe", "wrong-key", "platform", "variant", "unknown-platform", "role", "version", "id", "missing", "unavailable"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != "GET" {
|
|
||||||
t.Fatal("unexpected mutation")
|
|
||||||
}
|
|
||||||
if mode == "missing" {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "unavailable" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
parts := strings.Split(imagePlatform, "/")
|
|
||||||
labels := map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}
|
|
||||||
im := map[string]any{"Id": "image", "Os": parts[0], "Architecture": parts[1], "Config": map[string]any{"Labels": labels}}
|
|
||||||
switch mode {
|
|
||||||
case "recipe":
|
|
||||||
labels["dsh-mgr.recipe"] = "old"
|
|
||||||
case "wrong-key":
|
|
||||||
delete(labels, "dsh-mgr.recipe")
|
|
||||||
labels["dsh-mgr.recipe_rev"] = imageRecipeRev
|
|
||||||
case "platform":
|
|
||||||
im["Architecture"] = "other"
|
|
||||||
case "variant":
|
|
||||||
im["Variant"] = "v7"
|
|
||||||
case "unknown-platform":
|
|
||||||
delete(im, "Architecture")
|
|
||||||
case "role":
|
|
||||||
labels["dsh-mgr.role"] = "other"
|
|
||||||
case "version":
|
|
||||||
labels["dsh-mgr.version"] = "2.0.0"
|
|
||||||
case "id":
|
|
||||||
im["Id"] = "other"
|
|
||||||
}
|
|
||||||
writeJSON(w, im)
|
|
||||||
})
|
|
||||||
ok, err := a.imageMatches(context.Background(), "image", "0.2.0-rc.1")
|
|
||||||
if ok != (mode == "valid") || (err != nil) != (mode == "unavailable") {
|
|
||||||
t.Fatal(ok, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageIndexLocalInventory(t *testing.T) {
|
|
||||||
for _, mode := range []string{"external", "old-recipe", "other-platform", "inspect-failure"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
var deleted atomic.Bool
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
t.Errorf("unexpected Docker mutation: %s", r.Method)
|
|
||||||
}
|
|
||||||
if r.URL.Path == "/v1.47/images/json" {
|
|
||||||
if deleted.Load() {
|
|
||||||
io.WriteString(w, "[]")
|
|
||||||
} else {
|
|
||||||
io.WriteString(w, `[{"Id":"image","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.recipe":"2","dsh-mgr.version":"9.9.9"}}]`)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "inspect-failure" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if deleted.Load() {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
parts := strings.Split(imagePlatform, "/")
|
|
||||||
im := inspectedImage{ID: "image", Os: parts[0], Architecture: parts[1], Created: "2026-01-01T00:00:00Z", RepoTags: []string{"dsh-web:0.2.0-rc.1"}}
|
|
||||||
im.Config.Labels = map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}
|
|
||||||
if mode == "old-recipe" {
|
|
||||||
im.Config.Labels["dsh-mgr.recipe"] = "1"
|
|
||||||
}
|
|
||||||
if mode == "other-platform" {
|
|
||||||
im.Architecture, im.Variant = "arm", "v7"
|
|
||||||
}
|
|
||||||
writeJSON(w, im)
|
|
||||||
})
|
|
||||||
if mode != "external" {
|
|
||||||
seedVerifiedImage(t, a, "image", "0.2.0-rc.1")
|
|
||||||
}
|
|
||||||
if mode == "old-recipe" {
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE image_builds SET recipe_rev='1'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := imageRequest(a.images, sid, "GET", "")
|
|
||||||
if mode == "inspect-failure" {
|
|
||||||
var status string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE image_id='image'").Scan(&status); err != nil || status != "success" || w.Code != 502 {
|
|
||||||
t.Fatal(status, w.Code, err)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
wantStatus := "unverified"
|
|
||||||
if mode == "old-recipe" {
|
|
||||||
wantStatus = "existing_only"
|
|
||||||
}
|
|
||||||
var images []imageRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != wantStatus || images[0].Version != "0.2.0-rc.1" || images[0].Created != "2026-01-01T00:00:00Z" || images[0].Tag != "dsh-web:0.2.0-rc.1" {
|
|
||||||
t.Fatal(w.Body.String(), err)
|
|
||||||
}
|
|
||||||
if mode == "other-platform" && images[0].Platform != strings.Split(imagePlatform, "/")[0]+"/arm/v7" {
|
|
||||||
t.Fatal(images[0].Platform)
|
|
||||||
}
|
|
||||||
if mode == "old-recipe" && images[0].RecipeRev != "1" {
|
|
||||||
t.Fatal(images[0].RecipeRev)
|
|
||||||
}
|
|
||||||
if w := imageRequest(a.instances, sid, "POST", `{"name":"rejected","image_id":"image"}`); w.Code != 400 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
deleted.Store(true)
|
|
||||||
w = imageRequest(a.images, sid, "GET", "")
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != "missing" {
|
|
||||||
t.Fatal(w.Body.String(), err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageIndexUsageMissingAndDockerFailure(t *testing.T) {
|
|
||||||
var mode atomic.Int32
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != "GET" {
|
|
||||||
t.Error("unexpected mutation")
|
|
||||||
}
|
|
||||||
if r.URL.Path == "/v1.47/images/json" && mode.Load() != 1 {
|
|
||||||
io.WriteString(w, "[]")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch mode.Load() {
|
|
||||||
case 1:
|
|
||||||
w.WriteHeader(500)
|
|
||||||
case 2:
|
|
||||||
w.WriteHeader(404)
|
|
||||||
default:
|
|
||||||
writeImageInspect(w, "image", "0.2.0-rc.1")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "image", "0.2.0-rc.1")
|
|
||||||
if _, err := a.s.DB.Exec(`INSERT INTO instances(id,uid,name,image_id,status,created) VALUES('one','u','one','image','running',0),('two','other-user','two','image','stopped',0),('three','u','three','other','running',0)`); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
check := func(status string, count int) {
|
|
||||||
t.Helper()
|
|
||||||
w := imageRequest(a.images, sid, "GET", "")
|
|
||||||
var images []imageRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != status || images[0].UsageCount != count {
|
|
||||||
t.Fatal(w.Body.String(), err)
|
|
||||||
}
|
|
||||||
var storedStatus string
|
|
||||||
var storedCount int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status,usage_count FROM images WHERE image_id='image'").Scan(&storedStatus, &storedCount); err != nil || storedStatus != status || storedCount != count {
|
|
||||||
t.Fatal(storedStatus, storedCount, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
check("available", 2)
|
|
||||||
mode.Store(1)
|
|
||||||
if w := imageRequest(a.images, sid, "GET", ""); w.Code != 502 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status FROM image_builds WHERE image_id='image'").Scan(&status); err != nil || status != "success" {
|
|
||||||
t.Fatal(status, err)
|
|
||||||
}
|
|
||||||
mode.Store(2)
|
|
||||||
check("missing", 2)
|
|
||||||
if _, err := a.s.DB.Exec("DELETE FROM instances WHERE id='two'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
check("missing", 1)
|
|
||||||
verified, err := a.verifiedImages(context.Background())
|
|
||||||
if err != nil || len(verified) != 0 {
|
|
||||||
t.Fatal(verified, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildRateLimitDoesNotPenalizeSharedTask(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/verified/json" {
|
|
||||||
writeImageInspect(w, "verified", "0.2.0-rc.1")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
io.WriteString(w, `[{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "verified", "0.2.0-rc.1")
|
|
||||||
for _, id := range []string{"a", "b", "c"} {
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_build_requests VALUES('u',?,?)", id, time.Now().UTC().Format(time.RFC3339Nano)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`); w.Code != 202 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.2"}`); w.Code != 429 || w.Header().Get("Retry-After") == "" {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
a.s.DB.QueryRow("SELECT count(*) FROM image_build_requests").Scan(&n)
|
|
||||||
if n != 3 {
|
|
||||||
t.Fatal(n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildDiskPolicy(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected build") })
|
|
||||||
if err := a.checkBuildDisk(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
probeErr := errors.New("capacity probe failed")
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
available uint64
|
|
||||||
err error
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{"insufficient", minimumBuildDiskBytes - 1, nil, "insufficient"},
|
|
||||||
{"exact", minimumBuildDiskBytes, nil, ""},
|
|
||||||
{"probe-error", 0, probeErr, "unavailable"},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
buildDiskAvailable = func(path string) (uint64, error) {
|
|
||||||
if path != os.Getenv("DSH_DOCKER_ROOT_VISIBLE") {
|
|
||||||
t.Fatalf("unexpected probe path %q", path)
|
|
||||||
}
|
|
||||||
return tc.available, tc.err
|
|
||||||
}
|
|
||||||
err := a.checkBuildDisk(context.Background())
|
|
||||||
if tc.want == "" && err != nil || tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)) {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if tc.err != nil && !errors.Is(err, tc.err) {
|
|
||||||
t.Fatalf("probe error not preserved: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
buildDiskAvailable = func(string) (uint64, error) {
|
|
||||||
t.Fatal("unconfirmed Docker root must not be probed")
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
t.Setenv("DSH_DOCKER_ROOT_VISIBLE", "")
|
|
||||||
if err := a.checkBuildDisk(context.Background()); err == nil || !strings.Contains(err.Error(), "unknown") {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if requireBuildDisk(minimumBuildDiskBytes-1) == nil || requireBuildDisk(minimumBuildDiskBytes) != nil {
|
|
||||||
t.Fatal("disk threshold")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUnknownDockerDiskBlocksBuild(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path != "/v1.47/images/json" {
|
|
||||||
t.Errorf("unexpected Docker operation %s", r.URL.Path)
|
|
||||||
}
|
|
||||||
io.WriteString(w, "[]")
|
|
||||||
})
|
|
||||||
t.Setenv("DSH_DOCKER_ROOT_VISIBLE", "")
|
|
||||||
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
|
|
||||||
if w.Code != 202 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(3 * time.Second)
|
|
||||||
for {
|
|
||||||
var status, detail string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status,error FROM image_builds").Scan(&status, &detail); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if status == "failed" {
|
|
||||||
if !strings.Contains(detail, "capacity unknown") {
|
|
||||||
t.Fatal(detail)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal(status, detail)
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestExactNPMMetadata(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
body string
|
|
||||||
status int
|
|
||||||
ok bool
|
|
||||||
}{
|
|
||||||
{`{"version":"0.2.0-rc.1"}`, 200, true},
|
|
||||||
{`{"version":"0.2.0-rc.2"}`, 200, false},
|
|
||||||
{`{"version":"latest"}`, 200, false},
|
|
||||||
{`{}`, 404, false},
|
|
||||||
{`invalid`, 200, false},
|
|
||||||
} {
|
|
||||||
client := &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
|
|
||||||
if r.URL.Host != "registry.npmjs.org" || !strings.HasSuffix(r.URL.Path, "/0.2.0-rc.1") {
|
|
||||||
t.Error(r.URL)
|
|
||||||
}
|
|
||||||
return &http.Response{StatusCode: test.status, Body: io.NopCloser(strings.NewReader(test.body)), Header: make(http.Header)}, nil
|
|
||||||
})}
|
|
||||||
_, err := fetchNPMVersion(context.Background(), client, "0.2.0-rc.1")
|
|
||||||
if (err == nil) != test.ok {
|
|
||||||
t.Fatalf("%s: %v", test.body, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildListReconcilesMissingAndLatestTimestamp(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) })
|
|
||||||
seedVerifiedImage(t, a, "deleted", "0.2.0-rc.1")
|
|
||||||
w := imageRequest(a.builds, sid, "GET", "")
|
|
||||||
var records []buildRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &records); err != nil || len(records) != 1 || records[0].Status != "missing" {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
w = imageRequest(a.latest, sid, "GET", "")
|
|
||||||
var latest struct {
|
|
||||||
Version string `json:"version"`
|
|
||||||
CheckedAt string `json:"checked_at"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &latest); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := time.Parse(time.RFC3339, latest.CheckedAt); err != nil || latest.Version != "0.2.0-rc.1" {
|
|
||||||
t.Fatal(latest, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The one-second intent poller must reuse per-image evaluations instead of
|
|
||||||
// re-inspecting Docker every tick, and a generation bump (deletion or
|
|
||||||
// admission change) must invalidate the cache.
|
|
||||||
func TestIntentPollerCachesImageEvaluations(t *testing.T) {
|
|
||||||
inspects := 0
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
inspects++
|
|
||||||
writeImageInspect(w, "image", "0.2.0-rc.1")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(404)
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "image", "0.2.0-rc.1")
|
|
||||||
for n := 0; n < 3; n++ {
|
|
||||||
verified, err := a.verifiedImagesPoll(context.Background())
|
|
||||||
if err != nil || len(verified) != 1 || verified["image"] != "0.2.0-rc.1" {
|
|
||||||
t.Fatal(verified, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if inspects != 1 {
|
|
||||||
t.Fatalf("poller re-inspected Docker %d times for the same image", inspects)
|
|
||||||
}
|
|
||||||
buildQueue.Lock()
|
|
||||||
a.imageDeleteGeneration++
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if _, err := a.verifiedImagesPoll(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if inspects != 2 {
|
|
||||||
t.Fatalf("generation bump did not invalidate the cache: %d inspects", inspects)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,501 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"archive/tar"
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func imageTestAPI(t *testing.T, h http.HandlerFunc) (*API, string) {
|
|
||||||
t.Helper()
|
|
||||||
oldDiskAvailable := buildDiskAvailable
|
|
||||||
buildDiskAvailable = func(string) (uint64, error) { return minimumBuildDiskBytes, nil }
|
|
||||||
t.Cleanup(func() { buildDiskAvailable = oldDiskAvailable })
|
|
||||||
oldClient := npmBuildClient
|
|
||||||
npmBuildClient = &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
|
|
||||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"version":"0.2.0-rc.1"}`))}, nil
|
|
||||||
})}
|
|
||||||
t.Cleanup(func() { npmBuildClient = oldClient })
|
|
||||||
root := t.TempDir()
|
|
||||||
t.Setenv("DSH_DOCKER_ROOT_VISIBLE", root)
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/info" {
|
|
||||||
writeJSON(w, map[string]string{"DockerRootDir": root})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
h(w, r)
|
|
||||||
}))
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
s, err := store.Open(filepath.Join(t.TempDir(), "test.db"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
s.DB.SetMaxOpenConns(1)
|
|
||||||
a := New(s, t.TempDir())
|
|
||||||
a.docker = docker.New(srv.URL)
|
|
||||||
t.Cleanup(func() {
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
buildQueue.Lock()
|
|
||||||
active := buildQueue.active[s.DB]
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if !active {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Error("worker did not finish")
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
s.Close()
|
|
||||||
})
|
|
||||||
if err = s.CreateUser(store.User{ID: "u", Username: "user"}, "unused"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sid, _, err := s.Session("u")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
registerTestAccess(t, a, sid)
|
|
||||||
return a, sid
|
|
||||||
}
|
|
||||||
func imageRequest(h http.HandlerFunc, sid, method, body string) *httptest.ResponseRecorder {
|
|
||||||
r := httptest.NewRequest(method, "/", strings.NewReader(body))
|
|
||||||
if sid != "" {
|
|
||||||
r.AddCookie(&http.Cookie{Name: "dsh_session", Value: sid})
|
|
||||||
addTestAccess(r, sid)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h(w, r)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
func TestImageListContract(t *testing.T) {
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/unverified/json" {
|
|
||||||
writeImageInspect(w, "unverified", "0.2.0-rc.1")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.URL.Path == "/v1.47/images/sha256:abc/json" {
|
|
||||||
writeImageInspect(w, "sha256:abc", "0.2.0-rc.1")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !strings.Contains(r.URL.Query().Get("filters"), "dsh-mgr.role=runtime") {
|
|
||||||
t.Error("missing label filter")
|
|
||||||
}
|
|
||||||
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"},"RepoTags":["runtime:0.2.0-rc.1"]},{"Id":"other","Labels":{}},{"Id":"unverified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"9.9.9"}}]`)
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "sha256:abc", "0.2.0-rc.1")
|
|
||||||
if w := imageRequest(a.images, "", "GET", ""); w.Code != 401 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
w := imageRequest(a.images, sid, "GET", "")
|
|
||||||
var images []imageRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if len(images) != 2 || images[0].ID != "sha256:abc" || images[0].ImageID != images[0].ID || images[0].Status != "available" || images[1].Status != "unverified" {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
if w = imageRequest(a.images, sid, "POST", ""); w.Code != 405 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func seedVerifiedImage(t *testing.T, a *API, id, version string) {
|
|
||||||
t.Helper()
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
if err := a.ensureBuilds(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,image_id,created_at,recipe_rev,platform) VALUES(?,?,'success',?,'now',?,?)", store.ID(), version, id, imageRecipeRev, imagePlatform); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT OR REPLACE INTO image_admissions(image_id,version,recipe_rev,platform,state,evidence,admitted_by,admitted_at) VALUES(?,?,?,?,'compatible','test fixture explicit acceptance','test','now')", id, version, imageRecipeRev, imagePlatform); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestImageContextAndSanitizing(t *testing.T) {
|
|
||||||
r, err := imageBuildContext()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
tr := tar.NewReader(r)
|
|
||||||
n := 0
|
|
||||||
for {
|
|
||||||
h, err := tr.Next()
|
|
||||||
if err == io.EOF {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
data, _ := io.ReadAll(tr)
|
|
||||||
original, err := os.ReadFile(filepath.Join("..", "..", "image", h.Name))
|
|
||||||
if err != nil || string(data) != string(original) {
|
|
||||||
t.Fatalf("recipe drift %s: %v", h.Name, err)
|
|
||||||
}
|
|
||||||
if h.Name == "Dockerfile" && (!strings.Contains(string(data), "ARG RECIPE_REV="+imageRecipeRev+"\n") || !strings.Contains(string(data), "dsh-mgr.recipe=$RECIPE_REV")) {
|
|
||||||
t.Fatal("recipe revision or label differs from image policy")
|
|
||||||
}
|
|
||||||
n++
|
|
||||||
}
|
|
||||||
if n != 7 {
|
|
||||||
t.Fatal(n)
|
|
||||||
}
|
|
||||||
s := sanitizeBuildLog("token=topsecret Authorization: Bearer hidden password=private https://user:[email protected]/")
|
|
||||||
for _, secret := range []string{"topsecret", "hidden", "private", "user:pass"} {
|
|
||||||
if strings.Contains(s, secret) {
|
|
||||||
t.Fatal(s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(sanitizeBuildLog(strings.Repeat("x", 2*buildLogLimit))) > buildLogLimit {
|
|
||||||
t.Fatal("unbounded")
|
|
||||||
}
|
|
||||||
for _, v := range []string{"latest", "*", "^0.2.0-rc.1", "1.2", "0.2.0-rc.1;id", "https://example.com", ""} {
|
|
||||||
if validBuildVersion(v) {
|
|
||||||
t.Fatal(v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !validBuildVersion("0.2.0-rc.1-beta.1") {
|
|
||||||
t.Fatal("fixed prerelease rejected")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type imageRoundTrip func(*http.Request) (*http.Response, error)
|
|
||||||
|
|
||||||
func (f imageRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
|
||||||
func TestLatestOfficialRegistry(t *testing.T) {
|
|
||||||
client := &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
|
|
||||||
if r.URL.Host != "registry.npmjs.org" || r.URL.Scheme != "https" {
|
|
||||||
t.Fatal(r.URL)
|
|
||||||
}
|
|
||||||
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"version":"0.2.0-rc.1"}`)), Header: make(http.Header)}, nil
|
|
||||||
})}
|
|
||||||
v, err := fetchLatest(context.Background(), client)
|
|
||||||
if err != nil || v != "0.2.0-rc.1" {
|
|
||||||
t.Fatalf("%s %v", v, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestBuildDedupAndVerification(t *testing.T) {
|
|
||||||
var calls, deleted atomic.Int32
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
|
|
||||||
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
|
|
||||||
case "/v1.47/images/json":
|
|
||||||
if calls.Load() == 0 {
|
|
||||||
io.WriteString(w, "[]")
|
|
||||||
} else {
|
|
||||||
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
|
|
||||||
}
|
|
||||||
case "/v1.47/build":
|
|
||||||
calls.Add(1)
|
|
||||||
if r.URL.Query().Get("t") != "dsh-web:0.2.0-rc.1-"+imageRecipeRev {
|
|
||||||
t.Error("unexpected tag", r.URL.Query())
|
|
||||||
}
|
|
||||||
var args map[string]string
|
|
||||||
json.Unmarshal([]byte(r.URL.Query().Get("buildargs")), &args)
|
|
||||||
if args["DSH_VERSION"] != "0.2.0-rc.1" {
|
|
||||||
t.Error(args)
|
|
||||||
}
|
|
||||||
io.Copy(io.Discard, r.Body)
|
|
||||||
io.WriteString(w, `{"stream":"token=secretvalue"}`+string(rune(10))+`{"aux":{"ID":"sha256:abc"}}`)
|
|
||||||
case "/v1.47/containers/create":
|
|
||||||
var cfg struct {
|
|
||||||
Image string
|
|
||||||
HostConfig struct {
|
|
||||||
NetworkMode string
|
|
||||||
ReadonlyRootfs bool
|
|
||||||
Tmpfs map[string]string
|
|
||||||
}
|
|
||||||
}
|
|
||||||
json.NewDecoder(r.Body).Decode(&cfg)
|
|
||||||
if cfg.Image != "sha256:abc" || cfg.HostConfig.NetworkMode != "none" || !cfg.HostConfig.ReadonlyRootfs || !strings.Contains(cfg.HostConfig.Tmpfs["/tmp"], "exec") {
|
|
||||||
t.Errorf("unsafe config %+v", cfg)
|
|
||||||
}
|
|
||||||
io.WriteString(w, `{"Id":"check"}`)
|
|
||||||
case "/v1.47/containers/check/start", "/v1.47/exec/probe/start":
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case "/v1.47/containers/check/exec":
|
|
||||||
var cfg struct{ Cmd []string }
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil || len(cfg.Cmd) != 3 || !strings.Contains(cfg.Cmd[2], "execFileSync('dsh',['--version']") || !strings.Contains(cfg.Cmd[2], `actual!=="0.2.0-rc.1"`) {
|
|
||||||
t.Errorf("missing version verification: %+v %v", cfg, err)
|
|
||||||
}
|
|
||||||
io.WriteString(w, `{"Id":"probe"}`)
|
|
||||||
case "/v1.47/exec/probe/json":
|
|
||||||
io.WriteString(w, `{"Running":false,"ExitCode":0}`)
|
|
||||||
case "/v1.47/containers/check/json":
|
|
||||||
io.WriteString(w, `{"State":{"Running":true}}`)
|
|
||||||
case "/v1.47/containers/check":
|
|
||||||
deleted.Add(1)
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s", r.URL)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
responses := make(chan *httptest.ResponseRecorder, 8)
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
responses <- imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(responses)
|
|
||||||
first := ""
|
|
||||||
for w := range responses {
|
|
||||||
if w.Code != 202 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
var b buildRecord
|
|
||||||
json.Unmarshal(w.Body.Bytes(), &b)
|
|
||||||
if first == "" {
|
|
||||||
first = b.ID
|
|
||||||
}
|
|
||||||
if first != b.ID {
|
|
||||||
t.Fatal("duplicate job")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
w := imageRequest(a.builds, sid, "GET", "")
|
|
||||||
var jobs []buildRecord
|
|
||||||
json.Unmarshal(w.Body.Bytes(), &jobs)
|
|
||||||
if len(jobs) == 1 && jobs[0].Status == "success" {
|
|
||||||
if strings.Contains(jobs[0].Logs, "secretvalue") {
|
|
||||||
t.Fatal(jobs)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
if calls.Load() != 1 || deleted.Load() != 1 {
|
|
||||||
t.Fatalf("builds %d cleanup %d", calls.Load(), deleted.Load())
|
|
||||||
}
|
|
||||||
for _, body := range []string{`{"version":"latest"}`, `{"version":"0.2.0-rc.1","url":"bad"}`, `{"version":"0.2.0-rc.1"} {}`} {
|
|
||||||
if w := imageRequest(a.builds, sid, "POST", body); w.Code != 400 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestVerificationFailureCleansUp(t *testing.T) {
|
|
||||||
var deleted atomic.Bool
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/create"):
|
|
||||||
io.WriteString(w, `{"Id":"bad"}`)
|
|
||||||
case r.Method == "DELETE":
|
|
||||||
deleted.Store(true)
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
w.WriteHeader(500)
|
|
||||||
io.WriteString(w, `{"message":"cannot start"}`)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err := a.verifyBuiltImage(context.Background(), "bad", "0.2.0-rc.1"); err == nil {
|
|
||||||
t.Fatal("accepted failed start")
|
|
||||||
}
|
|
||||||
if !deleted.Load() {
|
|
||||||
t.Fatal("container leaked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildReuseAndMissingImage(t *testing.T) {
|
|
||||||
var exists atomic.Bool
|
|
||||||
exists.Store(true)
|
|
||||||
var builds, checks atomic.Int32
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
|
|
||||||
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
|
|
||||||
case "/v1.47/images/json":
|
|
||||||
if exists.Load() {
|
|
||||||
io.WriteString(w, `[{"Id":"sha256:reuse","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
|
|
||||||
} else {
|
|
||||||
io.WriteString(w, "[]")
|
|
||||||
}
|
|
||||||
case "/v1.47/build":
|
|
||||||
builds.Add(1)
|
|
||||||
exists.Store(true)
|
|
||||||
io.Copy(io.Discard, r.Body)
|
|
||||||
io.WriteString(w, `{"aux":{"ID":"sha256:reuse"}}`)
|
|
||||||
case "/v1.47/containers/create":
|
|
||||||
checks.Add(1)
|
|
||||||
io.WriteString(w, `{"Id":"c"}`)
|
|
||||||
case "/v1.47/containers/c/start", "/v1.47/exec/e/start", "/v1.47/containers/c":
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case "/v1.47/containers/c/exec":
|
|
||||||
io.WriteString(w, `{"Id":"e"}`)
|
|
||||||
case "/v1.47/exec/e/json":
|
|
||||||
io.WriteString(w, `{"Running":false,"ExitCode":0}`)
|
|
||||||
case "/v1.47/containers/c/json":
|
|
||||||
io.WriteString(w, `{"State":{"Running":true}}`)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s", r.URL)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
wait := func(id string) {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
var status, image string
|
|
||||||
err := a.s.DB.QueryRow("SELECT status,image_id FROM image_builds WHERE id=?", id).Scan(&status, &image)
|
|
||||||
if err == nil && status == "success" {
|
|
||||||
if image != "sha256:reuse" {
|
|
||||||
t.Fatal(image)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("status %s: %v", status, err)
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
post := func() buildRecord {
|
|
||||||
t.Helper()
|
|
||||||
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
|
|
||||||
if w.Code != 202 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
var b buildRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &b); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
first := post()
|
|
||||||
wait(first.ID)
|
|
||||||
if builds.Load() != 0 || checks.Load() != 1 {
|
|
||||||
t.Fatal("local image not reused and verified")
|
|
||||||
}
|
|
||||||
exists.Store(false)
|
|
||||||
second := post()
|
|
||||||
if second.ID == first.ID {
|
|
||||||
t.Fatal("missing image deduplicated")
|
|
||||||
}
|
|
||||||
wait(second.ID)
|
|
||||||
if builds.Load() != 1 || checks.Load() != 2 {
|
|
||||||
t.Fatal("missing image not rebuilt")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestBuildRestartRecovery(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) })
|
|
||||||
buildQueue.Lock()
|
|
||||||
defer buildQueue.Unlock()
|
|
||||||
if err := a.ensureBuilds(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('old','0.2.0-rc.1','running','now')"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
delete(buildQueue.initialized, a.s.DB)
|
|
||||||
if err := a.ensureBuilds(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var status, detail string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT status,error FROM image_builds WHERE id='old'").Scan(&status, &detail); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if status != "failed" || !strings.Contains(detail, "restart") {
|
|
||||||
t.Fatalf("%s %s", status, detail)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFailedVerificationNeverAvailable(t *testing.T) {
|
|
||||||
var deleted atomic.Bool
|
|
||||||
imageID := "sha256:" + strings.Repeat("c", 64)
|
|
||||||
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
|
|
||||||
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
|
|
||||||
case "/v1.47/images/json":
|
|
||||||
writeJSON(w, []any{map[string]any{"Id": imageID, "Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}}})
|
|
||||||
case "/v1.47/containers/create":
|
|
||||||
io.WriteString(w, `{"Id":"check"}`)
|
|
||||||
case "/v1.47/containers/check/start", "/v1.47/exec/probe/start":
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case "/v1.47/containers/check/exec":
|
|
||||||
io.WriteString(w, `{"Id":"probe"}`)
|
|
||||||
case "/v1.47/exec/probe/json":
|
|
||||||
io.WriteString(w, `{"Running":false,"ExitCode":1}`)
|
|
||||||
case "/v1.47/containers/check":
|
|
||||||
deleted.Store(true)
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case "/v1.47/images/" + imageID + "/json":
|
|
||||||
writeImageInspect(w, imageID, "0.2.0-rc.1")
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected Docker call %s", r.URL)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
|
|
||||||
if w.Code != 202 {
|
|
||||||
t.Fatal(w.Body.String())
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
var status, id, detail string
|
|
||||||
err := a.s.DB.QueryRow("SELECT status,image_id,error FROM image_builds").Scan(&status, &id, &detail)
|
|
||||||
if err == nil && status == "failed" {
|
|
||||||
if id != imageID || !strings.Contains(detail, "verification failed") || !deleted.Load() {
|
|
||||||
t.Fatalf("%s %s cleanup=%v", id, detail, deleted.Load())
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("status=%s error=%v", status, err)
|
|
||||||
}
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
w = imageRequest(a.images, sid, "GET", "")
|
|
||||||
var images []imageRecord
|
|
||||||
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != "unverified" {
|
|
||||||
t.Fatal("failed image advertised", w.Body.String())
|
|
||||||
}
|
|
||||||
w = imageRequest(a.instances, sid, "POST", fmt.Sprintf(`{"name":"rejected","image_id":%q}`, imageID))
|
|
||||||
if w.Code != 400 {
|
|
||||||
t.Fatal("unverified image accepted", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalBuildImagePrefersPersistedID(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/verified/json" {
|
|
||||||
writeImageInspect(w, "verified", "0.2.0-rc.1")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
io.WriteString(w, `[{"Id":"newer","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
|
|
||||||
})
|
|
||||||
id, err := a.localBuildImage(context.Background(), "0.2.0-rc.1", "verified")
|
|
||||||
if err != nil || id != "verified" {
|
|
||||||
t.Fatalf("id=%s err=%v", id, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeImageInspect(w http.ResponseWriter, id, version string) {
|
|
||||||
parts := strings.Split(imagePlatform, "/")
|
|
||||||
writeJSON(w, map[string]any{"Id": id, "Os": parts[0], "Architecture": parts[1], "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": version, "dsh-mgr.recipe": imageRecipeRev}}})
|
|
||||||
}
|
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *API) intents(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = a.s.EnsureIntents(); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method {
|
|
||||||
case http.MethodGet:
|
|
||||||
list, err := a.s.Intents(u.ID)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, list)
|
|
||||||
case http.MethodPost:
|
|
||||||
var in struct {
|
|
||||||
Name string `json:"name"`
|
|
||||||
Version string `json:"version"`
|
|
||||||
}
|
|
||||||
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
|
|
||||||
dec.DisallowUnknownFields()
|
|
||||||
if dec.Decode(&in) != nil || dec.Decode(new(any)) != io.EOF || strings.TrimSpace(in.Name) == "" || len(in.Name) > 200 || !validBuildVersion(in.Version) {
|
|
||||||
fail(w, 400, "name and fixed version required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !normalBuildAllowed(in.Version) {
|
|
||||||
fail(w, 400, errBuildUnsupported.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
i := store.Intent{ID: store.ID(), UID: u.ID, Name: strings.TrimSpace(in.Name), Version: in.Version, Host: a.runtimeHost(r), Status: "waiting"}
|
|
||||||
if err = a.s.ReserveIntent(i); err != nil {
|
|
||||||
if errors.Is(err, store.ErrInstanceLimit) {
|
|
||||||
fail(w, 409, err.Error())
|
|
||||||
} else {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(http.StatusAccepted)
|
|
||||||
writeJSON(w, i)
|
|
||||||
default:
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func (a *API) intent(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != http.MethodDelete {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err = a.s.EnsureIntents(); err != nil {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := strings.TrimPrefix(r.URL.Path, "/api/intents/")
|
|
||||||
if err = a.s.CancelIntent(id, u.ID); err != nil {
|
|
||||||
if errors.Is(err, store.ErrIntentState) {
|
|
||||||
fail(w, 409, "intent not found or no longer waiting")
|
|
||||||
} else {
|
|
||||||
fail(w, 500, "database error")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]string{"status": "cancelled"})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Call once at startup, after the build queue has recovered interrupted jobs.
|
|
||||||
func (a *API) StartIntents(ctx context.Context) error {
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := a.recoverIntents(ctx); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
ticker := time.NewTicker(time.Second)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
a.processIntents(ctx)
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) finishIntent(id, status, detail string) {
|
|
||||||
_, _ = a.s.DB.Exec(`UPDATE creation_intents SET status=?,error=? WHERE id=? AND status IN ('waiting','creating')`, status, detail, id)
|
|
||||||
}
|
|
||||||
func (a *API) recoverIntents(ctx context.Context) error {
|
|
||||||
list, err := a.s.Intents("")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, in := range list {
|
|
||||||
if in.Status != "creating" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
i, err := a.s.GetInstance(in.InstanceID)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
a.finishIntent(in.ID, "failed", "instance reservation missing after process restart")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err == nil && i.Status == "running" {
|
|
||||||
a.finishIntent(in.ID, "success", "")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
cleanup, cancel := context.WithTimeout(ctx, 30*time.Second)
|
|
||||||
if err == nil && i.ContainerID == "" {
|
|
||||||
err = a.recoverRuntimeContainer(cleanup, &i)
|
|
||||||
}
|
|
||||||
if err == nil && i.ContainerID != "" {
|
|
||||||
err = a.removeRuntimeContainer(cleanup, i.ContainerID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = a.removeRuntimeNetwork(cleanup, in.InstanceID)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
if err != nil {
|
|
||||||
a.s.SetInstance(in.InstanceID, "error")
|
|
||||||
a.finishIntent(in.ID, "failed", "instance creation requires recovery: "+err.Error())
|
|
||||||
a.s.AddAudit(in.UID, "instance.recovery_error", in.InstanceID, err.Error())
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
a.s.DeleteInstance(in.InstanceID)
|
|
||||||
_, _ = a.s.DB.Exec("DELETE FROM runtime_creates WHERE instance_id=?", in.InstanceID)
|
|
||||||
_, _ = a.s.DB.Exec("DELETE FROM runtime_networks WHERE instance_id=?", in.InstanceID)
|
|
||||||
a.finishIntent(in.ID, "failed", "instance creation interrupted by process restart")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) processIntents(ctx context.Context) {
|
|
||||||
list, err := a.s.Intents("")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, in := range list {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if in.Status != "waiting" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if in.BuildID == "" {
|
|
||||||
b, err := a.enqueueBuild(ctx, in.UID, in.Version)
|
|
||||||
if err != nil {
|
|
||||||
a.finishIntent(in.ID, "failed", "build submission failed")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec(`UPDATE creation_intents SET build_id=? WHERE id=? AND status='waiting'`, b.ID, in.ID); err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
in.BuildID = b.ID
|
|
||||||
}
|
|
||||||
var status, imageID, version string
|
|
||||||
err := a.s.DB.QueryRow(`SELECT status,image_id,version FROM image_builds WHERE id=?`, in.BuildID).Scan(&status, &imageID, &version)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if version != in.Version || status == "failed" || status == "cancelled" || status == "missing" {
|
|
||||||
a.finishIntent(in.ID, "failed", "build failed; requested version was not created")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if status != "success" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
work, cancel := context.WithTimeout(ctx, 90*time.Second)
|
|
||||||
verified, err := a.verifiedImagesPoll(work)
|
|
||||||
if err != nil {
|
|
||||||
cancel()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if !runtimeImageID.MatchString(imageID) || verified[imageID] != in.Version {
|
|
||||||
cancel()
|
|
||||||
a.finishIntent(in.ID, "failed", "verified image for requested version unavailable")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
u, err := a.s.UserByID(in.UID)
|
|
||||||
if err != nil {
|
|
||||||
cancel()
|
|
||||||
a.finishIntent(in.ID, "failed", "owner unavailable")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
i := store.Instance{ID: store.ID(), UID: in.UID, Name: in.Name, Version: in.Version, ImageID: imageID, Status: "creating", Created: time.Now()}
|
|
||||||
if err = a.s.ClaimIntent(in.ID, i); err != nil {
|
|
||||||
cancel()
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_, err = a.createRuntime(work, u, i, in.Host)
|
|
||||||
cancel()
|
|
||||||
if err != nil {
|
|
||||||
a.finishIntent(in.ID, "failed", err.Error())
|
|
||||||
} else {
|
|
||||||
a.finishIntent(in.ID, "success", "")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,166 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestIntentsReserveCancel(t *testing.T) {
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("reservation contacted Docker") })
|
|
||||||
if err := a.initAuth(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for n := 0; n < 3; n++ {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.intents(w, runtimeRequest("POST", "/api/intents", `{"name":"one","version":"0.2.0-rc.1"}`, sid))
|
|
||||||
want := 202
|
|
||||||
if n == 2 {
|
|
||||||
want = 409
|
|
||||||
}
|
|
||||||
if w.Code != want {
|
|
||||||
t.Fatalf("%d: %d %s", n, w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
list, err := a.s.Intents("owner")
|
|
||||||
if err != nil || len(list) != 2 {
|
|
||||||
t.Fatalf("%v %v", list, err)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.intent(w, runtimeRequest("DELETE", "/api/intents/"+list[0].ID, "", sid))
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: store.ID(), UID: "owner"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntentBuildSuccessUsesRealRuntime(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("runtime filesystem ownership requires root")
|
|
||||||
}
|
|
||||||
image := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
creates := 0
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.Contains(r.URL.Path, "/images/"):
|
|
||||||
writeJSON(w, map[string]any{"Id": image, "Os": "linux", "Architecture": "amd64", "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": "0.2.0-rc.1", "dsh-mgr.recipe": imageRecipeRev}}})
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
|
||||||
creates++
|
|
||||||
writeJSON(w, map[string]string{"Id": "runtime-created"})
|
|
||||||
default:
|
|
||||||
w.WriteHeader(204)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, image, "0.2.0-rc.1")
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var build string
|
|
||||||
if err := a.s.DB.QueryRow("SELECT id FROM image_builds WHERE version='0.2.0-rc.1'").Scan(&build); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
in := store.Intent{ID: store.ID(), UID: "owner", Name: "one", Version: "0.2.0-rc.1", Host: "panel.test"}
|
|
||||||
if err := a.s.ReserveIntent(in); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE creation_intents SET build_id=? WHERE id=?", build, in.ID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.processIntents(context.Background())
|
|
||||||
a.processIntents(context.Background())
|
|
||||||
list, err := a.s.Intents("owner")
|
|
||||||
if err != nil || len(list) != 1 || list[0].Status != "success" || creates != 1 {
|
|
||||||
t.Fatalf("%+v err=%v creates=%d", list, err, creates)
|
|
||||||
}
|
|
||||||
instances, err := a.s.Instances("owner")
|
|
||||||
if err != nil || len(instances) != 1 || instances[0].ImageID != image || instances[0].Version != "0.2.0-rc.1" {
|
|
||||||
t.Fatalf("%+v %v", instances, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntentFailedBuildReleasesAndCancelPreservesBuild(t *testing.T) {
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("failed intent contacted Docker") })
|
|
||||||
seedVerifiedImage(t, a, "sha256:"+strings.Repeat("b", 64), "0.2.0-rc.1")
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var build string
|
|
||||||
a.s.DB.QueryRow("SELECT id FROM image_builds LIMIT 1").Scan(&build)
|
|
||||||
for _, id := range []string{"cancel", "fail"} {
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: id, UID: "owner", Version: "0.2.0-rc.1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.s.DB.Exec("UPDATE creation_intents SET build_id=? WHERE id=?", build, id)
|
|
||||||
}
|
|
||||||
if err := a.s.CancelIntent("cancel", "owner"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id=?", build).Scan(&status)
|
|
||||||
if status != "success" {
|
|
||||||
t.Fatal(status)
|
|
||||||
}
|
|
||||||
a.s.DB.Exec("UPDATE image_builds SET status='failed' WHERE id=?", build)
|
|
||||||
a.processIntents(context.Background())
|
|
||||||
for n := 0; n < 2; n++ {
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: store.ID(), UID: "owner"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntentRestartCleansInterruptedCreationAndRetainsWaiting(t *testing.T) {
|
|
||||||
deletes := 0
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == "GET" && r.URL.Path == "/v1.47/networks/dsh-inst-reserved" {
|
|
||||||
w.Write([]byte(`{"Id":"network","Name":"dsh-inst-reserved","Driver":"bridge","Labels":{"dsh-mgr.instance":"reserved"}}`))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method == "GET" && r.URL.Path == "/v1.47/containers/dsh-inst-reserved/json" {
|
|
||||||
w.Write([]byte(`{"Id":"recovered","Name":"/dsh-inst-reserved","Config":{"Labels":{"dsh-mgr.instance":"reserved","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method == "DELETE" {
|
|
||||||
deletes++
|
|
||||||
}
|
|
||||||
w.WriteHeader(204)
|
|
||||||
})
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, id := range []string{"waiting", "creating"} {
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: id, UID: "owner", Version: "0.2.0-rc.1"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := a.s.ClaimIntent("creating", store.Instance{ID: "reserved", UID: "owner", Version: "0.2.0-rc.1", Status: "creating"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.recoverIntents(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if deletes != 2 {
|
|
||||||
t.Fatalf("cleanup calls=%d", deletes)
|
|
||||||
}
|
|
||||||
list, err := a.s.Intents("owner")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, in := range list {
|
|
||||||
if in.ID == "waiting" && in.Status != "waiting" {
|
|
||||||
t.Fatal(in)
|
|
||||||
}
|
|
||||||
if in.ID == "creating" && in.Status != "failed" {
|
|
||||||
t.Fatal(in)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: store.ID(), UID: "owner"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,56 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import "sync"
|
|
||||||
|
|
||||||
// keyedLocks hands out one mutex per string key so slow work for one instance
|
|
||||||
// or user never blocks unrelated work behind a shared global lock. Registry
|
|
||||||
// entries are removed once the last holder releases, keeping the map bounded.
|
|
||||||
type keyedLocks struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
locks map[string]*keyedLock
|
|
||||||
}
|
|
||||||
|
|
||||||
type keyedLock struct {
|
|
||||||
owner *keyedLocks
|
|
||||||
key string
|
|
||||||
mu sync.Mutex
|
|
||||||
refs int
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acquire registers interest in the key's lock (blocking callers keep the
|
|
||||||
// registry entry alive) and then blocks until the mutex is held.
|
|
||||||
func (k *keyedLocks) Acquire(key string) *keyedLock {
|
|
||||||
k.mu.Lock()
|
|
||||||
if k.locks == nil {
|
|
||||||
k.locks = make(map[string]*keyedLock)
|
|
||||||
}
|
|
||||||
l, ok := k.locks[key]
|
|
||||||
if !ok {
|
|
||||||
l = &keyedLock{owner: k, key: key}
|
|
||||||
k.locks[key] = l
|
|
||||||
}
|
|
||||||
l.refs++
|
|
||||||
k.mu.Unlock()
|
|
||||||
l.mu.Lock()
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
|
|
||||||
// Release frees the mutex and drops the registry entry when no holder or
|
|
||||||
// waiter remains.
|
|
||||||
func (l *keyedLock) Release() {
|
|
||||||
l.mu.Unlock()
|
|
||||||
k := l.owner
|
|
||||||
k.mu.Lock()
|
|
||||||
l.refs--
|
|
||||||
if l.refs == 0 {
|
|
||||||
delete(k.locks, l.key)
|
|
||||||
}
|
|
||||||
k.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// held reports whether any keyed lock is currently held or being waited for.
|
|
||||||
func (k *keyedLocks) held() bool {
|
|
||||||
k.mu.Lock()
|
|
||||||
defer k.mu.Unlock()
|
|
||||||
return len(k.locks) > 0
|
|
||||||
}
|
|
||||||
@@ -1,144 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var orphanID = regexp.MustCompile(`^[a-f0-9]{64}$`)
|
|
||||||
var orphanLabelID = regexp.MustCompile(`^[a-f0-9]{32}$`)
|
|
||||||
|
|
||||||
func (a *API) checkOrphan(ctx context.Context, id string) error {
|
|
||||||
var c struct {
|
|
||||||
ID, Name string
|
|
||||||
Config struct{ Labels map[string]string }
|
|
||||||
State struct {
|
|
||||||
Status string
|
|
||||||
Running, Paused, Restarting bool
|
|
||||||
}
|
|
||||||
Mounts []struct{ Type, Source, Destination string }
|
|
||||||
}
|
|
||||||
if !orphanID.MatchString(id) {
|
|
||||||
return errors.New("invalid container ID")
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, "GET", "/containers/"+id+"/json", nil, &c); err != nil {
|
|
||||||
return errors.New("container unavailable")
|
|
||||||
}
|
|
||||||
inst, owner := c.Config.Labels["dsh-mgr.instance"], c.Config.Labels["dsh-mgr.owner"]
|
|
||||||
if c.ID != id || c.Config.Labels["dsh-mgr.role"] != "dsh" || !orphanLabelID.MatchString(inst) || !orphanLabelID.MatchString(owner) || strings.TrimPrefix(c.Name, "/") != runtimeNetwork(inst) {
|
|
||||||
return errors.New("container identity not eligible")
|
|
||||||
}
|
|
||||||
if c.State.Running || c.State.Paused || c.State.Restarting || (c.State.Status != "exited" && c.State.Status != "created") {
|
|
||||||
return errors.New("only stopped containers can be reclaimed")
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM instances WHERE id=? OR container_id=?", inst, id).Scan(&n); err != nil || n != 0 {
|
|
||||||
return errors.New("instance reference or database unavailable")
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM users WHERE id=?", owner).Scan(&n); err != nil || n != 1 {
|
|
||||||
return errors.New("unknown owner")
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='upgrade_journal'").Scan(&n); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if n != 0 {
|
|
||||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM upgrade_journal WHERE instance_id=? OR old_container=? OR new_container=?", inst, id, id).Scan(&n); err != nil || n != 0 {
|
|
||||||
return errors.New("upgrade reference or database unavailable")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
host := os.Getenv("HOST_DATA_ROOT")
|
|
||||||
if host == "" {
|
|
||||||
host = a.root
|
|
||||||
}
|
|
||||||
if !filepath.IsAbs(host) {
|
|
||||||
return errors.New("host root unavailable")
|
|
||||||
}
|
|
||||||
expected := map[string]string{"/workspace": filepath.Join(host, "workspaces", owner, "instances", inst, "home")}
|
|
||||||
// Accept complete historical layouts, never arbitrary bind sources.
|
|
||||||
for _, m := range c.Mounts {
|
|
||||||
if m.Destination == "/data/dsh-home" {
|
|
||||||
expected = map[string]string{"/workspace": filepath.Join(host, "workspaces", owner, "files"), "/data/dsh-home": filepath.Join(host, "workspaces", owner, "instances", inst, "home")}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, m := range c.Mounts {
|
|
||||||
if m.Destination == "/shared" {
|
|
||||||
expected["/shared"] = filepath.Join(host, "workspaces", owner, "files")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, m := range c.Mounts {
|
|
||||||
if m.Type == "tmpfs" && m.Destination == "/tmp" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
want, ok := expected[m.Destination]
|
|
||||||
if !ok || m.Type != "bind" || filepath.Clean(m.Source) != want {
|
|
||||||
return errors.New("mount identity not eligible")
|
|
||||||
}
|
|
||||||
delete(expected, m.Destination)
|
|
||||||
}
|
|
||||||
if len(expected) != 0 {
|
|
||||||
return errors.New("required mounts missing")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) orphans(w http.ResponseWriter, r *http.Request) {
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !u.Admin {
|
|
||||||
fail(w, 403, "administrator required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
// checkOrphan re-verifies instance rows, upgrade journal entries and mount
|
|
||||||
// identities before removal, so no lifecycle lock is needed here: a
|
|
||||||
// container is only reclaimed when its instance row is already gone.
|
|
||||||
switch r.Method {
|
|
||||||
case "GET":
|
|
||||||
filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.role=dsh"}})
|
|
||||||
var list []struct {
|
|
||||||
ID string `json:"Id"`
|
|
||||||
}
|
|
||||||
if a.docker.Do(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), nil, &list) != nil {
|
|
||||||
fail(w, 502, "container inventory unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
eligible := []string{}
|
|
||||||
for _, c := range list {
|
|
||||||
if a.checkOrphan(ctx, c.ID) == nil {
|
|
||||||
eligible = append(eligible, c.ID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
writeJSON(w, eligible)
|
|
||||||
case "DELETE":
|
|
||||||
id := r.URL.Query().Get("id")
|
|
||||||
if r.URL.Query().Get("confirm") != id || id == "" {
|
|
||||||
fail(w, 400, "exact container ID confirmation required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := a.checkOrphan(ctx, id); err != nil {
|
|
||||||
fail(w, 409, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Never force removal or remove volumes. Docker rejects a concurrent start.
|
|
||||||
if a.docker.Do(ctx, "DELETE", "/containers/"+id+"?force=false&v=false", nil, nil) != nil {
|
|
||||||
fail(w, 409, "container changed or removal rejected")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
a.s.AddAudit(u.ID, "reconcile.reclaim", id, "stopped orphan removed; bind data retained")
|
|
||||||
writeJSON(w, map[string]bool{"ok": true})
|
|
||||||
default:
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestOrphanReclaimGuards(t *testing.T) {
|
|
||||||
for _, kind := range []string{"safe", "running", "foreign-mount", "reference", "upgrade", "confirmation", "nonadmin"} {
|
|
||||||
t.Run(kind, func(t *testing.T) {
|
|
||||||
t.Setenv("HOST_DATA_ROOT", "/host/data")
|
|
||||||
id, inst, owner := strings.Repeat("a", 64), strings.Repeat("b", 32), strings.Repeat("c", 32)
|
|
||||||
deleted := false
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == "DELETE" {
|
|
||||||
if r.URL.Query().Get("force") != "false" || r.URL.Query().Get("v") != "false" {
|
|
||||||
t.Fatal("unsafe deletion")
|
|
||||||
}
|
|
||||||
deleted = true
|
|
||||||
w.WriteHeader(204)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
source := "/host/data/workspaces/" + owner + "/instances/" + inst + "/home"
|
|
||||||
if kind == "foreign-mount" {
|
|
||||||
source = "/unrelated"
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]any{"Id": id, "Name": "/dsh-inst-" + inst, "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "dsh", "dsh-mgr.instance": inst, "dsh-mgr.owner": owner}}, "State": map[string]any{"Status": "exited", "Running": kind == "running"}, "Mounts": []map[string]string{{"Type": "bind", "Source": source, "Destination": "/workspace"}}})
|
|
||||||
})
|
|
||||||
if err := a.s.CreateUser(store.User{ID: owner, Username: "orphan-owner"}, "unused"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if kind != "nonadmin" {
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='owner'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if kind == "reference" {
|
|
||||||
if err := a.s.AddInstance(store.Instance{ID: inst, UID: owner, ContainerID: id}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if kind == "upgrade" {
|
|
||||||
if _, err := a.s.DB.Exec("CREATE TABLE upgrade_journal(instance_id TEXT,old_container TEXT,new_container TEXT); INSERT INTO upgrade_journal VALUES(?,?,'')", inst, id); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if kind == "safe" && a.checkOrphan(context.Background(), id) != nil {
|
|
||||||
t.Fatal("safe orphan rejected")
|
|
||||||
}
|
|
||||||
confirm := id
|
|
||||||
if kind == "confirmation" {
|
|
||||||
confirm = "wrong"
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.orphans(w, runtimeRequest("DELETE", "/api/orphans?id="+id+"&confirm="+confirm, "", sid))
|
|
||||||
if (w.Code == 200) != (kind == "safe") || deleted != (kind == "safe") {
|
|
||||||
t.Fatalf("status=%d deleted=%v body=%s", w.Code, deleted, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,75 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *API) changePassword(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
if r.Method != http.MethodPost {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if _, _, err := a.user(r); err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var in struct {
|
|
||||||
CurrentPassword string `json:"current_password"`
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, 4096))
|
|
||||||
decoder.DisallowUnknownFields()
|
|
||||||
if decoder.Decode(&in) != nil || decoder.Decode(new(any)) != io.EOF || in.CurrentPassword == "" || len(in.Password) < 12 || len(in.Password) > 72 {
|
|
||||||
fail(w, 400, "current password and new password 12–72 bytes required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Recheck credentials after waiting for concurrent logins or password resets.
|
|
||||||
a.authLoginMu.Lock()
|
|
||||||
defer a.authLoginMu.Unlock()
|
|
||||||
u, _, err := a.user(r)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 401, "authentication required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !a.reauthenticateLocked(w, u.ID, in.CurrentPassword, http.StatusBadRequest) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
hash, err := bcrypt.GenerateFromPassword([]byte(in.Password), bcrypt.DefaultCost)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "password error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tx, err := a.s.DB.BeginTx(r.Context(), nil)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "password change failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
_, err = tx.ExecContext(r.Context(), "UPDATE users SET pass_hash=? WHERE id=?", string(hash), u.ID)
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "DELETE FROM auth_refresh WHERE sid IN (SELECT id FROM sessions WHERE uid=?)", u.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "DELETE FROM sessions WHERE uid=?", u.ID)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "DELETE FROM auth_failures WHERE username=?", u.Username)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
_, err = tx.ExecContext(r.Context(), "INSERT INTO audit(uid,action,target,detail,at) VALUES(?,?,?,?,?)", u.ID, "user.change_password", u.ID, "", time.Now().Unix())
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = tx.Commit()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 500, "password change failed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]bool{"ok": true, "reauthenticate": true})
|
|
||||||
}
|
|
||||||
@@ -1,365 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const changePath = "/api/me/password"
|
|
||||||
|
|
||||||
func TestChangePasswordCurrentLength(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
password := strings.Repeat("a", 72)
|
|
||||||
hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE users SET pass_hash=? WHERE id='alice'", string(hash)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
cs := req("POST", "/api/login", resetLogin("alice", password), "", nil).Result().Cookies()
|
|
||||||
if w := req("POST", changePath, changeBody(password+"x", resetSecret), cookieValue(cs, "dsh_csrf"), cs); w.Code != 400 {
|
|
||||||
t.Fatal("accepted prefix", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", changePath, changeBody(password, resetSecret), cookieValue(cs, "dsh_csrf"), cs); w.Code != 200 {
|
|
||||||
t.Fatal("72-byte current rejected", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangePasswordConcurrentChanges(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
cs := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
start := make(chan struct{})
|
|
||||||
results := make(chan int, 2)
|
|
||||||
for i := 0; i < 2; i++ {
|
|
||||||
go func() {
|
|
||||||
<-start
|
|
||||||
results <- req("POST", changePath, changeBody("password-123", resetSecret), cookieValue(cs, "dsh_csrf"), cs).Code
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
close(start)
|
|
||||||
successes := 0
|
|
||||||
for i := 0; i < 2; i++ {
|
|
||||||
code := <-results
|
|
||||||
if code == 200 {
|
|
||||||
successes++
|
|
||||||
} else if code != 401 && code != 403 {
|
|
||||||
t.Fatal(code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if successes != 1 {
|
|
||||||
t.Fatal("multiple changes", successes)
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='user.change_password'").Scan(&n); err != nil || n != 1 {
|
|
||||||
t.Fatal(n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestChangePasswordRechecksWaitingCredentials(t *testing.T) {
|
|
||||||
for _, revoke := range []string{"session", "password"} {
|
|
||||||
t.Run(revoke, func(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
admin := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
reader := &resetSignalReader{reader: strings.NewReader(changeBody("password-123", resetSecret)), started: make(chan struct{})}
|
|
||||||
r := httptest.NewRequest("POST", changePath, reader)
|
|
||||||
r.Header.Set("Origin", a.origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", cookieValue(admin, "dsh_csrf"))
|
|
||||||
for _, c := range admin {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
newHash, err := bcrypt.GenerateFromPassword([]byte(resetSecret), bcrypt.MinCost)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE users SET pass_hash=? WHERE id='bob'", string(newHash)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.authLoginMu.Lock()
|
|
||||||
locked := true
|
|
||||||
defer func() {
|
|
||||||
if locked {
|
|
||||||
a.authLoginMu.Unlock()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
done := make(chan *httptest.ResponseRecorder, 1)
|
|
||||||
go func() { w := httptest.NewRecorder(); a.Handler().ServeHTTP(w, r); done <- w }()
|
|
||||||
select {
|
|
||||||
case <-reader.started:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("request did not reach decoder")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
t.Fatal("reset bypassed login lock", w.Code)
|
|
||||||
case <-time.After(150 * time.Millisecond):
|
|
||||||
}
|
|
||||||
query := "UPDATE users SET pass_hash=(SELECT pass_hash FROM users WHERE id='bob') WHERE id='alice'"
|
|
||||||
want := 400
|
|
||||||
if revoke == "session" {
|
|
||||||
query = "DELETE FROM sessions WHERE uid='alice'"
|
|
||||||
want = 401
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec(query); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.authLoginMu.Unlock()
|
|
||||||
locked = false
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
if w.Code != want {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("reset did not finish")
|
|
||||||
}
|
|
||||||
expected := resetSecret
|
|
||||||
if revoke == "session" {
|
|
||||||
expected = "password-123"
|
|
||||||
}
|
|
||||||
hash, err := a.s.Password("alice")
|
|
||||||
if err != nil || bcrypt.CompareHashAndPassword([]byte(hash), []byte(expected)) != nil {
|
|
||||||
t.Fatal("unauthorized reset", err)
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='user.change_password'").Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatal(n, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func changeBody(current, password string) string {
|
|
||||||
b, _ := json.Marshal(map[string]string{"current_password": current, "password": password})
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
func TestChangePasswordValidation(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
cs := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
csrf := cookieValue(cs, "dsh_csrf")
|
|
||||||
valid := changeBody("password-123", resetSecret)
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, method, body, csrf string
|
|
||||||
cookies []*http.Cookie
|
|
||||||
code int
|
|
||||||
}{
|
|
||||||
{"anonymous", "POST", valid, "", nil, 401},
|
|
||||||
{"csrf", "POST", valid, "bad", cs, 403},
|
|
||||||
{"method", "GET", "", csrf, cs, 405},
|
|
||||||
{"short", "POST", changeBody("password-123", strings.Repeat("a", 11)), csrf, cs, 400},
|
|
||||||
{"long", "POST", changeBody("password-123", strings.Repeat("a", 73)), csrf, cs, 400},
|
|
||||||
{"unicode long", "POST", changeBody("password-123", strings.Repeat("密", 25)), csrf, cs, 400},
|
|
||||||
{"missing current", "POST", changeBody("", resetSecret), csrf, cs, 400},
|
|
||||||
{"malformed", "POST", "{", csrf, cs, 400},
|
|
||||||
{"trailing", "POST", valid + "{}", csrf, cs, 400},
|
|
||||||
{"oversized", "POST", changeBody("password-123", strings.Repeat("a", 4096)), csrf, cs, 400},
|
|
||||||
{"target id", "POST", strings.TrimSuffix(valid, "}") + `,"id":"bob"}`, csrf, cs, 400},
|
|
||||||
{"wrong current", "POST", changeBody("wrong", resetSecret), csrf, cs, 400},
|
|
||||||
{"missing access", "POST", valid, csrf, []*http.Cookie{{Name: "dsh_session", Value: cookieValue(cs, "dsh_session")}}, 401},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
w := req(tc.method, changePath, tc.body, tc.csrf, tc.cookies)
|
|
||||||
if w.Code != tc.code {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for _, origin := range []string{"", "https://evil.test"} {
|
|
||||||
r := httptest.NewRequest("POST", changePath, strings.NewReader(valid))
|
|
||||||
r.Header.Set("Origin", origin)
|
|
||||||
r.Header.Set("X-CSRF-Token", csrf)
|
|
||||||
for _, c := range cs {
|
|
||||||
r.AddCookie(c)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.Handler().ServeHTTP(w, r)
|
|
||||||
if w.Code != 403 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", cs); w.Code != 200 {
|
|
||||||
t.Fatal("validation revoked session", w.Code)
|
|
||||||
}
|
|
||||||
for _, user := range []string{"alice", "bob"} {
|
|
||||||
if w := req("POST", "/api/login", resetLogin(user, "password-123"), "", nil); w.Code != 200 {
|
|
||||||
t.Fatal("validation changed password", user, w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangePasswordLifecycle(t *testing.T) {
|
|
||||||
for _, password := range []string{strings.Repeat("a", 12), strings.Repeat("密", 24)} {
|
|
||||||
t.Run(fmt.Sprint(len(password)), func(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
login := func(user string) []*http.Cookie {
|
|
||||||
w := req("POST", "/api/login", resetLogin(user, "password-123"), "", nil)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code)
|
|
||||||
}
|
|
||||||
return w.Result().Cookies()
|
|
||||||
}
|
|
||||||
cs, second, other := login("alice"), login("alice"), login("bob")
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO auth_failures VALUES('alice',2,0)"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := req("POST", changePath, changeBody("password-123", password), cookieValue(cs, "dsh_csrf"), cs)
|
|
||||||
var result map[string]bool
|
|
||||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || !result["ok"] || !result["reauthenticate"] || w.Header().Get("Cache-Control") != "no-store" {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for _, old := range [][]*http.Cookie{cs, second} {
|
|
||||||
if w := req("GET", "/api/me", "", "", old); w.Code != 401 {
|
|
||||||
t.Fatal("old access", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/refresh", "", cookieValue(old, "dsh_csrf"), old); w.Code != 403 {
|
|
||||||
t.Fatal("old refresh", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for query, want := range map[string]int{"SELECT count(*) FROM sessions WHERE uid='alice'": 0, "SELECT count(*) FROM auth_refresh": 1, "SELECT count(*) FROM auth_failures WHERE username='alice'": 0, "SELECT count(*) FROM audit WHERE action='user.change_password' AND uid='alice' AND target='alice' AND detail=''": 1} {
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow(query).Scan(&n); err != nil || n != want {
|
|
||||||
t.Fatal(query, n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", other); w.Code != 200 {
|
|
||||||
t.Fatal("other session", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/refresh", "", cookieValue(other, "dsh_csrf"), other); w.Code != 200 {
|
|
||||||
t.Fatal("other refresh", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil); w.Code != 401 {
|
|
||||||
t.Fatal("old password", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", resetLogin("alice", password), "", nil); w.Code != 200 {
|
|
||||||
t.Fatal("new password", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil); w.Code != 200 {
|
|
||||||
t.Fatal("other password", w.Code)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangePasswordConcurrentFailures(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
cs := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
results := make(chan int, 12)
|
|
||||||
start := make(chan struct{})
|
|
||||||
for i := 0; i < 12; i++ {
|
|
||||||
go func() {
|
|
||||||
<-start
|
|
||||||
results <- req("POST", changePath, changeBody("wrong", resetSecret), cookieValue(cs, "dsh_csrf"), cs).Code
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
close(start)
|
|
||||||
counts := map[int]int{}
|
|
||||||
for i := 0; i < 12; i++ {
|
|
||||||
counts[<-results]++
|
|
||||||
}
|
|
||||||
if counts[400] != 5 || counts[429] != 7 {
|
|
||||||
t.Fatal(counts)
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
var until int64
|
|
||||||
if err := a.s.DB.QueryRow("SELECT failures,locked_until FROM auth_failures WHERE username='alice'").Scan(&n, &until); err != nil || n != 5 || until <= time.Now().Unix() {
|
|
||||||
t.Fatal(n, until, err)
|
|
||||||
}
|
|
||||||
if w := req("POST", changePath, changeBody("password-123", resetSecret), cookieValue(cs, "dsh_csrf"), cs); w.Code != 429 {
|
|
||||||
t.Fatal("logged-in bypass", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil); w.Code != 429 {
|
|
||||||
t.Fatal("login bypass", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", cs); w.Code != 200 {
|
|
||||||
t.Fatal("wrong password logged out", w.Code)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE auth_failures SET locked_until=? WHERE username='alice'", time.Now().Add(-time.Second).Unix()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w := req("POST", changePath, changeBody("password-123", resetSecret), cookieValue(cs, "dsh_csrf"), cs); w.Code != 200 {
|
|
||||||
t.Fatal("expired lock", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangePasswordRollback(t *testing.T) {
|
|
||||||
for _, step := range []struct{ table, event string }{{"users", "UPDATE"}, {"auth_refresh", "DELETE"}, {"sessions", "DELETE"}, {"auth_failures", "DELETE"}, {"audit", "INSERT"}} {
|
|
||||||
t.Run(step.table, func(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = req("POST", "/api/login", resetLogin("alice", "password-123"), "", nil).Result().Cookies()
|
|
||||||
target := req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil).Result().Cookies()
|
|
||||||
before, err := a.s.Password("bob")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO auth_failures VALUES('bob',2,0)"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec(fmt.Sprintf("CREATE TRIGGER reject_reset BEFORE %s ON %s BEGIN SELECT RAISE(ABORT,'forced failure'); END", step.event, step.table)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := req("POST", changePath, changeBody("password-123", resetSecret), cookieValue(target, "dsh_csrf"), target)
|
|
||||||
if w.Code != 500 || strings.Contains(w.Body.String(), resetSecret) {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
after, err := a.s.Password("bob")
|
|
||||||
if err != nil || before != after {
|
|
||||||
t.Fatal("hash changed", err)
|
|
||||||
}
|
|
||||||
for query, want := range map[string]int{"SELECT count(*) FROM sessions WHERE uid='bob'": 1, "SELECT count(*) FROM auth_refresh": 2, "SELECT count(*) FROM auth_failures WHERE username='bob' AND failures=2 AND locked_until=0": 1, "SELECT count(*) FROM audit WHERE action='user.change_password'": 0} {
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow(query).Scan(&n); err != nil || n != want {
|
|
||||||
t.Fatal(query, n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if w := req("GET", "/api/me", "", "", target); w.Code != 200 {
|
|
||||||
t.Fatal("session changed", w.Code)
|
|
||||||
}
|
|
||||||
if w := req("POST", "/api/refresh", "", cookieValue(target, "dsh_csrf"), target); w.Code != 200 {
|
|
||||||
t.Fatal("refresh changed", w.Code)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestChangePasswordConcurrentLogin(t *testing.T) {
|
|
||||||
a, req := authFixture(t)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='alice'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
admin := req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil).Result().Cookies()
|
|
||||||
start := make(chan struct{})
|
|
||||||
results := make(chan *httptest.ResponseRecorder, 8)
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
go func() { <-start; results <- req("POST", "/api/login", resetLogin("bob", "password-123"), "", nil) }()
|
|
||||||
}
|
|
||||||
close(start)
|
|
||||||
w := req("POST", changePath, changeBody("password-123", resetSecret), cookieValue(admin, "dsh_csrf"), admin)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for i := 0; i < 8; i++ {
|
|
||||||
w := <-results
|
|
||||||
if w.Code != 200 && w.Code != 401 && w.Code != 429 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w.Code == 200 {
|
|
||||||
if check := req("GET", "/api/me", "", "", w.Result().Cookies()); check.Code != 401 {
|
|
||||||
t.Fatal("old-password login retained session", check.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM sessions WHERE uid='bob'").Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatal("old login sessions", n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
func toHostPath(containerRoot, hostRoot, containerPath string) (string, error) {
|
|
||||||
if !filepath.IsAbs(containerRoot) || !filepath.IsAbs(hostRoot) || !filepath.IsAbs(containerPath) {
|
|
||||||
return "", errors.New("absolute storage paths required")
|
|
||||||
}
|
|
||||||
rel, err := filepath.Rel(containerRoot, containerPath)
|
|
||||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
|
|
||||||
return "", errors.New("storage path escapes root")
|
|
||||||
}
|
|
||||||
return filepath.Join(hostRoot, rel), nil
|
|
||||||
}
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
func TestToHostPath(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
input, want string
|
|
||||||
bad bool
|
|
||||||
}{{"/data/workspaces/u/files", "/host/workspaces/u/files", false}, {"/data", "/host", false}, {"/data-other/x", "", true}, {"/data/../secret", "", true}, {"relative", "", true}} {
|
|
||||||
got, err := toHostPath("/data", "/host", tc.input)
|
|
||||||
if (err != nil) != tc.bad || got != tc.want {
|
|
||||||
t.Fatalf("%s: %q %v", tc.input, got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,226 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
const previewLimit = 1 << 20
|
|
||||||
|
|
||||||
var bundleReference = regexp.MustCompile("^(@[a-z0-9._-]+/)?[a-z0-9._-]+$")
|
|
||||||
|
|
||||||
// The caller must authenticate and check instance ownership before dispatching.
|
|
||||||
func (a *API) profilePreview(w http.ResponseWriter, r *http.Request, i store.Instance) {
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
if r.Method != http.MethodGet {
|
|
||||||
fail(w, 405, "method not allowed")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
bundles, err := a.profileBundles(i)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 400, "bundle manifest unavailable")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
result := map[string]any{"bundles": bundles, "config": "", "redacted": true}
|
|
||||||
if i.ContainerID == "" || i.Status != "running" {
|
|
||||||
result["preview_error"] = "启动实例后可预览已保存的配置。"
|
|
||||||
writeJSON(w, result)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
content, err := a.dumpProfile(ctx, i.ContainerID)
|
|
||||||
if err != nil {
|
|
||||||
result["preview_error"] = "配置预览不可用；命令输出和错误详情已隐藏。"
|
|
||||||
} else {
|
|
||||||
result["config"] = content
|
|
||||||
}
|
|
||||||
writeJSON(w, result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) profileBundles(i store.Instance) (any, error) {
|
|
||||||
parent, err := upgradeRoot(a.root, i)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer parent.Close()
|
|
||||||
if err = configCheckPath(parent, "home/profiles/web/package.json"); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
root, err := configProfileRoot(parent, false)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
f, err := root.OpenFile("package.json", os.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, 0)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
st, err := f.Stat()
|
|
||||||
if err != nil || !st.Mode().IsRegular() || st.Size() > previewLimit {
|
|
||||||
return nil, errors.New("invalid manifest")
|
|
||||||
}
|
|
||||||
data, err := io.ReadAll(io.LimitReader(f, previewLimit+1))
|
|
||||||
if err != nil || len(data) > previewLimit {
|
|
||||||
return nil, errors.New("invalid manifest")
|
|
||||||
}
|
|
||||||
var pkg map[string]json.RawMessage
|
|
||||||
if json.Unmarshal(data, &pkg) != nil {
|
|
||||||
return nil, errors.New("invalid manifest")
|
|
||||||
}
|
|
||||||
var profile any
|
|
||||||
raw := pkg["dsh.profile"]
|
|
||||||
if raw == nil {
|
|
||||||
var dsh map[string]json.RawMessage
|
|
||||||
if pkg["dsh"] != nil && json.Unmarshal(pkg["dsh"], &dsh) != nil {
|
|
||||||
return nil, errors.New("invalid manifest")
|
|
||||||
}
|
|
||||||
raw = dsh["profile"]
|
|
||||||
}
|
|
||||||
if raw == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if json.Unmarshal(raw, &profile) != nil {
|
|
||||||
return nil, errors.New("invalid manifest")
|
|
||||||
}
|
|
||||||
return redactPreview(profile, false), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) dumpProfile(ctx context.Context, container string) (string, error) {
|
|
||||||
var exec struct{ ID string }
|
|
||||||
err := a.docker.Do(ctx, "POST", "/containers/"+url.PathEscape(container)+"/exec", map[string]any{
|
|
||||||
"Cmd": []string{"dsh", "--profile", "web", "--dump-config"}, "AttachStdout": true, "AttachStderr": true, "Tty": false,
|
|
||||||
}, &exec)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if exec.ID == "" {
|
|
||||||
return "", errors.New("missing exec ID")
|
|
||||||
}
|
|
||||||
path := "/exec/" + url.PathEscape(exec.ID)
|
|
||||||
body, _ := json.Marshal(map[string]bool{"Detach": false, "Tty": false})
|
|
||||||
resp, err := a.docker.Stream(ctx, "POST", path+"/start", bytes.NewReader(body), "application/json")
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
data, err := previewStdout(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
var state struct {
|
|
||||||
Running bool
|
|
||||||
ExitCode int
|
|
||||||
}
|
|
||||||
if err = a.docker.Do(ctx, "GET", path+"/json", nil, &state); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if state.Running || state.ExitCode != 0 {
|
|
||||||
return "", errors.New("preview command failed")
|
|
||||||
}
|
|
||||||
var document any
|
|
||||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
|
||||||
if err = decoder.Decode(&document); err != nil {
|
|
||||||
return "", errors.New("invalid preview")
|
|
||||||
}
|
|
||||||
var extra any
|
|
||||||
if decoder.Decode(&extra) != io.EOF {
|
|
||||||
return "", errors.New("multiple preview documents")
|
|
||||||
}
|
|
||||||
switch document.(type) {
|
|
||||||
case map[string]any, []any:
|
|
||||||
// DSH exports its composed plugin tree as a YAML sequence.
|
|
||||||
default:
|
|
||||||
return "", errors.New("invalid preview structure")
|
|
||||||
}
|
|
||||||
safe, err := yaml.Marshal(redactPreview(document, true))
|
|
||||||
return string(safe), err
|
|
||||||
}
|
|
||||||
|
|
||||||
func previewStdout(r io.Reader) ([]byte, error) {
|
|
||||||
var output bytes.Buffer
|
|
||||||
total := 0
|
|
||||||
for {
|
|
||||||
var header [8]byte
|
|
||||||
_, err := io.ReadFull(r, header[:])
|
|
||||||
if err == io.EOF {
|
|
||||||
return output.Bytes(), nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if (header[0] != 1 && header[0] != 2) || header[1] != 0 || header[2] != 0 || header[3] != 0 {
|
|
||||||
return nil, errors.New("invalid exec stream")
|
|
||||||
}
|
|
||||||
size := int64(binary.BigEndian.Uint32(header[4:]))
|
|
||||||
total += 8
|
|
||||||
if size > int64(previewLimit-total) {
|
|
||||||
return nil, errors.New("preview too large")
|
|
||||||
}
|
|
||||||
total += int(size)
|
|
||||||
var dst io.Writer = io.Discard
|
|
||||||
if header[0] == 1 {
|
|
||||||
dst = &output
|
|
||||||
}
|
|
||||||
if _, err = io.CopyN(dst, r, size); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func redactPreview(value any, strict bool) any {
|
|
||||||
switch v := value.(type) {
|
|
||||||
case map[string]any:
|
|
||||||
out := make(map[string]any, len(v))
|
|
||||||
for key, item := range v {
|
|
||||||
normalized := strings.ToLower(strings.NewReplacer("_", "", "-", "", ".", "").Replace(key))
|
|
||||||
sensitive := false
|
|
||||||
for _, word := range []string{"key", "secret", "token", "password", "passwd", "credential", "authorization", "cookie", "env", "header"} {
|
|
||||||
sensitive = sensitive || strings.Contains(normalized, word)
|
|
||||||
}
|
|
||||||
if sensitive {
|
|
||||||
out[key] = "[REDACTED]"
|
|
||||||
} else {
|
|
||||||
out[key] = redactPreview(item, strict)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case []any:
|
|
||||||
out := make([]any, len(v))
|
|
||||||
for j, item := range v {
|
|
||||||
out[j] = redactPreview(item, strict)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
case string:
|
|
||||||
// Plugins can resolve credentials into arbitrary string fields, not just apiKey.
|
|
||||||
if strict || !bundleReference.MatchString(v) {
|
|
||||||
return "[REDACTED]"
|
|
||||||
}
|
|
||||||
return v
|
|
||||||
case map[any]any:
|
|
||||||
return "[REDACTED]"
|
|
||||||
default:
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,188 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"reflect"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
func previewFrame(stream byte, text string) []byte {
|
|
||||||
header := make([]byte, 8)
|
|
||||||
header[0] = stream
|
|
||||||
binary.BigEndian.PutUint32(header[4:], uint32(len(text)))
|
|
||||||
return append(header, []byte(text)...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProfileDumpFixedCommandAndRedaction(t *testing.T) {
|
|
||||||
daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.URL.Path {
|
|
||||||
case "/v1.47/containers/container/exec":
|
|
||||||
var request struct {
|
|
||||||
Cmd []string
|
|
||||||
AttachStdout, AttachStderr, Tty bool
|
|
||||||
}
|
|
||||||
if json.NewDecoder(r.Body).Decode(&request) != nil {
|
|
||||||
t.Error("invalid exec request")
|
|
||||||
}
|
|
||||||
if r.Method != "POST" || !reflect.DeepEqual(request.Cmd, []string{"dsh", "--profile", "web", "--dump-config"}) || !request.AttachStdout || !request.AttachStderr || request.Tty {
|
|
||||||
t.Errorf("unsafe command: %+v", request)
|
|
||||||
}
|
|
||||||
writeJSON(w, map[string]string{"Id": "exec"})
|
|
||||||
case "/v1.47/exec/exec/start":
|
|
||||||
w.Write(previewFrame(2, "stderr-secret"))
|
|
||||||
w.Write(previewFrame(1, "provider:\n apiKey: direct-secret\n API_KEY: underscore-secret\n password: numeric-secret\n nested:\n - token: nested-secret\n arbitrary: resolved-credential\n endpoint: https://name:[email protected]/?token=secret\n enabled: true\n"))
|
|
||||||
case "/v1.47/exec/exec/json":
|
|
||||||
writeJSON(w, map[string]any{"Running": false, "ExitCode": 0})
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected Docker call %s", r.URL.Path)
|
|
||||||
w.WriteHeader(404)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer daemon.Close()
|
|
||||||
a := &API{docker: docker.New(daemon.URL)}
|
|
||||||
out, err := a.dumpProfile(context.Background(), "container")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, secret := range []string{"secret", "resolved-credential", "https://"} {
|
|
||||||
if strings.Contains(out, secret) {
|
|
||||||
t.Fatalf("secret leaked: %s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !strings.Contains(out, "[REDACTED]") || !strings.Contains(out, "enabled: true") {
|
|
||||||
t.Fatalf("missing structure: %s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProfileDumpPluginSequence(t *testing.T) {
|
|
||||||
daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/exec"):
|
|
||||||
writeJSON(w, map[string]string{"Id": "exec"})
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/start"):
|
|
||||||
w.Write(previewFrame(1, "# bundle\n- id: plugin\n disabled: !!js '!ctx.get(\"profileContext\")'\n config:\n apiKey: private-secret\n enabled: true\n"))
|
|
||||||
default:
|
|
||||||
writeJSON(w, map[string]any{"Running": false, "ExitCode": 0})
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer daemon.Close()
|
|
||||||
a := &API{docker: docker.New(daemon.URL)}
|
|
||||||
out, err := a.dumpProfile(context.Background(), "container")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Contains(out, "private-secret") || strings.Contains(out, "ctx.get") || !strings.Contains(out, "enabled: true") || !strings.Contains(out, "[REDACTED]") {
|
|
||||||
t.Fatalf("invalid sequence redaction: %s", out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProfileDumpRejectsUnsafeOutput(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, output string
|
|
||||||
exit int
|
|
||||||
}{
|
|
||||||
{"invalid", "apiKey: [secret", 0}, {"plain", "secret", 0},
|
|
||||||
{"documents", "a: b\n---\napiKey: secret", 0}, {"failure", "a: b", 1},
|
|
||||||
{"empty", "", 0},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/exec"):
|
|
||||||
writeJSON(w, map[string]string{"Id": "exec"})
|
|
||||||
case strings.HasSuffix(r.URL.Path, "/start"):
|
|
||||||
w.Write(previewFrame(1, tc.output))
|
|
||||||
default:
|
|
||||||
writeJSON(w, map[string]any{"Running": false, "ExitCode": tc.exit})
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer daemon.Close()
|
|
||||||
a := &API{docker: docker.New(daemon.URL)}
|
|
||||||
if out, err := a.dumpProfile(context.Background(), "container"); err == nil || out != "" {
|
|
||||||
t.Fatalf("unsafe output returned: %q, %v", out, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPreviewStreamBounds(t *testing.T) {
|
|
||||||
for _, data := range [][]byte{
|
|
||||||
[]byte("raw secret"), {1, 0}, previewFrame(3, "secret"),
|
|
||||||
previewFrame(1, strings.Repeat("x", previewLimit)),
|
|
||||||
previewFrame(1, "secret")[:10],
|
|
||||||
} {
|
|
||||||
if _, err := previewStdout(bytes.NewReader(data)); err == nil {
|
|
||||||
t.Fatal("accepted invalid stream")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
data := append(previewFrame(2, "secret"), previewFrame(1, "safe")...)
|
|
||||||
got, err := previewStdout(bytes.NewReader(data))
|
|
||||||
if err != nil || string(got) != "safe" {
|
|
||||||
t.Fatalf("stdout: %q %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func previewFixture(t *testing.T) (*API, store.Instance, string) {
|
|
||||||
t.Helper()
|
|
||||||
root := t.TempDir()
|
|
||||||
i := store.Instance{ID: "instance", UID: "owner", ContainerID: "container", Status: "stopped"}
|
|
||||||
path := filepath.Join(root, "workspaces", i.UID, "instances", i.ID, "home", "profiles", "web")
|
|
||||||
if err := os.MkdirAll(path, 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return &API{root: root}, i, path
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProfilePreviewManifestAndErrors(t *testing.T) {
|
|
||||||
a, i, path := previewFixture(t)
|
|
||||||
if err := os.WriteFile(filepath.Join(path, "package.json"), []byte(`{"dsh":{"profile":{"bundles":["@deepseek/bundle"],"apiKey":"private-value"}},"scripts":{"secret":"ignored-value"}}`), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.profilePreview(w, httptest.NewRequest("GET", "/preview?cmd=cat&path=.credentials.yaml", nil), i)
|
|
||||||
if w.Code != 200 || !strings.Contains(w.Body.String(), "@deepseek/bundle") || strings.Contains(w.Body.String(), "private-value") || strings.Contains(w.Body.String(), "ignored-value") {
|
|
||||||
t.Fatalf("manifest response: %d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if w.Header().Get("Cache-Control") != "no-store" {
|
|
||||||
t.Fatal("preview must not be cached")
|
|
||||||
}
|
|
||||||
daemon := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "daemon-private-value", 500) }))
|
|
||||||
defer daemon.Close()
|
|
||||||
a.docker = docker.New(daemon.URL)
|
|
||||||
i.Status = "running"
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
a.profilePreview(w, httptest.NewRequest("GET", "/preview", nil), i)
|
|
||||||
if strings.Contains(w.Body.String(), "daemon-private-value") || !strings.Contains(w.Body.String(), "preview_error") {
|
|
||||||
t.Fatalf("unsafe error: %s", w.Body.String())
|
|
||||||
}
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
a.profilePreview(w, httptest.NewRequest("POST", "/preview", nil), i)
|
|
||||||
if w.Code != 405 {
|
|
||||||
t.Fatalf("method: %d", w.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProfileBundlesRejectsCredentialSymlink(t *testing.T) {
|
|
||||||
a, i, path := previewFixture(t)
|
|
||||||
credentials := filepath.Join(path, "..", "..", ".credentials.yaml")
|
|
||||||
if err := os.WriteFile(credentials, []byte("private-value"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.Symlink(credentials, filepath.Join(path, "package.json")); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.profileBundles(i); err == nil {
|
|
||||||
t.Fatal("accepted credential symlink")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"log"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RunQuotaMonitor reports soft quota crossings; it never claims to constrain
|
|
||||||
// runtime writes and never deletes user data to recover capacity.
|
|
||||||
func (a *API) RunQuotaMonitor(ctx context.Context) {
|
|
||||||
ticker := time.NewTicker(5 * time.Minute)
|
|
||||||
defer ticker.Stop()
|
|
||||||
exceeded := map[string]bool{}
|
|
||||||
for {
|
|
||||||
a.checkQuotas(ctx, exceeded)
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) checkQuotas(ctx context.Context, exceeded map[string]bool) {
|
|
||||||
rows, err := a.s.DB.QueryContext(ctx, "SELECT id,quota FROM users")
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var users []store.User
|
|
||||||
for rows.Next() {
|
|
||||||
var u store.User
|
|
||||||
if rows.Scan(&u.ID, &u.Quota) != nil {
|
|
||||||
rows.Close()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
users = append(users, u)
|
|
||||||
}
|
|
||||||
err = rows.Err()
|
|
||||||
rows.Close()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for _, u := range users {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
used, err := a.userUsage(u)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("quota usage unavailable for user %s", u.ID)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
over := used > u.Quota
|
|
||||||
if over && !exceeded[u.ID] {
|
|
||||||
a.s.AddAudit(u.ID, "quota.exceeded", u.ID, "soft quota exceeded; runtime writes remain unrestricted")
|
|
||||||
}
|
|
||||||
if !over && exceeded[u.ID] {
|
|
||||||
a.s.AddAudit(u.ID, "quota.recovered", u.ID, "")
|
|
||||||
}
|
|
||||||
exceeded[u.ID] = over
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestQuotaMonitorReportsTransitions(t *testing.T) {
|
|
||||||
a, _ := runtimeTestAPI(t, func(http.ResponseWriter, *http.Request) {})
|
|
||||||
dir := filepath.Join(a.root, "workspaces", "owner", "files")
|
|
||||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, "test"), []byte("over"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
state := map[string]bool{}
|
|
||||||
a.checkQuotas(context.Background(), state)
|
|
||||||
a.checkQuotas(context.Background(), state)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET quota=100 WHERE id='owner'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.checkQuotas(context.Background(), state)
|
|
||||||
var count int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action IN ('quota.exceeded','quota.recovered')").Scan(&count); err != nil || count != 2 {
|
|
||||||
t.Fatalf("transitions=%d error=%v", count, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Call with the owner's user lock held through the write. Runtime writes remain soft-limited.
|
|
||||||
func (a *API) quotaGrowth(uid string, growth int64) error {
|
|
||||||
var u store.User
|
|
||||||
u.ID = uid
|
|
||||||
if err := a.s.DB.QueryRow("SELECT quota FROM users WHERE id=?", uid).Scan(&u.Quota); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
used, err := a.userUsage(u)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if growth > 0 && (used > u.Quota || growth > u.Quota-used) {
|
|
||||||
return errors.New("quota exceeded")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func quotaTreeBytes(root *os.Root, name string) (int64, error) {
|
|
||||||
var total int64
|
|
||||||
entries := 0
|
|
||||||
err := fs.WalkDir(root.FS(), name, func(p string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
entries++
|
|
||||||
if entries > 100000 || strings.Count(p, "/") > 128 {
|
|
||||||
return errors.New("tree limit")
|
|
||||||
}
|
|
||||||
info, err := d.Info()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if info.Mode().IsRegular() {
|
|
||||||
if info.Size() > upgradeSnapshotLimit-total {
|
|
||||||
return errors.New("snapshot size limit")
|
|
||||||
}
|
|
||||||
total += info.Size()
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
return total, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) quotaSnapshot(uid string, root *os.Root, extra int64) (int64, error) {
|
|
||||||
size, err := quotaTreeBytes(root, "home")
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if err = a.quotaGrowth(uid, size+extra); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
return size, nil
|
|
||||||
}
|
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestProfileQuotaReplacement(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
profile := filepath.Join(home, "profiles/web/cordis.patch.yml")
|
|
||||||
if err := os.MkdirAll(filepath.Dir(profile), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
configWrite(t, profile, "foo: old")
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET quota=8 WHERE id=?", i.UID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
body string
|
|
||||||
code int
|
|
||||||
}{
|
|
||||||
{`{"content":"foo: biggg"}`, 413},
|
|
||||||
{`{"content":"foo: new"}`, 200},
|
|
||||||
{`{"content":"foo: x"}`, 200},
|
|
||||||
} {
|
|
||||||
w := configCall(a, i, "PUT", "profile", tc.body)
|
|
||||||
if w.Code != tc.code {
|
|
||||||
t.Fatalf("%d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
data, err := os.ReadFile(profile)
|
|
||||||
if err != nil || string(data) != "foo: x" {
|
|
||||||
t.Fatalf("%q %v", data, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSnapshotQuotaIncludesResourcesAndExistingSnapshots(t *testing.T) {
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
configWrite(t, filepath.Join(home, "state"), "1234")
|
|
||||||
if err := os.MkdirAll(filepath.Join(filepath.Dir(home), "pre-restore-old"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(filepath.Dir(home), "pre-restore-old/state"), "1234")
|
|
||||||
root, err := os.OpenRoot(filepath.Dir(home))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
if _, err = a.s.DB.Exec("UPDATE users SET quota=15 WHERE id=?", i.UID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err = a.quotaSnapshot(i.UID, root, 4); err == nil {
|
|
||||||
t.Fatal("resource bytes omitted")
|
|
||||||
}
|
|
||||||
if size, err := a.quotaSnapshot(i.UID, root, 3); err != nil || size != 4 {
|
|
||||||
t.Fatalf("size=%d err=%v", size, err)
|
|
||||||
}
|
|
||||||
if _, err := root.Lstat("pre-restore-old/state"); err != nil {
|
|
||||||
t.Fatal("existing snapshot changed")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestQuotaSerializedWriters(t *testing.T) {
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET quota=4 WHERE id=?", i.UID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
accepted := 0
|
|
||||||
for _, name := range []string{"one", "two"} {
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
user := a.userLocks.Acquire(i.UID)
|
|
||||||
defer user.Release()
|
|
||||||
if a.quotaGrowth(i.UID, 4) == nil {
|
|
||||||
if err := os.WriteFile(filepath.Join(home, name), []byte("1234"), 0600); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
accepted++
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
if accepted != 1 {
|
|
||||||
t.Fatalf("accepted %d writes", accepted)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (a *API) reauthenticate(w http.ResponseWriter, uid, password string) bool {
|
|
||||||
a.authLoginMu.Lock()
|
|
||||||
defer a.authLoginMu.Unlock()
|
|
||||||
return a.reauthenticateLocked(w, uid, password, http.StatusForbidden)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The caller holds authLoginMu; never acquire instance or user locks while holding it.
|
|
||||||
func (a *API) reauthenticateLocked(w http.ResponseWriter, uid, password string, invalidStatus int) bool {
|
|
||||||
u, err := a.s.UserByID(uid)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
locked, err := a.accountLocked(u.Username)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if locked {
|
|
||||||
fail(w, 429, "account temporarily locked; retry later")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
hash, err := a.s.Password(u.Username)
|
|
||||||
if err != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if len(password) > 72 || bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)) != nil {
|
|
||||||
if err = a.failedLogin(u.Username); err != nil {
|
|
||||||
fail(w, 503, "authentication unavailable")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
fail(w, invalidStatus, "invalid current password")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
@@ -1,335 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"mime/multipart"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"golang.org/x/crypto/bcrypt"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestReauthSlowUpgradeDoesNotBlockOtherLogin(t *testing.T) {
|
|
||||||
inspecting, release := make(chan struct{}), make(chan struct{})
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != http.MethodGet || !strings.HasSuffix(r.URL.Path, "/containers/old/json") {
|
|
||||||
t.Errorf("unexpected Docker request: %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
close(inspecting)
|
|
||||||
<-release
|
|
||||||
w.WriteHeader(503)
|
|
||||||
})
|
|
||||||
hash, err := bcrypt.GenerateFromPassword([]byte("original-password"), bcrypt.MinCost)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET pass_hash=? WHERE id='owner'", string(hash)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.s.CreateUser(store.User{ID: "other", Username: "other"}, string(hash)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
i := store.Instance{ID: "one", UID: "owner", ContainerID: "old", Status: "exited", Created: time.Now()}
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
upgradeDone := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(upgradeDone)
|
|
||||||
a.upgrade(httptest.NewRecorder(), runtimeRequest("POST", "/api/instances/one/upgrade", `{"image_id":"sha256:`+strings.Repeat("a", 64)+`","password":"original-password"}`, sid), i)
|
|
||||||
}()
|
|
||||||
defer func() {
|
|
||||||
close(release)
|
|
||||||
select {
|
|
||||||
case <-upgradeDone:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Error("upgrade did not finish after releasing Docker")
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-inspecting:
|
|
||||||
case <-upgradeDone:
|
|
||||||
t.Fatal("upgrade did not reach Docker after reauthentication")
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("upgrade did not reach inspection")
|
|
||||||
}
|
|
||||||
loginDone := make(chan *httptest.ResponseRecorder, 1)
|
|
||||||
go func() {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
r := runtimeRequest("POST", "/api/login", resetLogin("other", "original-password"), "")
|
|
||||||
r.Header.Set("Origin", a.origin)
|
|
||||||
a.login(w, r)
|
|
||||||
loginDone <- w
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case w := <-loginDone:
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("slow upgrade held authentication lock and blocked another user's login")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-upgradeDone:
|
|
||||||
t.Fatal("upgrade finished before Docker was released")
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReauthConcurrentFailures(t *testing.T) {
|
|
||||||
a, i, _ := configTestAPI(t, false)
|
|
||||||
start := make(chan struct{})
|
|
||||||
done := make(chan int, 10)
|
|
||||||
for n := 0; n < 10; n++ {
|
|
||||||
go func() {
|
|
||||||
<-start
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.reauthenticate(w, i.UID, "wrong")
|
|
||||||
done <- w.Code
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
close(start)
|
|
||||||
counts := map[int]int{}
|
|
||||||
for n := 0; n < 10; n++ {
|
|
||||||
select {
|
|
||||||
case code := <-done:
|
|
||||||
counts[code]++
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("reauthentication deadlocked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if counts[403] != 5 || counts[429] != 5 {
|
|
||||||
t.Fatal(counts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReauthOldBackupPasswordIndependent(t *testing.T) {
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
t.Skip("requires chown")
|
|
||||||
}
|
|
||||||
a, i, home := configTestAPI(t, false)
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "original")
|
|
||||||
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
|
|
||||||
var result struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
}
|
|
||||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &result) != nil || result.ID == "" {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET pass_hash='changed' WHERE id=?", i.UID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO auth_failures VALUES('owner',5,?)", time.Now().Add(time.Hour).Unix()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
configWrite(t, filepath.Join(home, ".credentials.yaml"), "changed")
|
|
||||||
w = configCall(a, i, "POST", "restore", `{"backup_id":"`+result.ID+`","password":"original-password"}`)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
content, err := os.ReadFile(filepath.Join(home, ".credentials.yaml"))
|
|
||||||
if err != nil || string(content) != "original" {
|
|
||||||
t.Fatal(string(content), err)
|
|
||||||
}
|
|
||||||
var failures int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT failures FROM auth_failures WHERE username='owner'").Scan(&failures); err != nil || failures != 5 {
|
|
||||||
t.Fatal(failures, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReauthSharedFailureLock(t *testing.T) {
|
|
||||||
a, i, _ := configTestAPI(t, false)
|
|
||||||
sid, _, err := a.s.Session(i.UID)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
registerTestAccess(t, a, sid)
|
|
||||||
backup := func(password string) *httptest.ResponseRecorder {
|
|
||||||
return configCall(a, i, "POST", "backups", `{"password":"`+password+`"}`)
|
|
||||||
}
|
|
||||||
upgrade := func(password string) *httptest.ResponseRecorder {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.upgrade(w, runtimeRequest("POST", "/api/instances/one/upgrade", `{"image_id":"sha256:`+strings.Repeat("a", 64)+`","password":"`+password+`"}`, sid), i)
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
for _, call := range []func(string) *httptest.ResponseRecorder{backup, upgrade, backup, upgrade} {
|
|
||||||
if w := call("wrong"); w.Code != 403 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.changePassword(w, runtimeRequest("POST", changePath, changeBody("wrong", resetSecret), sid))
|
|
||||||
if w.Code != 400 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
for _, call := range []func(string) *httptest.ResponseRecorder{backup, upgrade} {
|
|
||||||
if w := call("original-password"); w.Code != 429 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
r := runtimeRequest("POST", "/api/login", resetLogin("owner", "original-password"), "")
|
|
||||||
r.Header.Set("Origin", a.origin)
|
|
||||||
a.login(w, r)
|
|
||||||
if w.Code != 429 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
var failures int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT failures FROM auth_failures WHERE username='owner'").Scan(&failures); err != nil || failures != 5 {
|
|
||||||
t.Fatal(failures, err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE auth_failures SET locked_until=?", time.Now().Add(-time.Minute).Unix()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if w := upgrade("wrong"); w.Code != 403 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRow("SELECT failures FROM auth_failures WHERE username='owner'").Scan(&failures); err != nil || failures != 1 {
|
|
||||||
t.Fatal(failures, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReauthFailureStorageUnavailable(t *testing.T) {
|
|
||||||
a, i, _ := configTestAPI(t, false)
|
|
||||||
if _, err := a.s.DB.Exec("DROP TABLE auth_failures"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
w := configCall(a, i, "POST", "backups", `{"password":"original-password"}`)
|
|
||||||
if w.Code != 503 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type quotaPausedBody struct {
|
|
||||||
io.Reader
|
|
||||||
started chan struct{}
|
|
||||||
resume chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *quotaPausedBody) Read(p []byte) (int, error) {
|
|
||||||
if b.started != nil {
|
|
||||||
close(b.started)
|
|
||||||
b.started = nil
|
|
||||||
<-b.resume
|
|
||||||
}
|
|
||||||
return b.Reader.Read(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReauthQuotaInFlightUpload(t *testing.T) {
|
|
||||||
a, sid := runtimeTestAPI(t, func(http.ResponseWriter, *http.Request) {})
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1,quota=1048576 WHERE id='owner'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
u, err := a.s.UserByID("owner")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
dir := filepath.Join(a.root, "workspaces", u.ID)
|
|
||||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
root, err := os.OpenRoot(dir)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer root.Close()
|
|
||||||
var body bytes.Buffer
|
|
||||||
form := multipart.NewWriter(&body)
|
|
||||||
part, err := form.CreateFormFile("file", "large.txt")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
part.Write(bytes.Repeat([]byte("x"), 4096))
|
|
||||||
form.Close()
|
|
||||||
started, resume := make(chan struct{}), make(chan struct{})
|
|
||||||
r := httptest.NewRequest("POST", "/api/ws/upload", "aPausedBody{Reader: &body, started: started, resume: resume})
|
|
||||||
r.Header.Set("Content-Type", form.FormDataContentType())
|
|
||||||
done := make(chan *httptest.ResponseRecorder, 1)
|
|
||||||
go func() {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.wsUpload(w, r, root, u)
|
|
||||||
done <- w
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
close(resume)
|
|
||||||
t.Fatal("upload did not start reading")
|
|
||||||
}
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.users(w, runtimeRequest("PATCH", "/api/users", `{"id":"owner","quota_bytes":1024}`, sid))
|
|
||||||
close(resume)
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
if w.Code != 413 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("upload deadlocked")
|
|
||||||
}
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
if err != nil || len(entries) != 0 {
|
|
||||||
t.Fatal("rejected upload left files", entries, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReauthQuotaUpdateUsesUserLock(t *testing.T) {
|
|
||||||
a, sid := runtimeTestAPI(t, func(http.ResponseWriter, *http.Request) {})
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='owner'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
lock := a.userLocks.Acquire("owner")
|
|
||||||
defer func() {
|
|
||||||
if lock != nil {
|
|
||||||
lock.Release()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
done := make(chan *httptest.ResponseRecorder, 1)
|
|
||||||
go func() {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.users(w, runtimeRequest("PATCH", "/api/users", `{"id":"owner","quota_bytes":1024}`, sid))
|
|
||||||
done <- w
|
|
||||||
}()
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
a.userLocks.mu.Lock()
|
|
||||||
refs := lock.refs
|
|
||||||
a.userLocks.mu.Unlock()
|
|
||||||
if refs == 2 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
t.Fatal("quota update bypassed user lock", w.Code)
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("quota update did not wait on user lock")
|
|
||||||
}
|
|
||||||
runtime.Gosched()
|
|
||||||
}
|
|
||||||
lock.Release()
|
|
||||||
lock = nil
|
|
||||||
select {
|
|
||||||
case w := <-done:
|
|
||||||
if w.Code != 200 {
|
|
||||||
t.Fatal(w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("quota update deadlocked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Reconcile checks persisted identities against Docker without deleting unknown
|
|
||||||
// containers. Orphans require explicit administrator review, never blind adoption.
|
|
||||||
// Runs before the HTTP server starts, so no request can race this pass.
|
|
||||||
func (a *API) Reconcile(ctx context.Context) error {
|
|
||||||
filters, _ := json.Marshal(map[string][]string{"label": {"dsh-mgr.instance"}})
|
|
||||||
var containers []struct {
|
|
||||||
ID string
|
|
||||||
State string
|
|
||||||
Labels map[string]string
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, "GET", "/containers/json?all=1&filters="+url.QueryEscape(string(filters)), nil, &containers); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
rows, err := a.s.DB.QueryContext(ctx, "SELECT id,uid,container_id,status FROM instances")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
type record struct{ id, uid, container, status string }
|
|
||||||
var records []record
|
|
||||||
for rows.Next() {
|
|
||||||
var r record
|
|
||||||
if err = rows.Scan(&r.id, &r.uid, &r.container, &r.status); err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
records = append(records, r)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = rows.Err()
|
|
||||||
}
|
|
||||||
rows.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
known := make(map[string]record)
|
|
||||||
for _, r := range records {
|
|
||||||
if r.container == "" && (r.status == "creating" || r.status == "error") {
|
|
||||||
i, loadErr := a.s.GetInstance(r.id)
|
|
||||||
if loadErr != nil {
|
|
||||||
return loadErr
|
|
||||||
}
|
|
||||||
if recoveryErr := a.recoverRuntimeContainer(ctx, &i); recoveryErr != nil {
|
|
||||||
a.s.AddAudit(r.uid, "reconcile.recovery_error", r.id, recoveryErr.Error())
|
|
||||||
}
|
|
||||||
r.container = i.ContainerID
|
|
||||||
r.status = "error"
|
|
||||||
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET status='error' WHERE id=?", r.id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
known[r.id] = r
|
|
||||||
}
|
|
||||||
matched := make(map[string]bool)
|
|
||||||
panel := os.Getenv("PANEL_CONTAINER_ID")
|
|
||||||
var networkErrors []error
|
|
||||||
for _, c := range containers {
|
|
||||||
id := c.Labels["dsh-mgr.instance"]
|
|
||||||
r, ok := known[id]
|
|
||||||
if !ok || r.container != c.ID || r.uid != c.Labels["dsh-mgr.owner"] {
|
|
||||||
a.s.AddAudit("", "reconcile.orphan", c.ID, "container identity requires administrator review")
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
matched[id] = true
|
|
||||||
if panel != "" {
|
|
||||||
if networkErr := a.reconcilePanelNetwork(ctx, id, panel); networkErr != nil {
|
|
||||||
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET status='error' WHERE id=?", id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
a.s.AddAudit(r.uid, "reconcile.network_error", id, networkErr.Error())
|
|
||||||
networkErrors = append(networkErrors, fmt.Errorf("instance %s: %w", id, networkErr))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if r.status != "error" {
|
|
||||||
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET status=? WHERE id=?", c.State, id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, r := range records {
|
|
||||||
if r.container == "" || matched[r.id] || r.status == "error" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET status='error' WHERE id=?", r.id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
a.s.AddAudit(r.uid, "reconcile.missing", r.id, "recorded container is absent or identity mismatch")
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return fmt.Errorf("reconcile: %w", ctx.Err())
|
|
||||||
}
|
|
||||||
return errors.Join(networkErrors...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) reconcilePanelNetwork(ctx context.Context, id, panel string) error {
|
|
||||||
name := runtimeNetwork(id)
|
|
||||||
var network struct {
|
|
||||||
ID, Name, Driver string
|
|
||||||
Labels map[string]string
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, "GET", "/networks/"+url.PathEscape(name), nil, &network); err != nil {
|
|
||||||
return fmt.Errorf("inspect instance network: %w", err)
|
|
||||||
}
|
|
||||||
if network.ID == "" || network.Name != name || network.Driver != "bridge" || network.Labels["dsh-mgr.instance"] != id {
|
|
||||||
return fmt.Errorf("instance network identity mismatch")
|
|
||||||
}
|
|
||||||
container, err := a.docker.Inspect(ctx, panel)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("inspect panel: %w", err)
|
|
||||||
}
|
|
||||||
if container.ID == "" {
|
|
||||||
return fmt.Errorf("panel container identity missing")
|
|
||||||
}
|
|
||||||
for _, endpoint := range container.NetworkSettings.Networks {
|
|
||||||
if endpoint.NetworkID == network.ID {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, "POST", "/networks/"+url.PathEscape(network.ID)+"/connect", map[string]string{"Container": container.ID}, nil); err != nil {
|
|
||||||
// Another reconciliation may have connected the panel after inspection.
|
|
||||||
if current, inspectErr := a.docker.Inspect(ctx, container.ID); inspectErr == nil && current.ID == container.ID {
|
|
||||||
for _, endpoint := range current.NetworkSettings.Networks {
|
|
||||||
if endpoint.NetworkID == network.ID {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return fmt.Errorf("connect panel network: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,141 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestReconcileIdentityAndMissing(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != "GET" {
|
|
||||||
t.Fatalf("unexpected destructive request %s", r.Method)
|
|
||||||
}
|
|
||||||
w.Write([]byte(`[{"Id":"c1","State":"running","Labels":{"dsh-mgr.instance":"one","dsh-mgr.owner":"owner"}},{"Id":"c3","State":"running","Labels":{"dsh-mgr.instance":"orphan","dsh-mgr.owner":"owner"}}]`))
|
|
||||||
})
|
|
||||||
for _, i := range []store.Instance{{ID: "one", ContainerID: "c1", Status: "exited"}, {ID: "missing", ContainerID: "c2", Status: "running"}, {ID: "locked", ContainerID: "c4", Status: "error"}} {
|
|
||||||
i.UID = "owner"
|
|
||||||
i.Created = time.Now()
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := a.Reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for id, want := range map[string]string{"one": "running", "missing": "error", "locked": "error"} {
|
|
||||||
i, err := a.s.GetInstance(id)
|
|
||||||
if err != nil || i.Status != want {
|
|
||||||
t.Fatalf("%s: %+v %v", id, i, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var count int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='reconcile.orphan'").Scan(&count); err != nil || count != 1 {
|
|
||||||
t.Fatalf("orphan audit: %d %v", count, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReconcilePanelNetwork(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
connected, race, connectFail, missing, foreign, wrongDriver, panelFail bool
|
|
||||||
wantConnect int
|
|
||||||
wantError bool
|
|
||||||
}{
|
|
||||||
{name: "already connected", connected: true},
|
|
||||||
{name: "reconnect", wantConnect: 1},
|
|
||||||
{name: "concurrent connection", race: true, wantConnect: 1},
|
|
||||||
{name: "connect failure", connectFail: true, wantConnect: 1, wantError: true},
|
|
||||||
{name: "missing network", missing: true, wantError: true},
|
|
||||||
{name: "foreign network", foreign: true, wantError: true},
|
|
||||||
{name: "not bridge", wrongDriver: true, wantError: true},
|
|
||||||
{name: "panel inspect failure", panelFail: true, wantError: true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "configured-panel")
|
|
||||||
connects := 0
|
|
||||||
connected := tc.connected
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.Method + " " + r.URL.Path {
|
|
||||||
case "GET /v1.47/containers/json":
|
|
||||||
fmt.Fprint(w, `[{"Id":"c1","State":"running","Labels":{"dsh-mgr.instance":"one","dsh-mgr.owner":"owner"}},{"Id":"foreign","State":"running","Labels":{"dsh-mgr.instance":"foreign","dsh-mgr.owner":"other"}},{"Id":"unknown","State":"running","Labels":{"dsh-mgr.instance":"unknown","dsh-mgr.owner":"owner"}}]`)
|
|
||||||
case "GET /v1.47/networks/" + runtimeNetwork("one"):
|
|
||||||
if tc.missing {
|
|
||||||
http.Error(w, "network not found", http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
owner, driver := "one", "bridge"
|
|
||||||
if tc.foreign {
|
|
||||||
owner = "other"
|
|
||||||
}
|
|
||||||
if tc.wrongDriver {
|
|
||||||
driver = "host"
|
|
||||||
}
|
|
||||||
json.NewEncoder(w).Encode(map[string]any{"Id": "network-id", "Name": runtimeNetwork("one"), "Driver": driver, "Labels": map[string]string{"dsh-mgr.instance": owner}})
|
|
||||||
case "GET /v1.47/containers/configured-panel/json", "GET /v1.47/containers/panel-full-id/json":
|
|
||||||
if tc.panelFail {
|
|
||||||
http.Error(w, "panel missing", http.StatusNotFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
networkID := "unrelated-network"
|
|
||||||
if connected {
|
|
||||||
networkID = "network-id"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, `{"Id":"panel-full-id","NetworkSettings":{"Networks":{"bridge":{"NetworkID":%q}}}}`, networkID)
|
|
||||||
case "POST /v1.47/networks/network-id/connect":
|
|
||||||
connects++
|
|
||||||
var body map[string]string
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body) != 1 || body["Container"] != "panel-full-id" {
|
|
||||||
t.Errorf("unsafe connection body: %v, %v", body, err)
|
|
||||||
}
|
|
||||||
if tc.connectFail || tc.race {
|
|
||||||
connected = tc.race
|
|
||||||
http.Error(w, "connection failed", http.StatusForbidden)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
connected = true
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
|
|
||||||
http.Error(w, "unexpected request", http.StatusBadRequest)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
for _, i := range []store.Instance{
|
|
||||||
{ID: "one", UID: "owner", ContainerID: "c1", Status: "exited", Created: time.Now()},
|
|
||||||
{ID: "foreign", UID: "owner", ContainerID: "foreign", Status: "running", Created: time.Now()},
|
|
||||||
} {
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := a.Reconcile(context.Background()); (err != nil) != tc.wantError {
|
|
||||||
t.Fatalf("Reconcile error = %v, want error %v", err, tc.wantError)
|
|
||||||
}
|
|
||||||
if !tc.wantError {
|
|
||||||
if err := a.Reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if connects != tc.wantConnect {
|
|
||||||
t.Fatalf("connections = %d, want %d", connects, tc.wantConnect)
|
|
||||||
}
|
|
||||||
wantStatus, wantAudits := "running", 0
|
|
||||||
if tc.wantError {
|
|
||||||
wantStatus, wantAudits = "error", 1
|
|
||||||
}
|
|
||||||
i, err := a.s.GetInstance("one")
|
|
||||||
if err != nil || i.Status != wantStatus {
|
|
||||||
t.Fatalf("instance: %+v %v", i, err)
|
|
||||||
}
|
|
||||||
var audits int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='reconcile.network_error'").Scan(&audits); err != nil || audits != wantAudits {
|
|
||||||
t.Fatalf("network audits: %d %v", audits, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,305 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// pendingColumns whitelists the runtime_creates flag columns so no caller can
|
|
||||||
// concatenate an unvalidated column name into SQL.
|
|
||||||
type pendingColumn struct{ flag, stamp string }
|
|
||||||
|
|
||||||
var pendingColumns = map[string]pendingColumn{
|
|
||||||
"container_pending": {flag: "container_pending", stamp: "container_pending_at"},
|
|
||||||
"network_pending": {flag: "network_pending", stamp: "network_pending_at"},
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureColumn adds a TEXT column, tolerating databases that already have it.
|
|
||||||
func (a *API) ensureColumn(table, column string) error {
|
|
||||||
_, err := a.s.DB.Exec("ALTER TABLE " + table + " ADD COLUMN " + column + " TEXT NOT NULL DEFAULT ''")
|
|
||||||
if err != nil && strings.Contains(strings.ToLower(err.Error()), "duplicate column") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) recoverRuntimeContainer(ctx context.Context, i *store.Instance) error {
|
|
||||||
c, err := a.docker.Inspect(ctx, runtimeNetwork(i.ID))
|
|
||||||
if dockerNotFound(err) {
|
|
||||||
_, pending, checkErr := a.pendingSince(i.ID, "container_pending")
|
|
||||||
if checkErr != nil {
|
|
||||||
return checkErr
|
|
||||||
}
|
|
||||||
if pending {
|
|
||||||
return errors.New("container creation outcome remains uncertain")
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if c.ID == "" || strings.TrimPrefix(c.Name, "/") != runtimeNetwork(i.ID) || c.Config.Labels["dsh-mgr.instance"] != i.ID || c.Config.Labels["dsh-mgr.owner"] != i.UID || c.Config.Labels["dsh-mgr.role"] != "dsh" || c.Image != i.ImageID {
|
|
||||||
return errors.New("runtime container identity mismatch")
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.ExecContext(ctx, "UPDATE instances SET container_id=? WHERE id=? AND container_id=''", c.ID, i.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
i.ContainerID = c.ID
|
|
||||||
if err = a.setRuntimeCreatePending(i.ID, "container_pending", false); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (a *API) ensureRuntimeCreates() error {
|
|
||||||
_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_creates(instance_id TEXT PRIMARY KEY,container_pending INTEGER NOT NULL DEFAULT 0,network_pending INTEGER NOT NULL DEFAULT 0,container_pending_at TEXT NOT NULL DEFAULT '',network_pending_at TEXT NOT NULL DEFAULT '')")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err = a.ensureColumn("runtime_creates", "container_pending_at"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err = a.ensureColumn("runtime_creates", "network_pending_at"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = a.s.DB.Exec("CREATE TABLE IF NOT EXISTS runtime_networks(instance_id TEXT PRIMARY KEY,network_id TEXT NOT NULL)")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
func (a *API) setRuntimeCreatePending(id, column string, pending bool) error {
|
|
||||||
if err := a.ensureRuntimeCreates(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
c, ok := pendingColumns[column]
|
|
||||||
if !ok {
|
|
||||||
return errors.New("unknown pending column")
|
|
||||||
}
|
|
||||||
stamp := ""
|
|
||||||
if pending {
|
|
||||||
stamp = time.Now().UTC().Format(time.RFC3339Nano)
|
|
||||||
}
|
|
||||||
_, err := a.s.DB.Exec("INSERT INTO runtime_creates(instance_id,"+c.flag+","+c.stamp+") VALUES(?,?,?) ON CONFLICT(instance_id) DO UPDATE SET "+c.flag+"=excluded."+c.flag+","+c.stamp+"=excluded."+c.stamp, id, pending, stamp)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// pendingSince reports when a pending flag was raised. A legacy row without a
|
|
||||||
// timestamp reports a zero time without resolving the uncertainty.
|
|
||||||
func (a *API) pendingSince(id, column string) (time.Time, bool, error) {
|
|
||||||
if err := a.ensureRuntimeCreates(); err != nil {
|
|
||||||
return time.Time{}, false, err
|
|
||||||
}
|
|
||||||
c, ok := pendingColumns[column]
|
|
||||||
if !ok {
|
|
||||||
return time.Time{}, false, errors.New("unknown pending column")
|
|
||||||
}
|
|
||||||
var pending bool
|
|
||||||
var stamp string
|
|
||||||
err := a.s.DB.QueryRow("SELECT "+c.flag+",COALESCE("+c.stamp+",'') FROM runtime_creates WHERE instance_id=?", id).Scan(&pending, &stamp)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return time.Time{}, false, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return time.Time{}, false, err
|
|
||||||
}
|
|
||||||
if !pending {
|
|
||||||
return time.Time{}, false, nil
|
|
||||||
}
|
|
||||||
if stamp == "" {
|
|
||||||
return time.Time{}, true, nil
|
|
||||||
}
|
|
||||||
since, perr := time.Parse(time.RFC3339Nano, stamp)
|
|
||||||
if perr != nil {
|
|
||||||
return time.Time{}, true, nil
|
|
||||||
}
|
|
||||||
return since, true, nil
|
|
||||||
}
|
|
||||||
func (a *API) recoverRuntimeNetwork(ctx context.Context, id string) error {
|
|
||||||
_, pending, err := a.pendingSince(id, "network_pending")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var recorded string
|
|
||||||
err = a.s.DB.QueryRowContext(ctx, "SELECT network_id FROM runtime_networks WHERE instance_id=?", id).Scan(&recorded)
|
|
||||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
lookup := runtimeNetwork(id)
|
|
||||||
if recorded != "" {
|
|
||||||
lookup = recorded
|
|
||||||
}
|
|
||||||
var network struct {
|
|
||||||
ID, Name, Driver string
|
|
||||||
Labels map[string]string
|
|
||||||
Containers map[string]json.RawMessage
|
|
||||||
}
|
|
||||||
err = a.docker.Do(ctx, http.MethodGet, "/networks/"+url.PathEscape(lookup), nil, &network)
|
|
||||||
if dockerNotFound(err) {
|
|
||||||
if pending && recorded == "" {
|
|
||||||
return errors.New("network creation outcome remains uncertain")
|
|
||||||
}
|
|
||||||
return a.setRuntimeCreatePending(id, "network_pending", false)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if network.ID == "" || (recorded != "" && recorded != network.ID) || network.Name != runtimeNetwork(id) || network.Driver != "bridge" || network.Labels["dsh-mgr.instance"] != id {
|
|
||||||
return errors.New("runtime network identity mismatch")
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.ExecContext(ctx, "INSERT INTO runtime_networks(instance_id,network_id) VALUES(?,?) ON CONFLICT(instance_id) DO UPDATE SET network_id=excluded.network_id", id, network.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if panel := os.Getenv("PANEL_CONTAINER_ID"); panel != "" && len(network.Containers) != 0 {
|
|
||||||
if _, connected := network.Containers[panel]; !connected {
|
|
||||||
c, err := a.docker.Inspect(ctx, panel)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
panel = c.ID
|
|
||||||
}
|
|
||||||
if _, connected := network.Containers[panel]; connected {
|
|
||||||
err = a.docker.Do(ctx, http.MethodPost, "/networks/"+url.PathEscape(network.ID)+"/disconnect", map[string]any{"Container": panel, "Force": true}, nil)
|
|
||||||
if err != nil && !dockerNotFound(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
err = a.docker.Do(ctx, http.MethodDelete, "/networks/"+url.PathEscape(network.ID), nil, nil)
|
|
||||||
if err != nil && !dockerNotFound(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return a.setRuntimeCreatePending(id, "network_pending", false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *API) ensureBuildChecks() error {
|
|
||||||
_, err := a.s.DB.Exec("CREATE TABLE IF NOT EXISTS image_build_checks(build_id TEXT PRIMARY KEY,image_id TEXT NOT NULL)")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err = a.ensureColumn("image_build_checks", "container_id"); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return a.ensureColumn("image_build_checks", "created_at")
|
|
||||||
}
|
|
||||||
func buildCheckName(id string) string { return "dsh-build-check-" + id }
|
|
||||||
|
|
||||||
var errBuildCheckUncertain = errors.New("verification container creation outcome remains uncertain")
|
|
||||||
|
|
||||||
func (a *API) cleanupBuildCheck(ctx context.Context, id, image string) error {
|
|
||||||
var recorded string
|
|
||||||
if err := a.s.DB.QueryRowContext(ctx, "SELECT container_id FROM image_build_checks WHERE build_id=?", id).Scan(&recorded); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
lookup := buildCheckName(id)
|
|
||||||
if recorded != "" {
|
|
||||||
lookup = recorded
|
|
||||||
}
|
|
||||||
c, err := a.docker.Inspect(ctx, lookup)
|
|
||||||
if dockerNotFound(err) {
|
|
||||||
if recorded != "" {
|
|
||||||
_, err := a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// A name lookup cannot rule out a create still running in Docker.
|
|
||||||
return errBuildCheckUncertain
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if c.ID == "" || (recorded != "" && c.ID != recorded) || strings.TrimPrefix(c.Name, "/") != buildCheckName(id) || c.Image != image || c.Config.Labels["dsh-mgr.role"] != "build-check" || c.Config.Labels["dsh-mgr.build"] != id {
|
|
||||||
return errors.New("build verification container identity mismatch")
|
|
||||||
}
|
|
||||||
return a.deleteBuildCheck(ctx, id, c.ID)
|
|
||||||
}
|
|
||||||
func (a *API) deleteBuildCheck(ctx context.Context, id, container string) error {
|
|
||||||
if _, err := a.s.DB.ExecContext(ctx, "UPDATE image_build_checks SET container_id=? WHERE build_id=?", container, id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err := a.docker.Do(ctx, http.MethodDelete, "/containers/"+url.PathEscape(container)+"?force=1&v=1", nil, nil)
|
|
||||||
if err != nil && !dockerNotFound(err) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err = a.s.DB.ExecContext(ctx, "DELETE FROM image_build_checks WHERE build_id=?", id)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
func (a *API) recoverBuildChecks(ctx context.Context) error {
|
|
||||||
if err := a.ensureBuildChecks(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
rows, err := a.s.DB.QueryContext(ctx, "SELECT build_id,image_id,COALESCE(created_at,'') FROM image_build_checks")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
type check struct{ id, image, createdAt string }
|
|
||||||
var checks []check
|
|
||||||
for rows.Next() {
|
|
||||||
var c check
|
|
||||||
if err = rows.Scan(&c.id, &c.image, &c.createdAt); err != nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
checks = append(checks, c)
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
err = rows.Err()
|
|
||||||
}
|
|
||||||
rows.Close()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var failures []error
|
|
||||||
missing := false
|
|
||||||
for _, c := range checks {
|
|
||||||
err := a.cleanupBuildCheck(ctx, c.id, c.image)
|
|
||||||
if errors.Is(err, errBuildCheckUncertain) {
|
|
||||||
missing = true
|
|
||||||
} else {
|
|
||||||
failures = append(failures, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := errors.Join(failures...); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if missing {
|
|
||||||
return errBuildCheckUncertain
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (a *API) persistBuildResult(b buildRecord, status, logs, detail, image, finished string) error {
|
|
||||||
tx, err := a.s.DB.Begin()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer tx.Rollback()
|
|
||||||
if status == "success" && image != "" {
|
|
||||||
_, err = tx.Exec("INSERT INTO images(image_id,version,recipe_rev,platform,tag,status,built_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(image_id) DO UPDATE SET version=excluded.version,recipe_rev=excluded.recipe_rev,platform=excluded.platform,tag=excluded.tag,status=excluded.status,built_at=excluded.built_at", image, b.Version, imageRecipeRev, imagePlatform, "dsh-web:"+b.Version+"-"+imageRecipeRev, "unverified", finished)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err = tx.Exec("UPDATE image_builds SET status=?,logs=?,error=?,image_id=?,finished_at=? WHERE id=?", status, logs, detail, image, finished, b.ID); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return tx.Commit()
|
|
||||||
}
|
|
||||||
func (a *API) finishBuild(b buildRecord, status, logs, detail, image, finished string) {
|
|
||||||
for attempt := 0; ; attempt++ {
|
|
||||||
err := a.persistBuildResult(b, status, logs, detail, image, finished)
|
|
||||||
if err == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if attempt%60 == 0 {
|
|
||||||
log.Printf("build %s: terminal transaction failed; retrying: %s", b.ID, sanitizeBuildLog(err.Error()))
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-a.s.Done():
|
|
||||||
return
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,407 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
)
|
|
||||||
|
|
||||||
const liveRecoveryLabel = "dsh-test.recovery-run"
|
|
||||||
|
|
||||||
type liveRecoveryResource struct{ kind, id string }
|
|
||||||
type liveRecoveryFault struct {
|
|
||||||
method, path string
|
|
||||||
release chan struct{}
|
|
||||||
done chan struct{}
|
|
||||||
once sync.Once
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *liveRecoveryFault) unblock() { f.once.Do(func() { close(f.release) }) }
|
|
||||||
|
|
||||||
type liveRecoveryProxy struct {
|
|
||||||
t *testing.T
|
|
||||||
run string
|
|
||||||
transport *http.Transport
|
|
||||||
mu sync.Mutex
|
|
||||||
handlers sync.WaitGroup
|
|
||||||
resources []liveRecoveryResource
|
|
||||||
fault *liveRecoveryFault
|
|
||||||
}
|
|
||||||
|
|
||||||
func liveRequire(t *testing.T, err error) {
|
|
||||||
t.Helper()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *liveRecoveryProxy) arm(method, path string, delayed bool) *liveRecoveryFault {
|
|
||||||
f := &liveRecoveryFault{method: method, path: path, release: make(chan struct{}), done: make(chan struct{})}
|
|
||||||
if !delayed {
|
|
||||||
f.unblock()
|
|
||||||
}
|
|
||||||
p.mu.Lock()
|
|
||||||
p.fault = f
|
|
||||||
p.mu.Unlock()
|
|
||||||
p.t.Cleanup(f.unblock)
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *liveRecoveryProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
p.handlers.Add(1)
|
|
||||||
defer p.handlers.Done()
|
|
||||||
body, err := io.ReadAll(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
p.t.Error(err)
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
path := strings.TrimPrefix(r.URL.Path, "/v1.47")
|
|
||||||
create := r.Method == "POST" && (path == "/containers/create" || path == "/networks/create")
|
|
||||||
p.mu.Lock()
|
|
||||||
allowed := r.Method == "GET"
|
|
||||||
if create {
|
|
||||||
var spec struct {
|
|
||||||
Name string
|
|
||||||
Labels map[string]string
|
|
||||||
}
|
|
||||||
if json.Unmarshal(body, &spec) == nil {
|
|
||||||
name := spec.Name
|
|
||||||
if path == "/containers/create" {
|
|
||||||
name = r.URL.Query().Get("name")
|
|
||||||
}
|
|
||||||
allowed = spec.Labels[liveRecoveryLabel] == p.run && strings.Contains(name, p.run)
|
|
||||||
}
|
|
||||||
} else if !allowed {
|
|
||||||
for _, resource := range p.resources {
|
|
||||||
base := "/" + resource.kind + "/" + resource.id
|
|
||||||
if path == base || strings.HasPrefix(path, base+"/") {
|
|
||||||
allowed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if strings.HasSuffix(path, "/connect") || strings.HasSuffix(path, "/disconnect") {
|
|
||||||
var spec struct{ Container string }
|
|
||||||
_ = json.Unmarshal(body, &spec)
|
|
||||||
owned := false
|
|
||||||
for _, resource := range p.resources {
|
|
||||||
if resource.kind == "containers" && spec.Container == resource.id {
|
|
||||||
owned = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
allowed = allowed && owned
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var fault *liveRecoveryFault
|
|
||||||
if p.fault != nil && p.fault.method == r.Method && p.fault.path == path {
|
|
||||||
fault = p.fault
|
|
||||||
p.fault = nil
|
|
||||||
}
|
|
||||||
p.mu.Unlock()
|
|
||||||
if !allowed {
|
|
||||||
p.t.Errorf("blocked out-of-scope request: %s %s", r.Method, path)
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
if fault != nil {
|
|
||||||
defer close(fault.done)
|
|
||||||
select {
|
|
||||||
case <-fault.release:
|
|
||||||
default:
|
|
||||||
conn, _, hijackErr := w.(http.Hijacker).Hijack()
|
|
||||||
if hijackErr != nil {
|
|
||||||
p.t.Error(hijackErr)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_ = conn.Close()
|
|
||||||
<-fault.release
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A disconnected test client must not cancel the deliberately late daemon operation.
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
req := r.Clone(ctx)
|
|
||||||
req.URL.Scheme, req.URL.Host, req.RequestURI = "http", "docker", ""
|
|
||||||
req.Body = io.NopCloser(bytes.NewReader(body))
|
|
||||||
if len(body) == 0 {
|
|
||||||
req.Body = http.NoBody
|
|
||||||
}
|
|
||||||
resp, err := p.transport.RoundTrip(req)
|
|
||||||
if err != nil {
|
|
||||||
p.t.Error(err)
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
data, err := io.ReadAll(resp.Body)
|
|
||||||
if err != nil {
|
|
||||||
p.t.Error(err)
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
if create && resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
|
||||||
var result struct{ ID string }
|
|
||||||
if err := json.Unmarshal(data, &result); err != nil || result.ID == "" {
|
|
||||||
p.t.Errorf("missing real create ID: %s", data)
|
|
||||||
} else {
|
|
||||||
kind := "containers"
|
|
||||||
if path == "/networks/create" {
|
|
||||||
kind = "networks"
|
|
||||||
}
|
|
||||||
p.mu.Lock()
|
|
||||||
p.resources = append(p.resources, liveRecoveryResource{kind, result.ID})
|
|
||||||
p.mu.Unlock()
|
|
||||||
p.t.Logf("created %s ID=%s run=%s", kind, result.ID, p.run)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if fault != nil {
|
|
||||||
p.t.Logf("dropped real response: %s %s status=%d", r.Method, path, resp.StatusCode)
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
||||||
p.t.Errorf("fault did not follow successful daemon operation: %s", data)
|
|
||||||
}
|
|
||||||
panic(http.ErrAbortHandler)
|
|
||||||
}
|
|
||||||
for key, values := range resp.Header {
|
|
||||||
for _, value := range values {
|
|
||||||
w.Header().Add(key, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
w.WriteHeader(resp.StatusCode)
|
|
||||||
_, _ = w.Write(data)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newLiveRecovery(t *testing.T) (*API, *liveRecoveryProxy, string) {
|
|
||||||
t.Helper()
|
|
||||||
if os.Getenv("DSH_DOCKER_LIVE_TEST") != "1" {
|
|
||||||
t.Skip("opt in with DSH_DOCKER_LIVE_TEST=1; uses /var/run/docker.sock and local node:24-alpine only")
|
|
||||||
}
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
var nonce [12]byte
|
|
||||||
_, err := rand.Read(nonce[:])
|
|
||||||
liveRequire(t, err)
|
|
||||||
p := &liveRecoveryProxy{t: t, run: "live-" + hex.EncodeToString(nonce[:]), transport: &http.Transport{DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
|
||||||
return (&net.Dialer{}).DialContext(ctx, "unix", "/var/run/docker.sock")
|
|
||||||
}}}
|
|
||||||
real := docker.New("unix:///var/run/docker.sock")
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
var image struct{ ID string }
|
|
||||||
liveRequire(t, real.Do(ctx, "GET", "/images/node:24-alpine/json", nil, &image))
|
|
||||||
s, err := store.Open(filepath.Join(t.TempDir(), "live.db"))
|
|
||||||
liveRequire(t, err)
|
|
||||||
t.Cleanup(s.Close)
|
|
||||||
server := httptest.NewServer(p)
|
|
||||||
t.Cleanup(func() {
|
|
||||||
server.Close()
|
|
||||||
p.handlers.Wait()
|
|
||||||
p.transport.CloseIdleConnections()
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
p.mu.Lock()
|
|
||||||
resources := append([]liveRecoveryResource(nil), p.resources...)
|
|
||||||
p.mu.Unlock()
|
|
||||||
for _, kind := range []string{"containers", "networks"} {
|
|
||||||
for _, resource := range resources {
|
|
||||||
if resource.kind != kind {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
path := "/" + kind + "/" + resource.id
|
|
||||||
err := real.Do(ctx, "DELETE", path+"?force=1", nil, nil)
|
|
||||||
if err != nil && !dockerNotFound(err) {
|
|
||||||
t.Errorf("cleanup %s: %v", path, err)
|
|
||||||
}
|
|
||||||
if kind == "containers" {
|
|
||||||
path += "/json"
|
|
||||||
}
|
|
||||||
if err := real.Do(ctx, "GET", path, nil, nil); !dockerNotFound(err) {
|
|
||||||
t.Errorf("residual %s: %v", path, err)
|
|
||||||
} else {
|
|
||||||
t.Logf("cleanup confirmed absent %s %s", kind, resource.id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
filters, _ := json.Marshal(map[string][]string{"label": {liveRecoveryLabel + "=" + p.run}})
|
|
||||||
for _, path := range []string{"/containers/json?all=1&", "/networks?"} {
|
|
||||||
var residual []json.RawMessage
|
|
||||||
if err := real.Do(ctx, "GET", path+"filters="+url.QueryEscape(string(filters)), nil, &residual); err != nil || len(residual) != 0 {
|
|
||||||
t.Errorf("label residual check: %s count=%d err=%v", p.run, len(residual), err)
|
|
||||||
} else {
|
|
||||||
t.Logf("label residual check empty: %s %s", p.run, path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
return &API{s: s, root: t.TempDir(), docker: docker.New(server.URL)}, p, image.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
func liveContainerSpec(run, image string, labels map[string]string) map[string]any {
|
|
||||||
labels[liveRecoveryLabel] = run
|
|
||||||
return map[string]any{"Image": image, "Cmd": []string{"sleep", "600"}, "Labels": labels, "HostConfig": map[string]any{"NetworkMode": "none", "Memory": 64 << 20, "PidsLimit": 32}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDockerLiveRecoveryNetwork(t *testing.T) {
|
|
||||||
for _, panelMode := range []string{"unset", "unconnected", "connected-id", "connected-name"} {
|
|
||||||
t.Run(panelMode, func(t *testing.T) {
|
|
||||||
a, p, image := newLiveRecovery(t)
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
id := p.run
|
|
||||||
liveRequire(t, a.setRuntimeCreatePending(id, "network_pending", true))
|
|
||||||
p.arm("POST", "/networks/create", false)
|
|
||||||
err := a.docker.Do(ctx, "POST", "/networks/create", map[string]any{"Name": runtimeNetwork(id), "Driver": "bridge", "Labels": map[string]string{"dsh-mgr.instance": id, liveRecoveryLabel: p.run}}, nil)
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("create response was not lost")
|
|
||||||
}
|
|
||||||
var network struct{ ID string }
|
|
||||||
liveRequire(t, a.docker.Do(ctx, "GET", "/networks/"+runtimeNetwork(id), nil, &network))
|
|
||||||
if panelMode != "unset" {
|
|
||||||
name := p.run + "-panel"
|
|
||||||
var panel struct{ ID string }
|
|
||||||
spec := liveContainerSpec(p.run, image, map[string]string{})
|
|
||||||
if strings.HasPrefix(panelMode, "connected") {
|
|
||||||
spec["HostConfig"].(map[string]any)["NetworkMode"] = network.ID
|
|
||||||
}
|
|
||||||
liveRequire(t, a.docker.Do(ctx, "POST", "/containers/create?name="+name, spec, &panel))
|
|
||||||
liveRequire(t, a.docker.Do(ctx, "POST", "/containers/"+panel.ID+"/start", nil, nil))
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", panel.ID)
|
|
||||||
if panelMode == "connected-name" {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", name)
|
|
||||||
}
|
|
||||||
var endpoints struct{ Containers map[string]json.RawMessage }
|
|
||||||
liveRequire(t, a.docker.Do(ctx, "GET", "/networks/"+network.ID, nil, &endpoints))
|
|
||||||
_, connected := endpoints.Containers[panel.ID]
|
|
||||||
if connected != strings.HasPrefix(panelMode, "connected") {
|
|
||||||
t.Fatalf("unexpected real panel connection: %v", connected)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
p.arm("DELETE", "/networks/"+network.ID, false)
|
|
||||||
if err := a.removeRuntimeNetwork(ctx, id); err == nil {
|
|
||||||
t.Fatal("delete response was not lost")
|
|
||||||
}
|
|
||||||
var recorded string
|
|
||||||
liveRequire(t, a.s.DB.QueryRow("SELECT network_id FROM runtime_networks WHERE instance_id=?", id).Scan(&recorded))
|
|
||||||
if recorded != network.ID {
|
|
||||||
t.Fatalf("lost recorded identity: %s", recorded)
|
|
||||||
}
|
|
||||||
liveRequire(t, a.removeRuntimeNetwork(ctx, id))
|
|
||||||
_, pending, err := a.pendingSince(id, "network_pending")
|
|
||||||
liveRequire(t, err)
|
|
||||||
if pending {
|
|
||||||
t.Fatal("pending did not converge")
|
|
||||||
}
|
|
||||||
if err := a.docker.Do(ctx, "GET", "/networks/"+network.ID, nil, nil); !dockerNotFound(err) {
|
|
||||||
t.Fatalf("network still exists: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDockerLiveRecoveryLateContainer(t *testing.T) {
|
|
||||||
for _, build := range []bool{false, true} {
|
|
||||||
name := "runtime"
|
|
||||||
if build {
|
|
||||||
name = "build-check"
|
|
||||||
}
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
a, p, image := newLiveRecovery(t)
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
i := store.Instance{ID: p.run, UID: p.run, ImageID: image, Status: "error", Created: time.Now()}
|
|
||||||
name := runtimeNetwork(i.ID)
|
|
||||||
labels := map[string]string{"dsh-mgr.instance": i.ID, "dsh-mgr.owner": i.UID, "dsh-mgr.role": "dsh"}
|
|
||||||
if build {
|
|
||||||
name = buildCheckName(i.ID)
|
|
||||||
labels = map[string]string{"dsh-mgr.build": i.ID, "dsh-mgr.role": "build-check"}
|
|
||||||
liveRequire(t, a.ensureBuildChecks())
|
|
||||||
_, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES(?,?,?)", i.ID, image, time.Now().Add(-time.Hour).Format(time.RFC3339Nano))
|
|
||||||
liveRequire(t, err)
|
|
||||||
} else {
|
|
||||||
liveRequire(t, a.s.AddInstance(i))
|
|
||||||
liveRequire(t, a.setRuntimeCreatePending(i.ID, "container_pending", true))
|
|
||||||
}
|
|
||||||
fault := p.arm("POST", "/containers/create", true)
|
|
||||||
if err := a.docker.Do(ctx, "POST", "/containers/create?name="+name, liveContainerSpec(p.run, image, labels), nil); err == nil {
|
|
||||||
t.Fatal("expected disconnected create")
|
|
||||||
}
|
|
||||||
if _, err := a.docker.Inspect(ctx, name); !dockerNotFound(err) {
|
|
||||||
t.Fatalf("expected real initial 404: %v", err)
|
|
||||||
}
|
|
||||||
for attempt := 0; attempt < 2; attempt++ {
|
|
||||||
if build {
|
|
||||||
if err := a.recoverBuildChecks(ctx); !errors.Is(err, errBuildCheckUncertain) {
|
|
||||||
t.Fatalf("uncertainty lost: %v", err)
|
|
||||||
}
|
|
||||||
var count int
|
|
||||||
liveRequire(t, a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks WHERE build_id=?", i.ID).Scan(&count))
|
|
||||||
if count != 1 {
|
|
||||||
t.Fatal("build record lost")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err := a.recoverRuntimeContainer(ctx, &i); err == nil {
|
|
||||||
t.Fatal("uncertainty lost")
|
|
||||||
}
|
|
||||||
_, pending, err := a.pendingSince(i.ID, "container_pending")
|
|
||||||
liveRequire(t, err)
|
|
||||||
if !pending {
|
|
||||||
t.Fatal("pending lost")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fault.unblock()
|
|
||||||
select {
|
|
||||||
case <-fault.done:
|
|
||||||
case <-ctx.Done():
|
|
||||||
t.Fatal(ctx.Err())
|
|
||||||
}
|
|
||||||
container, err := a.docker.Inspect(ctx, name)
|
|
||||||
liveRequire(t, err)
|
|
||||||
p.arm("DELETE", "/containers/"+container.ID, false)
|
|
||||||
if build {
|
|
||||||
if err := a.recoverBuildChecks(ctx); err == nil {
|
|
||||||
t.Fatal("delete response was not lost")
|
|
||||||
}
|
|
||||||
var recorded string
|
|
||||||
liveRequire(t, a.s.DB.QueryRow("SELECT container_id FROM image_build_checks WHERE build_id=?", i.ID).Scan(&recorded))
|
|
||||||
if recorded != container.ID {
|
|
||||||
t.Fatal("build identity lost")
|
|
||||||
}
|
|
||||||
liveRequire(t, a.recoverBuildChecks(ctx))
|
|
||||||
var count int
|
|
||||||
liveRequire(t, a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count))
|
|
||||||
if count != 0 {
|
|
||||||
t.Fatal("build record retained")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
liveRequire(t, a.recoverRuntimeContainer(ctx, &i))
|
|
||||||
_, pending, err := a.pendingSince(i.ID, "container_pending")
|
|
||||||
liveRequire(t, err)
|
|
||||||
if pending {
|
|
||||||
t.Fatal("runtime pending did not converge")
|
|
||||||
}
|
|
||||||
stored, err := a.s.GetInstance(i.ID)
|
|
||||||
liveRequire(t, err)
|
|
||||||
if stored.ContainerID != container.ID {
|
|
||||||
t.Fatal("runtime identity lost")
|
|
||||||
}
|
|
||||||
if err := a.removeRuntimeContainer(ctx, i.ContainerID); err == nil {
|
|
||||||
t.Fatal("delete response was not lost")
|
|
||||||
}
|
|
||||||
liveRequire(t, a.removeRuntimeContainer(ctx, i.ContainerID))
|
|
||||||
}
|
|
||||||
if _, err := a.docker.Inspect(ctx, container.ID); !dockerNotFound(err) {
|
|
||||||
t.Fatalf("late resource not reclaimed: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,828 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/docker"
|
|
||||||
"git.misaka.ren/m1saka/dsh-web/internal/store"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"path/filepath"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestBuildCheckRenamedContainerRetainsIdentity(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method != "GET" || r.URL.Path != "/v1.47/containers/known/json" {
|
|
||||||
t.Errorf("unsafe lookup %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprint(w, `{"Id":"known","Name":"/renamed","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
|
|
||||||
})
|
|
||||||
if err := a.ensureBuildChecks(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.recoverBuildChecks(context.Background()); err == nil {
|
|
||||||
t.Fatal("renamed identity accepted")
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
|
|
||||||
t.Fatalf("record lost %d %v", n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestRuntimeNetworkDeletionLostResponseAndReplacement(t *testing.T) {
|
|
||||||
for _, foreign := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprint(foreign), func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
deleted := false
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method == "GET" {
|
|
||||||
if deleted {
|
|
||||||
if r.URL.Path != "/v1.47/networks/known" {
|
|
||||||
t.Error("did not use persisted ID")
|
|
||||||
}
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
owner := "net"
|
|
||||||
if foreign {
|
|
||||||
owner = "foreign"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, `{"Id":"known","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, owner)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if foreign {
|
|
||||||
t.Error("deleted foreign network")
|
|
||||||
}
|
|
||||||
deleted = true
|
|
||||||
w.WriteHeader(500)
|
|
||||||
})
|
|
||||||
if err := a.setRuntimeCreatePending("net", "network_pending", false); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.removeRuntimeNetwork(context.Background(), "net"); err == nil {
|
|
||||||
t.Fatal("first error missing")
|
|
||||||
}
|
|
||||||
if foreign {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := a.removeRuntimeNetwork(context.Background(), "net"); err != nil {
|
|
||||||
t.Fatalf("confirmed deletion did not converge: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
func TestStartIntentsUncertainCreationKeepsRecoveryAccessible(t *testing.T) {
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != "GET" {
|
|
||||||
t.Error("unexpected destructive call")
|
|
||||||
}
|
|
||||||
w.WriteHeader(404)
|
|
||||||
})
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: "intent", UID: "owner"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.s.ClaimIntent("intent", store.Instance{ID: "pending", UID: "owner", Status: "creating"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.setRuntimeCreatePending("pending", "container_pending", true); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
if err := a.StartIntents(ctx); err != nil {
|
|
||||||
t.Fatalf("uncertain creation blocked startup: %v", err)
|
|
||||||
}
|
|
||||||
i, err := a.s.GetInstance("pending")
|
|
||||||
if err != nil || i.Status != "error" {
|
|
||||||
t.Fatalf("recovery reservation missing %+v %v", i, err)
|
|
||||||
}
|
|
||||||
list, err := a.s.Intents("owner")
|
|
||||||
if err != nil || len(list) != 1 || list[0].Status != "failed" {
|
|
||||||
t.Fatalf("intent not terminated %+v %v", list, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkCreateLostResponse(t *testing.T) {
|
|
||||||
for _, mode := range []string{"present", "late", "foreign"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
late := false
|
|
||||||
deletes := 0
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method + " " + r.URL.Path {
|
|
||||||
case "POST /v1.47/networks/create":
|
|
||||||
w.WriteHeader(500)
|
|
||||||
case "GET /v1.47/containers/dsh-inst-net/json":
|
|
||||||
w.WriteHeader(404)
|
|
||||||
case "GET /v1.47/networks/dsh-inst-net":
|
|
||||||
if mode == "late" && !late {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := "net"
|
|
||||||
if mode == "foreign" {
|
|
||||||
id = "other"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q}}`, id)
|
|
||||||
case "DELETE /v1.47/networks/network-id":
|
|
||||||
deletes++
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "image", "1.2.3")
|
|
||||||
i := store.Instance{ID: "net", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
|
|
||||||
t.Fatal("lost response accepted")
|
|
||||||
}
|
|
||||||
if mode == "present" {
|
|
||||||
if deletes != 1 {
|
|
||||||
t.Fatal("network leaked")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
saved, err := a.s.GetInstance("net")
|
|
||||||
if err != nil || saved.Status != "error" {
|
|
||||||
t.Fatalf("lost reservation %+v %v", saved, err)
|
|
||||||
}
|
|
||||||
late = true
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instance(w, runtimeRequest("DELETE", "/api/instances/net", "", sid))
|
|
||||||
if mode == "foreign" {
|
|
||||||
if deletes != 0 || w.Code != 502 {
|
|
||||||
t.Fatal("foreign network touched")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if w.Code != 200 || deletes != 1 {
|
|
||||||
t.Fatalf("late network retry %d %d", w.Code, deletes)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildCheckReopenWorkerWithoutQueuedBuilds(t *testing.T) {
|
|
||||||
path := filepath.Join(t.TempDir(), "recovery.db")
|
|
||||||
s, err := store.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a := &API{s: s}
|
|
||||||
if err = a.ensureBuildChecks(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err = s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id) VALUES('task','image')"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
s.Close()
|
|
||||||
s, err = store.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer s.Close()
|
|
||||||
var attempts atomic.Int32
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == "GET" {
|
|
||||||
fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method == "DELETE" {
|
|
||||||
if attempts.Add(1) == 1 {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
} else {
|
|
||||||
w.WriteHeader(204)
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Errorf("unexpected %s", r.Method)
|
|
||||||
}))
|
|
||||||
defer srv.Close()
|
|
||||||
a = &API{s: s, docker: docker.New(srv.URL)}
|
|
||||||
if err = a.StartImageBuilds(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
deadline := time.Now().Add(4 * time.Second)
|
|
||||||
for {
|
|
||||||
var n int
|
|
||||||
if err = s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if n == 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("reopen worker did not retry")
|
|
||||||
}
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
if attempts.Load() != 2 {
|
|
||||||
t.Fatalf("attempts %d", attempts.Load())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildTerminalFailureDoesNotBlockNextTask(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) })
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, id := range []string{"a", "b"} {
|
|
||||||
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES(?,'1.2.3','queued',?)", id, id); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("CREATE TRIGGER fail_a BEFORE UPDATE ON image_builds WHEN NEW.id='a' AND NEW.status='failed' BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a.startBuildWorker()
|
|
||||||
deadline := time.Now().Add(4 * time.Second)
|
|
||||||
for {
|
|
||||||
var status string
|
|
||||||
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='b'").Scan(&status); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if status == "failed" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("A terminal retry blocked B")
|
|
||||||
}
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("DROP TRIGGER fail_a"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
var status string
|
|
||||||
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='a'").Scan(&status); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if status == "failed" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatal("A never converged")
|
|
||||||
}
|
|
||||||
time.Sleep(10 * time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCreateLostResponseRecovery(t *testing.T) {
|
|
||||||
for _, mode := range []string{"cleanup", "retry", "foreign", "inspect-failure", "late"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
deleted := 0
|
|
||||||
retry := false
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method + " " + r.URL.Path {
|
|
||||||
case "POST /v1.47/networks/create":
|
|
||||||
fmt.Fprint(w, `{"Id":"dsh-inst-lost"}`)
|
|
||||||
case "GET /v1.47/networks/dsh-inst-lost":
|
|
||||||
fmt.Fprint(w, `{"Id":"dsh-inst-lost","Name":"dsh-inst-lost","Driver":"bridge","Labels":{"dsh-mgr.instance":"lost"}}`)
|
|
||||||
case "POST /v1.47/containers/create":
|
|
||||||
fmt.Fprint(w, "{")
|
|
||||||
case "GET /v1.47/containers/dsh-inst-lost/json":
|
|
||||||
if mode == "late" && !retry {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "inspect-failure" && !retry {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
owner := "owner"
|
|
||||||
if mode == "foreign" {
|
|
||||||
owner = "another-panel"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":%q,"dsh-mgr.role":"dsh"}}}`, owner)
|
|
||||||
case "DELETE /v1.47/containers/created":
|
|
||||||
if mode == "retry" && !retry {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
deleted++
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case "DELETE /v1.47/networks/dsh-inst-lost":
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
seedVerifiedImage(t, a, "image", "1.2.3")
|
|
||||||
i := store.Instance{ID: "lost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "creating"}
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.createRuntime(context.Background(), store.User{ID: "owner"}, i, "panel.test"); err == nil {
|
|
||||||
t.Fatal("accepted lost response")
|
|
||||||
}
|
|
||||||
if mode == "cleanup" {
|
|
||||||
if _, err := a.s.GetInstance(i.ID); err == nil || deleted != 1 {
|
|
||||||
t.Fatal("cleanup incomplete")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
saved, err := a.s.GetInstance(i.ID)
|
|
||||||
if err != nil || saved.Status != "error" || deleted != 0 {
|
|
||||||
t.Fatalf("lost recovery record: %+v %v", saved, err)
|
|
||||||
}
|
|
||||||
retry = true
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instance(w, runtimeRequest("DELETE", "/api/instances/lost", "", sid))
|
|
||||||
if mode == "foreign" {
|
|
||||||
if w.Code != 502 || deleted != 0 {
|
|
||||||
t.Fatal("foreign resource touched")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if w.Code != 200 || deleted != 1 {
|
|
||||||
t.Fatalf("retry failed %d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildCheckRecoveryIdentityAndRetry(t *testing.T) {
|
|
||||||
for _, mode := range []string{"lost", "delete-failure", "foreign", "absent"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
deleted := 0
|
|
||||||
retry := false
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case r.Method == "POST" && r.URL.Path == "/v1.47/containers/create":
|
|
||||||
if r.URL.Query().Get("name") != buildCheckName("task") {
|
|
||||||
t.Error("missing task name")
|
|
||||||
}
|
|
||||||
if mode == "delete-failure" {
|
|
||||||
fmt.Fprint(w, `{"Id":"check"}`)
|
|
||||||
} else {
|
|
||||||
fmt.Fprint(w, "{")
|
|
||||||
}
|
|
||||||
case r.Method == "GET":
|
|
||||||
if mode == "absent" && !retry {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !retry && mode == "lost" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
id := "task"
|
|
||||||
if mode == "foreign" {
|
|
||||||
id = "another-panel"
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":%q}}}`, id)
|
|
||||||
case r.Method == "DELETE":
|
|
||||||
if !retry {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
deleted++
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
|
|
||||||
t.Fatal("accepted failed validation")
|
|
||||||
}
|
|
||||||
var count int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if count != 1 {
|
|
||||||
t.Fatal("lost cleanup record")
|
|
||||||
}
|
|
||||||
retry = true
|
|
||||||
err := a.recoverBuildChecks(context.Background())
|
|
||||||
if mode == "foreign" {
|
|
||||||
if err == nil || deleted != 0 {
|
|
||||||
t.Fatal("foreign container deleted")
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if deleted != 1 {
|
|
||||||
t.Fatal("not recovered")
|
|
||||||
}
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&count); err != nil || count != 0 {
|
|
||||||
t.Fatalf("record retained %d %v", count, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReconcileRecoversLostRuntimeIdentity(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if r.Method != "GET" {
|
|
||||||
t.Fatalf("destructive recovery: %s", r.Method)
|
|
||||||
}
|
|
||||||
if r.URL.Path == "/v1.47/containers/json" {
|
|
||||||
fmt.Fprint(w, `[{"Id":"created","State":"running","Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner"}}]`)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprint(w, `{"Id":"created","Name":"/dsh-inst-lost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"lost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
|
|
||||||
})
|
|
||||||
if err := a.s.AddInstance(store.Instance{ID: "lost", UID: "owner", ImageID: "image", Status: "creating"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.Reconcile(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
i, err := a.s.GetInstance("lost")
|
|
||||||
if err != nil || i.ContainerID != "created" || i.Status != "error" {
|
|
||||||
t.Fatalf("not recovered: %+v %v", i, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildResultTransactionAndRetry(t *testing.T) {
|
|
||||||
for _, table := range []string{"images", "image_builds"} {
|
|
||||||
t.Run(table, func(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker request") })
|
|
||||||
buildQueue.Lock()
|
|
||||||
err := a.ensureBuilds()
|
|
||||||
buildQueue.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
b := buildRecord{ID: "task", Version: "1.2.3"}
|
|
||||||
if _, err = a.s.DB.Exec("INSERT INTO image_builds(id,version,status,created_at) VALUES('task','1.2.3','validating','now')"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
event := "INSERT"
|
|
||||||
if table == "image_builds" {
|
|
||||||
event = "UPDATE"
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("CREATE TRIGGER fail_result BEFORE " + event + " ON " + table + " BEGIN SELECT RAISE(FAIL,'injected'); END"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err = a.persistBuildResult(b, "success", "logs", "", "image", "now"); err == nil {
|
|
||||||
t.Fatal("injection ignored")
|
|
||||||
}
|
|
||||||
var status string
|
|
||||||
var count int
|
|
||||||
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "validating" {
|
|
||||||
t.Fatalf("partial terminal write %s %v", status, err)
|
|
||||||
}
|
|
||||||
if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 0 {
|
|
||||||
t.Fatalf("partial index write %d %v", count, err)
|
|
||||||
}
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() { a.finishBuild(b, "success", "logs", "", "image", "now"); close(done) }()
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
t.Fatal("retry abandoned")
|
|
||||||
case <-time.After(30 * time.Millisecond):
|
|
||||||
}
|
|
||||||
if _, err = a.s.DB.Exec("DROP TRIGGER fail_result"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(4 * time.Second):
|
|
||||||
t.Fatal("terminal write not retried")
|
|
||||||
}
|
|
||||||
if err = a.s.DB.QueryRow("SELECT status FROM image_builds WHERE id='task'").Scan(&status); err != nil || status != "success" {
|
|
||||||
t.Fatalf("terminal %s %v", status, err)
|
|
||||||
}
|
|
||||||
if err = a.s.DB.QueryRow("SELECT count(*) FROM images").Scan(&count); err != nil || count != 1 {
|
|
||||||
t.Fatalf("index %d %v", count, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIntentRecoveryMissingReservationDoesNotTouchDocker(t *testing.T) {
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Error("missing reservation must not authorize Docker cleanup")
|
|
||||||
})
|
|
||||||
if err := a.s.EnsureIntents(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.s.ReserveIntent(store.Intent{ID: "missing", UID: "owner"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE creation_intents SET status='creating',instance_id='gone' WHERE id='missing'"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.recoverIntents(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
list, err := a.s.Intents("owner")
|
|
||||||
if err != nil || len(list) != 1 || list[0].Status != "failed" {
|
|
||||||
t.Fatalf("intent not terminated: %+v %v", list, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildTerminalRetryStopsWhenDatabaseCloses(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { t.Error("unexpected Docker call") })
|
|
||||||
a.s.Close()
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() { a.finishBuild(buildRecord{ID: "closed"}, "failed", "", "", "", "now"); close(done) }()
|
|
||||||
select {
|
|
||||||
case <-done:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("closed database retry did not stop")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildCheckDefiniteRejectionAndExternalDeletion(t *testing.T) {
|
|
||||||
for _, rejected := range []bool{true, false} {
|
|
||||||
t.Run(fmt.Sprint(rejected), func(t *testing.T) {
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.Method == "POST" && r.URL.Path == "/v1.47/containers/create" {
|
|
||||||
w.WriteHeader(400)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(404)
|
|
||||||
})
|
|
||||||
if rejected {
|
|
||||||
if err := a.verifyBuiltImage(context.Background(), "image", "1.2.3", "task"); err == nil {
|
|
||||||
t.Fatal("rejection accepted")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if err := a.ensureBuildChecks(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,container_id) VALUES('task','image','known')"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.recoverBuildChecks(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 {
|
|
||||||
t.Fatalf("definite outcome retained: %d %v", n, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUncertainPendingRetainsLateResources(t *testing.T) {
|
|
||||||
for _, column := range []string{"container_pending", "network_pending"} {
|
|
||||||
for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
|
|
||||||
t.Run(column+"/"+stamp, func(t *testing.T) {
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", "")
|
|
||||||
late := false
|
|
||||||
deletes := 0
|
|
||||||
a, sid := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method + " " + r.URL.Path {
|
|
||||||
case "GET /v1.47/containers/dsh-inst-ghost/json":
|
|
||||||
if !late || column != "container_pending" {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprint(w, `{"Id":"container-id","Name":"/dsh-inst-ghost","Image":"image","Config":{"Labels":{"dsh-mgr.instance":"ghost","dsh-mgr.owner":"owner","dsh-mgr.role":"dsh"}}}`)
|
|
||||||
case "GET /v1.47/networks/dsh-inst-ghost":
|
|
||||||
if !late || column != "network_pending" {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprint(w, `{"Id":"network-id","Name":"dsh-inst-ghost","Driver":"bridge","Labels":{"dsh-mgr.instance":"ghost"}}`)
|
|
||||||
case "DELETE /v1.47/containers/container-id", "DELETE /v1.47/networks/network-id":
|
|
||||||
deletes++
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
i := store.Instance{ID: "ghost", UID: "owner", Version: "1.2.3", ImageID: "image", Status: "error"}
|
|
||||||
if err := a.s.AddInstance(i); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := a.setRuntimeCreatePending(i.ID, column, true); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("UPDATE runtime_creates SET "+pendingColumns[column].stamp+"=? WHERE instance_id=?", stamp, i.ID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for attempt := 0; attempt < 2; attempt++ {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
|
|
||||||
if w.Code != 502 {
|
|
||||||
t.Fatalf("uncertainty discarded: %d %s", w.Code, w.Body.String())
|
|
||||||
}
|
|
||||||
if _, pending, err := a.pendingSince(i.ID, column); err != nil || !pending {
|
|
||||||
t.Fatalf("pending lost: %v %v", pending, err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.GetInstance(i.ID); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
late = true
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
a.instance(w, runtimeRequest("DELETE", "/api/instances/ghost", "", sid))
|
|
||||||
if w.Code != 200 || deletes != 1 {
|
|
||||||
t.Fatalf("late cleanup: %d %s deletes=%d", w.Code, w.Body.String(), deletes)
|
|
||||||
}
|
|
||||||
if _, err := a.s.GetInstance(i.ID); err == nil {
|
|
||||||
t.Fatal("instance retained")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBuildCheckUncertainRowRetainsLateContainer(t *testing.T) {
|
|
||||||
for _, stamp := range []string{time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), "", "invalid"} {
|
|
||||||
t.Run(stamp, func(t *testing.T) {
|
|
||||||
late := false
|
|
||||||
deletes := 0
|
|
||||||
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch r.Method + " " + r.URL.Path {
|
|
||||||
case "GET /v1.47/containers/dsh-build-check-task/json":
|
|
||||||
if !late {
|
|
||||||
w.WriteHeader(404)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprint(w, `{"Id":"check","Name":"/dsh-build-check-task","Image":"image","Config":{"Labels":{"dsh-mgr.role":"build-check","dsh-mgr.build":"task"}}}`)
|
|
||||||
case "DELETE /v1.47/containers/check":
|
|
||||||
deletes++
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err := a.ensureBuildChecks(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.s.DB.Exec("INSERT INTO image_build_checks(build_id,image_id,created_at) VALUES('task','image',?)", stamp); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for attempt := 0; attempt < 2; attempt++ {
|
|
||||||
if err := a.recoverBuildChecks(context.Background()); !errors.Is(err, errBuildCheckUncertain) {
|
|
||||||
t.Fatalf("uncertainty discarded: %v", err)
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 1 {
|
|
||||||
t.Fatalf("record lost: %d %v", n, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
late = true
|
|
||||||
if err := a.recoverBuildChecks(context.Background()); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var n int
|
|
||||||
if err := a.s.DB.QueryRow("SELECT count(*) FROM image_build_checks").Scan(&n); err != nil || n != 0 || deletes != 1 {
|
|
||||||
t.Fatalf("late cleanup: rows=%d deletes=%d err=%v", n, deletes, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRuntimeNetworkPanelDisconnect(t *testing.T) {
|
|
||||||
for _, mode := range []string{"not-connected", "connected", "panel-name", "other-connected", "lost-disconnect", "forbidden", "inspect-forbidden", "foreign"} {
|
|
||||||
t.Run(mode, func(t *testing.T) {
|
|
||||||
panel := "panel-id"
|
|
||||||
if mode == "panel-name" || mode == "inspect-forbidden" {
|
|
||||||
panel = "panel"
|
|
||||||
}
|
|
||||||
t.Setenv("PANEL_CONTAINER_ID", panel)
|
|
||||||
connected := mode != "not-connected"
|
|
||||||
disconnects, deletes := 0, 0
|
|
||||||
a, _ := runtimeTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/v1.47/images/image/json" {
|
|
||||||
writeImageInspect(w, "image", "1.2.3")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
switch r.Method + " " + r.URL.Path {
|
|
||||||
case "GET /v1.47/networks/dsh-inst-net", "GET /v1.47/networks/network-id":
|
|
||||||
owner := "net"
|
|
||||||
if mode == "foreign" {
|
|
||||||
owner = "other"
|
|
||||||
}
|
|
||||||
containers := "{}"
|
|
||||||
if connected {
|
|
||||||
containers = `{"panel-id":{}}`
|
|
||||||
}
|
|
||||||
if mode == "other-connected" {
|
|
||||||
containers = `{"other-id":{}}`
|
|
||||||
}
|
|
||||||
fmt.Fprintf(w, `{"Id":"network-id","Name":"dsh-inst-net","Driver":"bridge","Labels":{"dsh-mgr.instance":%q},"Containers":%s}`, owner, containers)
|
|
||||||
case "GET /v1.47/containers/panel/json", "GET /v1.47/containers/panel-id/json":
|
|
||||||
if mode == "inspect-forbidden" {
|
|
||||||
w.WriteHeader(403)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
fmt.Fprint(w, `{"Id":"panel-id"}`)
|
|
||||||
case "POST /v1.47/networks/network-id/disconnect":
|
|
||||||
disconnects++
|
|
||||||
var body struct {
|
|
||||||
Container string
|
|
||||||
Force bool
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body.Container != "panel-id" || !body.Force {
|
|
||||||
t.Errorf("disconnect identity: %+v %v", body, err)
|
|
||||||
}
|
|
||||||
if mode == "not-connected" || mode == "other-connected" || mode == "foreign" {
|
|
||||||
t.Error("unsafe disconnect")
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if mode == "forbidden" {
|
|
||||||
w.WriteHeader(403)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
connected = false
|
|
||||||
if mode == "lost-disconnect" {
|
|
||||||
w.WriteHeader(500)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.WriteHeader(204)
|
|
||||||
case "DELETE /v1.47/networks/network-id":
|
|
||||||
deletes++
|
|
||||||
w.WriteHeader(204)
|
|
||||||
default:
|
|
||||||
t.Errorf("unexpected %s %s", r.Method, r.URL.Path)
|
|
||||||
w.WriteHeader(500)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
if err := a.setRuntimeCreatePending("net", "network_pending", true); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
err := a.recoverRuntimeNetwork(context.Background(), "net")
|
|
||||||
wantError := mode == "lost-disconnect" || mode == "forbidden" || mode == "inspect-forbidden" || mode == "foreign"
|
|
||||||
if (err != nil) != wantError {
|
|
||||||
t.Fatalf("recovery error=%v wantError=%v", err, wantError)
|
|
||||||
}
|
|
||||||
if wantError {
|
|
||||||
if deletes != 0 {
|
|
||||||
t.Fatal("deleted after failure")
|
|
||||||
}
|
|
||||||
if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || !pending {
|
|
||||||
t.Fatalf("pending lost: %v %v", pending, err)
|
|
||||||
}
|
|
||||||
if mode != "lost-disconnect" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := a.recoverRuntimeNetwork(context.Background(), "net"); err != nil {
|
|
||||||
t.Fatalf("disconnect retry: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
wantDisconnects := 1
|
|
||||||
if mode == "not-connected" || mode == "other-connected" {
|
|
||||||
wantDisconnects = 0
|
|
||||||
}
|
|
||||||
if deletes != 1 || disconnects != wantDisconnects {
|
|
||||||
t.Fatalf("deletes=%d disconnects=%d", deletes, disconnects)
|
|
||||||
}
|
|
||||||
if _, pending, err := a.pendingSince("net", "network_pending"); err != nil || pending {
|
|
||||||
t.Fatalf("pending retained: %v %v", pending, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user