m1saka/dsh-web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor!: separate runtime releases from panel source

Browse Source
This commit is contained in:
m1saka
2026-10-14 15:05:10 +08:00
parent 20635ffaec
commit a2dd6ee27d
51 changed files with 221 additions and 2017 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/images_test.go
+24 -272
View File
@@ -1,7 +1,6 @@
package api
import (
"archive/tar"
"context"
"encoding/json"
"fmt"
@@ -10,10 +9,8 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -24,11 +21,6 @@ func imageTestAPI(t *testing.T, h http.HandlerFunc) (*API, string) {
oldDiskAvailable := buildDiskAvailable
buildDiskAvailable = func(string) (uint64, error) { return minimumBuildDiskBytes, nil }
t.Cleanup(func() { buildDiskAvailable = oldDiskAvailable })
oldClient := npmBuildClient
npmBuildClient = &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"version":"0.2.0-rc.1"}`))}, nil
})}
t.Cleanup(func() { npmBuildClient = oldClient })
root := t.TempDir()
t.Setenv("DSH_DOCKER_ROOT_VISIBLE", root)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -129,34 +121,7 @@ func seedVerifiedImage(t *testing.T, a *API, id, version string) {
t.Fatal(err)
}
}
func TestImageContextAndSanitizing(t *testing.T) {
r, err := imageBuildContext()
if err != nil {
t.Fatal(err)
}
tr := tar.NewReader(r)
n := 0
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
data, _ := io.ReadAll(tr)
original, err := os.ReadFile(filepath.Join("..", "..", "image", h.Name))
if err != nil || string(data) != string(original) {
t.Fatalf("recipe drift %s: %v", h.Name, err)
}
if h.Name == "Dockerfile" && (!strings.Contains(string(data), "ARG RECIPE_REV="+imageRecipeRev+"\n") || !strings.Contains(string(data), "dsh-mgr.recipe=$RECIPE_REV")) {
t.Fatal("recipe revision or label differs from image policy")
}
n++
}
if n != 7 {
t.Fatal(n)
}
func TestImageSanitizing(t *testing.T) {
s := sanitizeBuildLog("token=topsecret Authorization: Bearer hidden password=private https://user:[email protected]/")
for _, secret := range []string{"topsecret", "hidden", "private", "user:pass"} {
if strings.Contains(s, secret) {
@@ -176,129 +141,6 @@ func TestImageContextAndSanitizing(t *testing.T) {
}
}
type imageRoundTrip func(*http.Request) (*http.Response, error)
func (f imageRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestLatestOfficialRegistry(t *testing.T) {
client := &http.Client{Transport: imageRoundTrip(func(r *http.Request) (*http.Response, error) {
if r.URL.Host != "registry.npmjs.org" || r.URL.Scheme != "https" {
t.Fatal(r.URL)
}
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`{"version":"0.2.0-rc.1"}`)), Header: make(http.Header)}, nil
})}
v, err := fetchLatest(context.Background(), client)
if err != nil || v != "0.2.0-rc.1" {
t.Fatalf("%s %v", v, err)
}
}
func TestBuildDedupAndVerification(t *testing.T) {
var calls, deleted atomic.Int32
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
case "/v1.47/images/json":
if calls.Load() == 0 {
io.WriteString(w, "[]")
} else {
io.WriteString(w, `[{"Id":"sha256:abc","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
}
case "/v1.47/build":
calls.Add(1)
if r.URL.Query().Get("t") != "dsh-web:0.2.0-rc.1-"+imageRecipeRev {
t.Error("unexpected tag", r.URL.Query())
}
var args map[string]string
json.Unmarshal([]byte(r.URL.Query().Get("buildargs")), &args)
if args["DSH_VERSION"] != "0.2.0-rc.1" {
t.Error(args)
}
io.Copy(io.Discard, r.Body)
io.WriteString(w, `{"stream":"token=secretvalue"}`+string(rune(10))+`{"aux":{"ID":"sha256:abc"}}`)
case "/v1.47/containers/create":
var cfg struct {
Image string
HostConfig struct {
NetworkMode string
ReadonlyRootfs bool
Tmpfs map[string]string
}
}
json.NewDecoder(r.Body).Decode(&cfg)
if cfg.Image != "sha256:abc" || cfg.HostConfig.NetworkMode != "none" || !cfg.HostConfig.ReadonlyRootfs || !strings.Contains(cfg.HostConfig.Tmpfs["/tmp"], "exec") {
t.Errorf("unsafe config %+v", cfg)
}
io.WriteString(w, `{"Id":"check"}`)
case "/v1.47/containers/check/start", "/v1.47/exec/probe/start":
w.WriteHeader(204)
case "/v1.47/containers/check/exec":
var cfg struct{ Cmd []string }
if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil || len(cfg.Cmd) != 3 || !strings.Contains(cfg.Cmd[2], "execFileSync('dsh',['--version']") || !strings.Contains(cfg.Cmd[2], `actual!=="0.2.0-rc.1"`) {
t.Errorf("missing version verification: %+v %v", cfg, err)
}
io.WriteString(w, `{"Id":"probe"}`)
case "/v1.47/exec/probe/json":
io.WriteString(w, `{"Running":false,"ExitCode":0}`)
case "/v1.47/containers/check/json":
io.WriteString(w, `{"State":{"Running":true}}`)
case "/v1.47/containers/check":
deleted.Add(1)
w.WriteHeader(204)
default:
t.Errorf("unexpected %s", r.URL)
w.WriteHeader(500)
}
})
responses := make(chan *httptest.ResponseRecorder, 8)
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
responses <- imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
}()
}
wg.Wait()
close(responses)
first := ""
for w := range responses {
if w.Code != 202 {
t.Fatal(w.Body.String())
}
var b buildRecord
json.Unmarshal(w.Body.Bytes(), &b)
if first == "" {
first = b.ID
}
if first != b.ID {
t.Fatal("duplicate job")
}
}
deadline := time.Now().Add(5 * time.Second)
for {
w := imageRequest(a.builds, sid, "GET", "")
var jobs []buildRecord
json.Unmarshal(w.Body.Bytes(), &jobs)
if len(jobs) == 1 && jobs[0].Status == "success" {
if strings.Contains(jobs[0].Logs, "secretvalue") {
t.Fatal(jobs)
}
break
}
if time.Now().After(deadline) {
t.Fatal(w.Body.String())
}
time.Sleep(time.Millisecond)
}
if calls.Load() != 1 || deleted.Load() != 1 {
t.Fatalf("builds %d cleanup %d", calls.Load(), deleted.Load())
}
for _, body := range []string{`{"version":"latest"}`, `{"version":"0.2.0-rc.1","url":"bad"}`, `{"version":"0.2.0-rc.1"} {}`} {
if w := imageRequest(a.builds, sid, "POST", body); w.Code != 400 {
t.Fatal(w.Code)
}
}
}
func TestVerificationFailureCleansUp(t *testing.T) {
var deleted atomic.Bool
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
@@ -321,86 +163,6 @@ func TestVerificationFailureCleansUp(t *testing.T) {
}
}
func TestBuildReuseAndMissingImage(t *testing.T) {
var exists atomic.Bool
exists.Store(true)
var builds, checks atomic.Int32
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
writeImageInspect(w, strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/v1.47/images/"), "/json"), "0.2.0-rc.1")
case "/v1.47/images/json":
if exists.Load() {
io.WriteString(w, `[{"Id":"sha256:reuse","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
} else {
io.WriteString(w, "[]")
}
case "/v1.47/build":
builds.Add(1)
exists.Store(true)
io.Copy(io.Discard, r.Body)
io.WriteString(w, `{"aux":{"ID":"sha256:reuse"}}`)
case "/v1.47/containers/create":
checks.Add(1)
io.WriteString(w, `{"Id":"c"}`)
case "/v1.47/containers/c/start", "/v1.47/exec/e/start", "/v1.47/containers/c":
w.WriteHeader(204)
case "/v1.47/containers/c/exec":
io.WriteString(w, `{"Id":"e"}`)
case "/v1.47/exec/e/json":
io.WriteString(w, `{"Running":false,"ExitCode":0}`)
case "/v1.47/containers/c/json":
io.WriteString(w, `{"State":{"Running":true}}`)
default:
t.Errorf("unexpected %s", r.URL)
w.WriteHeader(500)
}
})
wait := func(id string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
var status, image string
err := a.s.DB.QueryRow("SELECT status,image_id FROM image_builds WHERE id=?", id).Scan(&status, &image)
if err == nil && status == "success" {
if image != "sha256:reuse" {
t.Fatal(image)
}
return
}
if time.Now().After(deadline) {
t.Fatalf("status %s: %v", status, err)
}
time.Sleep(time.Millisecond)
}
}
post := func() buildRecord {
t.Helper()
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
if w.Code != 202 {
t.Fatal(w.Body.String())
}
var b buildRecord
if err := json.Unmarshal(w.Body.Bytes(), &b); err != nil {
t.Fatal(err)
}
return b
}
first := post()
wait(first.ID)
if builds.Load() != 0 || checks.Load() != 1 {
t.Fatal("local image not reused and verified")
}
exists.Store(false)
second := post()
if second.ID == first.ID {
t.Fatal("missing image deduplicated")
}
wait(second.ID)
if builds.Load() != 1 || checks.Load() != 2 {
t.Fatal("missing image not rebuilt")
}
}
func TestBuildRestartRecovery(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) })
buildQueue.Lock()
@@ -427,6 +189,7 @@ func TestBuildRestartRecovery(t *testing.T) {
func TestFailedVerificationNeverAvailable(t *testing.T) {
var deleted atomic.Bool
imageID := "sha256:" + strings.Repeat("c", 64)
info := releaseFixture()
a, sid := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1.47/images/sha256:abc/json", "/v1.47/images/sha256:reuse/json":
@@ -444,33 +207,36 @@ func TestFailedVerificationNeverAvailable(t *testing.T) {
case "/v1.47/containers/check":
deleted.Store(true)
w.WriteHeader(204)
case "/v1.47/build":
writeJSON(w, map[string]any{"aux": map[string]string{"ID": imageID}})
case "/v1.47/images/" + imageID + "/json":
writeImageInspect(w, imageID, "0.2.0-rc.1")
parts := strings.Split(imagePlatform, "/")
writeJSON(w, map[string]any{"Id": imageID, "Os": parts[0], "Architecture": parts[1], "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": info.Version, "dsh-mgr.recipe": imageRecipeRev, "dsh-mgr.release": info.ReleaseID, "dsh-mgr.source": info.Repository, "dsh-mgr.commit": info.Commit}}})
default:
t.Errorf("unexpected Docker call %s", r.URL)
w.WriteHeader(500)
}
})
w := imageRequest(a.builds, sid, "POST", `{"version":"0.2.0-rc.1"}`)
if w.Code != 202 {
t.Fatal(w.Body.String())
if _, err := a.s.DB.Exec("UPDATE users SET admin=1 WHERE id='u'"); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for {
var status, id, detail string
err := a.s.DB.QueryRow("SELECT status,image_id,error FROM image_builds").Scan(&status, &id, &detail)
if err == nil && status == "failed" {
if id != imageID || !strings.Contains(detail, "verification failed") || !deleted.Load() {
t.Fatalf("%s %s cleanup=%v", id, detail, deleted.Load())
}
break
}
if time.Now().After(deadline) {
t.Fatalf("status=%s error=%v", status, err)
}
time.Sleep(time.Millisecond)
old := fetchRelease
t.Cleanup(func() { fetchRelease = old })
fetchRelease = func(context.Context, string, string) (releaseInfo, io.Reader, error) {
return info, strings.NewReader("context"), nil
}
w = imageRequest(a.images, sid, "GET", "")
b, err := a.enqueueRelease("u", info)
if err != nil {
t.Fatal(err)
}
a.runImageBuild(b)
if err := scanBuildSummary(a.s.DB.QueryRow("SELECT "+buildSummaryColumns+" FROM image_builds WHERE id=?", b.ID), &b); err != nil {
t.Fatal(err)
}
if b.Status != "failed" || b.ImageID != imageID || !strings.Contains(b.Error, "verification failed") || !deleted.Load() {
t.Fatalf("%+v cleanup=%v", b, deleted.Load())
}
w := imageRequest(a.images, sid, "GET", "")
var images []imageRecord
if err := json.Unmarshal(w.Body.Bytes(), &images); err != nil || w.Code != 200 || len(images) != 1 || images[0].Status != "unverified" {
t.Fatal("failed image advertised", w.Body.String())
@@ -481,20 +247,6 @@ func TestFailedVerificationNeverAvailable(t *testing.T) {
}
}
func TestLocalBuildImagePrefersPersistedID(t *testing.T) {
a, _ := imageTestAPI(t, func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1.47/images/verified/json" {
writeImageInspect(w, "verified", "0.2.0-rc.1")
return
}
io.WriteString(w, `[{"Id":"newer","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}},{"Id":"verified","Labels":{"dsh-mgr.role":"runtime","dsh-mgr.version":"0.2.0-rc.1","dsh-mgr.recipe":"`+imageRecipeRev+`"}}]`)
})
id, err := a.localBuildImage(context.Background(), "0.2.0-rc.1", "verified")
if err != nil || id != "verified" {
t.Fatalf("id=%s err=%v", id, err)
}
}
func writeImageInspect(w http.ResponseWriter, id, version string) {
parts := strings.Split(imagePlatform, "/")
writeJSON(w, map[string]any{"Id": id, "Os": parts[0], "Architecture": parts[1], "Config": map[string]any{"Labels": map[string]string{"dsh-mgr.role": "runtime", "dsh-mgr.version": version, "dsh-mgr.recipe": imageRecipeRev}}})