m1saka
602917698a
Add snapshot usage and protected cleanup, preserve unsaved profile edits, avoid holding SQL connections during snapshot deletion, and fix download filenames. Include regression tests and API documentation.
414 lines
13 KiB
Go
414 lines
13 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestSnapshotManagement(t *testing.T) {
|
|
a, i, home := configTestAPI(t, false)
|
|
a.docker = nil // These operations must never inspect or stop Docker.
|
|
parent := filepath.Dir(home)
|
|
suffix := strings.Repeat("a", 32)
|
|
names := []string{"pre-restore-" + suffix, "failed-restore-" + suffix, "upgrade-" + suffix + "-snapshot", "failed-home-" + suffix}
|
|
for _, name := range append(append([]string{}, names...), "pre-restore-no", "upgrade-"+suffix, "other") {
|
|
if err := os.Mkdir(filepath.Join(parent, name), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configWrite(t, filepath.Join(parent, name, "data"), "abc")
|
|
}
|
|
outside := t.TempDir()
|
|
configWrite(t, filepath.Join(outside, "keep"), "outside")
|
|
if err := os.Symlink(outside, filepath.Join(parent, names[0], "link")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Link(filepath.Join(parent, names[0], "data"), filepath.Join(parent, names[0], "hardlink")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := configCall(a, i, "GET", "snapshots", "")
|
|
var got []snapshotInfo
|
|
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &got) != nil || len(got) != 4 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
for _, s := range got {
|
|
want := int64(3)
|
|
if s.ID == names[0] {
|
|
want = 6
|
|
}
|
|
if s.Size != want || s.Protected || s.ProtectionReason != "" || s.SizeError != "" || s.CreatedAt.IsZero() {
|
|
t.Fatalf("%+v", s)
|
|
}
|
|
}
|
|
for _, name := range names {
|
|
w = configCall(a, i, "DELETE", "snapshots?id="+name, "")
|
|
if w.Code != 200 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if _, err := os.Stat(filepath.Join(parent, name)); !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, p := range []string{home, filepath.Join(parent, "other"), filepath.Join(outside, "keep")} {
|
|
if _, err := os.Stat(p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var count int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='snapshot.delete.completed' AND uid=? AND target=?", i.UID, i.ID).Scan(&count); err != nil || count != 4 {
|
|
t.Fatal(count, err)
|
|
}
|
|
if w = configCall(a, i, "DELETE", "snapshots?id="+names[0], ""); w.Code != 404 {
|
|
t.Fatal(w.Code)
|
|
}
|
|
}
|
|
|
|
func TestManagementInvalidPaths(t *testing.T) {
|
|
a, i, home := configTestAPI(t, false)
|
|
a.docker = nil
|
|
for _, action := range []string{"snapshots", "backups"} {
|
|
for _, id := range []string{"", "home", "..", "../home", "/home", strings.Repeat("A", 32), "pre-restore-" + strings.Repeat("a", 32) + "/../home"} {
|
|
if w := configCall(a, i, "DELETE", action+"?id="+url.QueryEscape(id), ""); w.Code != 400 {
|
|
t.Fatal(action, id, w.Code)
|
|
}
|
|
}
|
|
}
|
|
id := "failed-home-" + strings.Repeat("b", 32)
|
|
outside := t.TempDir()
|
|
configWrite(t, filepath.Join(outside, "keep"), "safe")
|
|
if err := os.Symlink(outside, filepath.Join(filepath.Dir(home), id)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if w := configCall(a, i, "DELETE", "snapshots?id="+id, ""); w.Code != 400 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if w := configCall(a, i, "GET", "snapshots", ""); w.Code != 200 || strings.TrimSpace(w.Body.String()) != "[]" {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if err := os.Rename(filepath.Dir(home), filepath.Dir(home)+"-saved"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.Symlink(outside, filepath.Dir(home)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, method := range []string{"GET", "DELETE"} {
|
|
if w := configCall(a, i, method, "snapshots?id="+id, ""); w.Code != 400 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(outside, "keep")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestManagementProtection(t *testing.T) {
|
|
for _, state := range []string{"error", "pending", "journal-error", "complete", "rolled_back"} {
|
|
t.Run(state, func(t *testing.T) {
|
|
a, i, home := configTestAPI(t, false)
|
|
id := "pre-restore-" + strings.Repeat("c", 32)
|
|
if err := os.Mkdir(filepath.Join(filepath.Dir(home), id), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var err error
|
|
if state == "error" {
|
|
_, err = a.s.DB.Exec("UPDATE instances SET status='error' WHERE id=?", i.ID)
|
|
} else if state == "journal-error" {
|
|
_, err = a.s.DB.Exec("CREATE TABLE upgrade_journal(broken TEXT)")
|
|
} else {
|
|
_, err = a.s.DB.Exec("CREATE TABLE upgrade_journal(instance_id TEXT,phase TEXT)")
|
|
if err == nil {
|
|
_, err = a.s.DB.Exec("INSERT INTO upgrade_journal VALUES(?,?)", i.ID, state)
|
|
}
|
|
}
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
protected := state != "complete" && state != "rolled_back"
|
|
w := configCall(a, i, "GET", "snapshots", "")
|
|
var got []snapshotInfo
|
|
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &got) != nil || len(got) != 1 || got[0].Protected != protected || (got[0].ProtectionReason != "") != protected {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
w = configCall(a, i, "DELETE", "snapshots?id="+id, "")
|
|
want := 200
|
|
if protected {
|
|
want = 409
|
|
}
|
|
if w.Code != want {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if protected {
|
|
w = configCall(a, i, "DELETE", "backups?id="+strings.Repeat("a", 32), "")
|
|
if w.Code != 409 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSnapshotSizeError(t *testing.T) {
|
|
a, i, home := configTestAPI(t, false)
|
|
id := "failed-restore-" + strings.Repeat("d", 32)
|
|
deep := filepath.Join(filepath.Dir(home), id)
|
|
for n := 0; n < 130; n++ {
|
|
deep = filepath.Join(deep, "d")
|
|
}
|
|
if err := os.MkdirAll(deep, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w := configCall(a, i, "GET", "snapshots", "")
|
|
var got []snapshotInfo
|
|
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &got) != nil || len(got) != 1 || got[0].Size != -1 || got[0].SizeError == "" {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestBackupDelete(t *testing.T) {
|
|
for _, mode := range []string{"indexed", "legacy", "stale", "missing-parent", "index-error", "symlink"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
a, i, _ := configTestAPI(t, false)
|
|
a.docker = nil
|
|
id := strings.Repeat("e", 32)
|
|
dir := filepath.Join(a.root, "backups", i.UID, i.ID)
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
file := filepath.Join(dir, id+".dshbak")
|
|
if mode != "stale" && mode != "symlink" {
|
|
configWrite(t, file, "encrypted")
|
|
}
|
|
if mode == "symlink" {
|
|
if err := os.Symlink(t.TempDir(), file); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if mode == "indexed" || mode == "stale" || mode == "missing-parent" {
|
|
if err := a.indexBackup(i, id, "manual", 2, 9); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if mode == "index-error" {
|
|
if _, err := a.s.DB.Exec("CREATE TABLE backups(broken TEXT)"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if mode == "missing-parent" {
|
|
if err := os.RemoveAll(filepath.Join(a.root, "backups")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
w := configCall(a, i, "DELETE", "backups?id="+id, "")
|
|
want := 200
|
|
if mode == "index-error" {
|
|
want = 500
|
|
}
|
|
if mode == "symlink" {
|
|
want = 400
|
|
}
|
|
if w.Code != want {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if want != 200 {
|
|
if _, err := os.Lstat(file); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return
|
|
}
|
|
if _, err := os.Stat(file); !os.IsNotExist(err) {
|
|
t.Fatal(err)
|
|
}
|
|
if mode != "legacy" {
|
|
var count int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM backups WHERE id=?", id).Scan(&count); err != nil || count != 0 {
|
|
t.Fatal(count, err)
|
|
}
|
|
}
|
|
var count int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action='backup.delete' AND target=? AND detail=?", i.ID, id).Scan(&count); err != nil || count != 1 {
|
|
t.Fatal(count, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagementOwnershipRecheck(t *testing.T) {
|
|
a, i, _ := configTestAPI(t, false)
|
|
if _, err := a.s.DB.Exec("UPDATE instances SET uid='someone-else' WHERE id=?", i.ID); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, action := range []string{"snapshots", "snapshots?id=pre-restore-" + strings.Repeat("a", 32), "backups?id=" + strings.Repeat("a", 32)} {
|
|
method := "DELETE"
|
|
if action == "snapshots" {
|
|
method = "GET"
|
|
}
|
|
if w := configCall(a, i, method, action, ""); w.Code != 404 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManagementAuditFailure(t *testing.T) {
|
|
a, i, home := configTestAPI(t, false)
|
|
id := strings.Repeat("f", 32)
|
|
snapshot := filepath.Join(filepath.Dir(home), "pre-restore-"+id)
|
|
if err := os.Mkdir(snapshot, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dir := filepath.Join(a.root, "backups", i.UID, i.ID)
|
|
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
file := filepath.Join(dir, id+".dshbak")
|
|
configWrite(t, file, "encrypted")
|
|
if err := a.indexBackup(i, id, "manual", 2, 9); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.s.DB.Exec("DROP TABLE audit"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, action := range []string{"snapshots?id=pre-restore-" + id, "backups?id=" + id} {
|
|
if w := configCall(a, i, "DELETE", action, ""); w.Code != 500 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
}
|
|
for _, p := range []string{snapshot, file} {
|
|
if _, err := os.Stat(p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
var count int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM backups WHERE id=?", id).Scan(&count); err != nil || count != 1 {
|
|
t.Fatal(count, err)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotScanCancelled(t *testing.T) {
|
|
root, err := os.OpenRoot(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer root.Close()
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := snapshotBytes(ctx, root, "."); err != context.Canceled {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotDeletionReleasesSQLConnection(t *testing.T) {
|
|
a, i, _ := configTestAPI(t, false)
|
|
a.s.DB.SetMaxOpenConns(1)
|
|
entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
|
w := httptest.NewRecorder()
|
|
go func() {
|
|
defer close(done)
|
|
a.deleteSnapshot(w, i, "snapshot", func() error {
|
|
close(entered)
|
|
<-release
|
|
return nil
|
|
})
|
|
}()
|
|
defer func() { close(release); <-done }()
|
|
select {
|
|
case <-entered:
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("removal did not start")
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
var count int
|
|
if err := a.s.DB.QueryRowContext(ctx, "SELECT count(*) FROM audit WHERE action='snapshot.delete.started'").Scan(&count); err != nil || count != 1 {
|
|
t.Fatalf("SQL blocked during removal or initial audit not persisted: count=%d err=%v", count, err)
|
|
}
|
|
}
|
|
|
|
func TestSnapshotDeletionAuditOutcomes(t *testing.T) {
|
|
for _, mode := range []string{"success", "remove-failed", "start-failed", "completion-audit-failed", "failure-audit-failed"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
a, i, _ := configTestAPI(t, false)
|
|
if mode == "start-failed" {
|
|
if _, err := a.s.DB.Exec("DROP TABLE audit"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
w := httptest.NewRecorder()
|
|
called := false
|
|
a.deleteSnapshot(w, i, "snapshot", func() error {
|
|
called = true
|
|
if strings.Contains(mode, "audit-failed") {
|
|
if _, err := a.s.DB.Exec(`CREATE TRIGGER reject_terminal BEFORE INSERT ON audit WHEN NEW.action != 'snapshot.delete.started' BEGIN SELECT RAISE(FAIL, 'audit unavailable'); END`); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if mode == "remove-failed" || mode == "failure-audit-failed" {
|
|
return os.ErrPermission
|
|
}
|
|
return nil
|
|
})
|
|
want := 500
|
|
if mode == "success" {
|
|
want = 200
|
|
}
|
|
if w.Code != want || called != (mode != "start-failed") {
|
|
t.Fatal(w.Code, w.Body.String(), called)
|
|
}
|
|
if mode == "start-failed" {
|
|
return
|
|
}
|
|
for _, state := range []string{"started", "completed", "failed"} {
|
|
wantCount := 0
|
|
if state == "started" || (state == "completed" && mode == "success") || (state == "failed" && mode == "remove-failed") {
|
|
wantCount = 1
|
|
}
|
|
var count int
|
|
if err := a.s.DB.QueryRow("SELECT count(*) FROM audit WHERE action=? AND uid=? AND target=? AND detail=?", "snapshot.delete."+state, i.UID, i.ID, "snapshot").Scan(&count); err != nil || count != wantCount {
|
|
t.Fatal(state, count, err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagementPathErrorClassification(t *testing.T) {
|
|
for _, tc := range []struct {
|
|
name string
|
|
err error
|
|
want int
|
|
}{
|
|
{"unsafe", errUnsafeManagementPath, 400},
|
|
{"wrapped-unsafe", fmt.Errorf("path: %w", errUnsafeManagementPath), 400},
|
|
{"permission", &os.PathError{Op: "lstat", Path: "snapshot", Err: os.ErrPermission}, 500},
|
|
{"closed", os.ErrClosed, 500},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
w := httptest.NewRecorder()
|
|
managementPathError(w, tc.err)
|
|
if w.Code != tc.want {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
})
|
|
}
|
|
a, i, _ := configTestAPI(t, false)
|
|
base := t.TempDir()
|
|
configWrite(t, filepath.Join(base, "file"), "not a directory")
|
|
a.root = filepath.Join(base, "file", "child")
|
|
if w := configCall(a, i, "GET", "snapshots", ""); w.Code != 500 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
a.root = filepath.Join(base, "missing")
|
|
if w := configCall(a, i, "GET", "snapshots", ""); w.Code != 200 || strings.TrimSpace(w.Body.String()) != "[]" {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
if w := configCall(a, i, "DELETE", "snapshots?id=pre-restore-"+strings.Repeat("a", 32), ""); w.Code != 404 {
|
|
t.Fatal(w.Code, w.Body.String())
|
|
}
|
|
}
|