add chunked upload to bypass 100MB proxy body limit

Resumable /api/uploads flow (init -> append -> complete -> abort) so a
large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default
50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would
reject. The web console auto-routes files larger than one slice through the
chunked path with cross-slice progress and transient-error retry; smaller
files keep the single-shot upload.

- store: in-memory upload sessions + .part reassembly (offset writes,
  out-of-order/idempotent), reuse upload id as task id, orphan sweep on
  start, idle-session GC
- fix limitedReader rejecting bodies exactly at the cap (broke full chunks)
- fix drag-drop double-firing when a drop lands on the dropzone (its drop
  handler now stopPropagation so the window listener doesn't re-upload)
- tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
root
2026-09-10 15:12:15 +08:00
parent ab6bf8b370
commit e5a2dc0a8d
11 changed files with 1144 additions and 35 deletions
+29 -16
View File
@@ -55,8 +55,9 @@ type Store struct {
tasksFile string
uploadsDir string
mu sync.Mutex
tasks map[string]*Task
mu sync.Mutex
tasks map[string]*Task
sessions map[string]*UploadSession // in-progress chunked uploads
}
// Open initializes the store at dir, creating directories and loading any
@@ -75,10 +76,14 @@ func Open(dir string) (*Store, error) {
tasksFile: filepath.Join(dir, "tasks.json"),
uploadsDir: uploads,
tasks: map[string]*Task{},
sessions: map[string]*UploadSession{},
}
if err := s.load(); err != nil {
return nil, err
}
// In-memory upload sessions do not survive a restart; sweep any .part
// files left behind by an interrupted chunked upload so they don't leak.
s.sweepParts()
return s, nil
}
@@ -130,7 +135,9 @@ func (s *Store) Create(originalName string, r io.Reader, maxBytes int64, maxRetr
rel := filepath.Join("uploads", id+ext)
dst := filepath.Join(s.dir, rel)
// Copy with a size cap so a malicious upload can't exhaust disk.
// Copy with a size cap so a malicious upload can't exhaust disk. The cap
// is inclusive: a body of exactly maxBytes is accepted, and anything past
// it is rejected.
tmp := dst + ".tmp"
f, err := os.Create(tmp)
if err != nil {
@@ -142,10 +149,7 @@ func (s *Store) Create(originalName string, r io.Reader, maxBytes int64, maxRetr
os.Remove(tmp)
return nil, fmt.Errorf("read upload: %w", err)
}
if n >= maxBytes {
// Could be exactly at the limit; treat hitting the cap as too large
// unless the reader is now exhausted. We err on the side of caution.
}
_ = n // bytes copied (<= maxBytes); the cap above is the real guard
if err := os.Rename(tmp, dst); err != nil {
os.Remove(tmp)
return nil, fmt.Errorf("save upload: %w", err)
@@ -364,8 +368,9 @@ func newID() string {
return hex.EncodeToString(b)
}
// limitedReader wraps io.Reader with a hard byte cap; exceeding it returns an
// error so the upload is rejected before it fills the disk.
// limitedReader wraps io.Reader with an inclusive byte cap: a body that ends
// at exactly max is accepted (max is the allowed maximum), while any bytes
// past it are rejected so an oversized body can't fill the disk.
type limitedReader struct {
r io.Reader
max int64
@@ -373,14 +378,22 @@ type limitedReader struct {
}
func (l *limitedReader) Read(p []byte) (int, error) {
if l.n >= l.max {
return 0, fmt.Errorf("upload exceeds %d bytes", l.max)
}
remain := l.max - l.n
if int64(len(p)) > remain {
p = p[:remain]
if l.n < l.max {
remain := l.max - l.n
if int64(len(p)) > remain {
p = p[:remain]
}
n, err := l.r.Read(p)
l.n += int64(n)
return n, err
}
// Already at the cap. Tell an exactly-at-cap body (reader now exhausted)
// from one that still has bytes (over the cap): read once — any non-zero
// return means the body exceeded the cap.
n, err := l.r.Read(p)
l.n += int64(n)
return n, err
if n > 0 {
return n, fmt.Errorf("upload exceeds %d bytes", l.max)
}
return 0, err
}