add chunked upload to bypass 100MB proxy body limit
Resumable /api/uploads flow (init -> append -> complete -> abort) so a large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default 50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would reject. The web console auto-routes files larger than one slice through the chunked path with cross-slice progress and transient-error retry; smaller files keep the single-shot upload. - store: in-memory upload sessions + .part reassembly (offset writes, out-of-order/idempotent), reuse upload id as task id, orphan sweep on start, idle-session GC - fix limitedReader rejecting bodies exactly at the cap (broke full chunks) - fix drag-drop double-firing when a drop lands on the dropzone (its drop handler now stopPropagation so the window listener doesn't re-upload) - tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
+29
-16
@@ -55,8 +55,9 @@ type Store struct {
|
||||
tasksFile string
|
||||
uploadsDir string
|
||||
|
||||
mu sync.Mutex
|
||||
tasks map[string]*Task
|
||||
mu sync.Mutex
|
||||
tasks map[string]*Task
|
||||
sessions map[string]*UploadSession // in-progress chunked uploads
|
||||
}
|
||||
|
||||
// Open initializes the store at dir, creating directories and loading any
|
||||
@@ -75,10 +76,14 @@ func Open(dir string) (*Store, error) {
|
||||
tasksFile: filepath.Join(dir, "tasks.json"),
|
||||
uploadsDir: uploads,
|
||||
tasks: map[string]*Task{},
|
||||
sessions: map[string]*UploadSession{},
|
||||
}
|
||||
if err := s.load(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// In-memory upload sessions do not survive a restart; sweep any .part
|
||||
// files left behind by an interrupted chunked upload so they don't leak.
|
||||
s.sweepParts()
|
||||
return s, nil
|
||||
}
|
||||
|
||||
@@ -130,7 +135,9 @@ func (s *Store) Create(originalName string, r io.Reader, maxBytes int64, maxRetr
|
||||
rel := filepath.Join("uploads", id+ext)
|
||||
dst := filepath.Join(s.dir, rel)
|
||||
|
||||
// Copy with a size cap so a malicious upload can't exhaust disk.
|
||||
// Copy with a size cap so a malicious upload can't exhaust disk. The cap
|
||||
// is inclusive: a body of exactly maxBytes is accepted, and anything past
|
||||
// it is rejected.
|
||||
tmp := dst + ".tmp"
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
@@ -142,10 +149,7 @@ func (s *Store) Create(originalName string, r io.Reader, maxBytes int64, maxRetr
|
||||
os.Remove(tmp)
|
||||
return nil, fmt.Errorf("read upload: %w", err)
|
||||
}
|
||||
if n >= maxBytes {
|
||||
// Could be exactly at the limit; treat hitting the cap as too large
|
||||
// unless the reader is now exhausted. We err on the side of caution.
|
||||
}
|
||||
_ = n // bytes copied (<= maxBytes); the cap above is the real guard
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
os.Remove(tmp)
|
||||
return nil, fmt.Errorf("save upload: %w", err)
|
||||
@@ -364,8 +368,9 @@ func newID() string {
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// limitedReader wraps io.Reader with a hard byte cap; exceeding it returns an
|
||||
// error so the upload is rejected before it fills the disk.
|
||||
// limitedReader wraps io.Reader with an inclusive byte cap: a body that ends
|
||||
// at exactly max is accepted (max is the allowed maximum), while any bytes
|
||||
// past it are rejected so an oversized body can't fill the disk.
|
||||
type limitedReader struct {
|
||||
r io.Reader
|
||||
max int64
|
||||
@@ -373,14 +378,22 @@ type limitedReader struct {
|
||||
}
|
||||
|
||||
func (l *limitedReader) Read(p []byte) (int, error) {
|
||||
if l.n >= l.max {
|
||||
return 0, fmt.Errorf("upload exceeds %d bytes", l.max)
|
||||
}
|
||||
remain := l.max - l.n
|
||||
if int64(len(p)) > remain {
|
||||
p = p[:remain]
|
||||
if l.n < l.max {
|
||||
remain := l.max - l.n
|
||||
if int64(len(p)) > remain {
|
||||
p = p[:remain]
|
||||
}
|
||||
n, err := l.r.Read(p)
|
||||
l.n += int64(n)
|
||||
return n, err
|
||||
}
|
||||
// Already at the cap. Tell an exactly-at-cap body (reader now exhausted)
|
||||
// from one that still has bytes (over the cap): read once — any non-zero
|
||||
// return means the body exceeded the cap.
|
||||
n, err := l.r.Read(p)
|
||||
l.n += int64(n)
|
||||
return n, err
|
||||
if n > 0 {
|
||||
return n, fmt.Errorf("upload exceeds %d bytes", l.max)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user