add chunked upload to bypass 100MB proxy body limit

Resumable /api/uploads flow (init -> append -> complete -> abort) so a
large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default
50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would
reject. The web console auto-routes files larger than one slice through the
chunked path with cross-slice progress and transient-error retry; smaller
files keep the single-shot upload.

- store: in-memory upload sessions + .part reassembly (offset writes,
  out-of-order/idempotent), reuse upload id as task id, orphan sweep on
  start, idle-session GC
- fix limitedReader rejecting bodies exactly at the cap (broke full chunks)
- fix drag-drop double-firing when a drop lands on the dropzone (its drop
  handler now stopPropagation so the window listener doesn't re-upload)
- tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
root
2026-09-10 15:12:15 +08:00
parent ab6bf8b370
commit e5a2dc0a8d
11 changed files with 1144 additions and 35 deletions
+105
View File
@@ -3,6 +3,7 @@ package server
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"git.misaka.ren/m1saka/record-asr/internal/audio"
@@ -23,6 +24,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
"models": s.manager != nil && s.manager.Ready(),
"version": s.modelVersion(),
"max_upload_bytes": s.cfg.MaxUploadBytes,
"chunk_size": s.cfg.ChunkSize,
})
}
@@ -82,6 +84,109 @@ func (s *Server) handleCreateTask(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusAccepted, t)
}
// --- resumable chunked uploads ---
//
// Files too large to fit a single request through a size-capping proxy (e.g.
// Cloudflare's 100MB body limit) are uploaded as many small slices instead.
// handleCreateUpload starts a session, handleUploadChunk appends one slice,
// handleCompleteUpload finalizes it into a task, and handleAbortUpload drops
// it. Each slice is <= ASR_CHUNK_SIZE_MB, safely under the proxy cap.
// handleCreateUpload starts a resumable chunked upload session. The request
// body is a small JSON {filename, size}; the response carries the upload id,
// the chunk size, and the total chunk count the client should send.
func (s *Server) handleCreateUpload(w http.ResponseWriter, r *http.Request) {
var req struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
}
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20))
if err := dec.Decode(&req); err != nil {
renderError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
return
}
if req.Filename == "" {
renderError(w, http.StatusBadRequest, "missing 'filename'")
return
}
if req.Size <= 0 {
renderError(w, http.StatusBadRequest, "missing or invalid 'size'")
return
}
if !audio.IsSupported(req.Filename) {
renderError(w, http.StatusUnsupportedMediaType,
"unsupported extension; supported: "+audio.SupportedExtensions())
return
}
sess, err := s.store.CreateUpload(req.Filename, req.Size, s.cfg.ChunkSize, s.cfg.MaxUploadBytes)
if err != nil {
renderError(w, http.StatusBadRequest, "init upload: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{
"upload_id": sess.UploadID,
"chunk_size": sess.ChunkSize,
"total_chunks": sess.TotalChunks,
})
}
// handleUploadChunk appends one chunk (#index, the 0-based query param) of a
// chunked upload session. The request body is the raw chunk bytes
// (application/octet-stream); each slice is capped at ASR_CHUNK_SIZE_MB.
func (s *Server) handleUploadChunk(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
idx, err := strconv.Atoi(r.URL.Query().Get("index"))
if err != nil || idx < 0 {
renderError(w, http.StatusBadRequest, "missing or invalid 'index' query param")
return
}
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.ChunkSize)
if err := s.store.AppendChunk(id, idx, r.Body, s.cfg.ChunkSize); err != nil {
if err == store.ErrUploadNotFound {
renderError(w, http.StatusNotFound, "upload session not found or expired")
return
}
renderError(w, http.StatusBadRequest, "chunk: "+err.Error())
return
}
received, total, ok := s.store.UploadStatus(id)
if !ok {
renderError(w, http.StatusNotFound, "upload session not found")
return
}
writeJSON(w, http.StatusOK, map[string]any{
"index": idx,
"received": received,
"total": total,
})
}
// handleCompleteUpload finalizes a chunked upload and enqueues the task. The
// response mirrors POST /api/tasks (202 + the task) so the client treats the
// finalized upload exactly like a single-shot upload from this point on.
func (s *Server) handleCompleteUpload(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
t, err := s.store.CompleteUpload(id, s.cfg.MaxRetries)
if err != nil {
if err == store.ErrUploadNotFound {
renderError(w, http.StatusNotFound, "upload session not found or expired")
return
}
renderError(w, http.StatusBadRequest, "complete: "+err.Error())
return
}
w.Header().Set("Location", "/api/tasks/"+t.ID)
writeJSON(w, http.StatusAccepted, t)
}
// handleAbortUpload cancels a chunked upload session and removes its part
// file. Idempotent: a missing session still returns 204.
func (s *Server) handleAbortUpload(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
_ = s.store.AbortUpload(id)
w.WriteHeader(http.StatusNoContent)
}
// handleListTasks returns all tasks newest-first.
func (s *Server) handleListTasks(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"tasks": s.store.List()})