add chunked upload to bypass 100MB proxy body limit
Resumable /api/uploads flow (init -> append -> complete -> abort) so a large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default 50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would reject. The web console auto-routes files larger than one slice through the chunked path with cross-slice progress and transient-error retry; smaller files keep the single-shot upload. - store: in-memory upload sessions + .part reassembly (offset writes, out-of-order/idempotent), reuse upload id as task id, orphan sweep on start, idle-session GC - fix limitedReader rejecting bodies exactly at the cap (broke full chunks) - fix drag-drop double-firing when a drop lands on the dropzone (its drop handler now stopPropagation so the window listener doesn't re-upload) - tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.misaka.ren/m1saka/record-asr/internal/audio"
|
||||
@@ -23,6 +24,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
||||
"models": s.manager != nil && s.manager.Ready(),
|
||||
"version": s.modelVersion(),
|
||||
"max_upload_bytes": s.cfg.MaxUploadBytes,
|
||||
"chunk_size": s.cfg.ChunkSize,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -82,6 +84,109 @@ func (s *Server) handleCreateTask(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, t)
|
||||
}
|
||||
|
||||
// --- resumable chunked uploads ---
|
||||
//
|
||||
// Files too large to fit a single request through a size-capping proxy (e.g.
|
||||
// Cloudflare's 100MB body limit) are uploaded as many small slices instead.
|
||||
// handleCreateUpload starts a session, handleUploadChunk appends one slice,
|
||||
// handleCompleteUpload finalizes it into a task, and handleAbortUpload drops
|
||||
// it. Each slice is <= ASR_CHUNK_SIZE_MB, safely under the proxy cap.
|
||||
|
||||
// handleCreateUpload starts a resumable chunked upload session. The request
|
||||
// body is a small JSON {filename, size}; the response carries the upload id,
|
||||
// the chunk size, and the total chunk count the client should send.
|
||||
func (s *Server) handleCreateUpload(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20))
|
||||
if err := dec.Decode(&req); err != nil {
|
||||
renderError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Filename == "" {
|
||||
renderError(w, http.StatusBadRequest, "missing 'filename'")
|
||||
return
|
||||
}
|
||||
if req.Size <= 0 {
|
||||
renderError(w, http.StatusBadRequest, "missing or invalid 'size'")
|
||||
return
|
||||
}
|
||||
if !audio.IsSupported(req.Filename) {
|
||||
renderError(w, http.StatusUnsupportedMediaType,
|
||||
"unsupported extension; supported: "+audio.SupportedExtensions())
|
||||
return
|
||||
}
|
||||
sess, err := s.store.CreateUpload(req.Filename, req.Size, s.cfg.ChunkSize, s.cfg.MaxUploadBytes)
|
||||
if err != nil {
|
||||
renderError(w, http.StatusBadRequest, "init upload: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"upload_id": sess.UploadID,
|
||||
"chunk_size": sess.ChunkSize,
|
||||
"total_chunks": sess.TotalChunks,
|
||||
})
|
||||
}
|
||||
|
||||
// handleUploadChunk appends one chunk (#index, the 0-based query param) of a
|
||||
// chunked upload session. The request body is the raw chunk bytes
|
||||
// (application/octet-stream); each slice is capped at ASR_CHUNK_SIZE_MB.
|
||||
func (s *Server) handleUploadChunk(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
idx, err := strconv.Atoi(r.URL.Query().Get("index"))
|
||||
if err != nil || idx < 0 {
|
||||
renderError(w, http.StatusBadRequest, "missing or invalid 'index' query param")
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.ChunkSize)
|
||||
if err := s.store.AppendChunk(id, idx, r.Body, s.cfg.ChunkSize); err != nil {
|
||||
if err == store.ErrUploadNotFound {
|
||||
renderError(w, http.StatusNotFound, "upload session not found or expired")
|
||||
return
|
||||
}
|
||||
renderError(w, http.StatusBadRequest, "chunk: "+err.Error())
|
||||
return
|
||||
}
|
||||
received, total, ok := s.store.UploadStatus(id)
|
||||
if !ok {
|
||||
renderError(w, http.StatusNotFound, "upload session not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"index": idx,
|
||||
"received": received,
|
||||
"total": total,
|
||||
})
|
||||
}
|
||||
|
||||
// handleCompleteUpload finalizes a chunked upload and enqueues the task. The
|
||||
// response mirrors POST /api/tasks (202 + the task) so the client treats the
|
||||
// finalized upload exactly like a single-shot upload from this point on.
|
||||
func (s *Server) handleCompleteUpload(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
t, err := s.store.CompleteUpload(id, s.cfg.MaxRetries)
|
||||
if err != nil {
|
||||
if err == store.ErrUploadNotFound {
|
||||
renderError(w, http.StatusNotFound, "upload session not found or expired")
|
||||
return
|
||||
}
|
||||
renderError(w, http.StatusBadRequest, "complete: "+err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/api/tasks/"+t.ID)
|
||||
writeJSON(w, http.StatusAccepted, t)
|
||||
}
|
||||
|
||||
// handleAbortUpload cancels a chunked upload session and removes its part
|
||||
// file. Idempotent: a missing session still returns 204.
|
||||
func (s *Server) handleAbortUpload(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
_ = s.store.AbortUpload(id)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleListTasks returns all tasks newest-first.
|
||||
func (s *Server) handleListTasks(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"tasks": s.store.List()})
|
||||
|
||||
Reference in New Issue
Block a user