add chunked upload to bypass 100MB proxy body limit

Resumable /api/uploads flow (init -> append -> complete -> abort) so a
large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default
50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would
reject. The web console auto-routes files larger than one slice through the
chunked path with cross-slice progress and transient-error retry; smaller
files keep the single-shot upload.

- store: in-memory upload sessions + .part reassembly (offset writes,
  out-of-order/idempotent), reuse upload id as task id, orphan sweep on
  start, idle-session GC
- fix limitedReader rejecting bodies exactly at the cap (broke full chunks)
- fix drag-drop double-firing when a drop lands on the dropzone (its drop
  handler now stopPropagation so the window listener doesn't re-upload)
- tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
root
2026-09-10 15:12:15 +08:00
parent ab6bf8b370
commit e5a2dc0a8d
11 changed files with 1144 additions and 35 deletions
+105
View File
@@ -3,6 +3,7 @@ package server
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"git.misaka.ren/m1saka/record-asr/internal/audio"
@@ -23,6 +24,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
"models": s.manager != nil && s.manager.Ready(),
"version": s.modelVersion(),
"max_upload_bytes": s.cfg.MaxUploadBytes,
"chunk_size": s.cfg.ChunkSize,
})
}
@@ -82,6 +84,109 @@ func (s *Server) handleCreateTask(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusAccepted, t)
}
// --- resumable chunked uploads ---
//
// Files too large to fit a single request through a size-capping proxy (e.g.
// Cloudflare's 100MB body limit) are uploaded as many small slices instead.
// handleCreateUpload starts a session, handleUploadChunk appends one slice,
// handleCompleteUpload finalizes it into a task, and handleAbortUpload drops
// it. Each slice is <= ASR_CHUNK_SIZE_MB, safely under the proxy cap.
// handleCreateUpload starts a resumable chunked upload session. The request
// body is a small JSON {filename, size}; the response carries the upload id,
// the chunk size, and the total chunk count the client should send.
func (s *Server) handleCreateUpload(w http.ResponseWriter, r *http.Request) {
var req struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
}
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20))
if err := dec.Decode(&req); err != nil {
renderError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
return
}
if req.Filename == "" {
renderError(w, http.StatusBadRequest, "missing 'filename'")
return
}
if req.Size <= 0 {
renderError(w, http.StatusBadRequest, "missing or invalid 'size'")
return
}
if !audio.IsSupported(req.Filename) {
renderError(w, http.StatusUnsupportedMediaType,
"unsupported extension; supported: "+audio.SupportedExtensions())
return
}
sess, err := s.store.CreateUpload(req.Filename, req.Size, s.cfg.ChunkSize, s.cfg.MaxUploadBytes)
if err != nil {
renderError(w, http.StatusBadRequest, "init upload: "+err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]any{
"upload_id": sess.UploadID,
"chunk_size": sess.ChunkSize,
"total_chunks": sess.TotalChunks,
})
}
// handleUploadChunk appends one chunk (#index, the 0-based query param) of a
// chunked upload session. The request body is the raw chunk bytes
// (application/octet-stream); each slice is capped at ASR_CHUNK_SIZE_MB.
func (s *Server) handleUploadChunk(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
idx, err := strconv.Atoi(r.URL.Query().Get("index"))
if err != nil || idx < 0 {
renderError(w, http.StatusBadRequest, "missing or invalid 'index' query param")
return
}
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.ChunkSize)
if err := s.store.AppendChunk(id, idx, r.Body, s.cfg.ChunkSize); err != nil {
if err == store.ErrUploadNotFound {
renderError(w, http.StatusNotFound, "upload session not found or expired")
return
}
renderError(w, http.StatusBadRequest, "chunk: "+err.Error())
return
}
received, total, ok := s.store.UploadStatus(id)
if !ok {
renderError(w, http.StatusNotFound, "upload session not found")
return
}
writeJSON(w, http.StatusOK, map[string]any{
"index": idx,
"received": received,
"total": total,
})
}
// handleCompleteUpload finalizes a chunked upload and enqueues the task. The
// response mirrors POST /api/tasks (202 + the task) so the client treats the
// finalized upload exactly like a single-shot upload from this point on.
func (s *Server) handleCompleteUpload(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
t, err := s.store.CompleteUpload(id, s.cfg.MaxRetries)
if err != nil {
if err == store.ErrUploadNotFound {
renderError(w, http.StatusNotFound, "upload session not found or expired")
return
}
renderError(w, http.StatusBadRequest, "complete: "+err.Error())
return
}
w.Header().Set("Location", "/api/tasks/"+t.ID)
writeJSON(w, http.StatusAccepted, t)
}
// handleAbortUpload cancels a chunked upload session and removes its part
// file. Idempotent: a missing session still returns 204.
func (s *Server) handleAbortUpload(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
_ = s.store.AbortUpload(id)
w.WriteHeader(http.StatusNoContent)
}
// handleListTasks returns all tasks newest-first.
func (s *Server) handleListTasks(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, map[string]any{"tasks": s.store.List()})
+8
View File
@@ -64,6 +64,14 @@ func (s *Server) Router() http.Handler {
mux.HandleFunc("POST /api/tasks/{id}/skip", s.handleSkipTask)
mux.HandleFunc("GET /api/tasks/{id}/transcript", s.handleTranscript)
// Resumable chunked uploads (also authed). Lets a large file slip past
// size-capping proxies (e.g. Cloudflare's 100MB body limit) as many
// small slices, finalized into a task indistinguishable from /api/tasks.
mux.HandleFunc("POST /api/uploads", s.handleCreateUpload)
mux.HandleFunc("POST /api/uploads/{id}", s.handleUploadChunk)
mux.HandleFunc("POST /api/uploads/{id}/complete", s.handleCompleteUpload)
mux.HandleFunc("DELETE /api/uploads/{id}", s.handleAbortUpload)
// OpenAI-compatible synchronous transcription (also authed).
mux.HandleFunc("POST /v1/audio/transcriptions", s.handleOpenAITranscription)
+114 -17
View File
@@ -311,6 +311,26 @@
</table>
</div>
<div class="api-section">
<h3>分片上传 <span class="tag">绕过代理体积上限(如 Cloudflare 100MB)</span></h3>
<table class="api-tbl">
<tr><th>方法</th><th>路径</th><th>说明</th></tr>
<tr><td class="m post">POST</td><td><code>/api/uploads</code></td><td>开启上传会话(JSON:<code>{filename,size}</code>)→ <code>{upload_id, chunk_size, total_chunks}</code></td></tr>
<tr><td class="m post">POST</td><td><code>/api/uploads/{id}?index=N</code></td><td>上传第 N 片(0 起,原始字节,≤ chunk_size)。乱序/重发幂等</td></tr>
<tr><td class="m post">POST</td><td><code>/api/uploads/{id}/complete</code></td><td>收齐后合并并入队 → 202 + 任务(等同 <code>/api/tasks</code>)</td></tr>
<tr><td class="m del">DELETE</td><td><code>/api/uploads/{id}</code></td><td>放弃会话并删除分片 → 204</td></tr>
</table>
<pre class="code"><span class="c"># 大文件分片上传(每片 50MB,默认)</span>
curl -H "X-Api-Key: $PW" -H "Content-Type: application/json" \
-d '{"filename":"big.mp4","size":123456789}' http://localhost:8080/api/uploads
# → {"upload_id":"ab12cd34","chunk_size":52428800,"total_chunks":3}
for i in 0 1 2; do
curl -H "X-Api-Key: $PW" --data-binary @"part_$i.bin" \
"http://localhost:8080/api/uploads/ab12cd34?index=$i"
done
curl -H "X-Api-Key: $PW" -X POST http://localhost:8080/api/uploads/ab12cd34/complete</pre>
</div>
<div class="api-section">
<h3>OpenAI 兼容 <span class="tag">同步</span></h3>
<table class="api-tbl">
@@ -368,7 +388,7 @@ const fmtTime = iso=>{ if(!iso) return "—"; const d=new Date(iso); return isNa
const fmtDur = s=> s==null?"—": (s<60?s.toFixed(1)+"s" : Math.floor(s/60)+"m"+String(Math.round(s%60)).padStart(2,"0")+"s");
const escapeHtml = s=>(s||"").replace(/[&<>]/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;"}[c]));
const state = { pw: localStorage.getItem("asr_pw")||"", tasks:[], sel:null, busy:false, authing:false, uploading:false, maxUpload:0 };
const state = { pw: localStorage.getItem("asr_pw")||"", tasks:[], sel:null, busy:false, authing:false, uploading:false, maxUpload:0, chunkSize:0 };
// --- password modal (shown only when verification is needed) ---
const authModal=$("#authModal"), authPw=$("#authPw"), authMsg=$("#authMsg"), authOk=$("#authOk"), authCancel=$("#authCancel");
@@ -480,6 +500,7 @@ async function loadHealth(){
try{
const h = await api("/healthz");
state.maxUpload = h.max_upload_bytes || 0;
state.chunkSize = h.chunk_size || 0;
$("#htext").textContent = h.models?"模型就绪":"模型加载中";
$("#hdot").className = "dot "+(h.models?"ok":"down");
}catch(e){ $("#htext").textContent="服务不可达"; $("#hdot").className="dot down"; }
@@ -606,9 +627,12 @@ function renderDropzone(c){
const dz=el("div",{class:"dropzone"});
dz.innerHTML=`<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M12 16V4M5 11l7-7 7 7M4 20h16"/></svg><h3>拖拽音频或视频到这里,或点击上传</h3><p>音频直接转写;视频会自动提取音轨。约 1~3 秒出结果。</p><div class="ext">音频 .wav/.mp3/.flac/.ogg/.m4a/.aac/.opus · 视频 .mp4/.mkv/.mov/.avi/.ts…</div>`;
dz.onclick=()=>$("#file").click();
dz.addEventListener("dragover",e=>{e.preventDefault();dz.classList.add("drag")});
dz.addEventListener("dragover",e=>{e.preventDefault();e.stopPropagation();dz.classList.add("drag")});
dz.addEventListener("dragleave",()=>dz.classList.remove("drag"));
dz.addEventListener("drop",e=>{e.preventDefault();dz.classList.remove("drag"); if(e.dataTransfer.files.length) uploadFiles(e.dataTransfer.files);});
// stopPropagation: the global window 'drop' listener also uploads, so
// without this a drop landing on the dropzone would bubble up and upload
// the same file twice. The dropzone owns drops on itself.
dz.addEventListener("drop",e=>{e.preventDefault();e.stopPropagation();dz.classList.remove("drag"); if(e.dataTransfer.files.length) uploadFiles(e.dataTransfer.files);});
w.appendChild(dz); c.appendChild(w);
}
@@ -627,21 +651,12 @@ async function uploadFiles(files){
item.querySelector(".upload-name").textContent=f.name;
list.appendChild(item);
const bar=item.querySelector(".upload-bar>div"), pct=item.querySelector(".upload-pct"), st=item.querySelector(".upload-status");
const fd=new FormData(); fd.append("file",f);
// Large files go through the chunked path so each request stays under
// size-capping proxies' body limits (e.g. Cloudflare's 100MB). Files no
// bigger than one chunk use the simple single-shot multipart upload.
const useChunked = state.chunkSize && f.size>state.chunkSize;
try{
const t=await new Promise((res,rej)=>{
const x=new XMLHttpRequest();
x.open("POST","/api/tasks");
x.setRequestHeader("X-Api-Key",state.pw);
x.upload.onprogress=e=>{ if(e.lengthComputable){ const p=Math.round(e.loaded/e.total*100); bar.style.width=p+"%"; pct.textContent=p+"%"; } };
x.onload=()=>{
if(x.status>=200&&x.status<300){ try{res(JSON.parse(x.responseText));}catch(e){res(null);} }
else if(x.status===401){ state.pw=""; localStorage.removeItem("asr_pw"); openAuthModal("密码不正确或服务已重启,请重新输入。",true); rej(new Error("unauthorized")); }
else { let m="HTTP "+x.status; try{m=(JSON.parse(x.responseText).error)||m;}catch(e){} rej(new Error(m)); }
};
x.onerror=()=>rej(new Error("网络错误,上传失败"));
x.send(fd);
});
const t = useChunked ? await uploadChunked(f,bar,pct,st) : await uploadSingle(f,bar,pct,st);
bar.style.width="100%"; pct.textContent="100%";
st.textContent="已入队,转写排队中…"; item.classList.add("done");
toast("已加入队列: "+(t.original_name||t.id),"ok");
@@ -651,6 +666,88 @@ async function uploadFiles(files){
// Let the user see the finished state briefly, then refresh the queue.
setTimeout(()=>{ state.uploading=false; loadTasks(); }, 600);
}
// uploadSingle sends a whole file as one multipart request (POST /api/tasks)
// with XHR so the progress bar reflects bytes sent.
function uploadSingle(f,bar,pct,st){
return new Promise((res,rej)=>{
const fd=new FormData(); fd.append("file",f);
const x=new XMLHttpRequest();
x.open("POST","/api/tasks");
x.setRequestHeader("X-Api-Key",state.pw);
x.upload.onprogress=e=>{ if(e.lengthComputable){ const p=Math.round(e.loaded/e.total*100); bar.style.width=p+"%"; pct.textContent=p+"%"; } };
x.onload=()=>{
if(x.status>=200&&x.status<300){ try{res(JSON.parse(x.responseText));}catch(e){res(null);} }
else if(x.status===401){ state.pw=""; localStorage.removeItem("asr_pw"); openAuthModal("密码不正确或服务已重启,请重新输入。",true); rej(new Error("unauthorized")); }
else { let m="HTTP "+x.status; try{m=(JSON.parse(x.responseText).error)||m;}catch(e){} rej(new Error(m)); }
};
x.onerror=()=>rej(new Error("网络错误,上传失败"));
x.send(fd);
});
}
// uploadChunked sends a file as ceil(size/chunkSize) slices to the resumable
// upload API (POST /api/uploads -> /api/uploads/{id}?index=N -> /complete),
// tracking overall progress across slices. A failing slice is retried a few
// times on transient errors; on give-up the partial session is aborted.
async function uploadChunked(f,bar,pct,st){
let uid=null; // declared outside try so the catch can reference it safely
try{
const init = await api("/api/uploads", {
method:"POST",
headers:{"Content-Type":"application/json"},
body: JSON.stringify({filename:f.name, size:f.size})
});
uid=init.upload_id;
const cs=+init.chunk_size, total=+init.total_chunks;
const setPct=bytes=>{ const p=Math.min(100,Math.round(bytes/f.size*100)); bar.style.width=p+"%"; pct.textContent=p+"%"; };
let uploaded=0;
for(let i=0;i<total;i++){
const start=i*cs, end=Math.min(start+cs,f.size);
const blob=f.slice(start,end);
let done=false, lastErr;
for(let attempt=0; attempt<3 && !done; attempt++){
try{
await new Promise((res,rej)=>{
const x=new XMLHttpRequest();
x.open("POST", `/api/uploads/${uid}?index=${i}`);
x.setRequestHeader("X-Api-Key",state.pw);
x.upload.onprogress=e=>{ if(e.lengthComputable) setPct(uploaded+e.loaded); };
x.onload=()=>{
if(x.status>=200&&x.status<300){ try{res(JSON.parse(x.responseText));}catch(e){res(null);} return; }
const m=(()=>{ let s="HTTP "+x.status; try{s=(JSON.parse(x.responseText).error)||s;}catch(e){} return s; })();
const e=new Error(m); e.status=x.status; rej(e);
};
x.onerror=()=>{ const e=new Error("网络错误"); e.status=0; rej(e); };
x.send(blob);
});
done=true;
}catch(e){
lastErr=e;
// Auth failure: surface immediately (the modal reopens); don't retry.
if(e.status===401){ state.pw=""; localStorage.removeItem("asr_pw"); openAuthModal("密码不正确或服务已重启,请重新输入。",true); throw e; }
// 4xx (bad request / session gone): retrying won't help; give up.
if(e.status>=400 && e.status<500 && e.status!==0) throw e;
// Transient (network/5xx): back off and retry unless last attempt.
if(attempt<2){ await new Promise(r=>setTimeout(r, 800*(attempt+1))); }
}
}
if(!done) throw lastErr || new Error("上传分片失败");
uploaded += (end-start);
setPct(uploaded);
}
const task = await api(`/api/uploads/${uid}/complete`, {method:"POST"});
return task;
}catch(e){
// Best-effort cleanup of the partial session — but only if one was
// actually created (init may have failed) and not on an auth failure
// (the password is already cleared and a retry would just 401 again).
if(uid && e.status!==401 && state.pw){
try{ await fetch(`/api/uploads/${uid}`,{method:"DELETE",headers:{"X-Api-Key":state.pw}}); }catch(_){}
}
throw e;
}
}
$("#file").addEventListener("change",e=>{ if(e.target.files.length) uploadFiles(e.target.files); e.target.value=""; });
$("#uploadBtn").addEventListener("click",()=>$("#file").click());
+153
View File
@@ -0,0 +1,153 @@
package server
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"testing"
"git.misaka.ren/m1saka/record-asr/internal/config"
"git.misaka.ren/m1saka/record-asr/internal/store"
)
// posReader emits a deterministic, position-dependent byte stream so that no
// two slices of the generated file are identical — a reorder would then
// change the reassembled hash and be caught.
type posReader struct{ pos int64 }
func (r *posReader) Read(p []byte) (int, error) {
for i := range p {
x := r.pos + int64(i)
p[i] = byte((x*127 ^ (x >> 7)) & 0xff)
}
r.pos += int64(len(p))
return len(p), nil
}
// sha256sum returns the hex SHA-256 of a file, streaming so 120MB never sits
// in memory at once.
func sha256sum(t *testing.T, path string) string {
t.Helper()
f, err := os.Open(path)
if err != nil {
t.Fatalf("open %s: %v", path, err)
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
t.Fatalf("hash %s: %v", path, err)
}
return hex.EncodeToString(h.Sum(nil))
}
// TestChunkedUploadLargeFile uploads a 120MB file — larger than the 100MB CDN
// body cap the feature exists to bypass — as 25MB slices and checks that the
// reassembled file is byte-identical to the original. Guarded so the default
// test suite stays fast; run with:
//
// ASR_TEST_LARGE=1 go test -run Large ./internal/server/...
func TestChunkedUploadLargeFile(t *testing.T) {
if os.Getenv("ASR_TEST_LARGE") == "" {
t.Skip("set ASR_TEST_LARGE=1 to run the large-file chunked upload test (~120MB on disk)")
}
const total = 120 << 20 // 120 MiB — over the 100MB cap, no per-request limit here
const chunk = 25 << 20 // 25 MiB slices → 5 chunks (last is 20 MiB, partial)
dir := t.TempDir()
st, err := store.Open(dir)
if err != nil {
t.Fatalf("store.Open: %v", err)
}
cfg := config.Config{DataDir: dir, ChunkSize: chunk, MaxUploadBytes: 512 << 20, MaxRetries: 3}
srv := New(cfg, st, nil, nil) // upload path never touches the transcribe/model stack
ts := httptest.NewServer(srv.Router())
t.Cleanup(ts.Close)
// 1. Generate the 120MB original on disk, hashing as we go.
origPath := filepath.Join(dir, "original.wav")
of, err := os.Create(origPath)
if err != nil {
t.Fatalf("create orig: %v", err)
}
if _, err := io.Copy(of, io.LimitReader(&posReader{}, total)); err != nil {
t.Fatalf("write orig: %v", err)
}
of.Close()
wantHash := sha256sum(t, origPath)
// 2. Open a chunked-upload session.
body, _ := json.Marshal(map[string]any{"filename": "large.wav", "size": total})
resp, err := http.Post(ts.URL+"/api/uploads", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("init: %v", err)
}
var init struct {
UploadID string `json:"upload_id"`
ChunkSize int64 `json:"chunk_size"`
TotalChunks int `json:"total_chunks"`
}
json.NewDecoder(resp.Body).Decode(&init)
resp.Body.Close()
if init.UploadID == "" || init.ChunkSize != chunk || init.TotalChunks != 5 {
t.Fatalf("init = %+v", init)
}
// 3. Stream each slice straight from the original file (no full copy in RAM).
src, err := os.Open(origPath)
if err != nil {
t.Fatalf("open src: %v", err)
}
defer src.Close()
for i := 0; i < init.TotalChunks; i++ {
offset := int64(i) * chunk
size := int64(chunk)
if offset+size > total {
size = total - offset
}
req, _ := http.NewRequest("POST",
ts.URL+"/api/uploads/"+init.UploadID+"?index="+strconv.Itoa(i),
io.NewSectionReader(src, offset, size))
req.Header.Set("Content-Type", "application/octet-stream")
cresp, cerr := http.DefaultClient.Do(req)
if cerr != nil {
t.Fatalf("chunk %d: %v", i, cerr)
}
if cresp.StatusCode != http.StatusOK {
t.Fatalf("chunk %d status = %d", i, cresp.StatusCode)
}
cresp.Body.Close()
}
// 4. Complete → 202 + pending task.
req, _ := http.NewRequest("POST", ts.URL+"/api/uploads/"+init.UploadID+"/complete", nil)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("complete: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("complete status = %d", resp.StatusCode)
}
var task store.Task
json.NewDecoder(resp.Body).Decode(&task)
if task.Status != store.StatusPending {
t.Fatalf("task status = %q, want %q", task.Status, store.StatusPending)
}
// 5. The reassembled file must be byte-identical (same SHA-256) and exact size.
gotHash := sha256sum(t, st.AudioPath(&task))
if gotHash != wantHash {
t.Fatalf("reassembled file hash mismatch:\n want %s\n got %s", wantHash, gotHash)
}
fi, err := os.Stat(st.AudioPath(&task))
if err != nil || fi.Size() != total {
t.Fatalf("final size = %d (%v), want %d", fi.Size(), err, total)
}
}
+200
View File
@@ -0,0 +1,200 @@
package server
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strconv"
"testing"
"git.misaka.ren/m1saka/record-asr/internal/config"
"git.misaka.ren/m1saka/record-asr/internal/store"
)
// newUploadTestServer wires a Server backed by a real store + a minimal config
// (auth disabled, tiny 100B chunks) with nil pipeline/manager — the upload
// handlers never touch the transcribe/model stack, so this is safe.
func newUploadTestServer(t *testing.T) (*store.Store, *httptest.Server) {
t.Helper()
dir := t.TempDir()
st, err := store.Open(dir)
if err != nil {
t.Fatalf("store.Open: %v", err)
}
cfg := config.Config{
DataDir: dir,
ChunkSize: 100,
MaxUploadBytes: 1 << 20,
MaxRetries: 3,
}
srv := New(cfg, st, nil, nil)
ts := httptest.NewServer(srv.Router())
t.Cleanup(ts.Close)
return st, ts
}
// initSession opens a chunked-upload session and returns the upload id.
func initSession(t *testing.T, ts *httptest.Server, name string, size int64) string {
t.Helper()
body, _ := json.Marshal(map[string]any{"filename": name, "size": size})
resp, err := http.Post(ts.URL+"/api/uploads", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("init: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("init status = %d", resp.StatusCode)
}
var r struct {
UploadID string `json:"upload_id"`
ChunkSize int64 `json:"chunk_size"`
TotalChunks int `json:"total_chunks"`
}
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
t.Fatalf("init decode: %v", err)
}
if r.UploadID == "" || r.ChunkSize != 100 {
t.Fatalf("init resp = %+v", r)
}
return r.UploadID
}
func sendChunk(t *testing.T, ts, id string, idx int, data []byte) int {
t.Helper()
req, _ := http.NewRequest("POST", ts+"/api/uploads/"+id+"?index="+strconv.Itoa(idx), bytes.NewReader(data))
req.Header.Set("Content-Type", "application/octet-stream")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("chunk %d: %v", idx, err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("chunk %d status = %d", idx, resp.StatusCode)
}
var r struct {
Received int `json:"received"`
Total int `json:"total"`
}
json.NewDecoder(resp.Body).Decode(&r)
return r.Received
}
// TestChunkedUploadHTTP exercises the full endpoint flow end-to-end.
func TestChunkedUploadHTTP(t *testing.T) {
st, ts := newUploadTestServer(t)
// /healthz advertises the chunk size the client uses as its threshold.
hresp, _ := http.Get(ts.URL + "/healthz")
var hz struct {
ChunkSize int64 `json:"chunk_size"`
MaxUploadBytes int64 `json:"max_upload_bytes"`
}
json.NewDecoder(hresp.Body).Decode(&hz)
hresp.Body.Close()
if hz.ChunkSize != 100 || hz.MaxUploadBytes != 1<<20 {
t.Fatalf("healthz = %+v", hz)
}
// 250B file -> 3 chunks (100, 100, 50). Send out of order.
uid := initSession(t, ts, "clip.wav", 250)
c0 := bytes.Repeat([]byte{0xAA}, 100)
c1 := bytes.Repeat([]byte{0xBB}, 100)
c2 := bytes.Repeat([]byte{0xCC}, 50)
want := append(append([]byte{}, c0...), c1...)
want = append(want, c2...)
if got := sendChunk(t, ts.URL, uid, 1, c1); got != 1 {
t.Fatalf("after chunk 1 received=%d want 1", got)
}
if got := sendChunk(t, ts.URL, uid, 0, c0); got != 2 {
t.Fatalf("after chunk 0 received=%d want 2", got)
}
if got := sendChunk(t, ts.URL, uid, 2, c2); got != 3 {
t.Fatalf("after chunk 2 received=%d want 3", got)
}
// Complete -> 202 + task (same shape as /api/tasks).
req, _ := http.NewRequest("POST", ts.URL+"/api/uploads/"+uid+"/complete", nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("complete: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusAccepted {
t.Fatalf("complete status = %d", resp.StatusCode)
}
var task store.Task
json.NewDecoder(resp.Body).Decode(&task)
if task.Status != store.StatusPending || task.OriginalName != "clip.wav" {
t.Fatalf("task = %+v", task)
}
// The reassembled file matches the original bytes exactly.
got, err := os.ReadFile(st.AudioPath(&task))
if err != nil {
t.Fatalf("read final file: %v", err)
}
if !bytes.Equal(got, want) {
t.Fatalf("reassembled bytes mismatch:\n got %x\n want %x", got, want)
}
}
// TestChunkedUploadHTTPErrors checks the error paths: missing index, unknown
// session, oversized chunk, and idempotent abort.
func TestChunkedUploadHTTPErrors(t *testing.T) {
_, ts := newUploadTestServer(t)
// Missing ?index -> 400.
resp, _ := http.Post(ts.URL+"/api/uploads/someid", "application/octet-stream", bytes.NewReader([]byte("x")))
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("missing-index status = %d, want 400", resp.StatusCode)
}
// Complete on an unknown session -> 404.
req, _ := http.NewRequest("POST", ts.URL+"/api/uploads/doesnotexist/complete", nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("unknown complete: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("unknown complete status = %d, want 404", resp.StatusCode)
}
// Oversized chunk (101B > 100B cap) -> 400.
uid := initSession(t, ts, "big.wav", 250)
req, _ = http.NewRequest("POST", ts.URL+"/api/uploads/"+uid+"?index=0", bytes.NewReader(bytes.Repeat([]byte{0x9}, 101)))
req.Header.Set("Content-Type", "application/octet-stream")
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("oversized chunk: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("oversized chunk status = %d, want 400", resp.StatusCode)
}
// Abort a fresh session -> 204, and the id is gone afterwards.
uid2 := initSession(t, ts, "abort.wav", 100)
req, _ = http.NewRequest("DELETE", ts.URL+"/api/uploads/"+uid2, nil)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("abort: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNoContent {
t.Fatalf("abort status = %d, want 204", resp.StatusCode)
}
// Completing an aborted session -> 404.
req, _ = http.NewRequest("POST", ts.URL+"/api/uploads/"+uid2+"/complete", nil)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post-abort complete: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("post-abort complete status = %d, want 404", resp.StatusCode)
}
}