add chunked upload to bypass 100MB proxy body limit
Resumable /api/uploads flow (init -> append -> complete -> abort) so a large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default 50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would reject. The web console auto-routes files larger than one slice through the chunked path with cross-slice progress and transient-error retry; smaller files keep the single-shot upload. - store: in-memory upload sessions + .part reassembly (offset writes, out-of-order/idempotent), reuse upload id as task id, orphan sweep on start, idle-session GC - fix limitedReader rejecting bodies exactly at the cap (broke full chunks) - fix drag-drop double-firing when a drop lands on the dropzone (its drop handler now stopPropagation so the window listener doesn't re-upload) - tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.misaka.ren/m1saka/record-asr/internal/audio"
|
||||
@@ -23,6 +24,7 @@ func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
|
||||
"models": s.manager != nil && s.manager.Ready(),
|
||||
"version": s.modelVersion(),
|
||||
"max_upload_bytes": s.cfg.MaxUploadBytes,
|
||||
"chunk_size": s.cfg.ChunkSize,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -82,6 +84,109 @@ func (s *Server) handleCreateTask(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, t)
|
||||
}
|
||||
|
||||
// --- resumable chunked uploads ---
|
||||
//
|
||||
// Files too large to fit a single request through a size-capping proxy (e.g.
|
||||
// Cloudflare's 100MB body limit) are uploaded as many small slices instead.
|
||||
// handleCreateUpload starts a session, handleUploadChunk appends one slice,
|
||||
// handleCompleteUpload finalizes it into a task, and handleAbortUpload drops
|
||||
// it. Each slice is <= ASR_CHUNK_SIZE_MB, safely under the proxy cap.
|
||||
|
||||
// handleCreateUpload starts a resumable chunked upload session. The request
|
||||
// body is a small JSON {filename, size}; the response carries the upload id,
|
||||
// the chunk size, and the total chunk count the client should send.
|
||||
func (s *Server) handleCreateUpload(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20))
|
||||
if err := dec.Decode(&req); err != nil {
|
||||
renderError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Filename == "" {
|
||||
renderError(w, http.StatusBadRequest, "missing 'filename'")
|
||||
return
|
||||
}
|
||||
if req.Size <= 0 {
|
||||
renderError(w, http.StatusBadRequest, "missing or invalid 'size'")
|
||||
return
|
||||
}
|
||||
if !audio.IsSupported(req.Filename) {
|
||||
renderError(w, http.StatusUnsupportedMediaType,
|
||||
"unsupported extension; supported: "+audio.SupportedExtensions())
|
||||
return
|
||||
}
|
||||
sess, err := s.store.CreateUpload(req.Filename, req.Size, s.cfg.ChunkSize, s.cfg.MaxUploadBytes)
|
||||
if err != nil {
|
||||
renderError(w, http.StatusBadRequest, "init upload: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"upload_id": sess.UploadID,
|
||||
"chunk_size": sess.ChunkSize,
|
||||
"total_chunks": sess.TotalChunks,
|
||||
})
|
||||
}
|
||||
|
||||
// handleUploadChunk appends one chunk (#index, the 0-based query param) of a
|
||||
// chunked upload session. The request body is the raw chunk bytes
|
||||
// (application/octet-stream); each slice is capped at ASR_CHUNK_SIZE_MB.
|
||||
func (s *Server) handleUploadChunk(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
idx, err := strconv.Atoi(r.URL.Query().Get("index"))
|
||||
if err != nil || idx < 0 {
|
||||
renderError(w, http.StatusBadRequest, "missing or invalid 'index' query param")
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, s.cfg.ChunkSize)
|
||||
if err := s.store.AppendChunk(id, idx, r.Body, s.cfg.ChunkSize); err != nil {
|
||||
if err == store.ErrUploadNotFound {
|
||||
renderError(w, http.StatusNotFound, "upload session not found or expired")
|
||||
return
|
||||
}
|
||||
renderError(w, http.StatusBadRequest, "chunk: "+err.Error())
|
||||
return
|
||||
}
|
||||
received, total, ok := s.store.UploadStatus(id)
|
||||
if !ok {
|
||||
renderError(w, http.StatusNotFound, "upload session not found")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
"index": idx,
|
||||
"received": received,
|
||||
"total": total,
|
||||
})
|
||||
}
|
||||
|
||||
// handleCompleteUpload finalizes a chunked upload and enqueues the task. The
|
||||
// response mirrors POST /api/tasks (202 + the task) so the client treats the
|
||||
// finalized upload exactly like a single-shot upload from this point on.
|
||||
func (s *Server) handleCompleteUpload(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
t, err := s.store.CompleteUpload(id, s.cfg.MaxRetries)
|
||||
if err != nil {
|
||||
if err == store.ErrUploadNotFound {
|
||||
renderError(w, http.StatusNotFound, "upload session not found or expired")
|
||||
return
|
||||
}
|
||||
renderError(w, http.StatusBadRequest, "complete: "+err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/api/tasks/"+t.ID)
|
||||
writeJSON(w, http.StatusAccepted, t)
|
||||
}
|
||||
|
||||
// handleAbortUpload cancels a chunked upload session and removes its part
|
||||
// file. Idempotent: a missing session still returns 204.
|
||||
func (s *Server) handleAbortUpload(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
_ = s.store.AbortUpload(id)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleListTasks returns all tasks newest-first.
|
||||
func (s *Server) handleListTasks(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"tasks": s.store.List()})
|
||||
|
||||
@@ -64,6 +64,14 @@ func (s *Server) Router() http.Handler {
|
||||
mux.HandleFunc("POST /api/tasks/{id}/skip", s.handleSkipTask)
|
||||
mux.HandleFunc("GET /api/tasks/{id}/transcript", s.handleTranscript)
|
||||
|
||||
// Resumable chunked uploads (also authed). Lets a large file slip past
|
||||
// size-capping proxies (e.g. Cloudflare's 100MB body limit) as many
|
||||
// small slices, finalized into a task indistinguishable from /api/tasks.
|
||||
mux.HandleFunc("POST /api/uploads", s.handleCreateUpload)
|
||||
mux.HandleFunc("POST /api/uploads/{id}", s.handleUploadChunk)
|
||||
mux.HandleFunc("POST /api/uploads/{id}/complete", s.handleCompleteUpload)
|
||||
mux.HandleFunc("DELETE /api/uploads/{id}", s.handleAbortUpload)
|
||||
|
||||
// OpenAI-compatible synchronous transcription (also authed).
|
||||
mux.HandleFunc("POST /v1/audio/transcriptions", s.handleOpenAITranscription)
|
||||
|
||||
|
||||
+114
-17
@@ -311,6 +311,26 @@
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div class="api-section">
|
||||
<h3>分片上传 <span class="tag">绕过代理体积上限(如 Cloudflare 100MB)</span></h3>
|
||||
<table class="api-tbl">
|
||||
<tr><th>方法</th><th>路径</th><th>说明</th></tr>
|
||||
<tr><td class="m post">POST</td><td><code>/api/uploads</code></td><td>开启上传会话(JSON:<code>{filename,size}</code>)→ <code>{upload_id, chunk_size, total_chunks}</code></td></tr>
|
||||
<tr><td class="m post">POST</td><td><code>/api/uploads/{id}?index=N</code></td><td>上传第 N 片(0 起,原始字节,≤ chunk_size)。乱序/重发幂等</td></tr>
|
||||
<tr><td class="m post">POST</td><td><code>/api/uploads/{id}/complete</code></td><td>收齐后合并并入队 → 202 + 任务(等同 <code>/api/tasks</code>)</td></tr>
|
||||
<tr><td class="m del">DELETE</td><td><code>/api/uploads/{id}</code></td><td>放弃会话并删除分片 → 204</td></tr>
|
||||
</table>
|
||||
<pre class="code"><span class="c"># 大文件分片上传(每片 50MB,默认)</span>
|
||||
curl -H "X-Api-Key: $PW" -H "Content-Type: application/json" \
|
||||
-d '{"filename":"big.mp4","size":123456789}' http://localhost:8080/api/uploads
|
||||
# → {"upload_id":"ab12cd34","chunk_size":52428800,"total_chunks":3}
|
||||
for i in 0 1 2; do
|
||||
curl -H "X-Api-Key: $PW" --data-binary @"part_$i.bin" \
|
||||
"http://localhost:8080/api/uploads/ab12cd34?index=$i"
|
||||
done
|
||||
curl -H "X-Api-Key: $PW" -X POST http://localhost:8080/api/uploads/ab12cd34/complete</pre>
|
||||
</div>
|
||||
|
||||
<div class="api-section">
|
||||
<h3>OpenAI 兼容 <span class="tag">同步</span></h3>
|
||||
<table class="api-tbl">
|
||||
@@ -368,7 +388,7 @@ const fmtTime = iso=>{ if(!iso) return "—"; const d=new Date(iso); return isNa
|
||||
const fmtDur = s=> s==null?"—": (s<60?s.toFixed(1)+"s" : Math.floor(s/60)+"m"+String(Math.round(s%60)).padStart(2,"0")+"s");
|
||||
const escapeHtml = s=>(s||"").replace(/[&<>]/g,c=>({"&":"&","<":"<",">":">"}[c]));
|
||||
|
||||
const state = { pw: localStorage.getItem("asr_pw")||"", tasks:[], sel:null, busy:false, authing:false, uploading:false, maxUpload:0 };
|
||||
const state = { pw: localStorage.getItem("asr_pw")||"", tasks:[], sel:null, busy:false, authing:false, uploading:false, maxUpload:0, chunkSize:0 };
|
||||
|
||||
// --- password modal (shown only when verification is needed) ---
|
||||
const authModal=$("#authModal"), authPw=$("#authPw"), authMsg=$("#authMsg"), authOk=$("#authOk"), authCancel=$("#authCancel");
|
||||
@@ -480,6 +500,7 @@ async function loadHealth(){
|
||||
try{
|
||||
const h = await api("/healthz");
|
||||
state.maxUpload = h.max_upload_bytes || 0;
|
||||
state.chunkSize = h.chunk_size || 0;
|
||||
$("#htext").textContent = h.models?"模型就绪":"模型加载中";
|
||||
$("#hdot").className = "dot "+(h.models?"ok":"down");
|
||||
}catch(e){ $("#htext").textContent="服务不可达"; $("#hdot").className="dot down"; }
|
||||
@@ -606,9 +627,12 @@ function renderDropzone(c){
|
||||
const dz=el("div",{class:"dropzone"});
|
||||
dz.innerHTML=`<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path d="M12 16V4M5 11l7-7 7 7M4 20h16"/></svg><h3>拖拽音频或视频到这里,或点击上传</h3><p>音频直接转写;视频会自动提取音轨。约 1~3 秒出结果。</p><div class="ext">音频 .wav/.mp3/.flac/.ogg/.m4a/.aac/.opus · 视频 .mp4/.mkv/.mov/.avi/.ts…</div>`;
|
||||
dz.onclick=()=>$("#file").click();
|
||||
dz.addEventListener("dragover",e=>{e.preventDefault();dz.classList.add("drag")});
|
||||
dz.addEventListener("dragover",e=>{e.preventDefault();e.stopPropagation();dz.classList.add("drag")});
|
||||
dz.addEventListener("dragleave",()=>dz.classList.remove("drag"));
|
||||
dz.addEventListener("drop",e=>{e.preventDefault();dz.classList.remove("drag"); if(e.dataTransfer.files.length) uploadFiles(e.dataTransfer.files);});
|
||||
// stopPropagation: the global window 'drop' listener also uploads, so
|
||||
// without this a drop landing on the dropzone would bubble up and upload
|
||||
// the same file twice. The dropzone owns drops on itself.
|
||||
dz.addEventListener("drop",e=>{e.preventDefault();e.stopPropagation();dz.classList.remove("drag"); if(e.dataTransfer.files.length) uploadFiles(e.dataTransfer.files);});
|
||||
w.appendChild(dz); c.appendChild(w);
|
||||
}
|
||||
|
||||
@@ -627,21 +651,12 @@ async function uploadFiles(files){
|
||||
item.querySelector(".upload-name").textContent=f.name;
|
||||
list.appendChild(item);
|
||||
const bar=item.querySelector(".upload-bar>div"), pct=item.querySelector(".upload-pct"), st=item.querySelector(".upload-status");
|
||||
const fd=new FormData(); fd.append("file",f);
|
||||
// Large files go through the chunked path so each request stays under
|
||||
// size-capping proxies' body limits (e.g. Cloudflare's 100MB). Files no
|
||||
// bigger than one chunk use the simple single-shot multipart upload.
|
||||
const useChunked = state.chunkSize && f.size>state.chunkSize;
|
||||
try{
|
||||
const t=await new Promise((res,rej)=>{
|
||||
const x=new XMLHttpRequest();
|
||||
x.open("POST","/api/tasks");
|
||||
x.setRequestHeader("X-Api-Key",state.pw);
|
||||
x.upload.onprogress=e=>{ if(e.lengthComputable){ const p=Math.round(e.loaded/e.total*100); bar.style.width=p+"%"; pct.textContent=p+"%"; } };
|
||||
x.onload=()=>{
|
||||
if(x.status>=200&&x.status<300){ try{res(JSON.parse(x.responseText));}catch(e){res(null);} }
|
||||
else if(x.status===401){ state.pw=""; localStorage.removeItem("asr_pw"); openAuthModal("密码不正确或服务已重启,请重新输入。",true); rej(new Error("unauthorized")); }
|
||||
else { let m="HTTP "+x.status; try{m=(JSON.parse(x.responseText).error)||m;}catch(e){} rej(new Error(m)); }
|
||||
};
|
||||
x.onerror=()=>rej(new Error("网络错误,上传失败"));
|
||||
x.send(fd);
|
||||
});
|
||||
const t = useChunked ? await uploadChunked(f,bar,pct,st) : await uploadSingle(f,bar,pct,st);
|
||||
bar.style.width="100%"; pct.textContent="100%";
|
||||
st.textContent="已入队,转写排队中…"; item.classList.add("done");
|
||||
toast("已加入队列: "+(t.original_name||t.id),"ok");
|
||||
@@ -651,6 +666,88 @@ async function uploadFiles(files){
|
||||
// Let the user see the finished state briefly, then refresh the queue.
|
||||
setTimeout(()=>{ state.uploading=false; loadTasks(); }, 600);
|
||||
}
|
||||
|
||||
// uploadSingle sends a whole file as one multipart request (POST /api/tasks)
|
||||
// with XHR so the progress bar reflects bytes sent.
|
||||
function uploadSingle(f,bar,pct,st){
|
||||
return new Promise((res,rej)=>{
|
||||
const fd=new FormData(); fd.append("file",f);
|
||||
const x=new XMLHttpRequest();
|
||||
x.open("POST","/api/tasks");
|
||||
x.setRequestHeader("X-Api-Key",state.pw);
|
||||
x.upload.onprogress=e=>{ if(e.lengthComputable){ const p=Math.round(e.loaded/e.total*100); bar.style.width=p+"%"; pct.textContent=p+"%"; } };
|
||||
x.onload=()=>{
|
||||
if(x.status>=200&&x.status<300){ try{res(JSON.parse(x.responseText));}catch(e){res(null);} }
|
||||
else if(x.status===401){ state.pw=""; localStorage.removeItem("asr_pw"); openAuthModal("密码不正确或服务已重启,请重新输入。",true); rej(new Error("unauthorized")); }
|
||||
else { let m="HTTP "+x.status; try{m=(JSON.parse(x.responseText).error)||m;}catch(e){} rej(new Error(m)); }
|
||||
};
|
||||
x.onerror=()=>rej(new Error("网络错误,上传失败"));
|
||||
x.send(fd);
|
||||
});
|
||||
}
|
||||
|
||||
// uploadChunked sends a file as ceil(size/chunkSize) slices to the resumable
|
||||
// upload API (POST /api/uploads -> /api/uploads/{id}?index=N -> /complete),
|
||||
// tracking overall progress across slices. A failing slice is retried a few
|
||||
// times on transient errors; on give-up the partial session is aborted.
|
||||
async function uploadChunked(f,bar,pct,st){
|
||||
let uid=null; // declared outside try so the catch can reference it safely
|
||||
try{
|
||||
const init = await api("/api/uploads", {
|
||||
method:"POST",
|
||||
headers:{"Content-Type":"application/json"},
|
||||
body: JSON.stringify({filename:f.name, size:f.size})
|
||||
});
|
||||
uid=init.upload_id;
|
||||
const cs=+init.chunk_size, total=+init.total_chunks;
|
||||
const setPct=bytes=>{ const p=Math.min(100,Math.round(bytes/f.size*100)); bar.style.width=p+"%"; pct.textContent=p+"%"; };
|
||||
let uploaded=0;
|
||||
for(let i=0;i<total;i++){
|
||||
const start=i*cs, end=Math.min(start+cs,f.size);
|
||||
const blob=f.slice(start,end);
|
||||
let done=false, lastErr;
|
||||
for(let attempt=0; attempt<3 && !done; attempt++){
|
||||
try{
|
||||
await new Promise((res,rej)=>{
|
||||
const x=new XMLHttpRequest();
|
||||
x.open("POST", `/api/uploads/${uid}?index=${i}`);
|
||||
x.setRequestHeader("X-Api-Key",state.pw);
|
||||
x.upload.onprogress=e=>{ if(e.lengthComputable) setPct(uploaded+e.loaded); };
|
||||
x.onload=()=>{
|
||||
if(x.status>=200&&x.status<300){ try{res(JSON.parse(x.responseText));}catch(e){res(null);} return; }
|
||||
const m=(()=>{ let s="HTTP "+x.status; try{s=(JSON.parse(x.responseText).error)||s;}catch(e){} return s; })();
|
||||
const e=new Error(m); e.status=x.status; rej(e);
|
||||
};
|
||||
x.onerror=()=>{ const e=new Error("网络错误"); e.status=0; rej(e); };
|
||||
x.send(blob);
|
||||
});
|
||||
done=true;
|
||||
}catch(e){
|
||||
lastErr=e;
|
||||
// Auth failure: surface immediately (the modal reopens); don't retry.
|
||||
if(e.status===401){ state.pw=""; localStorage.removeItem("asr_pw"); openAuthModal("密码不正确或服务已重启,请重新输入。",true); throw e; }
|
||||
// 4xx (bad request / session gone): retrying won't help; give up.
|
||||
if(e.status>=400 && e.status<500 && e.status!==0) throw e;
|
||||
// Transient (network/5xx): back off and retry unless last attempt.
|
||||
if(attempt<2){ await new Promise(r=>setTimeout(r, 800*(attempt+1))); }
|
||||
}
|
||||
}
|
||||
if(!done) throw lastErr || new Error("上传分片失败");
|
||||
uploaded += (end-start);
|
||||
setPct(uploaded);
|
||||
}
|
||||
const task = await api(`/api/uploads/${uid}/complete`, {method:"POST"});
|
||||
return task;
|
||||
}catch(e){
|
||||
// Best-effort cleanup of the partial session — but only if one was
|
||||
// actually created (init may have failed) and not on an auth failure
|
||||
// (the password is already cleared and a retry would just 401 again).
|
||||
if(uid && e.status!==401 && state.pw){
|
||||
try{ await fetch(`/api/uploads/${uid}`,{method:"DELETE",headers:{"X-Api-Key":state.pw}}); }catch(_){}
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
$("#file").addEventListener("change",e=>{ if(e.target.files.length) uploadFiles(e.target.files); e.target.value=""; });
|
||||
$("#uploadBtn").addEventListener("click",()=>$("#file").click());
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"git.misaka.ren/m1saka/record-asr/internal/config"
|
||||
"git.misaka.ren/m1saka/record-asr/internal/store"
|
||||
)
|
||||
|
||||
// posReader emits a deterministic, position-dependent byte stream so that no
|
||||
// two slices of the generated file are identical — a reorder would then
|
||||
// change the reassembled hash and be caught.
|
||||
type posReader struct{ pos int64 }
|
||||
|
||||
func (r *posReader) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
x := r.pos + int64(i)
|
||||
p[i] = byte((x*127 ^ (x >> 7)) & 0xff)
|
||||
}
|
||||
r.pos += int64(len(p))
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// sha256sum returns the hex SHA-256 of a file, streaming so 120MB never sits
|
||||
// in memory at once.
|
||||
func sha256sum(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
t.Fatalf("hash %s: %v", path, err)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))
|
||||
}
|
||||
|
||||
// TestChunkedUploadLargeFile uploads a 120MB file — larger than the 100MB CDN
|
||||
// body cap the feature exists to bypass — as 25MB slices and checks that the
|
||||
// reassembled file is byte-identical to the original. Guarded so the default
|
||||
// test suite stays fast; run with:
|
||||
//
|
||||
// ASR_TEST_LARGE=1 go test -run Large ./internal/server/...
|
||||
func TestChunkedUploadLargeFile(t *testing.T) {
|
||||
if os.Getenv("ASR_TEST_LARGE") == "" {
|
||||
t.Skip("set ASR_TEST_LARGE=1 to run the large-file chunked upload test (~120MB on disk)")
|
||||
}
|
||||
const total = 120 << 20 // 120 MiB — over the 100MB cap, no per-request limit here
|
||||
const chunk = 25 << 20 // 25 MiB slices → 5 chunks (last is 20 MiB, partial)
|
||||
|
||||
dir := t.TempDir()
|
||||
st, err := store.Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("store.Open: %v", err)
|
||||
}
|
||||
cfg := config.Config{DataDir: dir, ChunkSize: chunk, MaxUploadBytes: 512 << 20, MaxRetries: 3}
|
||||
srv := New(cfg, st, nil, nil) // upload path never touches the transcribe/model stack
|
||||
ts := httptest.NewServer(srv.Router())
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
// 1. Generate the 120MB original on disk, hashing as we go.
|
||||
origPath := filepath.Join(dir, "original.wav")
|
||||
of, err := os.Create(origPath)
|
||||
if err != nil {
|
||||
t.Fatalf("create orig: %v", err)
|
||||
}
|
||||
if _, err := io.Copy(of, io.LimitReader(&posReader{}, total)); err != nil {
|
||||
t.Fatalf("write orig: %v", err)
|
||||
}
|
||||
of.Close()
|
||||
wantHash := sha256sum(t, origPath)
|
||||
|
||||
// 2. Open a chunked-upload session.
|
||||
body, _ := json.Marshal(map[string]any{"filename": "large.wav", "size": total})
|
||||
resp, err := http.Post(ts.URL+"/api/uploads", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
var init struct {
|
||||
UploadID string `json:"upload_id"`
|
||||
ChunkSize int64 `json:"chunk_size"`
|
||||
TotalChunks int `json:"total_chunks"`
|
||||
}
|
||||
json.NewDecoder(resp.Body).Decode(&init)
|
||||
resp.Body.Close()
|
||||
if init.UploadID == "" || init.ChunkSize != chunk || init.TotalChunks != 5 {
|
||||
t.Fatalf("init = %+v", init)
|
||||
}
|
||||
|
||||
// 3. Stream each slice straight from the original file (no full copy in RAM).
|
||||
src, err := os.Open(origPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open src: %v", err)
|
||||
}
|
||||
defer src.Close()
|
||||
for i := 0; i < init.TotalChunks; i++ {
|
||||
offset := int64(i) * chunk
|
||||
size := int64(chunk)
|
||||
if offset+size > total {
|
||||
size = total - offset
|
||||
}
|
||||
req, _ := http.NewRequest("POST",
|
||||
ts.URL+"/api/uploads/"+init.UploadID+"?index="+strconv.Itoa(i),
|
||||
io.NewSectionReader(src, offset, size))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
cresp, cerr := http.DefaultClient.Do(req)
|
||||
if cerr != nil {
|
||||
t.Fatalf("chunk %d: %v", i, cerr)
|
||||
}
|
||||
if cresp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("chunk %d status = %d", i, cresp.StatusCode)
|
||||
}
|
||||
cresp.Body.Close()
|
||||
}
|
||||
|
||||
// 4. Complete → 202 + pending task.
|
||||
req, _ := http.NewRequest("POST", ts.URL+"/api/uploads/"+init.UploadID+"/complete", nil)
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("complete: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("complete status = %d", resp.StatusCode)
|
||||
}
|
||||
var task store.Task
|
||||
json.NewDecoder(resp.Body).Decode(&task)
|
||||
if task.Status != store.StatusPending {
|
||||
t.Fatalf("task status = %q, want %q", task.Status, store.StatusPending)
|
||||
}
|
||||
|
||||
// 5. The reassembled file must be byte-identical (same SHA-256) and exact size.
|
||||
gotHash := sha256sum(t, st.AudioPath(&task))
|
||||
if gotHash != wantHash {
|
||||
t.Fatalf("reassembled file hash mismatch:\n want %s\n got %s", wantHash, gotHash)
|
||||
}
|
||||
fi, err := os.Stat(st.AudioPath(&task))
|
||||
if err != nil || fi.Size() != total {
|
||||
t.Fatalf("final size = %d (%v), want %d", fi.Size(), err, total)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"git.misaka.ren/m1saka/record-asr/internal/config"
|
||||
"git.misaka.ren/m1saka/record-asr/internal/store"
|
||||
)
|
||||
|
||||
// newUploadTestServer wires a Server backed by a real store + a minimal config
|
||||
// (auth disabled, tiny 100B chunks) with nil pipeline/manager — the upload
|
||||
// handlers never touch the transcribe/model stack, so this is safe.
|
||||
func newUploadTestServer(t *testing.T) (*store.Store, *httptest.Server) {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
st, err := store.Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("store.Open: %v", err)
|
||||
}
|
||||
cfg := config.Config{
|
||||
DataDir: dir,
|
||||
ChunkSize: 100,
|
||||
MaxUploadBytes: 1 << 20,
|
||||
MaxRetries: 3,
|
||||
}
|
||||
srv := New(cfg, st, nil, nil)
|
||||
ts := httptest.NewServer(srv.Router())
|
||||
t.Cleanup(ts.Close)
|
||||
return st, ts
|
||||
}
|
||||
|
||||
// initSession opens a chunked-upload session and returns the upload id.
|
||||
func initSession(t *testing.T, ts *httptest.Server, name string, size int64) string {
|
||||
t.Helper()
|
||||
body, _ := json.Marshal(map[string]any{"filename": name, "size": size})
|
||||
resp, err := http.Post(ts.URL+"/api/uploads", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("init: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("init status = %d", resp.StatusCode)
|
||||
}
|
||||
var r struct {
|
||||
UploadID string `json:"upload_id"`
|
||||
ChunkSize int64 `json:"chunk_size"`
|
||||
TotalChunks int `json:"total_chunks"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
|
||||
t.Fatalf("init decode: %v", err)
|
||||
}
|
||||
if r.UploadID == "" || r.ChunkSize != 100 {
|
||||
t.Fatalf("init resp = %+v", r)
|
||||
}
|
||||
return r.UploadID
|
||||
}
|
||||
|
||||
func sendChunk(t *testing.T, ts, id string, idx int, data []byte) int {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest("POST", ts+"/api/uploads/"+id+"?index="+strconv.Itoa(idx), bytes.NewReader(data))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("chunk %d: %v", idx, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("chunk %d status = %d", idx, resp.StatusCode)
|
||||
}
|
||||
var r struct {
|
||||
Received int `json:"received"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
json.NewDecoder(resp.Body).Decode(&r)
|
||||
return r.Received
|
||||
}
|
||||
|
||||
// TestChunkedUploadHTTP exercises the full endpoint flow end-to-end.
|
||||
func TestChunkedUploadHTTP(t *testing.T) {
|
||||
st, ts := newUploadTestServer(t)
|
||||
|
||||
// /healthz advertises the chunk size the client uses as its threshold.
|
||||
hresp, _ := http.Get(ts.URL + "/healthz")
|
||||
var hz struct {
|
||||
ChunkSize int64 `json:"chunk_size"`
|
||||
MaxUploadBytes int64 `json:"max_upload_bytes"`
|
||||
}
|
||||
json.NewDecoder(hresp.Body).Decode(&hz)
|
||||
hresp.Body.Close()
|
||||
if hz.ChunkSize != 100 || hz.MaxUploadBytes != 1<<20 {
|
||||
t.Fatalf("healthz = %+v", hz)
|
||||
}
|
||||
|
||||
// 250B file -> 3 chunks (100, 100, 50). Send out of order.
|
||||
uid := initSession(t, ts, "clip.wav", 250)
|
||||
c0 := bytes.Repeat([]byte{0xAA}, 100)
|
||||
c1 := bytes.Repeat([]byte{0xBB}, 100)
|
||||
c2 := bytes.Repeat([]byte{0xCC}, 50)
|
||||
want := append(append([]byte{}, c0...), c1...)
|
||||
want = append(want, c2...)
|
||||
if got := sendChunk(t, ts.URL, uid, 1, c1); got != 1 {
|
||||
t.Fatalf("after chunk 1 received=%d want 1", got)
|
||||
}
|
||||
if got := sendChunk(t, ts.URL, uid, 0, c0); got != 2 {
|
||||
t.Fatalf("after chunk 0 received=%d want 2", got)
|
||||
}
|
||||
if got := sendChunk(t, ts.URL, uid, 2, c2); got != 3 {
|
||||
t.Fatalf("after chunk 2 received=%d want 3", got)
|
||||
}
|
||||
|
||||
// Complete -> 202 + task (same shape as /api/tasks).
|
||||
req, _ := http.NewRequest("POST", ts.URL+"/api/uploads/"+uid+"/complete", nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("complete: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusAccepted {
|
||||
t.Fatalf("complete status = %d", resp.StatusCode)
|
||||
}
|
||||
var task store.Task
|
||||
json.NewDecoder(resp.Body).Decode(&task)
|
||||
if task.Status != store.StatusPending || task.OriginalName != "clip.wav" {
|
||||
t.Fatalf("task = %+v", task)
|
||||
}
|
||||
|
||||
// The reassembled file matches the original bytes exactly.
|
||||
got, err := os.ReadFile(st.AudioPath(&task))
|
||||
if err != nil {
|
||||
t.Fatalf("read final file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("reassembled bytes mismatch:\n got %x\n want %x", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunkedUploadHTTPErrors checks the error paths: missing index, unknown
|
||||
// session, oversized chunk, and idempotent abort.
|
||||
func TestChunkedUploadHTTPErrors(t *testing.T) {
|
||||
_, ts := newUploadTestServer(t)
|
||||
|
||||
// Missing ?index -> 400.
|
||||
resp, _ := http.Post(ts.URL+"/api/uploads/someid", "application/octet-stream", bytes.NewReader([]byte("x")))
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("missing-index status = %d, want 400", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Complete on an unknown session -> 404.
|
||||
req, _ := http.NewRequest("POST", ts.URL+"/api/uploads/doesnotexist/complete", nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("unknown complete: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("unknown complete status = %d, want 404", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Oversized chunk (101B > 100B cap) -> 400.
|
||||
uid := initSession(t, ts, "big.wav", 250)
|
||||
req, _ = http.NewRequest("POST", ts.URL+"/api/uploads/"+uid+"?index=0", bytes.NewReader(bytes.Repeat([]byte{0x9}, 101)))
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("oversized chunk: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("oversized chunk status = %d, want 400", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Abort a fresh session -> 204, and the id is gone afterwards.
|
||||
uid2 := initSession(t, ts, "abort.wav", 100)
|
||||
req, _ = http.NewRequest("DELETE", ts.URL+"/api/uploads/"+uid2, nil)
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("abort: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("abort status = %d, want 204", resp.StatusCode)
|
||||
}
|
||||
// Completing an aborted session -> 404.
|
||||
req, _ = http.NewRequest("POST", ts.URL+"/api/uploads/"+uid2+"/complete", nil)
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("post-abort complete: %v", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("post-abort complete status = %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user