add chunked upload to bypass 100MB proxy body limit
Resumable /api/uploads flow (init -> append -> complete -> abort) so a large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default 50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would reject. The web console auto-routes files larger than one slice through the chunked path with cross-slice progress and transient-error retry; smaller files keep the single-shot upload. - store: in-memory upload sessions + .part reassembly (offset writes, out-of-order/idempotent), reuse upload id as task id, orphan sweep on start, idle-session GC - fix limitedReader rejecting bodies exactly at the cap (broke full chunks) - fix drag-drop double-firing when a drop lands on the dropzone (its drop handler now stopPropagation so the window listener doesn't re-upload) - tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
root
@@ -0,0 +1,203 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// chunkBytes returns a deterministic slice of n bytes whose value is b.
|
||||
func chunkBytes(n int, b byte) []byte {
|
||||
out := make([]byte, n)
|
||||
for i := range out {
|
||||
out[i] = b
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// newTestStore opens a store in a fresh temp dir.
|
||||
func newTestStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
s, err := Open(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// TestChunkedUploadReassemble verifies that out-of-order, idempotent chunks
|
||||
// reassemble into the exact original bytes and enqueue a pending task.
|
||||
func TestChunkedUploadReassemble(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
const chunk = 10
|
||||
const total = 25 // 3 chunks: 10 + 10 + 5
|
||||
|
||||
sess, err := s.CreateUpload("note.wav", total, chunk, 512)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateUpload: %v", err)
|
||||
}
|
||||
if sess.TotalChunks != 3 {
|
||||
t.Fatalf("TotalChunks = %d, want 3", sess.TotalChunks)
|
||||
}
|
||||
|
||||
// Expected file content: three slices with distinct byte values.
|
||||
c0 := chunkBytes(10, 0xAA)
|
||||
c1 := chunkBytes(10, 0xBB)
|
||||
c2 := chunkBytes(5, 0xCC)
|
||||
want := append(append([]byte{}, c0...), c1...)
|
||||
want = append(want, c2...)
|
||||
|
||||
// Send out of order: 1, 0, 2.
|
||||
for _, step := range []struct {
|
||||
idx int
|
||||
data []byte
|
||||
}{
|
||||
{1, c1}, {0, c0}, {2, c2},
|
||||
} {
|
||||
if err := s.AppendChunk(sess.UploadID, step.idx, bytes.NewReader(step.data), chunk); err != nil {
|
||||
t.Fatalf("AppendChunk(%d): %v", step.idx, err)
|
||||
}
|
||||
got, total, ok := s.UploadStatus(sess.UploadID)
|
||||
if !ok {
|
||||
t.Fatalf("UploadStatus: session gone")
|
||||
}
|
||||
_ = total
|
||||
// After the last append all 3 should be received.
|
||||
if step.idx == 2 && got != 3 {
|
||||
t.Fatalf("after last chunk received=%d want 3", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Re-send chunk 0 (idempotent): content must be unchanged.
|
||||
if err := s.AppendChunk(sess.UploadID, 0, bytes.NewReader(c0), chunk); err != nil {
|
||||
t.Fatalf("AppendChunk(0) re-send: %v", err)
|
||||
}
|
||||
|
||||
task, err := s.CompleteUpload(sess.UploadID, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("CompleteUpload: %v", err)
|
||||
}
|
||||
if task.Status != StatusPending {
|
||||
t.Fatalf("task status = %q, want %q", task.Status, StatusPending)
|
||||
}
|
||||
if task.OriginalName != "note.wav" {
|
||||
t.Fatalf("original_name = %q", task.OriginalName)
|
||||
}
|
||||
|
||||
// The part file is gone; the final audio file holds the exact bytes.
|
||||
if _, err := os.Stat(filepath.Join(s.uploadsDir, sess.UploadID+".part")); !os.IsNotExist(err) {
|
||||
t.Fatalf("part file still present: %v", err)
|
||||
}
|
||||
got, err := os.ReadFile(filepath.Join(s.dir, task.Filename))
|
||||
if err != nil {
|
||||
t.Fatalf("read final file: %v", err)
|
||||
}
|
||||
if !bytes.Equal(got, want) {
|
||||
t.Fatalf("final bytes mismatch:\n got %x\n want %x", got, want)
|
||||
}
|
||||
if int64(len(got)) != total {
|
||||
t.Fatalf("final size = %d, want %d", len(got), total)
|
||||
}
|
||||
|
||||
// The session became a task the store knows about.
|
||||
if s.Get(task.ID) == nil {
|
||||
t.Fatalf("Get(task) returned nil after complete")
|
||||
}
|
||||
// ...and is no longer an upload session.
|
||||
if _, _, ok := s.UploadStatus(sess.UploadID); ok {
|
||||
t.Fatalf("session still present after complete")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunkedUploadMissingChunk ensures complete fails when a chunk is absent.
|
||||
func TestChunkedUploadMissingChunk(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
sess, _ := s.CreateUpload("x.wav", 20, 10, 512) // 2 chunks
|
||||
_ = s.AppendChunk(sess.UploadID, 0, bytes.NewReader(chunkBytes(10, 1)), 10)
|
||||
if _, err := s.CompleteUpload(sess.UploadID, 3); err == nil {
|
||||
t.Fatalf("CompleteUpload succeeded with a missing chunk")
|
||||
}
|
||||
// The session survives a failed complete so the client can send the rest.
|
||||
if _, _, ok := s.UploadStatus(sess.UploadID); !ok {
|
||||
t.Fatalf("session dropped after failed complete")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunkedUploadLengthMismatch rejects a chunk whose size is wrong.
|
||||
func TestChunkedUploadLengthMismatch(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
sess, _ := s.CreateUpload("x.wav", 20, 10, 512) // 2 chunks of 10
|
||||
// Send only 7 bytes for a chunk that should be 10.
|
||||
if err := s.AppendChunk(sess.UploadID, 0, bytes.NewReader(chunkBytes(7, 1)), 10); err == nil {
|
||||
t.Fatalf("AppendChunk accepted a short chunk")
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunkedUploadAbort removes the part file and drops the session.
|
||||
func TestChunkedUploadAbort(t *testing.T) {
|
||||
s := newTestStore(t)
|
||||
sess, _ := s.CreateUpload("x.wav", 10, 10, 512)
|
||||
_ = s.AppendChunk(sess.UploadID, 0, bytes.NewReader(chunkBytes(10, 1)), 10)
|
||||
part := filepath.Join(s.uploadsDir, sess.UploadID+".part")
|
||||
if _, err := os.Stat(part); err != nil {
|
||||
t.Fatalf("part file missing before abort: %v", err)
|
||||
}
|
||||
if err := s.AbortUpload(sess.UploadID); err != nil {
|
||||
t.Fatalf("AbortUpload: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(part); !os.IsNotExist(err) {
|
||||
t.Fatalf("part file present after abort: %v", err)
|
||||
}
|
||||
if _, _, ok := s.UploadStatus(sess.UploadID); ok {
|
||||
t.Fatalf("session still present after abort")
|
||||
}
|
||||
// Abort of an unknown session is idempotent, not an error.
|
||||
if err := s.AbortUpload("nope"); err != nil {
|
||||
t.Fatalf("AbortUpload(unknown) = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSweepPartsOnOpen confirms a leftover .part file from a previous run is
|
||||
// removed when the store reopens (in-memory sessions don't survive restarts).
|
||||
func TestSweepPartsOnOpen(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s1, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open: %v", err)
|
||||
}
|
||||
sess, _ := s1.CreateUpload("x.wav", 10, 10, 512)
|
||||
part := filepath.Join(s1.uploadsDir, sess.UploadID+".part")
|
||||
if _, err := os.Stat(part); err != nil {
|
||||
t.Fatalf("part missing: %v", err)
|
||||
}
|
||||
s1.gcUploads() // no-op: session not idle yet; part must survive
|
||||
if _, err := os.Stat(part); err != nil {
|
||||
t.Fatalf("part swept too early: %v", err)
|
||||
}
|
||||
|
||||
// Reopen the same dir: the orphan .part file must be swept.
|
||||
s2, err := Open(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("Open(2): %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(s2.uploadsDir, sess.UploadID+".part")); !os.IsNotExist(err) {
|
||||
t.Fatalf("orphan .part file survived reopen: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSweepPartsPreservesTaskAudio ensures the startup sweep only touches
|
||||
// .part files, never finalized task audio (uploads/<id>.<ext>).
|
||||
func TestSweepPartsPreservesTaskAudio(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
s1, _ := Open(dir)
|
||||
task, err := s1.Create("kept.wav", bytes.NewReader(chunkBytes(10, 7)), 512, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("Create: %v", err)
|
||||
}
|
||||
audioPath := filepath.Join(s1.dir, task.Filename)
|
||||
s1.sweepParts() // must not delete the task's audio
|
||||
if _, err := os.Stat(audioPath); err != nil {
|
||||
t.Fatalf("task audio swept by sweepParts: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user