m1saka/record-asr
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add chunked upload to bypass 100MB proxy body limit

Browse Source 
Resumable /api/uploads flow (init -> append -> complete -> abort) so a
large recording uploads as many small slices (<= ASR_CHUNK_SIZE_MB, default
50MB) instead of one request a size-capping proxy (Cloudflare's 100MB) would
reject. The web console auto-routes files larger than one slice through the
chunked path with cross-slice progress and transient-error retry; smaller
files keep the single-shot upload.

- store: in-memory upload sessions + .part reassembly (offset writes,
  out-of-order/idempotent), reuse upload id as task id, orphan sweep on
  start, idle-session GC
- fix limitedReader rejecting bodies exactly at the cap (broke full chunks)
- fix drag-drop double-firing when a drop lands on the dropzone (its drop
  handler now stopPropagation so the window listener doesn't re-upload)
- tests: store + HTTP flow, plus a 120MB reassembly test (ASR_TEST_LARGE)
This commit is contained in:
root
2026-09-10 15:12:15 +08:00
parent ab6bf8b370
commit e5a2dc0a8d
11 changed files with 1144 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/config/config.go
+16 -2
View File
@@ -36,6 +36,11 @@ type Config struct {
MaxRetries int
// ASR_MAX_UPLOAD_MB: per-upload size cap (default 512).
MaxUploadBytes int64
// ASR_CHUNK_SIZE_MB: size of each slice in a resumable chunked upload,
// kept well under size-capping proxies' body limits so a large file can
// be uploaded as many small requests instead of one that a CDN (e.g.
// Cloudflare's 100MB cap) would reject. Default 50.
ChunkSize int64
// ASR_WORKERS: concurrent transcription workers (default 1).
Workers int
// ASR_MAX_CONCURRENT: cap on simultaneously in-flight synchronous
@@ -73,6 +78,7 @@ func Load() (Config, error) {
UseITN: envBool("ASR_USE_ITN", true),
MaxRetries: envInt("ASR_MAX_RETRIES", 3),
MaxUploadBytes: int64(envInt("ASR_MAX_UPLOAD_MB", 512)) * 1024 * 1024,
ChunkSize: int64(envInt("ASR_CHUNK_SIZE_MB", 50)) * 1024 * 1024,
Workers: envInt("ASR_WORKERS", 1),
MaxConcurrent: envInt("ASR_MAX_CONCURRENT", 0),
VadThreshold: float32(envFloat("ASR_VAD_THRESHOLD", 0.5)),
@@ -111,6 +117,14 @@ func Load() (Config, error) {
if c.MaxUploadBytes < 1<<20 {
c.MaxUploadBytes = 512 << 20
}
// Chunk size must be > 1MB to be useful and < 100MB so each slice clears
// common CDN body caps (e.g. Cloudflare's 100MB) with room to spare.
if c.ChunkSize < 1<<20 {
c.ChunkSize = 50 << 20
}
if c.ChunkSize > 100<<20 {
c.ChunkSize = 100 << 20
}
if c.DownloadTimeout < time.Minute {
c.DownloadTimeout = 30 * time.Minute
}
@@ -130,9 +144,9 @@ func Load() (Config, error) {
func (c Config) AuthEnabled() bool { return c.Password != "" }
func (c Config) String() string {
return fmt.Sprintf("addr=%s data=%s models=%s threads=%d lang=%s itn=%t workers=%d maxconcurrent=%d retries=%d vad{thr=%.2f max=%.0fs win=%d}",
return fmt.Sprintf("addr=%s data=%s models=%s threads=%d lang=%s itn=%t workers=%d maxconcurrent=%d retries=%d chunk=%dB vad{thr=%.2f max=%.0fs win=%d}",
c.Addr, c.DataDir, c.ModelDir, c.NumThreads, c.Language, c.UseITN, c.Workers,
c.MaxConcurrent, c.MaxRetries, c.VadThreshold, c.VadMaxSpeechDuration, c.VadWindowSize)
c.MaxConcurrent, c.MaxRetries, c.ChunkSize, c.VadThreshold, c.VadMaxSpeechDuration, c.VadWindowSize)
}
func defaultThreads() int {