add full source: transcribe pipeline, HTTP API, OpenAI endpoint, web UI, Docker

2026-09-09 16:04:10 +08:00
parent 757861f1dd
commit 7010da8855
19 changed files with 2918 additions and 0 deletions
@@ -0,0 +1,190 @@
+// Package transcribe ties the silero VAD and the SenseVoice recognizer together
+// into a single "audio file -> transcript" pipeline.
+//
+// The VAD is stateful, so segmentation of one file cannot interleave with
+// another; the manager guards VAD access with a mutex. Recognition of the
+// resulting speech segments is stateless and could be parallelized.
+package transcribe
+
+import (
+	"fmt"
+	"strings"
+
+	sherpa "github.com/k2-fsa/sherpa-onnx-go/sherpa_onnx"
+
+	"git.misaka.ren/m1saka/record-asr/internal/audio"
+	"git.misaka.ren/m1saka/record-asr/internal/models"
+)
+
+// Segment is one VAD-detected speech region with its transcript.
+type Segment struct {
+	Text  string  `json:"text"`
+	Start float64 `json:"start"` // seconds from file start
+	End   float64 `json:"end"`   // seconds from file start
+	Lang  string  `json:"lang"`
+}
+
+// Result is the full transcript for one audio file.
+type Result struct {
+	Text     string    `json:"text"`      // all segments joined by newline
+	Lang     string    `json:"lang"`      // dominant detected language
+	Emotion  string    `json:"emotion"`   // dominant detected emotion
+	Duration float64   `json:"duration"`  // audio duration in seconds
+	Segments []Segment `json:"segments"`
+}
+
+// New returns a Pipeline bound to the given model manager.
+func New(m *models.Manager) *Pipeline {
+	return &Pipeline{m: m}
+}
+
+// Pipeline wraps a model manager to provide file transcription.
+type Pipeline struct {
+	m *models.Manager
+}
+
+// Transcribe decodes the audio file at path, runs VAD to isolate speech, and
+// recognizes each segment with SenseVoice.
+func (p *Pipeline) Transcribe(path string) (*Result, error) {
+	samples, sampleRate, err := audio.Decode(path)
+	if err != nil {
+		return nil, err
+	}
+	if sampleRate != audio.TargetSampleRate {
+		return nil, fmt.Errorf("internal: expected 16kHz samples, got %d", sampleRate)
+	}
+	duration := float64(len(samples)) / float64(audio.TargetSampleRate)
+
+	segments, err := p.segment(samples)
+	if err != nil {
+		return nil, err
+	}
+
+	res := &Result{
+		Duration: duration,
+		Segments: make([]Segment, 0, len(segments)),
+	}
+	var texts []string
+	for _, seg := range segments {
+		// Recognize, splitting extremely long segments (should not happen
+		// given the VAD MaxSpeechDuration cap, but guards against pathological
+		// inputs that exceed SenseVoice's ~30s comfort window).
+		parts := splitForRecognize(seg.Samples, 28*audio.TargetSampleRate)
+		var segTexts []string
+		var segLang, segEmotion string
+		segStart := float64(seg.Start) / float64(audio.TargetSampleRate)
+		for _, part := range parts {
+			txt, lang, emo, perr := p.recognize(part)
+			if perr != nil {
+				return nil, fmt.Errorf("recognize segment at %.2fs: %w", segStart, perr)
+			}
+			if txt != "" {
+				segTexts = append(segTexts, txt)
+			}
+			if segLang == "" {
+				segLang, segEmotion = lang, emo
+			}
+		}
+		segText := strings.Join(segTexts, " ")
+		segEnd := segStart + float64(len(seg.Samples))/float64(audio.TargetSampleRate)
+		if segText == "" {
+			segText = " "
+		}
+		res.Segments = append(res.Segments, Segment{
+			Text:  segText,
+			Start: segStart,
+			End:   segEnd,
+			Lang:  segLang,
+		})
+		texts = append(texts, segText)
+		if res.Lang == "" {
+			res.Lang, res.Emotion = segLang, segEmotion
+		}
+	}
+	res.Text = strings.Join(texts, "\n")
+	return res, nil
+}
+
+// segment runs the silero VAD over the full sample buffer and returns the
+// completed speech segments. The VAD is stateful and shared, so this is the
+// only place that touches it, under the manager's lock, and it resets the
+// detector before use so no state leaks between files.
+func (p *Pipeline) segment(samples []float32) ([]sherpa.SpeechSegment, error) {
+	if p.m.VAD == nil {
+		return nil, fmt.Errorf("VAD not loaded")
+	}
+	p.m.VadMu().Lock()
+	defer p.m.VadMu().Unlock()
+
+	vad := p.m.VAD
+	vad.Reset()
+
+	// Feed in window-sized chunks; drain completed segments after each feed.
+	w := 512 // silero window at 16kHz
+	if len(samples) > 0 && w > len(samples) {
+		w = len(samples)
+	}
+	var out []sherpa.SpeechSegment
+	drain := func() {
+		for !vad.IsEmpty() {
+			seg := vad.Front()
+			if seg == nil {
+				break
+			}
+			// Copy because Front()'s backing memory is freed on Pop().
+			cp := sherpa.SpeechSegment{Start: seg.Start, Samples: append([]float32(nil), seg.Samples...)}
+			out = append(out, cp)
+			vad.Pop()
+		}
+	}
+	for i := 0; i < len(samples); i += w {
+		end := i + w
+		if end > len(samples) {
+			end = len(samples)
+		}
+		vad.AcceptWaveform(samples[i:end])
+		drain()
+	}
+	vad.Flush()
+	drain()
+	return out, nil
+}
+
+// recognize runs SenseVoice over a single contiguous speech chunk.
+func (p *Pipeline) recognize(samples []float32) (text, lang, emotion string, err error) {
+	if len(samples) == 0 {
+		return "", "", "", nil
+	}
+	if p.m.Recognizer == nil {
+		return "", "", "", fmt.Errorf("recognizer not loaded")
+	}
+	stream := sherpa.NewOfflineStream(p.m.Recognizer)
+	if stream == nil {
+		return "", "", "", fmt.Errorf("create offline stream failed")
+	}
+	defer sherpa.DeleteOfflineStream(stream)
+	stream.AcceptWaveform(audio.TargetSampleRate, samples)
+	p.m.Recognizer.Decode(stream)
+	r := stream.GetResult()
+	if r == nil {
+		return "", "", "", nil
+	}
+	return r.Text, r.Lang, r.Emotion, nil
+}
+
+// splitForRecognize breaks a sample slice into chunks no longer than maxLen to
+// stay within the recognizer's comfortable input window.
+func splitForRecognize(samples []float32, maxLen int) [][]float32 {
+	if maxLen <= 0 || len(samples) <= maxLen {
+		return [][]float32{samples}
+	}
+	var parts [][]float32
+	for i := 0; i < len(samples); i += maxLen {
+		end := i + maxLen
+		if end > len(samples) {
+			end = len(samples)
+		}
+		parts = append(parts, samples[i:end])
+	}
+	return parts
+}