test: prove logger shutdown deadline behavior
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -43,6 +44,26 @@ type fakePool struct {
|
||||
scripts []batchScript
|
||||
}
|
||||
|
||||
type queueBackend struct {
|
||||
prepare func(context.Context, string) (logger.Batch, error)
|
||||
healthy atomic.Bool
|
||||
calls atomic.Int32
|
||||
}
|
||||
|
||||
func newQueueBackend(prepare func(context.Context, string) (logger.Batch, error)) *queueBackend {
|
||||
b := &queueBackend{prepare: prepare}
|
||||
b.healthy.Store(true)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *queueBackend) PrepareBatch(ctx context.Context, query string) (logger.Batch, error) {
|
||||
b.calls.Add(1)
|
||||
return b.prepare(ctx, query)
|
||||
}
|
||||
|
||||
func (b *queueBackend) Healthy() bool { return b.healthy.Load() }
|
||||
func (b *queueBackend) MarkUnhealthy() { b.healthy.Store(false) }
|
||||
|
||||
func (p *fakePool) PrepareBatch(_ context.Context, query string) (logger.Batch, error) {
|
||||
idx := p.calls
|
||||
p.calls++
|
||||
@@ -360,6 +381,117 @@ func TestStopIsIndependentFromStartContext(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueStopDeadlineCancelsBlockedPrepare(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
backend := newQueueBackend(func(ctx context.Context, _ string) (logger.Batch, error) {
|
||||
close(started)
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
})
|
||||
q := logger.NewQueueWithBackend(backend, 1, 1, 1, time.Hour)
|
||||
q.Start(context.Background())
|
||||
q.Submit(&logger.LogEntry{RequestID: "blocked-prepare"})
|
||||
<-started
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond)
|
||||
defer cancel()
|
||||
startedAt := time.Now()
|
||||
if err := q.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Stop error=%v want deadline exceeded", err)
|
||||
}
|
||||
if elapsed := time.Since(startedAt); elapsed > 250*time.Millisecond {
|
||||
t.Fatalf("Stop exceeded total deadline: %v", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueStopDeadlineCancelsBlockedSend(t *testing.T) {
|
||||
started := make(chan struct{})
|
||||
backend := newQueueBackend(func(ctx context.Context, _ string) (logger.Batch, error) {
|
||||
return &blockingSendBatch{ctx: ctx, started: started}, nil
|
||||
})
|
||||
q := logger.NewQueueWithBackend(backend, 1, 1, 1, time.Hour)
|
||||
q.Start(context.Background())
|
||||
q.Submit(&logger.LogEntry{RequestID: "blocked-send"})
|
||||
<-started
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := q.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Stop error=%v want deadline exceeded", err)
|
||||
}
|
||||
deadline := time.Now().Add(250 * time.Millisecond)
|
||||
for q.Stats().Bytes != 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
if stats := q.Stats(); stats.Bytes != 0 || stats.Ambiguous != 1 || stats.Failed != 1 {
|
||||
t.Fatalf("stats after canceled Send=%+v", stats)
|
||||
}
|
||||
if backend.Healthy() {
|
||||
t.Fatal("Send failure did not mark backend unhealthy")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueStopDeadlineCancelsPrepareRetryBackoff(t *testing.T) {
|
||||
backend := newQueueBackend(func(context.Context, string) (logger.Batch, error) {
|
||||
return nil, errors.New("prepare failed")
|
||||
})
|
||||
q := logger.NewQueueWithBackend(backend, 1, 1, 1, time.Hour)
|
||||
q.Start(context.Background())
|
||||
q.Submit(&logger.LogEntry{RequestID: "backoff"})
|
||||
|
||||
deadline := time.Now().Add(time.Second)
|
||||
for backend.calls.Load() == 0 && time.Now().Before(deadline) {
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := q.Stop(ctx); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("Stop error=%v want deadline exceeded", err)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
if calls := backend.calls.Load(); calls != 1 {
|
||||
t.Fatalf("PrepareBatch calls=%d want 1 before deadline cancels backoff", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueExhaustsPrepareRetriesAndMarksBackendUnhealthy(t *testing.T) {
|
||||
backend := newQueueBackend(func(context.Context, string) (logger.Batch, error) {
|
||||
return nil, errors.New("prepare failed")
|
||||
})
|
||||
q := logger.NewQueueWithBackend(backend, 2, 2, 1, time.Hour)
|
||||
q.Start(context.Background())
|
||||
q.Submit(&logger.LogEntry{RequestID: "a"})
|
||||
q.Submit(&logger.LogEntry{RequestID: "b"})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := q.Stop(ctx); err != nil {
|
||||
t.Fatalf("Stop: %v", err)
|
||||
}
|
||||
if calls := backend.calls.Load(); calls != 3 {
|
||||
t.Fatalf("PrepareBatch calls=%d want 3", calls)
|
||||
}
|
||||
if stats := q.Stats(); stats.Failed != 2 || stats.Ambiguous != 0 || stats.Bytes != 0 {
|
||||
t.Fatalf("stats=%+v", stats)
|
||||
}
|
||||
if backend.Healthy() {
|
||||
t.Fatal("exhausted Prepare retries did not mark backend unhealthy")
|
||||
}
|
||||
}
|
||||
|
||||
type blockingSendBatch struct {
|
||||
ctx context.Context
|
||||
started chan struct{}
|
||||
}
|
||||
|
||||
func (*blockingSendBatch) Append(...any) error { return nil }
|
||||
func (b *blockingSendBatch) Send() error {
|
||||
close(b.started)
|
||||
<-b.ctx.Done()
|
||||
return b.ctx.Err()
|
||||
}
|
||||
func (*blockingSendBatch) Abort() error { return nil }
|
||||
|
||||
func contains(s, substr string) bool {
|
||||
for i := 0; i+len(substr) <= len(s); i++ {
|
||||
if s[i:i+len(substr)] == substr {
|
||||
|
||||
Reference in New Issue
Block a user