feat(streaming): restore preset controls and add separate FPS/bitrate tuning
Restore working output resolution presets for the sidecar service path and extend the video streaming UI/backend with separate user-controlled FPS and bitrate settings. This change: - wires resolution presets back through the sidecar service mode - adds a dedicated framerate selection independent from resolution - adds a dedicated video bitrate input independent from resolution - passes resolution, FPS, and bitrate through frontend, API, backend, and sidecar paths - makes local mode and service mode use the same explicit stream settings - exposes active framerate and bitrate in stream status - keeps preset values only as fallback defaults where appropriate Tested successfully across multiple combinations, including: - 480p / 720p / 1080p - 24 FPS / 30 FPS / 60 FPS - local files and YouTube sources - high bitrate cases up to 1080p@60fps with 6000k video bitrate
This commit is contained in:
Philipp
@@ -241,6 +241,20 @@ The Docker images handle migrations automatically on startup.
|
||||
| `SIDECAR_URL` | — | Optional. Full URL of the WebRTC sidecar service (e.g. `http://ts6-sidecar:9800`). Set in Docker when sidecar runs as a separate container. |
|
||||
| `YT_COOKIE_FILE` | — | Optional. Path to a Netscape-format cookies.txt file for yt-dlp. Can also be managed via **Settings → YouTube** in the UI. |
|
||||
|
||||
## Environment Variables Sidecar(VideoStreaming)
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `VIDEO_QUEUE_SIZE` | `2048` | Size of the video RTP queue |
|
||||
| `AUDIO_QUEUE_SIZE` | `4096` | Size of the audio RTP queue |
|
||||
| `SYNC_PLAYOUT_BUFFER_MS` | `4` | Small playout buffer used by the adaptive pacing logic |
|
||||
| `SYNC_VIDEO_BIAS_MS` | `4` | Optional extra holdback for video to fine-tune sync |
|
||||
| `AUDIO_DELAY_MS` | `0` | Legacy / manual audio delay option With the current pacing logic this is typically expected to stay at 0 |
|
||||
| `SIDECAR_DEBUG_LOGS` | `1` | Enables verbose debug logging for high-frequency runtime details |
|
||||
| `VIDEO_READ_RTP_BUFFER` | `4194304` | UDP OS-socketbuffer for video port |
|
||||
| `AUDIO_READ_RTP_BUFFER` | `1048576` | UDP OS-socketbuffer for audio port |
|
||||
| `VIDEO_BUFSIZE` | `1M` | FFmpeg Video Buffer |
|
||||
|
||||
## Music Bot Text Commands
|
||||
|
||||
When a music bot is connected to a channel, users in that channel can control it via chat:
|
||||
|
||||
@@ -583,9 +583,9 @@ musicBotRoutes.post('/:id/stream/start', async (req: Request, res: Response, nex
|
||||
const manager: VoiceBotManager = req.app.locals.voiceBotManager;
|
||||
const bot = manager.getBot(parseInt(req.params.id as string));
|
||||
if (!bot) throw new AppError(404, 'Music bot not found');
|
||||
const { source, preset } = req.body;
|
||||
const { source, preset, framerate, bitrate } = req.body;
|
||||
if (!source) throw new AppError(400, 'source is required');
|
||||
await bot.startVideoStream(source, preset);
|
||||
await bot.startVideoStream(source, preset, framerate, bitrate);
|
||||
res.json({ success: true, status: bot.videoStreamStatus });
|
||||
} catch (err) { next(err); }
|
||||
});
|
||||
|
||||
@@ -49,8 +49,14 @@ export class SidecarClient {
|
||||
await this.call('POST', '/peer/close', { id });
|
||||
}
|
||||
|
||||
async setSource(source: string): Promise<void> {
|
||||
await this.call('POST', '/source', { source });
|
||||
async setSource(source: string, width?: number, height?: number, framerate?: number, bitrate?: string): Promise<void> {
|
||||
await this.call('POST', '/source', {
|
||||
source,
|
||||
width,
|
||||
height,
|
||||
framerate,
|
||||
bitrate,
|
||||
});
|
||||
}
|
||||
|
||||
async stopSource(): Promise<void> {
|
||||
|
||||
@@ -29,6 +29,8 @@ export interface VideoStreamStatus {
|
||||
streamId: string | null;
|
||||
source: string | null;
|
||||
preset: string;
|
||||
framerate: number;
|
||||
bitrate: string;
|
||||
startedAt: number | null;
|
||||
viewerCount: number;
|
||||
viewers: VideoViewerInfo[];
|
||||
|
||||
@@ -127,6 +127,8 @@ export class VoiceBot extends EventEmitter {
|
||||
private _activeStreamId: string | null = null;
|
||||
private _videoSource: string | null = null;
|
||||
private _videoPreset: string = DEFAULT_PRESET;
|
||||
private _videoFramerate: number = STREAM_PRESETS[DEFAULT_PRESET]?.framerate ?? 30;
|
||||
private _videoBitrate: string = STREAM_PRESETS[DEFAULT_PRESET]?.bitrate ?? '2500k';
|
||||
private _videoStartedAt: number | null = null;
|
||||
private _viewers: Map<number, VideoViewerInfo> = new Map();
|
||||
|
||||
@@ -750,6 +752,8 @@ export class VoiceBot extends EventEmitter {
|
||||
streamId: this._activeStreamId,
|
||||
source: this._videoSource,
|
||||
preset: this._videoPreset,
|
||||
framerate: this._videoFramerate,
|
||||
bitrate: this._videoBitrate,
|
||||
startedAt: this._videoStartedAt,
|
||||
viewerCount: this._viewers.size,
|
||||
viewers: Array.from(this._viewers.values()),
|
||||
@@ -758,7 +762,7 @@ export class VoiceBot extends EventEmitter {
|
||||
}
|
||||
|
||||
/** Start video streaming to TS6 via WebRTC */
|
||||
async startVideoStream(source: string, preset?: string): Promise<void> {
|
||||
async startVideoStream(source: string, preset?: string, framerate?: number, bitrate?: string): Promise<void> {
|
||||
if (this._status !== 'connected' && this._status !== 'playing' && this._status !== 'paused') {
|
||||
throw new Error('Bot is not connected');
|
||||
}
|
||||
@@ -768,8 +772,17 @@ export class VoiceBot extends EventEmitter {
|
||||
|
||||
const sidecarBinary = this.config.sidecarBinaryPath || process.env.SIDECAR_BINARY_PATH || 'sidecar';
|
||||
const sidecarPort = this.config.sidecarPort || 9800;
|
||||
this._videoPreset = preset || this.config.streamPreset || DEFAULT_PRESET;
|
||||
this._videoPreset = preset ?? this.config.streamPreset ?? DEFAULT_PRESET;
|
||||
const presetConfig = STREAM_PRESETS[this._videoPreset] || STREAM_PRESETS[DEFAULT_PRESET];
|
||||
const effectiveFramerate = framerate && framerate > 0
|
||||
? framerate
|
||||
: presetConfig.framerate;
|
||||
const effectiveBitrate = bitrate?.trim()
|
||||
? bitrate.trim()
|
||||
: presetConfig.bitrate;
|
||||
|
||||
this._videoFramerate = effectiveFramerate;
|
||||
this._videoBitrate = effectiveBitrate;
|
||||
|
||||
// Check if sidecar URL is set (Docker mode — sidecar runs as separate container)
|
||||
const sidecarUrl = process.env.SIDECAR_URL;
|
||||
@@ -782,9 +795,9 @@ export class VoiceBot extends EventEmitter {
|
||||
const sidecarConfig: SidecarConfig = {
|
||||
binaryPath: sidecarBinary,
|
||||
port: sidecarPort,
|
||||
videoBitrate: presetConfig.bitrate,
|
||||
videoBitrate: effectiveBitrate,
|
||||
videoResolution: { width: presetConfig.width, height: presetConfig.height },
|
||||
videoFramerate: presetConfig.framerate,
|
||||
videoFramerate: effectiveFramerate,
|
||||
};
|
||||
|
||||
this.sidecarProc = new SidecarProcess(sidecarConfig);
|
||||
@@ -848,7 +861,13 @@ export class VoiceBot extends EventEmitter {
|
||||
|
||||
// Resolve YouTube/streaming URLs via yt-dlp, then start ffmpeg
|
||||
const resolvedSource = await resolveVideoUrl(source, presetConfig.height);
|
||||
await this.sidecarHttp.setSource(resolvedSource);
|
||||
await this.sidecarHttp.setSource(
|
||||
resolvedSource,
|
||||
presetConfig.width,
|
||||
presetConfig.height,
|
||||
effectiveFramerate,
|
||||
effectiveBitrate,
|
||||
);
|
||||
|
||||
console.log(`[VoiceBot ${this.config.id}] Video stream started: ${stream.id}, source: ${source}`);
|
||||
this.emit('videoStreamStarted', { streamId: stream.id, source, preset: this._videoPreset });
|
||||
@@ -907,7 +926,14 @@ export class VoiceBot extends EventEmitter {
|
||||
this._videoSource = source;
|
||||
const currentPreset = STREAM_PRESETS[this._videoPreset] || STREAM_PRESETS[DEFAULT_PRESET];
|
||||
const resolvedSource = await resolveVideoUrl(source, currentPreset.height);
|
||||
await this.sidecarHttp.setSource(resolvedSource);
|
||||
|
||||
await this.sidecarHttp.setSource(
|
||||
resolvedSource,
|
||||
currentPreset.width,
|
||||
currentPreset.height,
|
||||
this._videoFramerate,
|
||||
this._videoBitrate,
|
||||
);
|
||||
console.log(`[VoiceBot ${this.config.id}] Video source changed: ${source}`);
|
||||
this.emit('videoSourceChanged', source);
|
||||
}
|
||||
@@ -960,7 +986,7 @@ export class VoiceBot extends EventEmitter {
|
||||
const clid = parseInt(params.clid) || 0;
|
||||
if (this._viewers.has(clid)) {
|
||||
console.log(`[VoiceBot ${this.config.id}] Viewer left: clid=${clid}`);
|
||||
this.sidecarHttp?.closePeer(String(clid)).catch(() => {});
|
||||
this.sidecarHttp?.closePeer(String(clid)).catch(() => { });
|
||||
this._viewers.delete(clid);
|
||||
this.emit('videoViewerLeft', clid);
|
||||
}
|
||||
|
||||
@@ -39,8 +39,8 @@ export const musicBotsApi = {
|
||||
playerWidgetToken: (id: number) => api.get(`/music-bots/${id}/player-widget-token`).then((r) => r.data),
|
||||
|
||||
// Video Streaming
|
||||
startStream: (id: number, source: string, preset?: string) =>
|
||||
api.post(`/music-bots/${id}/stream/start`, { source, preset }).then((r) => r.data),
|
||||
startStream: (id: number, source: string, preset?: string, framerate?: number, bitrate?: string) =>
|
||||
api.post(`/music-bots/${id}/stream/start`, { source, preset, framerate, bitrate }).then((r) => r.data),
|
||||
stopStream: (id: number) => api.post(`/music-bots/${id}/stream/stop`).then((r) => r.data),
|
||||
setStreamSource: (id: number, source: string) =>
|
||||
api.post(`/music-bots/${id}/stream/source`, { source }).then((r) => r.data),
|
||||
|
||||
@@ -25,6 +25,12 @@ const PRESETS = [
|
||||
{ value: '1080p', label: '1080p (1920x1080, 4.5 Mbps)' },
|
||||
];
|
||||
|
||||
const FPS_OPTIONS = [
|
||||
{ value: '24', label: '24 FPS' },
|
||||
{ value: '30', label: '30 FPS' },
|
||||
{ value: '60', label: '60 FPS' },
|
||||
];
|
||||
|
||||
interface VideoStreamTabProps {
|
||||
botId: number;
|
||||
botStatus: string;
|
||||
@@ -33,6 +39,8 @@ interface VideoStreamTabProps {
|
||||
export function VideoStreamTab({ botId, botStatus }: VideoStreamTabProps) {
|
||||
const [sourceUrl, setSourceUrl] = useState('');
|
||||
const [preset, setPreset] = useState('720p');
|
||||
const [framerate, setFramerate] = useState('30');
|
||||
const [bitrate, setBitrate] = useState('2500k');
|
||||
|
||||
const { data: streamStatus } = useVideoStreamStatus(botId);
|
||||
const startStream = useStartVideoStream();
|
||||
@@ -45,7 +53,13 @@ export function VideoStreamTab({ botId, botStatus }: VideoStreamTabProps) {
|
||||
|
||||
const handleStart = () => {
|
||||
if (!sourceUrl.trim()) return;
|
||||
startStream.mutate({ botId, source: sourceUrl.trim(), preset });
|
||||
startStream.mutate({
|
||||
botId,
|
||||
source: sourceUrl.trim(),
|
||||
preset,
|
||||
framerate: Number(framerate),
|
||||
bitrate: bitrate.trim(),
|
||||
});
|
||||
};
|
||||
|
||||
const handleStop = () => {
|
||||
@@ -117,19 +131,48 @@ export function VideoStreamTab({ botId, botStatus }: VideoStreamTabProps) {
|
||||
</div>
|
||||
|
||||
{!isStreaming && (
|
||||
<div className="space-y-2">
|
||||
<Label>Quality Preset</Label>
|
||||
<div className="flex gap-2">
|
||||
{PRESETS.map((p) => (
|
||||
<Button
|
||||
key={p.value}
|
||||
variant={preset === p.value ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setPreset(p.value)}
|
||||
>
|
||||
{p.value}
|
||||
</Button>
|
||||
))}
|
||||
<div className="space-y-4">
|
||||
<div className="space-y-2">
|
||||
<Label>Quality Preset</Label>
|
||||
<div className="flex gap-2">
|
||||
{PRESETS.map((p) => (
|
||||
<Button
|
||||
key={p.value}
|
||||
variant={preset === p.value ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setPreset(p.value)}
|
||||
>
|
||||
{p.value}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label>Frame Rate (FPS)</Label>
|
||||
<div className="flex gap-2">
|
||||
{FPS_OPTIONS.map((fps) => (
|
||||
<Button
|
||||
key={fps.value}
|
||||
variant={framerate === fps.value ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setFramerate(fps.value)}
|
||||
>
|
||||
{fps.label}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>Video Bitrate</Label>
|
||||
<Input
|
||||
value={bitrate}
|
||||
onChange={(e) => setBitrate(e.target.value)}
|
||||
placeholder="e.g. 1500k, 2500k, 4500k"
|
||||
/>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Examples: 1500k, 2500k, 4500k, 6000k
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
@@ -176,6 +219,8 @@ export function VideoStreamTab({ botId, botStatus }: VideoStreamTabProps) {
|
||||
{isStreaming && streamStatus && (
|
||||
<div className="mt-3 flex flex-wrap gap-3 text-xs text-muted-foreground">
|
||||
<span>Preset: <strong>{streamStatus.preset}</strong></span>
|
||||
<span>FPS: <strong>{streamStatus.framerate}</strong></span>
|
||||
<span>Bitrate: <strong>{streamStatus.bitrate}</strong></span>
|
||||
{streamStatus.source && (
|
||||
<span className="truncate max-w-xs">
|
||||
Source: <strong>{streamStatus.source}</strong>
|
||||
|
||||
@@ -235,8 +235,19 @@ export function useVideoStreamStatus(botId: number | null) {
|
||||
export function useStartVideoStream() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation({
|
||||
mutationFn: ({ botId, source, preset }: { botId: number; source: string; preset?: string }) =>
|
||||
musicBotsApi.startStream(botId, source, preset),
|
||||
mutationFn: ({
|
||||
botId,
|
||||
source,
|
||||
preset,
|
||||
framerate,
|
||||
bitrate,
|
||||
}: {
|
||||
botId: number;
|
||||
source: string;
|
||||
preset?: string;
|
||||
framerate?: number;
|
||||
bitrate?: string;
|
||||
}) => musicBotsApi.startStream(botId, source, preset, framerate, bitrate),
|
||||
onSuccess: (_, { botId }) => {
|
||||
qc.invalidateQueries({ queryKey: ['video-stream-status', botId] });
|
||||
qc.invalidateQueries({ queryKey: ['music-bot-state', botId] });
|
||||
|
||||
+40
-16
@@ -316,12 +316,14 @@ func (s *Sidecar) StartRTP() error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("bind video UDP: %w", err)
|
||||
}
|
||||
_ = s.videoConn.SetReadBuffer(envIntOrDefault("VIDEO_RTP_READ_BUFFER", 4*1024*1024))
|
||||
s.videoPort = s.videoConn.LocalAddr().(*net.UDPAddr).Port
|
||||
|
||||
s.audioConn, err = net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0})
|
||||
if err != nil {
|
||||
return fmt.Errorf("bind audio UDP: %w", err)
|
||||
}
|
||||
_ = s.audioConn.SetReadBuffer(envIntOrDefault("AUDIO_RTP_READ_BUFFER", 1*1024*1024))
|
||||
s.audioPort = s.audioConn.LocalAddr().(*net.UDPAddr).Port
|
||||
|
||||
log.Printf("[RTP] Video port: %d, Audio port: %d", s.videoPort, s.audioPort)
|
||||
@@ -815,14 +817,32 @@ func (s *Sidecar) ClosePeer(id string) {
|
||||
s.peersLock.Unlock()
|
||||
}
|
||||
|
||||
func (s *Sidecar) StartFFmpeg(source string) {
|
||||
func (s *Sidecar) StartFFmpeg(source string, width int, height int, framerate int, bitrate string) {
|
||||
s.resetSyncTiming()
|
||||
s.ffmpegLock.Lock()
|
||||
|
||||
defer s.ffmpegLock.Unlock()
|
||||
|
||||
s.StopFFmpegLocked()
|
||||
|
||||
s.source = source
|
||||
|
||||
w := width
|
||||
h := height
|
||||
fps := framerate
|
||||
|
||||
if w <= 0 {
|
||||
w = envIntOrDefault("VIDEO_WIDTH", 1280)
|
||||
}
|
||||
|
||||
if h <= 0 {
|
||||
h = envIntOrDefault("VIDEO_HEIGHT", 720)
|
||||
}
|
||||
|
||||
if fps <= 0 {
|
||||
fps = envIntOrDefault("VIDEO_FRAMERATE", 30)
|
||||
}
|
||||
|
||||
args := []string{}
|
||||
|
||||
if source != "" {
|
||||
@@ -831,22 +851,22 @@ func (s *Sidecar) StartFFmpeg(source string) {
|
||||
} else {
|
||||
args = append(args, "-stream_loop", "-1")
|
||||
}
|
||||
|
||||
args = append(args, "-fflags", "+genpts+discardcorrupt", "-re", "-i", source)
|
||||
} else {
|
||||
w := envIntOrDefault("VIDEO_WIDTH", 1280)
|
||||
h := envIntOrDefault("VIDEO_HEIGHT", 720)
|
||||
args = append(args, "-re", "-f", "lavfi", "-i", fmt.Sprintf("color=c=black:s=%dx%d:r=1", w, h))
|
||||
}
|
||||
|
||||
w := envIntOrDefault("VIDEO_WIDTH", 1280)
|
||||
h := envIntOrDefault("VIDEO_HEIGHT", 720)
|
||||
vBitrate := envOrDefault("VIDEO_BITRATE", "1500k")
|
||||
vBitrate := strings.TrimSpace(bitrate)
|
||||
if vBitrate == "" {
|
||||
vBitrate = envOrDefault("VIDEO_BITRATE", "1500k")
|
||||
}
|
||||
audioDelayMs := envIntOrDefault("AUDIO_DELAY_MS", 0)
|
||||
|
||||
if source != "" {
|
||||
vf := fmt.Sprintf(
|
||||
"fps=30,scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2,format=yuv420p",
|
||||
w, h, w, h,
|
||||
"fps=%d,scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2,format=yuv420p",
|
||||
fps, w, h, w, h,
|
||||
)
|
||||
args = append(args,
|
||||
"-map", "0:v:0",
|
||||
@@ -856,15 +876,15 @@ func (s *Sidecar) StartFFmpeg(source string) {
|
||||
args = append(args,
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-c:v", "libvpx",
|
||||
"-cpu-used", "8",
|
||||
"-cpu-used", "6",
|
||||
"-deadline", "realtime",
|
||||
"-lag-in-frames", "0",
|
||||
"-error-resilient", "1",
|
||||
"-b:v", vBitrate,
|
||||
"-maxrate", "2M",
|
||||
"-bufsize", "100k",
|
||||
"-keyint_min", "15",
|
||||
"-g", "15",
|
||||
"-maxrate", vBitrate,
|
||||
"-bufsize", envOrDefault("VIDEO_BUFSIZE", "500k"),
|
||||
"-keyint_min", "12",
|
||||
"-g", "12",
|
||||
"-auto-alt-ref", "0",
|
||||
"-payload_type", "96",
|
||||
"-ssrc", "11111111",
|
||||
@@ -1054,14 +1074,18 @@ func main() {
|
||||
|
||||
mux.HandleFunc("POST /source", func(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Source string `json:"source"`
|
||||
Source string `json:"source"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Framerate int `json:"framerate"`
|
||||
Bitrate string `json:"bitrate"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
log.Printf("[API] Setting source: %s", req.Source)
|
||||
sidecar.StartFFmpeg(req.Source)
|
||||
log.Printf("[API] Setting source: %s (%dx%d @ %dfps)", req.Source, req.Width, req.Height, req.Framerate, req.Bitrate)
|
||||
sidecar.StartFFmpeg(req.Source, req.Width, req.Height, req.Framerate, req.Bitrate)
|
||||
json.NewEncoder(w).Encode(map[string]string{"status": "ok"})
|
||||
})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user