From b949951ef90b8e9283e691fe93d8703da4a3f1d7 Mon Sep 17 00:00:00 2001 From: foxyz Date: Thu, 15 Oct 2026 17:18:39 +0300 Subject: [PATCH] Sounds play on flutter_soloud instead of three mpv players SoundEngine (platform/audio) opens the device once (2 s after start, or on the first sound), loads each synthesized sound into memory (no temp files), plays overlapping voices and is disposed on window close, before leaving the server. The SfxService interface and its callers are unchanged; loudness matches the old mpv path (v^6). WAV only, so no codec libraries are bundled and no build packages are needed. Co-Authored-By: Claude Opus 5.5 --- apps/desktop/lib/main.dart | 7 + .../lib/src/platform/audio/sound_engine.dart | 133 ++++++++++++++++ apps/desktop/lib/src/sfx/synth_sfx.dart | 96 ++++------- apps/desktop/lib/src/shell/settings_tab.dart | 6 +- apps/desktop/lib/src/state/session_state.dart | 3 +- .../linux/flutter/generated_plugins.cmake | 1 + apps/desktop/pubspec.lock | 44 ++++- apps/desktop/pubspec.yaml | 9 ++ apps/desktop/test/sound_device_e2e_test.dart | 35 ++++ apps/desktop/test/sound_engine_test.dart | 150 ++++++++++++++++++ .../windows/flutter/generated_plugins.cmake | 1 + docs/ARCHITECTURE.md | 17 +- docs/DEVELOPMENT.md | 11 +- 13 files changed, 436 insertions(+), 77 deletions(-) create mode 100644 apps/desktop/lib/src/platform/audio/sound_engine.dart create mode 100644 apps/desktop/test/sound_device_e2e_test.dart create mode 100644 apps/desktop/test/sound_engine_test.dart diff --git a/apps/desktop/lib/main.dart b/apps/desktop/lib/main.dart index 5be3e96..1e1241e 100644 --- a/apps/desktop/lib/main.dart +++ b/apps/desktop/lib/main.dart @@ -73,6 +73,8 @@ Future main(List args) async { // Linux: the launcher entry and icon Wayland needs (release installs). unawaited(installDesktopEntry()); runApp(const WatchTogetherApp()); + // The sound device opens once the window is up, not on the first sound. + Future.delayed(const Duration(seconds: 2), SessionState.instance.sfx.warmUp); } /// Close button / Alt+F4: leave gracefully first (others see "left", a @@ -88,6 +90,11 @@ class _CloseGuard with WindowListener { if (_closing) return; _closing = true; await memory.save(); + try { + await SessionState.instance.sfx.dispose().timeout(const Duration(milliseconds: 500)); + } catch (e) { + debugPrint('sound shutdown on close failed: $e'); + } try { await SessionState.instance.leave().timeout(const Duration(seconds: 2)); } catch (e) { diff --git a/apps/desktop/lib/src/platform/audio/sound_engine.dart b/apps/desktop/lib/src/platform/audio/sound_engine.dart new file mode 100644 index 0000000..b0cd922 --- /dev/null +++ b/apps/desktop/lib/src/platform/audio/sound_engine.dart @@ -0,0 +1,133 @@ +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:flutter_soloud/flutter_soloud.dart'; +import 'package:sync_relay_core/core.dart'; + +final _log = createLogger('sound'); + +/// What [SoundEngine] needs from an audio library: [SoLoudBackend] in the +/// app, a fake in tests. A sound handle is whatever [load] returns. +abstract class AudioBackend { + Future init(); + /// Decodes [wav] (a whole WAV file) into memory under [name]. + Future load(String name, Uint8List wav); + /// Starts one voice of [sound] at [volume] (linear gain, 0..1); voices + /// overlap. + void play(Object sound, double volume); + Future unload(Object sound); + Future deinit(); +} + +/// flutter_soloud (SoLoud mixing into miniaudio's output device). +class SoLoudBackend implements AudioBackend { + SoLoud get _s => SoLoud.instance; + + @override + Future init() => _s.init(); + + @override + Future load(String name, Uint8List wav) => _s.loadMem(name, wav); + + @override + void play(Object sound, double volume) => _s.play(sound as AudioSource, volume: volume); + + @override + Future unload(Object sound) => _s.disposeSource(sound as AudioSource); + + @override + Future deinit() => _s.deinitAsync(); +} + +/// The app's short in-memory sounds. The audio device opens on [warmUp] or +/// the first [play]; if it can't (no device, a broken driver) the engine +/// logs once and every play is silently dropped. Each sound is loaded once +/// per slot from the WAV bytes its renderer gives; a new key in a slot +/// replaces (and frees) what the slot held. +class SoundEngine { + final AudioBackend _backend; + SoundEngine({AudioBackend? backend}) : _backend = backend ?? SoLoudBackend(); + + Future? _ready; + var _disposed = false; + /// Per slot: the loaded key and its sound (null: failed to load). + final Map)> _slots = {}; + /// Load names are unique: SoLoud hands back the old sound for a name it + /// already holds, which would defeat a re-render. + var _loads = 0; + + /// Opens the audio device now, so the first sound has no wait. + Future warmUp() => _ready ??= _init(); + + Future _init() async { + if (_disposed) return false; + try { + await _backend.init(); + return true; + } catch (e) { + _log.warning('no sounds: audio init failed: $e'); + return false; + } + } + + /// Plays the sound of [slot] at [volume] (linear gain). [key] names what + /// [wav] renders: the same key reuses the loaded sound; another key, or + /// [fresh], renders and loads it again. + Future play(String slot, String key, Future Function() wav, + {required double volume, bool fresh = false}) async { + if (_disposed || volume <= 0) return; + if (!await warmUp() || _disposed) return; + var held = _slots[slot]; + if (held == null || held.$1 != key || fresh) { + if (held != null) unawaited(_unload(held.$2)); + held = (key, _load(key, wav)); + _slots[slot] = held; + } + final sound = await held.$2; + if (sound == null) { + // Retry on the next play. + if (identical(_slots[slot], held)) _slots.remove(slot); + return; + } + if (_disposed || !identical(_slots[slot], held)) return; + try { + _backend.play(sound, volume.clamp(0.0, 1.0)); + } catch (e) { + _log.warning('sound $key: $e'); + } + } + + Future _load(String key, Future Function() wav) async { + try { + return await _backend.load('$key#${_loads++}', await wav()); + } catch (e) { + _log.warning('sound $key: $e'); + return null; + } + } + + Future _unload(Future sound) async { + try { + final s = await sound; + if (s != null && !_disposed) await _backend.unload(s); + } catch (e) { + _log.fine('unload: $e'); + } + } + + /// Frees every sound and closes the audio device. Plays after it are + /// dropped. + Future dispose() async { + if (_disposed) return; + _disposed = true; + _slots.clear(); + final ready = _ready; + if (ready == null || !await ready) return; + try { + // Deinit frees the loaded sounds with the engine. + await _backend.deinit(); + } catch (e) { + _log.warning('audio deinit: $e'); + } + } +} diff --git a/apps/desktop/lib/src/sfx/synth_sfx.dart b/apps/desktop/lib/src/sfx/synth_sfx.dart index dfab9d8..b7146f3 100644 --- a/apps/desktop/lib/src/sfx/synth_sfx.dart +++ b/apps/desktop/lib/src/sfx/synth_sfx.dart @@ -1,88 +1,60 @@ import 'dart:async'; -import 'dart:io'; +import 'dart:math' as math; import 'package:flutter/foundation.dart'; -import 'package:media_kit/media_kit.dart'; -import 'package:path_provider/path_provider.dart'; +import '../platform/audio/sound_engine.dart'; import 'sfx.dart'; import 'sound_bank.dart'; import 'synth.dart'; -/// Plays [SoundSpec]s: each is rendered once to a WAV in the temp dir and -/// played on a small pool of audio-only mpv players (so two sounds can -/// overlap). [volume] (0..1) is read per play and applied on a square -/// curve (ears are log); [enabled] filters cues. +/// Plays [SoundSpec]s on a [SoundEngine]: each is rendered once (off the +/// UI isolate) to WAV bytes and loaded into memory; sounds overlap. +/// [volume] (0..1) is read per play and applied as v^6: the square curve +/// (ears are log) the app always used, times mpv's own cubic volume it +/// used to play through, so the slider sounds as it did; [enabled] +/// filters cues. class SynthSfx implements SfxService { final double Function() volume; final bool Function(SfxKind kind) enabled; /// Which sound set each sound plays from. final SoundPreset Function(SfxKind kind) presetFor; - SynthSfx({required this.volume, bool Function(SfxKind kind)? enabled, SoundPreset Function(SfxKind kind)? presetFor}) + final SoundEngine engine; + SynthSfx( + {required this.volume, + bool Function(SfxKind kind)? enabled, + SoundPreset Function(SfxKind kind)? presetFor, + SoundEngine? engine}) : enabled = enabled ?? ((_) => true), - presetFor = presetFor ?? ((_) => defaultPreset); - - static const _poolSize = 3; - final List _pool = []; - var _next = 0; - final Map> _files = {}; - Directory? _dir; + presetFor = presetFor ?? ((_) => defaultPreset), + engine = engine ?? SoundEngine(); @override - Future play(SfxCue cue) async { - if (!enabled(cue.kind)) return; - final preset = presetFor(cue.kind); - await playSpec('${preset.name}-${cue.kind.name}-${cue.readyIndex}', - soundFor(cue.kind, readyIndex: cue.readyIndex, preset: preset)); + Future play(SfxCue cue) { + if (!enabled(cue.kind)) return Future.value(); + return _play(cue.kind, cue.readyIndex, presetFor(cue.kind)); } /// Plays [kind] even when it is switched off (Settings' preview), always /// freshly rendered: an edited spec plays after a hot reload. - Future preview(SfxKind kind, {int readyIndex = 1, SoundPreset? preset}) { - final p = preset ?? presetFor(kind); - final key = '${p.name}-${kind.name}-$readyIndex'; - _files.remove(key); - return playSpec(key, soundFor(kind, readyIndex: readyIndex, preset: p)); - } + Future preview(SfxKind kind, {int readyIndex = 1, SoundPreset? preset}) => + _play(kind, readyIndex, preset ?? presetFor(kind), fresh: true); - /// Plays [spec], cached under [key] (same key = same sound). - Future playSpec(String key, SoundSpec spec) async { + /// One slot per sound and ready step: a new preset replaces it. + Future _play(SfxKind kind, int readyIndex, SoundPreset preset, {bool fresh = false}) { final v = volume(); - if (v <= 0) return; - try { - final path = await (_files[key] ??= _write(key, spec)); - final player = await _player(); - await player.setVolume(100 * v * v); - await player.open(Media(path)); - } catch (e) { - _files.remove(key); - debugPrint('sfx $key: $e'); - } + return engine.play('${kind.name}-$readyIndex', '${preset.name}-${kind.name}-$readyIndex', + () => renderWav(soundFor(kind, readyIndex: readyIndex, preset: preset)), + volume: math.pow(v, 6).toDouble(), fresh: fresh); } - Future _write(String key, SoundSpec spec) async { - _dir ??= await Directory('${(await getTemporaryDirectory()).path}/watch_together_sfx').create(recursive: true); - final f = File('${_dir!.path}/$key.wav'); - await f.writeAsBytes(wav(render(spec))); - return f.path; - } + /// Opens the audio device ahead of the first sound. + Future warmUp() => engine.warmUp(); - Future _player() async { - if (_pool.length < _poolSize) { - final p = Player( - configuration: const PlayerConfiguration(vo: 'null', osc: false, libass: false, title: 'sfx'), - ); - await (p.platform as NativePlayer).setProperty('audio-display', 'no'); - _pool.add(p); - return p; - } - return _pool[_next++ % _poolSize]; - } - - Future dispose() async { - for (final p in _pool) { - await p.dispose(); - } - _pool.clear(); - } + Future dispose() => engine.dispose(); } + +/// [spec] as WAV bytes, rendered on a worker isolate. +Future renderWav(SoundSpec spec) => compute(_renderWav, spec, debugLabel: 'sfx render'); + +Uint8List _renderWav(SoundSpec spec) => wav(render(spec)); diff --git a/apps/desktop/lib/src/shell/settings_tab.dart b/apps/desktop/lib/src/shell/settings_tab.dart index c153714..9189af0 100644 --- a/apps/desktop/lib/src/shell/settings_tab.dart +++ b/apps/desktop/lib/src/shell/settings_tab.dart @@ -18,7 +18,6 @@ import '../screens/home/about_page.dart'; import '../screens/home/settings_page.dart'; import '../sfx/sfx.dart'; import '../sfx/sound_bank.dart'; -import '../sfx/synth_sfx.dart'; import '../state/session_state.dart'; import '../state/settings.dart'; import '../update/update_controller.dart'; @@ -798,10 +797,7 @@ class NotificationsInfo extends StatelessWidget { class SoundSettings extends StatelessWidget { const SoundSettings({super.key}); - void _preview(SfxKind kind) { - final sfx = SessionState.instance.sfx; - if (sfx is SynthSfx) unawaited(sfx.preview(kind, readyIndex: 2)); - } + void _preview(SfxKind kind) => unawaited(SessionState.instance.sfx.preview(kind, readyIndex: 2)); @override Widget build(BuildContext context) { diff --git a/apps/desktop/lib/src/state/session_state.dart b/apps/desktop/lib/src/state/session_state.dart index 89c2e70..3834ae0 100644 --- a/apps/desktop/lib/src/state/session_state.dart +++ b/apps/desktop/lib/src/state/session_state.dart @@ -160,7 +160,8 @@ class SessionState { final route = signal(AppRoute.home); final recents = signal>(const []); final status = signal(const HealthStatus(HealthSeverity.info, 'idle')); - final SfxService sfx = SynthSfx( + /// Owns the app's audio device ([SynthSfx.dispose] on close). + final SynthSfx sfx = SynthSfx( volume: () => AppSettings.instance.soundVolume.peek(), // A nudge is an explicit poke: it ignores its switch (not the volume). enabled: (k) => k == SfxKind.nudge || !AppSettings.instance.soundsOff.peek().contains(k.name), diff --git a/apps/desktop/linux/flutter/generated_plugins.cmake b/apps/desktop/linux/flutter/generated_plugins.cmake index 93ca350..4c318c8 100644 --- a/apps/desktop/linux/flutter/generated_plugins.cmake +++ b/apps/desktop/linux/flutter/generated_plugins.cmake @@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + flutter_soloud jni ) diff --git a/apps/desktop/pubspec.lock b/apps/desktop/pubspec.lock index 2346329..f01fe2f 100644 --- a/apps/desktop/pubspec.lock +++ b/apps/desktop/pubspec.lock @@ -153,6 +153,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.1.2" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" file_picker: dependency: "direct main" description: @@ -227,6 +235,14 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_soloud: + dependency: "direct main" + description: + name: flutter_soloud + sha256: "1c297e52182c5e8fd447ae8430bf0b598445dc586e72863ceeef0416242b2ae8" + url: "https://pub.dev" + source: hosted + version: "5.2.1" flutter_test: dependency: "direct dev" description: flutter @@ -246,6 +262,14 @@ packages: url: "https://git.senkochan.moe/Foxyz/fox_utils.git" source: git version: "0.1.9" + glob: + dependency: transitive + description: + name: glob + sha256: "218aeb56050c714f62a3182775320dfa04602b55074873e24e31bbd39bda96fb" + url: "https://pub.dev" + source: hosted + version: "2.2.0" hive_ce: dependency: "direct main" description: @@ -495,6 +519,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" + native_toolchain_c: + dependency: transitive + description: + name: native_toolchain_c + sha256: "90eaad8059ebc2004a63810a4696ea9acdb02473e47ed3b6b3bafb100ac77e31" + url: "https://pub.dev" + source: hosted + version: "0.19.5" objective_c: dependency: transitive description: @@ -623,6 +655,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.0" + process: + dependency: transitive + description: + name: process + sha256: "4242ba3508d37e01808bdf71ad1d5bb93a8d671bf2e7450e6b1b353fb0808891" + url: "https://pub.dev" + source: hosted + version: "5.0.6" pub_semver: dependency: transitive description: @@ -786,14 +826,14 @@ packages: path: "../../packages/core" relative: true source: path - version: "1.1.2" + version: "1.2.0" sync_relay_server: dependency: "direct main" description: path: "../../packages/server" relative: true source: path - version: "1.1.2" + version: "1.2.0" synchronized: dependency: transitive description: diff --git a/apps/desktop/pubspec.yaml b/apps/desktop/pubspec.yaml index 966a67f..525f9a0 100644 --- a/apps/desktop/pubspec.yaml +++ b/apps/desktop/pubspec.yaml @@ -36,6 +36,15 @@ dependencies: git: url: https://git.senkochan.moe/Foxyz/leaf_widgets.git ref: v0.1.0 + # App sounds (sfx/, platform/audio/): SoLoud + miniaudio, built from + # source by its build hook (needs only a C++ toolchain). + flutter_soloud: ^5.2.1 +# The sounds are WAVs made in memory: no Ogg/Opus/FLAC codecs, so the +# build hook downloads and bundles no Xiph libraries. +hooks: + user_defines: + flutter_soloud: + no_xiph_libs: true dev_dependencies: flutter_test: sdk: flutter diff --git a/apps/desktop/test/sound_device_e2e_test.dart b/apps/desktop/test/sound_device_e2e_test.dart new file mode 100644 index 0000000..fda3da7 --- /dev/null +++ b/apps/desktop/test/sound_device_e2e_test.dart @@ -0,0 +1,35 @@ +import 'dart:io'; + +import 'package:flutter/services.dart'; +import 'package:flutter_soloud/flutter_soloud.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:watch_together/src/platform/audio/sound_engine.dart'; +import 'package:watch_together/src/sfx/sfx.dart'; +import 'package:watch_together/src/sfx/sound_bank.dart'; +import 'package:watch_together/src/sfx/synth_sfx.dart'; + +// Needs a real audio output: opens the device through flutter_soloud and +// plays every sound of every preset once, quietly (WT_SFX_VOLUME, default +// 0.1, on the square curve like the app). + +void main() { + // flutter_soloud's init asks path_provider for a temp dir. + TestWidgetsFlutterBinding.ensureInitialized().defaultBinaryMessenger.setMockMethodCallHandler( + const MethodChannel('plugins.flutter.io/path_provider'), (_) async => Directory.systemTemp.path); + test('every sound loads and plays on the real device', timeout: const Timeout(Duration(minutes: 2)), () async { + final volume = double.tryParse(Platform.environment['WT_SFX_VOLUME'] ?? '') ?? 0.1; + final engine = SoundEngine(); + expect(await engine.warmUp(), isTrue, reason: 'audio device opened'); + final sfx = SynthSfx(volume: () => volume, engine: engine); + for (final p in SoundPreset.values) { + for (final k in SfxKind.values) { + await sfx.preview(k, readyIndex: 2, preset: p); + expect(SoLoud.instance.getActiveVoiceCount(), greaterThan(0), reason: '${p.name}/${k.name}'); + await Future.delayed(const Duration(milliseconds: 250)); + } + } + await Future.delayed(const Duration(seconds: 2)); + await sfx.dispose(); + expect(SoLoud.instance.isInitialized, isFalse); + }); +} diff --git a/apps/desktop/test/sound_engine_test.dart b/apps/desktop/test/sound_engine_test.dart new file mode 100644 index 0000000..2ccbc42 --- /dev/null +++ b/apps/desktop/test/sound_engine_test.dart @@ -0,0 +1,150 @@ +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:watch_together/src/platform/audio/sound_engine.dart'; +import 'package:watch_together/src/sfx/sfx.dart'; +import 'package:watch_together/src/sfx/sound_bank.dart'; +import 'package:watch_together/src/sfx/synth.dart'; +import 'package:watch_together/src/sfx/synth_sfx.dart'; + +class _FakeBackend implements AudioBackend { + final bool failInit; + _FakeBackend({this.failInit = false}); + + var inits = 0, deinits = 0; + final loads = {}; + final unloaded = []; + final plays = <(String, double)>[]; + + @override + Future init() async { + inits++; + if (failInit) throw StateError('no audio device'); + } + + @override + Future load(String name, Uint8List wav) async { + loads[name] = wav; + return name; + } + + @override + void play(Object sound, double volume) => plays.add((sound as String, volume)); + + @override + Future unload(Object sound) async => unloaded.add(sound); + + @override + Future deinit() async => deinits++; +} + +Future _bytes() async => Uint8List.fromList([1, 2, 3]); + +void main() { + test('loads a sound once per key and lets plays overlap', () async { + final b = _FakeBackend(); + final e = SoundEngine(backend: b); + await Future.wait([ + e.play('chat', 'abyss-chat', _bytes, volume: 0.5), + e.play('chat', 'abyss-chat', _bytes, volume: 0.5), + ]); + await e.play('chat', 'abyss-chat', _bytes, volume: 0.25); + expect(b.inits, 1); + expect(b.loads.length, 1); + expect(b.plays.map((p) => p.$2), [0.5, 0.5, 0.25]); + }); + + test('a new key or fresh replaces and frees the slot', () async { + final b = _FakeBackend(); + final e = SoundEngine(backend: b); + await e.play('chat', 'abyss-chat', _bytes, volume: 1); + await e.play('chat', 'cosmic-chat', _bytes, volume: 1); + await e.play('chat', 'cosmic-chat', _bytes, volume: 1, fresh: true); + await Future.delayed(Duration.zero); + expect(b.loads.length, 3); + expect(b.loads.keys.toSet().length, 3, reason: 'load names are unique'); + expect(b.unloaded.length, 2); + expect(b.plays.map((p) => p.$1).toSet().length, 3); + }); + + test('silence and zero volume load nothing', () async { + final b = _FakeBackend(); + final e = SoundEngine(backend: b); + await e.play('chat', 'abyss-chat', _bytes, volume: 0); + expect(b.inits, 0); + expect(b.loads, isEmpty); + }); + + test('a failed init means no sounds, no throw, one try', () async { + final b = _FakeBackend(failInit: true); + final e = SoundEngine(backend: b); + expect(await e.warmUp(), isFalse); + await e.play('chat', 'abyss-chat', _bytes, volume: 1); + await e.play('join', 'abyss-join', _bytes, volume: 1); + expect(b.inits, 1); + expect(b.loads, isEmpty); + expect(b.plays, isEmpty); + await e.dispose(); + expect(b.deinits, 0); + }); + + test('a failed render is retried on the next play', () async { + final b = _FakeBackend(); + final e = SoundEngine(backend: b); + await e.play('chat', 'abyss-chat', () async => throw StateError('render'), volume: 1); + expect(b.plays, isEmpty); + await e.play('chat', 'abyss-chat', _bytes, volume: 1); + expect(b.plays.length, 1); + }); + + test('dispose deinits once and drops later plays', () async { + final b = _FakeBackend(); + final e = SoundEngine(backend: b); + await e.play('chat', 'abyss-chat', _bytes, volume: 1); + await e.dispose(); + await e.dispose(); + await e.play('chat', 'abyss-chat', _bytes, volume: 1); + expect(b.deinits, 1); + expect(b.plays.length, 1); + }); + + test('dispose without a sound ever played opens nothing', () async { + final b = _FakeBackend(); + await SoundEngine(backend: b).dispose(); + expect(b.inits + b.deinits, 0); + }); + + group('SynthSfx', () { + test('renders a WAV per sound, volume as before (v^6), switches and preview', () async { + final b = _FakeBackend(); + var volume = 0.5; + final off = {SfxKind.join}; + final sfx = SynthSfx( + volume: () => volume, + enabled: (k) => !off.contains(k), + presetFor: (_) => SoundPreset.woodAndGlass, + engine: SoundEngine(backend: b), + ); + for (final k in SfxKind.values) { + await sfx.play(SfxCue(k, readyIndex: 2)); + } + expect(b.plays.length, SfxKind.values.length - 1); + expect(b.plays.every((p) => p.$2 == 0.015625), isTrue); // 0.5^6 + for (final MapEntry(:key, :value) in b.loads.entries) { + expect(String.fromCharCodes(value.sublist(0, 4)), 'RIFF', reason: key); + expect(key, startsWith('woodAndGlass-')); + } + final chat = b.loads.entries.firstWhere((e) => e.key.startsWith('woodAndGlass-chat-2')); + expect(chat.value, wav(render(soundFor(SfxKind.chat, readyIndex: 2, preset: SoundPreset.woodAndGlass)))); + + await sfx.preview(SfxKind.join, readyIndex: 2); + expect(b.plays.last.$1, startsWith('woodAndGlass-join-2')); + + volume = 0; + await sfx.play(const SfxCue(SfxKind.chat)); + expect(b.plays.length, SfxKind.values.length); + await sfx.dispose(); + expect(b.deinits, 1); + }); + }); +} diff --git a/apps/desktop/windows/flutter/generated_plugins.cmake b/apps/desktop/windows/flutter/generated_plugins.cmake index cc9f746..ab1568f 100644 --- a/apps/desktop/windows/flutter/generated_plugins.cmake +++ b/apps/desktop/windows/flutter/generated_plugins.cmake @@ -11,6 +11,7 @@ list(APPEND FLUTTER_PLUGIN_LIST ) list(APPEND FLUTTER_FFI_PLUGIN_LIST + flutter_soloud jni ) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 428c80b..78bbb53 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -335,7 +335,7 @@ menu and Settings say so). Otherwise the session player logs at `warn` on media_kit's handle): media_kit drains mpv's log on the UI thread, one message at a time, so a chatty level, or ffmpeg logging every retry of a refused request, could starve the UI. Shader and yt-dlp hook errors are -`error` lines and still arrive. The thumbnail and sound players keep +`error` lines and still arrive. The thumbnail players keep media_kit's default (`error`) and nothing listens to their log. The hardware path is fragile: media_kit borrows whatever EGL context GDK left current on the main thread (upstream media-kit#1404), so an X11 session or a Flutter/GDK change @@ -1261,10 +1261,17 @@ panel) and chat messages (Copy message, and a link in it). dark space reverb) to quiet WAVs; `sound_bank.dart` voices every event in three presets (`SoundPreset`: Cosmic, Abyss, Wood & glass) in one shared key, so mixed presets still agree. `SynthSfx` renders a sound on - first use into the temp dir (cache key: preset, sound, ready index) and - plays it on a pool of three audio-only mpv players, at - `AppSettings.soundVolume` (square curve) unless switched off in - `soundsOff`. Settings → Notifications & sounds has the volume, the + first use (on a worker isolate, to WAV bytes in memory) and hands it to + `SoundEngine` (`platform/audio/sound_engine.dart`), which loads it into + flutter_soloud (SoLoud mixing into a miniaudio output device; voices + overlap) once per slot (sound and ready index); a different preset in a + slot replaces and frees the old sound. Each play is at + `AppSettings.soundVolume` (as v^6, SoLoud's linear gain: the square curve times mpv's cubic volume the sounds used to play through, so the slider sounds as before) unless + switched off in `soundsOff`. The device opens 2 s after startup + (`warmUp`), so neither startup nor the first sound waits on it; if it + can't open, the engine logs `no sounds: audio init failed` once and drops + every play. The close path disposes the engine (deinit, 500 ms bound) + before leaving the server. Settings → Notifications & sounds has the volume, the overall sound set (`soundPreset`), and per sound a preview, its own set (`soundPresetFor`, else the overall one) and a switch; a preview always re-renders, so an edited spec plays after a hot reload. diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d2ce63f..aeefeec 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -6,7 +6,11 @@ Layout and rules: [ARCHITECTURE.md](ARCHITECTURE.md). Needs the Flutter SDK (beta channel, Dart 3.13+) and, for the Linux desktop build, the usual GTK toolchain (clang, cmake, ninja, pkg-config, libgtk-3-dev). -mpv libraries come from `media_kit_libs_video`. `fox_utils` (drag & drop, +mpv libraries come from `media_kit_libs_video`. `flutter_soloud` (the app's +sounds) compiles SoLoud and miniaudio from source in its build hook with +that C++ toolchain (MSVC on Windows); it needs no audio dev headers (miniaudio +loads ALSA / PulseAudio / JACK at run time) and, with `no_xiph_libs` set in +the desktop pubspec, downloads nothing. `fox_utils` (drag & drop, clipboard files and images; native Linux/Windows plugin) is the owner's shared package, a git dependency pinned to a tag of `git.senkochan.moe/Foxyz/fox_utils`: bump it by tagging there and changing @@ -67,6 +71,9 @@ and `e2e_check` (simulated viewers against a live server). media_kit "S/W rendering" because they have no display; that says nothing about the app. Run only tests related to what you changed. +- `sound_device_e2e` opens the real audio device (flutter_soloud) and plays + every sound of every preset quietly (`WT_SFX_VOLUME`, default 0.1); + `sound_engine_test` covers the engine with a fake backend. - CI (`.gitea/workflows/ci.yml`): analyze + test for core and server, plus `dart compile exe` of the dedicated server (fails if Flutter creeps into `packages/server`). Desktop tests are not in CI. @@ -184,7 +191,7 @@ slow to come up shows mpv's error with "no HTTP error logged". - media_kit has no public `Player.setProperty`; the code reaches mpv through a dynamic call on `player.platform`. Cache size goes through `PlayerConfiguration(bufferSize:)`. -- Hot restart with any media_kit `Player` alive (session, thumbnails, sfx) +- Hot restart with any media_kit `Player` alive (session, thumbnails) used to SIGSEGV on mpv's `core` thread: each mpv handle's wakeup callback is a `NativeCallable` of the old isolate, freed by the restart, and media_kit then sends the leftover handles `quit`, which fires it. `fox_utils`' Linux