From ad53b6822a23967ec6a298c01c567b83f7a150b4 Mon Sep 17 00:00:00 2001 From: foxyz Date: Sun, 18 Oct 2026 17:29:37 +0300 Subject: [PATCH] Shaders: Anime4K at full precision on OpenGL ES; sharp video renders the fitted picture Colours and grid: media_kit renders through OpenGL ES, where mpv's mediump samplers made Anime4K's networks compute at half precision (fp16 on NVIDIA); the VL networks of HQ overflow and round there, strongest once the upscalers run at screen size. The extracted copies route every network read through a highp identity function; on GLES the result now equals desktop OpenGL's. Resize: the output is the picture fitted in the view (no bars, centred on whole pixels), so Flutter scales the last texture to the new view without the shrink-then-jump; the resize wait is 120 ms. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + .../src/platform/desktop/media_kit_host.dart | 5 +- .../lib/src/platform/ports/player_host.dart | 32 +++++++-- .../lib/src/shaders/shader_assets.dart | 66 ++++++++++++++++++- apps/desktop/lib/src/shell/video_view.dart | 5 +- apps/desktop/test/shaders_test.dart | 37 ++++++++++- apps/desktop/test/video_view_test.dart | 23 ++++++- docs/ARCHITECTURE.md | 36 +++++++--- 8 files changed, 184 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e9fd6e7..6dc947dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,8 @@ Links you put up are downloaded once for everyone, quick chat and danmaku in the ### Fixed +- Upscaling shaders (Anime4K, the HQ modes) no longer shift the colours or draw a grid over the picture, which showed most in a large window or fullscreen +- Sharp video: resizing the window or opening the chat panel no longer shrinks the picture for a moment before it jumps back - Joining while something plays on Windows: the video opens again by itself when it was too early - Relay: players no longer get stuck after a slow or stalled source - Relay: X (Twitter) clips play through the relay diff --git a/apps/desktop/lib/src/platform/desktop/media_kit_host.dart b/apps/desktop/lib/src/platform/desktop/media_kit_host.dart index 99358f7f..122483e0 100644 --- a/apps/desktop/lib/src/platform/desktop/media_kit_host.dart +++ b/apps/desktop/lib/src/platform/desktop/media_kit_host.dart @@ -569,7 +569,10 @@ class _MkVideo implements HostedVideo { // NativeVideoController (Linux, Windows): not exported, reached // through its public fields. final dynamic native = await controller.platform.future; - final want = player.host._outputSize; + final view = player.host._outputSize; + final params = player.mk.state.videoParams; + final dw = params.dw ?? 0, dh = params.dh ?? 0; + final want = view == null ? null : fitOutputSize(view, dw > 0 && dh > 0 ? (dw, dh) : null); if (_sized && want == _sent && !again) return; if (want == null) { if (!_sized) return; // never resized: already the video's size diff --git a/apps/desktop/lib/src/platform/ports/player_host.dart b/apps/desktop/lib/src/platform/ports/player_host.dart index 7333c72c..1f936d52 100644 --- a/apps/desktop/lib/src/platform/ports/player_host.dart +++ b/apps/desktop/lib/src/platform/ports/player_host.dart @@ -148,10 +148,11 @@ abstract interface class PlayerHost { /// start). HostedVideo video(HostedPlayer player, {bool probe = false}); - /// The pixel size every video output renders at: the view's, so mpv - /// scales the picture and draws subtitles at screen resolution - /// (Settings → Player, "Sharp video"). Null: each output at its video's - /// own size. Applies to live outputs and to new ones once they are up. + /// The view's pixel size: every video output renders the picture fitted + /// in it ([fitOutputSize]), so mpv scales the picture and draws + /// subtitles at screen resolution (Settings → Player, "Sharp video"). + /// Null: each output at its video's own size. Applies to live outputs + /// and to new ones once they are up. void setOutputSize(int? width, int? height); /// How video renders, as last learned (kept after leaving). @@ -203,3 +204,26 @@ bool mpvLoadFailed(PlayerLogLine log, Duration duration) { return duration == Duration.zero && ((log.prefix == 'cplayer' && log.level == 'fatal') || (log.prefix == 'demux' && log.level == 'error')); } + +/// The size a video output renders at for a [view] of that many pixels: +/// the [video]'s picture (its display size, aspect included) fitted +/// inside, no letterbox. The texture then has the picture's aspect, so +/// while a resize settles Flutter scales the last one to the new view +/// with no jump (a texture of the whole view would shrink inside its own +/// black bars until the new one came). Drawn 1:1: the fitted side is the +/// view's, the other is rounded down to the view's parity, so the +/// picture sits on whole pixels when centred (a half-pixel offset would +/// blur every line). Unknown video: the view. +(int, int) fitOutputSize((int, int) view, (int, int)? video) { + final (vw, vh) = view; + if (video == null || video.$1 <= 0 || video.$2 <= 0) return view; + final aspect = video.$1 / video.$2; + int under(double x, int full) { + var n = x.floor().clamp(1, full); + if ((full - n).isOdd) n = n > 1 ? n - 1 : n; + return n; + } + + if (vw / vh > aspect) return (under(vh * aspect, vw), vh); // pillarbox + return (vw, under(vw / aspect, vh)); // letterbox (or exact) +} diff --git a/apps/desktop/lib/src/shaders/shader_assets.dart b/apps/desktop/lib/src/shaders/shader_assets.dart index 332ec335..f05b81ed 100644 --- a/apps/desktop/lib/src/shaders/shader_assets.dart +++ b/apps/desktop/lib/src/shaders/shader_assets.dart @@ -1,3 +1,4 @@ +import 'dart:convert'; import 'dart:io'; import 'package:flutter/services.dart'; @@ -7,8 +8,13 @@ import 'shader_presets.dart'; const anime4kAssetDir = 'assets/shaders/anime4k'; +/// The extracted copy's layout: bumped when [highpAnime4k] changes what +/// is written, so an older copy is written again. +const anime4kCopyRevision = 'hp1'; + /// mpv needs real files: copies the shipped Anime4K shaders out of the -/// asset bundle into `/shaders/anime4k-`, once per +/// asset bundle into `/shaders/anime4k--`, +/// each made full precision on OpenGL ES ([highpAnime4k]), once per /// version (a `.complete` marker is written last, so a cut-short copy is /// redone). Returns that directory. Future extractAnime4k( @@ -17,7 +23,7 @@ Future extractAnime4k( }) async { final dir = Directory( '${supportDir.path}${Platform.pathSeparator}shaders' - '${Platform.pathSeparator}anime4k-$anime4kVersion'); + '${Platform.pathSeparator}anime4k-$anime4kVersion-$anime4kCopyRevision'); final marker = File('${dir.path}${Platform.pathSeparator}.complete'); final files = anime4kAllFiles; if (await marker.exists() && @@ -28,14 +34,68 @@ Future extractAnime4k( final b = bundle ?? rootBundle; for (final name in [...files, 'LICENSE']) { final data = await b.load('$anime4kAssetDir/$name'); + final bytes = data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes); await File(_join(dir, name)).writeAsBytes( - data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes), + name.endsWith('.glsl') ? utf8.encode(highpAnime4k(utf8.decode(bytes))) : bytes, flush: true); } await marker.writeAsString(anime4kVersion, flush: true); + // Older copies (another version or revision) are not read again. + try { + await for (final e in dir.parent.list()) { + final name = e.uri.pathSegments.where((s) => s.isNotEmpty).last; + if (e is Directory && name.startsWith('anime4k-') && e.path != dir.path) { + await e.delete(recursive: true); + } + } + } catch (_) {} return dir; } +/// The name of the helper [highpAnime4k] adds to each pass. +const _hp = 'starsync_hp'; + +/// [glsl] (an Anime4K file) with its convolutions computed at full +/// precision on an OpenGL ES context, as they are on desktop OpenGL. +/// +/// media_kit renders through OpenGL ES (EGL on Linux, ANGLE on Windows), +/// where mpv declares `precision mediump sampler2D`: a texture read is +/// mediump, and Anime4K's weights are bare literals, so every +/// `mat4(...) * go_0(...)` of a network ran at half precision (fp16 on +/// NVIDIA). The large (VL) networks overflow and round there: shifted +/// colours, posterised areas and a regular grid, worse once the +/// upscalers run at screen size. Each pass gets an identity function +/// with a highp parameter and result, and every `go_N` / `g_N` read goes +/// through it, so the products and sums that follow are highp. Desktop +/// GLSL ignores precision qualifiers, so nothing changes there. Anything +/// that is not such a read is left as it is. +String highpAnime4k(String glsl) { + final read = RegExp(r'\b\w+_tex(?:Off)?\((?:[^()]|\([^()]*\))*\)'); + final define = RegExp(r'^#define g(?:o)?_\d+\b'); + final out = []; + var inHeader = false, helper = false; + for (final line in const LineSplitter().convert(glsl)) { + if (line.startsWith('//!')) { + // A new pass starts at the first `//!` line after code. + if (!inHeader) helper = false; + inHeader = true; + out.add(line); + continue; + } + inHeader = false; + if (define.hasMatch(line)) { + if (!helper) { + out.add('highp vec4 $_hp(highp vec4 v) { return v; }'); + helper = true; + } + out.add(line.replaceAllMapped(read, (m) => '$_hp(${m[0]})')); + continue; + } + out.add(line); + } + return '${out.join('\n')}\n'; +} + /// Absolute paths of [preset]'s Anime4K files inside [dir]. List anime4kPaths( Directory dir, ShaderPreset preset, ShaderQuality quality) => diff --git a/apps/desktop/lib/src/shell/video_view.dart b/apps/desktop/lib/src/shell/video_view.dart index 67d042f0..073c01d9 100644 --- a/apps/desktop/lib/src/shell/video_view.dart +++ b/apps/desktop/lib/src/shell/video_view.dart @@ -12,7 +12,8 @@ import '../playback/playback_controller.dart'; // bottom subtitles move up while something covers the bottom of the // picture (the quick chat box; the control bar when chosen). -/// The video output's size for a view of [logical] size at [dpr]: its +/// The view size the host fits each video output into ([fitOutputSize]) +/// for a view of [logical] size at [dpr]: its /// physical pixels when [on] and video renders on the GPU, else null (the /// video's own size). Software rendering keeps the video's size: every /// pixel there is CPU work. @@ -26,7 +27,7 @@ import '../playback/playback_controller.dart'; /// How long a view size must hold before the output follows it (a /// window drag resizes many times a second; each resize reallocates the /// output's texture). -const outputResizeDelay = Duration(milliseconds: 180); +const outputResizeDelay = Duration(milliseconds: 120); /// Sends the size of the view around [child] to the player host (see /// [videoOutputSize]): at once the first time, then once a resize has diff --git a/apps/desktop/test/shaders_test.dart b/apps/desktop/test/shaders_test.dart index 78d8f0c1..86a83781 100644 --- a/apps/desktop/test/shaders_test.dart +++ b/apps/desktop/test/shaders_test.dart @@ -103,10 +103,11 @@ void main() { test('copies the bundle once per version, redoes a broken copy', () async { final support = _fresh('shaders_support'); final dir = await extractAnime4k(support); - expect(dir.path, endsWith('shaders/anime4k-$anime4kVersion')); + expect(dir.path, endsWith('shaders/anime4k-$anime4kVersion-$anime4kCopyRevision')); for (final f in [...anime4kAllFiles, 'LICENSE']) { - expect(File('${dir.path}/$f').readAsBytesSync(), - File('$anime4kAssetDir/$f').readAsBytesSync(), + final shipped = File('$anime4kAssetDir/$f'); + expect(File('${dir.path}/$f').readAsStringSync(), + f.endsWith('.glsl') ? highpAnime4k(shipped.readAsStringSync()) : shipped.readAsStringSync(), reason: f); } // Second run is a no-op: a marker-tagged copy stays untouched. @@ -132,6 +133,36 @@ void main() { }); }); + test('Anime4K reads go through a highp helper, once per pass', () { + const src = '''// header +//!DESC pass 1 +//!HOOK MAIN +//!BIND MAIN +#define go_0(x_off, y_off) (MAIN_texOff(vec2(x_off, y_off))) +vec4 hook() { return mat4(1.0) * go_0(0.0, 0.0); } +//!DESC pass 2 +//!HOOK MAIN +//!BIND conv2d_tf +#define go_0(x_off, y_off) (max((conv2d_tf_texOff(vec2(x_off, y_off))), 0.0)) +#define go_1(x_off, y_off) (max(-(conv2d_tf_texOff(vec2(x_off, y_off))), 0.0)) +#define g_0 (max((conv2d_tf_tex(conv2d_tf_pos)), 0.0)) +vec4 hook() { return go_0(0.0, 0.0) + go_1(0.0, 0.0) + g_0 + MAIN_tex(MAIN_pos); } +'''; + final out = highpAnime4k(src); + expect('highp vec4 starsync_hp('.allMatches(out), hasLength(2), reason: 'one helper per pass'); + expect(out, contains('#define go_0(x_off, y_off) (starsync_hp(MAIN_texOff(vec2(x_off, y_off))))')); + expect(out, contains('(max(-(starsync_hp(conv2d_tf_texOff(vec2(x_off, y_off)))), 0.0))')); + expect(out, contains('#define g_0 (max((starsync_hp(conv2d_tf_tex(conv2d_tf_pos))), 0.0))')); + expect(out, contains('+ g_0 + MAIN_tex(MAIN_pos); }'), reason: 'other code stays as it is'); + // Every shipped file: every network read wrapped, the helper before its first use. + for (final f in anime4kAllFiles) { + final o = highpAnime4k(File('$anime4kAssetDir/$f').readAsStringSync()); + for (final line in o.split('\n').where((l) => RegExp(r'^#define g(o)?_\d').hasMatch(l))) { + expect(line, contains('starsync_hp('), reason: '$f: $line'); + } + } + }); + test('shader settings persist across a reload', () async { Hive.init(_fresh('shaders_hive').path); final a = AppSettings(); diff --git a/apps/desktop/test/video_view_test.dart b/apps/desktop/test/video_view_test.dart index 732b4ca3..89b7c2db 100644 --- a/apps/desktop/test/video_view_test.dart +++ b/apps/desktop/test/video_view_test.dart @@ -1,6 +1,6 @@ import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:starsync/src/platform/ports/player_host.dart' show RenderMode; +import 'package:starsync/src/platform/ports/player_host.dart' show RenderMode, fitOutputSize; import 'package:starsync/src/screens/home/ui.dart' show devToolsFor; import 'package:starsync/src/shell/video_view.dart'; @@ -23,6 +23,27 @@ void main() { }); }); + group('fitOutputSize', () { + test('the picture fitted in the view, no bars', () { + expect(fitOutputSize((1920, 1080), (1280, 720)), (1920, 1080)); + // 16:9 on a 2560x1080 screen: pillarboxed, full height. + expect(fitOutputSize((2560, 1080), (1280, 720)), (1920, 1080)); + // The chat panel narrows the view: letterboxed, full width. + expect(fitOutputSize((880, 720), (1280, 720)), (880, 494)); + expect(fitOutputSize((1280, 720), null), (1280, 720)); + expect(fitOutputSize((1280, 720), (0, 0)), (1280, 720)); + }); + + test('the bars split evenly: the picture sits on whole pixels', () { + for (final (view, video) in [((881, 720), (1280, 720)), ((1000, 999), (1920, 800)), ((1366, 768), (720, 576))]) { + final (w, h) = fitOutputSize(view, video); + expect(w <= view.$1 && h <= view.$2, isTrue); + expect((view.$1 - w).isEven && (view.$2 - h).isEven, isTrue, reason: '$view $video -> ($w, $h)'); + expect(w == view.$1 || h == view.$2, isTrue, reason: 'one side fills the view'); + } + }); + }); + group('subtitleLiftPos', () { test('a picture filling the view lifts by the controls', () { // 108 px of controls + 8 px gap in a 1080 px picture: 10.74 %. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 17d56c82..9179f459 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -575,8 +575,20 @@ controls' ⚙ → Upscaling shader menu, Ctrl+0..6, both also save it). The Anim v4.0.1 files live in `apps/desktop/assets/shaders/anime4k/` (only the 14 the presets use, plus its MIT LICENSE); mpv needs real paths, so `extractAnime4k` copies them once per version to -`/shaders/anime4k-/` (a `.complete` marker is written -last). `anime4kFiles` maps each mode to the file list of Anime4K's mpv +`/shaders/anime4k--/` (a `.complete` +marker is written last; other `anime4k-*` copies are deleted), each +rewritten by `highpAnime4k`. media_kit renders through OpenGL ES (EGL +on Linux, ANGLE on Windows), where mpv declares `precision mediump +sampler2D`: a texture read is mediump and Anime4K's weights are bare +literals, so a network's `mat4(...) * go_0(...)` sums ran at half +precision (fp16 on NVIDIA). The VL networks of HQ overflow and round +there: shifted, posterised colours and a regular grid, strongest once +the upscalers run (output over 1.2x the video, so mostly with "Sharp +video" at a large size). The rewrite adds a highp identity function to +each pass and routes every `go_N` / `g_N` read through it; the result on +OpenGL ES then equals desktop OpenGL's (an mpv CLI frame grab, +`--opengl-es=yes` vs `no`, identical). Desktop GLSL ignores the +qualifiers. Custom shaders are passed as they are. `anime4kFiles` maps each mode to the file list of Anime4K's mpv templates: HQ = the high-end GPU set (VL + M networks), Fast = the low-end set (M + S). A missing file, a path containing the list separator, a read-back mismatch, or an mpv `libmpv_render` error while shaders are on @@ -608,12 +620,20 @@ a 720p image for a 720p file. With the setting on and the GPU renderer work, nor before the probe knows), the session view (`shell/video_view.dart`, `VideoOutputSizer` around the `Video`) sends its size in physical pixels to the host (`PlayerHost.setOutputSize`): at -once the first time, then once a resize has held 180 ms -(`outputResizeDelay`: a window drag, fullscreen, full ↔ mini). The host -applies it to every live output and to each new one once it is up -(`NativeVideoController.setSize`); mpv then letterboxes inside it and -draws subtitles at that size (text subtitles may sit in the black bars, -as in mpv). media_kit's controller sets an output back to the video's +once the first time, then once a resize has held 120 ms +(`outputResizeDelay`: a window drag, the chat panel, fullscreen, full ↔ +mini). The host renders each output at the picture fitted in that view +(`fitOutputSize` over the video's display size, re-fitted when it +changes), not the whole view: the texture has the picture's aspect, so +while a resize settles Flutter scales the last texture to the new view +without a jump (a texture of the whole view, bars included, shrank +inside its own bars until the new one came, about 0.2 s later), and the +shaders work on no bar pixels. The side that fills the view is exact and +the other is rounded down to the view's parity, so the centred picture +sits on whole pixels and is drawn 1:1. It applies to every live output +and to each new one once it is up (`NativeVideoController.setSize`, +which only changes the size the next frame is drawn at); subtitles are +drawn at that size, inside the picture. media_kit's controller sets an output back to the video's size whenever the video's parameters change (each new file), so the host sends the size again after it, under the controller's own lock; a fresh player's first frame or two show at the video's size. A paused player