Restructure step 5: one media pipeline on a player host

- PlayerHost port (MediaKitHost): players by role, pins, hwdec policy,
  render probe, log routing, disposal order; the only media_kit user.
- MediaPipeline: plan (one pure switch over state, origin, caps,
  presence) -> resolve -> open (prepared player promoted, else fresh)
  -> ready; commands queued during a switch apply once (P15); viewers
  play the buffer and wait for a dropped source without seek loops (D6).
- Every web page resolved by our yt-dlp into an EDL with headers and
  cookies (danmaku excluded); mpv's ytdl_hook no longer used (ytdl=no).
- D11: sign-in errors say which cookies were used and whether they hold
  any for the site.
- Relay: 256 KiB slices read as they fill, superseded pulls released
  (seek-to-play ~4.2 s -> ~2.8 s, P29); own Jellyfin relay channel only
  when someone needs it (P28).
- STARSYNC_SMOKE run and a Windows CI smoke step after the build.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
foxyzandClaude Opus 5.5 committed 2026-10-16 01:08:54 +03:00
1 parent 8970f2694b
commit 25b19f2d6a
47 files changed
+4083 -1728

No files matched your search

+1 -1
View File
@@ -9,7 +9,7 @@ abstract class RelayConstants {
static const int protoMax = 16;
// Relay (`/r/...`): a member's bytes through the server's buffer.
static const int relaySliceBytes = 1 << 20;
static const int relaySliceBytes = 256 * 1024;
static const int relayFrameBytes = 64 * 1024;
/// Defaults for the buffer; they mirror a viewer's mpv buffer (150 s /
/// 256 MiB ahead, 64 MiB back), so whatever a viewer may seek to soon is
+39 -11
View File
@@ -164,6 +164,45 @@ Map<String, List<NetscapeCookie>> cookiesBySite(Iterable<NetscapeCookie> cookies
return {for (final k in m.keys.toList()..sort()) k: m[k]!};
}
/// [parseCookies], or none for a file without any.
List<NetscapeCookie> parseCookiesOrEmpty(String text) {
try {
return parseCookies(text);
} on FormatException {
return const [];
}
}
/// "Firefox", "Chrome", "Firefox (work)": a browser and its profile.
String browserLabel(CookieBrowser b) {
final name = b.browser.isEmpty ? b.browser : '${b.browser[0].toUpperCase()}${b.browser.substring(1)}';
return b.profile.isEmpty ? name : '$name (${b.profile})';
}
/// The cookies part of a sign-in error (D11): which cookies were used and
/// how many of them are for [site] ([count] -1: unknown; [read]: what
/// reading a browser's gave).
String cookieNoteText(CookieSource? source, String site, int count, {({int count, String? error})? read}) {
switch (source) {
case null:
return 'no cookies were used';
case CookieFile():
if (count < 0) return 'used the cookies file';
return count == 0
? 'used the cookies file, which has none for $site'
: 'used the cookies file ($count for $site: signed out or expired?)';
case CookieBrowser():
final who = "${browserLabel(source)}'s cookies";
final error = read?.error;
if (error != null) return "used $who, but yt-dlp couldn't read them: $error";
final n = read?.count ?? count;
if (n < 0) return 'used $who';
return n == 0
? 'used $who, which have none for $site: sign in to $site there'
: 'used $who ($n for $site: that account may not see it)';
}
}
/// Sites ticked by default after an import: those yt-dlp is usually given
/// cookies for.
const suggestedCookieSites = {'bilibili.com', 'youtube.com'};
@@ -233,17 +272,6 @@ class YtDlpCookies {
null => const [],
};
/// The same as mpv's `ytdl-raw-options` value ('' for none); each value
/// length-quoted (`%N%`) so commas and '=' in paths survive.
String mpvRawOptions([CookieSource? source]) {
String q(String v) => '%${utf8.encode(v).length}%$v';
return switch (source ?? read(count: false)) {
CookieFile(:final path) => 'cookies=${q(path)}',
CookieBrowser b => 'cookies-from-browser=${q(b.spec)}',
null => '',
};
}
/// Keeps [cookies] as the private copy, replacing any browser setting.
Future<CookieFile> saveFile(List<NetscapeCookie> cookies) async {
await Directory(dir).create(recursive: true);
+27 -4
View File
@@ -95,6 +95,22 @@ final _unreachable = RegExp(
r'network is unreachable|no route to host|name or service not known|getaddrinfo|unable to download webpage',
caseSensitive: false);
/// yt-dlp's [error] (plain words or raw) may be about signing in: worth
/// saying which cookies were used (D11).
bool mayNeedSignIn(String error) {
final t = playErrorText(error);
return t == needsCookiesText || t == noFormatsText;
}
/// [plain] ([playErrorText]) with the cookies [note] (D11), for the two
/// texts about signing in; anything else as is.
String withCookieNote(String plain, String? note) {
if (note == null || note.isEmpty) return plain;
if (plain == needsCookiesText) return 'This site needs cookies ($note) — Settings → Player → yt-dlp → Cookies';
if (plain == noFormatsText) return 'No video found: it may need signing in ($note)';
return plain;
}
/// Why a link or media cannot be played, in plain words, from the raw
/// error of yt-dlp, ffmpeg/mpv or the link probe; anything it does not
/// know comes back as is (one line, at most 100 characters).
@@ -131,6 +147,10 @@ class LinkProbe {
final String? pageTitle;
/// Why yt-dlp could not read the page.
final String? ytDlpError;
/// Which cookies that run used ([YtDlp.cookieNote]), when it may be about
/// signing in.
final String? cookies;
/// The HTTP status of the probe's GET; 0 when unknown.
final int status;
const LinkProbe.file(this.url, this.size)
@@ -138,8 +158,9 @@ class LinkProbe {
info = null,
pageTitle = null,
ytDlpError = null,
cookies = null,
status = 0;
const LinkProbe.page(this.url, {this.info, this.pageTitle, this.ytDlpError, this.status = 0})
const LinkProbe.page(this.url, {this.info, this.pageTitle, this.ytDlpError, this.cookies, this.status = 0})
: isPage = true,
size = -1;
@@ -152,7 +173,7 @@ class LinkProbe {
// A 4xx says more than yt-dlp's words it has no plain text for.
final known = err != null && _plainTexts.contains(err);
if (!known && status >= 400 && status < 500) return playErrorText('HTTP Error $status');
return err;
return err == null ? null : withCookieNote(err, cookies);
}
/// The name to show: yt-dlp's title, the page's, else the host; null
@@ -219,12 +240,13 @@ Future<LinkProbe> _probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async
} catch (e) {
getError = e;
}
String? ytErr;
String? ytErr, cookies;
if (ytDlp != null) {
try {
return LinkProbe.page(url, info: await ytDlp.resolve(url.toString()), status: status);
} catch (e) {
ytErr = '$e';
if (e is YtDlpException) cookies = e.cookies;
}
}
if (getError != null) {
@@ -238,7 +260,8 @@ Future<LinkProbe> _probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async
body = await _readText(await req.close().timeout(const Duration(seconds: 15)));
} catch (_) {}
}
return LinkProbe.page(url, pageTitle: body == null ? null : htmlTitle(body), ytDlpError: ytErr, status: status);
return LinkProbe.page(url,
pageTitle: body == null ? null : htmlTitle(body), ytDlpError: ytErr, cookies: cookies, status: status);
} finally {
if (client == null) c.close(force: true);
}
+173 -4
View File
@@ -8,7 +8,9 @@ import 'package:crypto/crypto.dart';
import 'package:path/path.dart' as p;
import 'cookies.dart';
import 'link_probe.dart' show mayNeedSignIn;
import 'logging.dart';
import 'relay.dart' show mpvEdl;
// yt-dlp: turns a web page (YouTube, Twitter/X, ...) into media URLs. The
// app and the server each keep their own copy, the official standalone
@@ -19,7 +21,11 @@ final _log = createLogger('yt-dlp');
class YtDlpException implements Exception {
final String message;
const YtDlpException(this.message);
/// Which cookies the run used and what they hold for the site, when the
/// failure may be about signing in ([YtDlp.cookieNote], D11).
final String? cookies;
const YtDlpException(this.message, {this.cookies});
@override
String toString() => message;
}
@@ -121,6 +127,10 @@ class YtDlpInfo {
/// Every relayable format by id, to find [tracks] again after a
/// re-resolve.
final Map<String, YtDlpFormat> formats;
/// What a player opens for this page ([playFormat]); null when yt-dlp
/// picked nothing a player can open.
final YtDlpPlay? play;
const YtDlpInfo({
required this.title,
this.uploader = '',
@@ -131,6 +141,7 @@ class YtDlpInfo {
this.isLive = false,
this.tracks = const [],
this.formats = const {},
this.play,
});
/// One line for the log: extractor, duration, live, and each picked
@@ -163,6 +174,7 @@ class YtDlpInfo {
duration: j['duration'] is num ? (j['duration'] as num).toDouble() : null,
thumbnail: str('thumbnail').isEmpty ? null : str('thumbnail'),
isLive: isLive,
play: YtDlpPlay.parse(j),
tracks: isLive ? const [] : pickTracks(formats),
formats: {
for (final f in formats)
@@ -341,6 +353,100 @@ Map<String, String> parseSha256Sums(String text) => {
m.group(2)!: m.group(1)!.toLowerCase(),
};
/// The formats a player opens for a page: the best video up to 1080p with
/// the best audio, else the best single file (yt-dlp's `-f`).
const playFormat = 'bv*[height<=1080]+ba/b';
/// The subtitles a player gets: every language but Bilibili's danmaku.
const playSubLangs = 'all,-danmaku';
/// A page's subtitle track for a player (`requested_subtitles`).
class YtDlpSubtitle {
final String lang;
final String name;
final String url;
final String ext;
const YtDlpSubtitle(this.lang, this.url, {this.name = '', this.ext = ''});
}
/// What a player opens for a page, as yt-dlp resolved it ([playFormat]):
/// one file, or a video and an audio stream played together, with the
/// headers its CDN wants (the first format's `http_headers`, plus its
/// `cookies` as a Cookie header) and the subtitles ([playSubLangs]).
/// URLs are bound to the resolving machine (its address, its cookies).
class YtDlpPlay {
final List<String> urls;
final Map<String, String> headers;
final List<YtDlpSubtitle> subtitles;
const YtDlpPlay(this.urls, {this.headers = const {}, this.subtitles = const []});
/// The URL mpv opens: the file, or an EDL of the streams.
String get mpvUrl => urls.length == 1 ? urls.single : mpvEdl(urls);
/// From `yt-dlp -J`: its `requested_formats`, else the top-level format;
/// null when none of them is something a player can open.
static YtDlpPlay? parse(Map<String, Object?> j) {
final picked = j['requested_formats'] is List
? [for (final f in j['requested_formats'] as List) if (f is Map) Map<String, Object?>.from(f)]
: [if (j['url'] is String) j];
if (picked.isEmpty) return null;
final urls = <String>[];
for (final f in picked) {
final url = _playUrl(f);
if (url == null) return null;
urls.add(url);
}
final first = picked.first;
final headers = <String, String>{
if (first['http_headers'] is Map)
for (final e in (first['http_headers'] as Map).entries)
if (e.key is String && e.value is String) e.key as String: e.value as String,
};
final cookie = cookieHeaderOf(first['cookies']);
if (cookie.isNotEmpty) headers['Cookie'] = cookie;
final subs = <YtDlpSubtitle>[];
final requested = j['requested_subtitles'];
if (requested is Map) {
for (final e in requested.entries) {
final lang = '${e.key}', v = e.value;
if (lang == 'danmaku' || v is! Map || v['url'] is! String) continue;
final ext = v['ext'] is String ? v['ext'] as String : '';
if (ext == 'xml' || ext == 'json') continue;
subs.add(YtDlpSubtitle(lang, v['url'] as String,
name: v['name'] is String ? v['name'] as String : '', ext: ext));
}
}
return YtDlpPlay(urls, headers: headers, subtitles: subs);
}
static String? _playUrl(Map<String, Object?> f) {
final protocol = f['protocol'] is String ? f['protocol'] as String : 'https';
if (protocol.startsWith('http_dash')) {
final manifest = f['manifest_url'];
return manifest is String && manifest.startsWith('http') ? manifest : null;
}
final url = f['url'];
if (url is! String || !(url.startsWith('http://') || url.startsWith('https://'))) return null;
return const {'http', 'https', 'm3u8', 'm3u8_native'}.contains(protocol) ? url : null;
}
}
/// A Cookie header from yt-dlp's `cookies` field (`name=value; Domain=...;
/// Path=/; name2=value2; ...`): the name=value pairs, attributes dropped.
String cookieHeaderOf(Object? cookies) {
if (cookies is! String || cookies.trim().isEmpty) return '';
const attributes = {'domain', 'path', 'expires', 'max-age', 'secure', 'httponly', 'samesite', 'comment', 'version'};
final pairs = <String>[];
for (final part in cookies.split(';')) {
final t = part.trim();
final i = t.indexOf('=');
final name = (i < 0 ? t : t.substring(0, i)).trim();
if (i <= 0 || attributes.contains(name.toLowerCase())) continue;
pairs.add('$name=${t.substring(i + 1).trim()}');
}
return pairs.join('; ');
}
/// The last `ERROR:` line of yt-dlp's stderr without its prefixes
/// (`ERROR: [twitter] 123: No video` -> `No video`), else the last line.
String ytDlpErrorText(String stderr) {
@@ -602,7 +708,11 @@ class YtDlp {
final e = await err;
_log.warning('resolve ${logUrl(url)}: exit $code after ${sw.elapsedMilliseconds} ms; stderr tail:\n'
'${_tail(e)}');
throw YtDlpException(ytDlpErrorText(e));
final text = ytDlpErrorText(e);
// A site that hides the video from guests: say which cookies went.
final note = mayNeedSignIn(text) ? await cookieNote(url) : null;
if (note != null) _log.info('resolve ${logUrl(url)}: $note');
throw YtDlpException(text, cookies: note);
}
try {
final j = jsonDecode(await out);
@@ -616,6 +726,59 @@ class YtDlp {
}
}
/// Which cookies a run for [url] uses and what they hold for its site
/// ([cookieSite]), in plain words (D11): none; the private file and how
/// many of its cookies are for the site; a browser, and how many cookies
/// for the site yt-dlp could read from it (one run of yt-dlp that writes
/// the browser's cookies to a private file in [dir], counted and deleted
/// at once; values are never logged).
Future<String> cookieNote(String url) async {
final site = cookieSite(Uri.tryParse(url)?.host ?? '');
final source = cookies.read(count: false);
switch (source) {
case null:
return 'no cookies were used';
case CookieFile(:final path):
try {
final n = parseCookies(await File(path).readAsString()).where((c) => cookieSite(c.host) == site).length;
return cookieNoteText(source, site, n);
} catch (_) {
return cookieNoteText(source, site, -1);
}
case CookieBrowser():
return cookieNoteText(source, site, -1, read: await _browserCookies(source, site));
}
}
/// How many cookies for [site] yt-dlp reads from [browser], or why it
/// cannot.
Future<({int count, String? error})> _browserCookies(CookieBrowser browser, String site) async {
final exe = await ensure();
if (exe == null) return (count: -1, error: lastError ?? 'yt-dlp is not available');
final jar = File(p.join(dir, '.cookie-check-$pid.txt'));
try {
// Private before yt-dlp writes into it (it keeps the mode).
await jar.writeAsString('# Netscape HTTP Cookie File\n');
if (!Platform.isWindows) await Process.run('chmod', ['600', jar.path]);
// No URL: yt-dlp loads the browser's cookies, saves the jar and stops.
final r = await Process.run(
exe, ['--ignore-config', '--cookies-from-browser', browser.spec, '--cookies', jar.path])
.timeout(const Duration(seconds: 30));
final cookies = parseCookiesOrEmpty(await jar.readAsString());
final err = '${r.stderr}';
if (cookies.isEmpty && err.contains('ERROR:') && !err.contains('provide at least one URL')) {
return (count: -1, error: ytDlpErrorText(err));
}
return (count: cookies.where((c) => cookieSite(c.host) == site).length, error: null);
} on TimeoutException {
return (count: -1, error: 'yt-dlp took longer than 30 s reading them');
} catch (e) {
return (count: -1, error: '$e');
} finally {
if (await jar.exists()) await jar.delete().catchError((_) => jar);
}
}
/// [args] for the log: the cookie source named, not its path or spec.
static String _argsForLog(List<String> args) {
final out = <String>[];
@@ -640,8 +803,14 @@ class YtDlp {
}
/// The command line [resolve] runs.
List<String> resolveArgs(String url) =>
['-J', '--no-playlist', '--no-warnings', '--ignore-config', ...cookies.args(), '--', url];
List<String> resolveArgs(String url) => [
'-J', '--no-playlist', '--no-warnings', '--ignore-config',
// What a player opens ([YtDlpPlay]): yt-dlp's own pick at most
// 1080p, and the page's subtitles but Bilibili's danmaku (an XML
// comment feed, not subtitles).
'-f', playFormat, '--write-subs', '--sub-langs', playSubLangs,
...cookies.args(), '--', url,
];
Future<void> close() => _changes.close();
}
+73 -6
View File
@@ -30,26 +30,93 @@ void main() {
expect(again.map((c) => c.toLine()), cookies.map((c) => c.toLine()));
});
test('kept sites become a private file yt-dlp and mpv are pointed at', () async {
test('kept sites become a private file yt-dlp is pointed at', () async {
final dir = await Directory.systemTemp.createTemp('cookies_test');
addTearDown(() => dir.delete(recursive: true));
final src = File('${dir.path}/export.json')..writeAsStringSync(_json);
final y = YtDlp('${dir.path}/yt-dlp, a=b');
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', 'U']);
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '-f', playFormat, '--write-subs', '--sub-langs', playSubLangs, '--', 'U']);
final kept = await y.cookies.import(src.path, sites: {'bilibili.com'});
expect([kept.count, ...kept.sites], [1, 'bilibili.com']);
final path = y.cookies.filePath;
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--cookies', path, '--', 'U']);
expect(y.cookies.mpvRawOptions(), 'cookies=%${path.length}%$path');
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '-f', playFormat, '--write-subs', '--sub-langs', playSubLangs, '--cookies', path, '--', 'U']);
if (!Platform.isWindows) expect(File(path).statSync().modeString(), 'rw-------');
await y.cookies.saveBrowser('firefox', 'work');
expect(y.resolveArgs('U').sublist(4, 6), ['--cookies-from-browser', 'firefox:work']);
expect(y.resolveArgs('U').sublist(9, 11), ['--cookies-from-browser', 'firefox:work']);
expect(File(path).existsSync(), isFalse);
expect(y.cookies.mpvRawOptions(), 'cookies-from-browser=%12%firefox:work');
await y.cookies.remove();
expect(y.cookies.read(), isNull);
});
// D11: a sign-in error says which cookies were used and what they hold
// for the site.
group('cookie notes', () {
test('the texts', () {
expect(cookieNoteText(null, 'bilibili.com', -1), 'no cookies were used');
const file = CookieFile('/x', 3, ['youtube.com']);
expect(cookieNoteText(file, 'bilibili.com', 0), 'used the cookies file, which has none for bilibili.com');
expect(cookieNoteText(file, 'bilibili.com', 4),
'used the cookies file (4 for bilibili.com: signed out or expired?)');
const ff = CookieBrowser('firefox', 'work');
expect(cookieNoteText(ff, 'bilibili.com', -1, read: (count: 0, error: null)),
"used Firefox (work)'s cookies, which have none for bilibili.com: sign in to bilibili.com there");
expect(cookieNoteText(ff, 'bilibili.com', -1, read: (count: 12, error: null)),
"used Firefox (work)'s cookies (12 for bilibili.com: that account may not see it)");
expect(cookieNoteText(const CookieBrowser('chrome'), 'x.com', -1, read: (count: -1, error: 'locked')),
"used Chrome's cookies, but yt-dlp couldn't read them: locked");
expect(withCookieNote(noFormatsText, 'no cookies were used'),
'No video found: it may need signing in (no cookies were used)');
expect(withCookieNote(needsCookiesText, 'x'), 'This site needs cookies (x) — Settings → Player → yt-dlp → Cookies');
expect(withCookieNote(notFoundText, 'x'), notFoundText);
expect(mayNeedSignIn('No video formats found!'), isTrue);
expect(mayNeedSignIn('HTTP Error 404'), isFalse);
});
test("a failed resolve names the file's cookies for the site; a browser's are counted by yt-dlp",
() async {
final dir = await Directory.systemTemp.createTemp('cookie_note');
addTearDown(() => dir.delete(recursive: true));
// A stand-in yt-dlp: a version, a resolve that finds no formats, and
// the browser read (no URL) that writes the jar it is given.
final fake = File('${dir.path}/yt-dlp');
await fake.writeAsString(r'''#!/bin/sh
case "$1" in --version) echo 2099.01.01; exit 0;; esac
jar=""; prev=""
for a in "$@"; do [ "$prev" = "--cookies" ] && jar="$a"; prev="$a"; done
case "$*" in *-J*) echo "ERROR: [BiliBili] BV1: No video formats found!" >&2; exit 1;; esac
printf '# Netscape HTTP Cookie File\n.bilibili.com\tTRUE\t/\tTRUE\t2000000000\tSESSDATA\tx\n.bilibili.com\tTRUE\t/\tFALSE\t2000000000\tbuvid3\ty\n.other.org\tTRUE\t/\tFALSE\t0\ta\tb\n' > "$jar"
echo "Extracted 3 cookies from firefox" >&2
echo "yt-dlp: error: You must provide at least one URL." >&2; exit 2
''');
await Process.run('chmod', ['+x', fake.path]);
final y = YtDlp(dir.path, updateEvery: const Duration(days: 9999));
await File('${dir.path}/last-check').writeAsString(DateTime.now().toIso8601String());
const page = 'https://www.bilibili.com/video/BV1';
Future<YtDlpException> fail() async {
try {
await y.resolve(page);
} on YtDlpException catch (e) {
return e;
}
throw StateError('resolved');
}
expect((await fail()).cookies, 'no cookies were used');
await y.cookies.saveFile(parseCookies('.youtube.com\tTRUE\t/\tFALSE\t0\tSID\tv\n'));
expect((await fail()).cookies, 'used the cookies file, which has none for bilibili.com');
await y.cookies.saveBrowser('firefox');
final e = await fail();
expect(e.message, contains('No video formats found'));
expect(e.cookies, "used Firefox's cookies (2 for bilibili.com: that account may not see it)");
expect(dir.listSync().map((f) => f.path).where((p) => p.contains('cookie-check')), isEmpty,
reason: 'the browser jar is deleted at once');
// The probe carries it into the plain words.
expect(LinkProbe.page(Uri.parse(page), ytDlpError: e.message, cookies: e.cookies).problem,
"No video found: it may need signing in (used Firefox's cookies (2 for bilibili.com: that account may not see it))");
}, skip: Platform.isWindows ? 'the stand-in yt-dlp is a shell script' : false);
});
}
+42
View File
@@ -261,4 +261,46 @@ void main() {
expect(pick(['avc1.640033', 'vp09.00.40.08', 'av01.0.08M.08']), 'v2');
expect(pick(['avc1.640033', 'vp09.00.40.08']), 'v1');
});
test('what a player opens: yt-dlp\'s pick, an EDL for video + audio, headers with the cookies, subtitles but danmaku', () {
final split = YtDlpInfo.parse({
'title': 'Page',
'formats': <Object>[],
'requested_formats': [
{
'format_id': 'v', 'url': 'https://cdn/v.m4s', 'protocol': 'https',
'http_headers': {'Referer': 'https://www.bilibili.com/', 'User-Agent': 'UA'},
'cookies': 'SESSDATA=abc; Domain=.bilibili.com; Path=/; Secure; Expires=1999999999; buvid3=xyz; Domain=.bilibili.com',
},
{'format_id': 'a', 'url': 'https://cdn/a.m4s', 'protocol': 'https', 'http_headers': {'Referer': 'other'}},
],
'requested_subtitles': {
'zh-CN': {'url': 'https://subs/zh.vtt', 'ext': 'vtt', 'name': 'Chinese'},
'danmaku': {'url': 'https://comment/1.xml', 'ext': 'xml'},
},
}).play!;
expect(split.urls, ['https://cdn/v.m4s', 'https://cdn/a.m4s']);
expect(split.mpvUrl, mpvEdl(['https://cdn/v.m4s', 'https://cdn/a.m4s']));
expect(split.headers, {'Referer': 'https://www.bilibili.com/', 'User-Agent': 'UA', 'Cookie': 'SESSDATA=abc; buvid3=xyz'});
expect([for (final s in split.subtitles) '${s.lang} ${s.name} ${s.url}'], ['zh-CN Chinese https://subs/zh.vtt']);
final single = YtDlpInfo.parse({
'title': 'Tweet', 'url': 'https://video.twimg.com/v.mp4', 'protocol': 'https', 'ext': 'mp4',
'http_headers': {'User-Agent': 'UA'},
}).play!;
expect(single.mpvUrl, 'https://video.twimg.com/v.mp4');
expect(single.headers, {'User-Agent': 'UA'});
final hls = YtDlpInfo.parse({
'title': 'Live', 'is_live': true,
'requested_formats': [{'url': 'https://x/live.m3u8', 'protocol': 'm3u8_native'}],
}).play!;
expect(hls.mpvUrl, 'https://x/live.m3u8', reason: 'a live stream plays, it is only not relayed');
expect(YtDlpInfo.parse({'title': 'DASH', 'requested_formats': [{'url': 'https://x/f', 'protocol': 'http_dash_segments', 'manifest_url': 'https://x/m.mpd'}]}).play?.mpvUrl,
'https://x/m.mpd');
expect(YtDlpInfo.parse({'title': 'none', 'formats': <Object>[]}).play, isNull);
expect(YtDlpInfo.parse({'title': 'rtmp', 'url': 'rtmp://x/y', 'protocol': 'rtmp'}).play, isNull);
expect(cookieHeaderOf(null), '');
expect(cookieHeaderOf('a=1; Path=/; HttpOnly; b=2=3'), 'a=1; b=2=3');
});
}
+101 -38
View File
@@ -112,14 +112,41 @@ class ChannelRelayFeed extends RelayFeed {
class _Pull {
final int id, start, end;
int pos;
final buf = BytesBuilder(copy: false);
// The slice being filled: its chunks so far (readers take them as they
// come, see [RelayBuffer.read]).
final parts = <Uint8List>[];
int filled = 0;
_Pull(this.id, this.start, this.end) : pos = start;
/// Bytes of the slice being filled from [from] (offset in the slice).
Uint8List? partFrom(int from) {
if (from >= filled) return null;
final out = BytesBuilder(copy: false);
var at = 0;
for (final p in parts) {
final end = at + p.length;
if (end > from) out.add(at >= from ? p : Uint8List.sublistView(p, from - at));
at = end;
}
return out.takeBytes();
}
Uint8List takeSlice() {
final out = BytesBuilder(copy: false);
for (final p in parts) {
out.add(p);
}
parts.clear();
filled = 0;
return out.takeBytes();
}
}
class _Viewer {
int pos = 0;
int open = 0; // requests in flight
int newest = 0; // sequence of its newest request
(int slice, int req)? blocking; // where its newest request waits
DateTime lastActive = DateTime.now();
}
@@ -134,9 +161,11 @@ class _Viewer {
/// slowest and [ahead] past the furthest viewer. Evicted by distance from
/// the viewers, never LRU.
/// - Filled only while someone watches: a viewer blocked on a slice comes
/// first (oldest wait first); then read-ahead.
/// first (oldest wait first); then read-ahead. A blocked viewer reads the
/// slice being filled as its bytes arrive, not once it is whole.
/// - Only a viewer's newest request moves its position or asks the feed
/// for anything; older ones still get bytes once they are buffered.
/// for anything; older ones still get bytes once they are buffered, and
/// stop holding the feed once superseded (a seek elsewhere).
/// Everything lives in memory: dropping the buffer is the cleanup.
class RelayBuffer implements TrackBuffer {
/// Set by the relay when a media's tracks share the budget.
@@ -166,6 +195,8 @@ class RelayBuffer implements TrackBuffer {
final _slices = <int, Uint8List>{};
final _waiters = <int, List<Completer<void>>>{};
final _blocked = <int, int>{}; // slice -> wait start (us)
// The driving requests waiting on each blocked slice.
final _blockedBy = <int, Set<int>>{};
final _viewers = <String, _Viewer>{};
@override
FeedClock clock = FeedClock();
@@ -256,18 +287,19 @@ class RelayBuffer implements TrackBuffer {
_endPull(pl);
return;
}
pl.buf.add(Uint8List.sublistView(data, off, off + take));
pl.parts.add(Uint8List.sublistView(data, off, off + take));
pl.filled += take;
off += take;
pl.pos += take;
if (pl.pos == _sliceEnd(s) + 1) {
_slices[s] = pl.buf.takeBytes();
for (final c in _waiters.remove(s) ?? const <Completer<void>>[]) {
c.complete();
}
if (_shouldStop(pl, s)) {
_endPull(pl);
return;
}
final done = pl.pos == _sliceEnd(s) + 1;
if (done) _slices[s] = pl.takeSlice();
// Whole, or more of it: whoever waits on it reads on.
for (final c in _waiters.remove(s) ?? const <Completer<void>>[]) {
c.complete();
}
if (done && _shouldStop(pl, s)) {
_endPull(pl);
return;
}
}
}
@@ -405,25 +437,54 @@ class RelayBuffer implements TrackBuffer {
await _kick.future.timeout(const Duration(seconds: 5), onTimeout: () {});
}
Future<bool> _waitSlice(int s, bool drives) async {
/// The bytes of slice [s] from [from] (offset in the slice) as soon as
/// there are any: the whole slice's rest, or what the pull filling it has
/// so far. Null when they did not come within [waitLimit]. [drives]: the
/// request is its viewer's newest ([driving] says while it still is), so
/// it may move the feed.
Future<Uint8List?> _waitBytes(_Viewer v, int s, int from, int req, bool Function() driving) async {
final deadline = DateTime.now().add(waitLimit);
while (!_slices.containsKey(s)) {
if (_disposed || DateTime.now().isAfter(deadline)) return false;
if (drives) {
_blocked.putIfAbsent(s, () => DateTime.now().microsecondsSinceEpoch);
// A pull nobody waits on gives way at once; one serving another
// blocked viewer hands over at its next slice boundary.
try {
while (true) {
final whole = _slices[s];
if (whole != null) return from == 0 ? whole : Uint8List.sublistView(whole, from);
final pl = _pull;
final at = pl == null ? -1 : pl.pos ~/ slice;
if (pl != null && at != s && !_blocked.containsKey(at)) _endPull(pl);
_wake();
if (pl != null && pl.pos ~/ slice == s && pl.start <= s * slice) {
final part = pl.partFrom(from);
if (part != null) return part;
}
if (_disposed || DateTime.now().isAfter(deadline)) return null;
if (driving()) {
_blocked.putIfAbsent(s, () => DateTime.now().microsecondsSinceEpoch);
_blockedBy.putIfAbsent(s, () => {}).add(req);
v.blocking = (s, req);
// A pull nobody waits on gives way at once; one serving another
// blocked viewer hands over at its next slice boundary.
final at = pl == null ? -1 : pl.pos ~/ slice;
if (pl != null && at != s && !_blocked.containsKey(at)) _endPull(pl);
_wake();
} else {
_unblock(s, req);
}
final c = Completer<void>();
_waiters.putIfAbsent(s, () => []).add(c);
await c.future.timeout(const Duration(seconds: 2), onTimeout: () {});
}
final c = Completer<void>();
_waiters.putIfAbsent(s, () => []).add(c);
await c.future.timeout(const Duration(seconds: 2), onTimeout: () {});
} finally {
_unblock(s, req);
}
}
// [req] no longer waits on slice [s]; the slice stops counting as
// blocked once nobody driving does.
void _unblock(int s, int req) {
final by = _blockedBy[s];
if (by == null || !by.remove(req)) return;
if (by.isEmpty) {
_blockedBy.remove(s);
_blocked.remove(s);
_wake();
}
_blocked.remove(s);
return true;
}
// ---------------------------------------------------------------- viewers
@@ -444,6 +505,10 @@ class RelayBuffer implements TrackBuffer {
final tail = start >= size - pin;
final me = ++_reqSeq;
if (!tail) {
// A seek: the request it supersedes stops holding the feed at once.
final was = v.blocking;
if (was != null) _unblock(was.$1, was.$2);
v.blocking = null;
v.newest = me;
v.pos = start;
}
@@ -454,16 +519,14 @@ class RelayBuffer implements TrackBuffer {
var off = start;
while (off <= end) {
final s = off ~/ slice;
if (!await _waitSlice(s, tail || v.newest == me)) {
throw StateError('relay: bytes at $off did not arrive');
}
final data = _slices[s];
if (data == null) continue; // evicted between wake and read
final from = off - s * slice, to = min(data.length, end - s * slice + 1);
yield Uint8List.sublistView(data, from, to);
bytesToViewers += to - from;
onSent?.call(to - from);
off += to - from;
final data = await _waitBytes(v, s, off - s * slice, me, () => tail || v.newest == me);
if (data == null) throw StateError('relay: bytes at $off did not arrive');
final n = min(data.length, end - off + 1);
if (n <= 0) continue;
yield n == data.length ? data : Uint8List.sublistView(data, 0, n);
bytesToViewers += n;
onSent?.call(n);
off += n;
v.lastActive = DateTime.now();
if (!tail && v.newest == me) v.pos = off;
_wake();