Restructure step 5: one media pipeline on a player host
- PlayerHost port (MediaKitHost): players by role, pins, hwdec policy, render probe, log routing, disposal order; the only media_kit user. - MediaPipeline: plan (one pure switch over state, origin, caps, presence) -> resolve -> open (prepared player promoted, else fresh) -> ready; commands queued during a switch apply once (P15); viewers play the buffer and wait for a dropped source without seek loops (D6). - Every web page resolved by our yt-dlp into an EDL with headers and cookies (danmaku excluded); mpv's ytdl_hook no longer used (ytdl=no). - D11: sign-in errors say which cookies were used and whether they hold any for the site. - Relay: 256 KiB slices read as they fill, superseded pulls released (seek-to-play ~4.2 s -> ~2.8 s, P29); own Jellyfin relay channel only when someone needs it (P28). - STARSYNC_SMOKE run and a Windows CI smoke step after the build. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8970f2694b
commit
25b19f2d6a
47 files changed
+4083
-1728
No files matched your search
@@ -9,7 +9,7 @@ abstract class RelayConstants {
|
||||
static const int protoMax = 16;
|
||||
|
||||
// Relay (`/r/...`): a member's bytes through the server's buffer.
|
||||
static const int relaySliceBytes = 1 << 20;
|
||||
static const int relaySliceBytes = 256 * 1024;
|
||||
static const int relayFrameBytes = 64 * 1024;
|
||||
/// Defaults for the buffer; they mirror a viewer's mpv buffer (150 s /
|
||||
/// 256 MiB ahead, 64 MiB back), so whatever a viewer may seek to soon is
|
||||
|
||||
@@ -164,6 +164,45 @@ Map<String, List<NetscapeCookie>> cookiesBySite(Iterable<NetscapeCookie> cookies
|
||||
return {for (final k in m.keys.toList()..sort()) k: m[k]!};
|
||||
}
|
||||
|
||||
/// [parseCookies], or none for a file without any.
|
||||
List<NetscapeCookie> parseCookiesOrEmpty(String text) {
|
||||
try {
|
||||
return parseCookies(text);
|
||||
} on FormatException {
|
||||
return const [];
|
||||
}
|
||||
}
|
||||
|
||||
/// "Firefox", "Chrome", "Firefox (work)": a browser and its profile.
|
||||
String browserLabel(CookieBrowser b) {
|
||||
final name = b.browser.isEmpty ? b.browser : '${b.browser[0].toUpperCase()}${b.browser.substring(1)}';
|
||||
return b.profile.isEmpty ? name : '$name (${b.profile})';
|
||||
}
|
||||
|
||||
/// The cookies part of a sign-in error (D11): which cookies were used and
|
||||
/// how many of them are for [site] ([count] -1: unknown; [read]: what
|
||||
/// reading a browser's gave).
|
||||
String cookieNoteText(CookieSource? source, String site, int count, {({int count, String? error})? read}) {
|
||||
switch (source) {
|
||||
case null:
|
||||
return 'no cookies were used';
|
||||
case CookieFile():
|
||||
if (count < 0) return 'used the cookies file';
|
||||
return count == 0
|
||||
? 'used the cookies file, which has none for $site'
|
||||
: 'used the cookies file ($count for $site: signed out or expired?)';
|
||||
case CookieBrowser():
|
||||
final who = "${browserLabel(source)}'s cookies";
|
||||
final error = read?.error;
|
||||
if (error != null) return "used $who, but yt-dlp couldn't read them: $error";
|
||||
final n = read?.count ?? count;
|
||||
if (n < 0) return 'used $who';
|
||||
return n == 0
|
||||
? 'used $who, which have none for $site: sign in to $site there'
|
||||
: 'used $who ($n for $site: that account may not see it)';
|
||||
}
|
||||
}
|
||||
|
||||
/// Sites ticked by default after an import: those yt-dlp is usually given
|
||||
/// cookies for.
|
||||
const suggestedCookieSites = {'bilibili.com', 'youtube.com'};
|
||||
@@ -233,17 +272,6 @@ class YtDlpCookies {
|
||||
null => const [],
|
||||
};
|
||||
|
||||
/// The same as mpv's `ytdl-raw-options` value ('' for none); each value
|
||||
/// length-quoted (`%N%`) so commas and '=' in paths survive.
|
||||
String mpvRawOptions([CookieSource? source]) {
|
||||
String q(String v) => '%${utf8.encode(v).length}%$v';
|
||||
return switch (source ?? read(count: false)) {
|
||||
CookieFile(:final path) => 'cookies=${q(path)}',
|
||||
CookieBrowser b => 'cookies-from-browser=${q(b.spec)}',
|
||||
null => '',
|
||||
};
|
||||
}
|
||||
|
||||
/// Keeps [cookies] as the private copy, replacing any browser setting.
|
||||
Future<CookieFile> saveFile(List<NetscapeCookie> cookies) async {
|
||||
await Directory(dir).create(recursive: true);
|
||||
|
||||
@@ -95,6 +95,22 @@ final _unreachable = RegExp(
|
||||
r'network is unreachable|no route to host|name or service not known|getaddrinfo|unable to download webpage',
|
||||
caseSensitive: false);
|
||||
|
||||
/// yt-dlp's [error] (plain words or raw) may be about signing in: worth
|
||||
/// saying which cookies were used (D11).
|
||||
bool mayNeedSignIn(String error) {
|
||||
final t = playErrorText(error);
|
||||
return t == needsCookiesText || t == noFormatsText;
|
||||
}
|
||||
|
||||
/// [plain] ([playErrorText]) with the cookies [note] (D11), for the two
|
||||
/// texts about signing in; anything else as is.
|
||||
String withCookieNote(String plain, String? note) {
|
||||
if (note == null || note.isEmpty) return plain;
|
||||
if (plain == needsCookiesText) return 'This site needs cookies ($note) — Settings → Player → yt-dlp → Cookies';
|
||||
if (plain == noFormatsText) return 'No video found: it may need signing in ($note)';
|
||||
return plain;
|
||||
}
|
||||
|
||||
/// Why a link or media cannot be played, in plain words, from the raw
|
||||
/// error of yt-dlp, ffmpeg/mpv or the link probe; anything it does not
|
||||
/// know comes back as is (one line, at most 100 characters).
|
||||
@@ -131,6 +147,10 @@ class LinkProbe {
|
||||
final String? pageTitle;
|
||||
/// Why yt-dlp could not read the page.
|
||||
final String? ytDlpError;
|
||||
|
||||
/// Which cookies that run used ([YtDlp.cookieNote]), when it may be about
|
||||
/// signing in.
|
||||
final String? cookies;
|
||||
/// The HTTP status of the probe's GET; 0 when unknown.
|
||||
final int status;
|
||||
const LinkProbe.file(this.url, this.size)
|
||||
@@ -138,8 +158,9 @@ class LinkProbe {
|
||||
info = null,
|
||||
pageTitle = null,
|
||||
ytDlpError = null,
|
||||
cookies = null,
|
||||
status = 0;
|
||||
const LinkProbe.page(this.url, {this.info, this.pageTitle, this.ytDlpError, this.status = 0})
|
||||
const LinkProbe.page(this.url, {this.info, this.pageTitle, this.ytDlpError, this.cookies, this.status = 0})
|
||||
: isPage = true,
|
||||
size = -1;
|
||||
|
||||
@@ -152,7 +173,7 @@ class LinkProbe {
|
||||
// A 4xx says more than yt-dlp's words it has no plain text for.
|
||||
final known = err != null && _plainTexts.contains(err);
|
||||
if (!known && status >= 400 && status < 500) return playErrorText('HTTP Error $status');
|
||||
return err;
|
||||
return err == null ? null : withCookieNote(err, cookies);
|
||||
}
|
||||
|
||||
/// The name to show: yt-dlp's title, the page's, else the host; null
|
||||
@@ -219,12 +240,13 @@ Future<LinkProbe> _probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async
|
||||
} catch (e) {
|
||||
getError = e;
|
||||
}
|
||||
String? ytErr;
|
||||
String? ytErr, cookies;
|
||||
if (ytDlp != null) {
|
||||
try {
|
||||
return LinkProbe.page(url, info: await ytDlp.resolve(url.toString()), status: status);
|
||||
} catch (e) {
|
||||
ytErr = '$e';
|
||||
if (e is YtDlpException) cookies = e.cookies;
|
||||
}
|
||||
}
|
||||
if (getError != null) {
|
||||
@@ -238,7 +260,8 @@ Future<LinkProbe> _probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async
|
||||
body = await _readText(await req.close().timeout(const Duration(seconds: 15)));
|
||||
} catch (_) {}
|
||||
}
|
||||
return LinkProbe.page(url, pageTitle: body == null ? null : htmlTitle(body), ytDlpError: ytErr, status: status);
|
||||
return LinkProbe.page(url,
|
||||
pageTitle: body == null ? null : htmlTitle(body), ytDlpError: ytErr, cookies: cookies, status: status);
|
||||
} finally {
|
||||
if (client == null) c.close(force: true);
|
||||
}
|
||||
|
||||
@@ -8,7 +8,9 @@ import 'package:crypto/crypto.dart';
|
||||
import 'package:path/path.dart' as p;
|
||||
|
||||
import 'cookies.dart';
|
||||
import 'link_probe.dart' show mayNeedSignIn;
|
||||
import 'logging.dart';
|
||||
import 'relay.dart' show mpvEdl;
|
||||
|
||||
// yt-dlp: turns a web page (YouTube, Twitter/X, ...) into media URLs. The
|
||||
// app and the server each keep their own copy, the official standalone
|
||||
@@ -19,7 +21,11 @@ final _log = createLogger('yt-dlp');
|
||||
|
||||
class YtDlpException implements Exception {
|
||||
final String message;
|
||||
const YtDlpException(this.message);
|
||||
|
||||
/// Which cookies the run used and what they hold for the site, when the
|
||||
/// failure may be about signing in ([YtDlp.cookieNote], D11).
|
||||
final String? cookies;
|
||||
const YtDlpException(this.message, {this.cookies});
|
||||
@override
|
||||
String toString() => message;
|
||||
}
|
||||
@@ -121,6 +127,10 @@ class YtDlpInfo {
|
||||
/// Every relayable format by id, to find [tracks] again after a
|
||||
/// re-resolve.
|
||||
final Map<String, YtDlpFormat> formats;
|
||||
|
||||
/// What a player opens for this page ([playFormat]); null when yt-dlp
|
||||
/// picked nothing a player can open.
|
||||
final YtDlpPlay? play;
|
||||
const YtDlpInfo({
|
||||
required this.title,
|
||||
this.uploader = '',
|
||||
@@ -131,6 +141,7 @@ class YtDlpInfo {
|
||||
this.isLive = false,
|
||||
this.tracks = const [],
|
||||
this.formats = const {},
|
||||
this.play,
|
||||
});
|
||||
|
||||
/// One line for the log: extractor, duration, live, and each picked
|
||||
@@ -163,6 +174,7 @@ class YtDlpInfo {
|
||||
duration: j['duration'] is num ? (j['duration'] as num).toDouble() : null,
|
||||
thumbnail: str('thumbnail').isEmpty ? null : str('thumbnail'),
|
||||
isLive: isLive,
|
||||
play: YtDlpPlay.parse(j),
|
||||
tracks: isLive ? const [] : pickTracks(formats),
|
||||
formats: {
|
||||
for (final f in formats)
|
||||
@@ -341,6 +353,100 @@ Map<String, String> parseSha256Sums(String text) => {
|
||||
m.group(2)!: m.group(1)!.toLowerCase(),
|
||||
};
|
||||
|
||||
/// The formats a player opens for a page: the best video up to 1080p with
|
||||
/// the best audio, else the best single file (yt-dlp's `-f`).
|
||||
const playFormat = 'bv*[height<=1080]+ba/b';
|
||||
|
||||
/// The subtitles a player gets: every language but Bilibili's danmaku.
|
||||
const playSubLangs = 'all,-danmaku';
|
||||
|
||||
/// A page's subtitle track for a player (`requested_subtitles`).
|
||||
class YtDlpSubtitle {
|
||||
final String lang;
|
||||
final String name;
|
||||
final String url;
|
||||
final String ext;
|
||||
const YtDlpSubtitle(this.lang, this.url, {this.name = '', this.ext = ''});
|
||||
}
|
||||
|
||||
/// What a player opens for a page, as yt-dlp resolved it ([playFormat]):
|
||||
/// one file, or a video and an audio stream played together, with the
|
||||
/// headers its CDN wants (the first format's `http_headers`, plus its
|
||||
/// `cookies` as a Cookie header) and the subtitles ([playSubLangs]).
|
||||
/// URLs are bound to the resolving machine (its address, its cookies).
|
||||
class YtDlpPlay {
|
||||
final List<String> urls;
|
||||
final Map<String, String> headers;
|
||||
final List<YtDlpSubtitle> subtitles;
|
||||
const YtDlpPlay(this.urls, {this.headers = const {}, this.subtitles = const []});
|
||||
|
||||
/// The URL mpv opens: the file, or an EDL of the streams.
|
||||
String get mpvUrl => urls.length == 1 ? urls.single : mpvEdl(urls);
|
||||
|
||||
/// From `yt-dlp -J`: its `requested_formats`, else the top-level format;
|
||||
/// null when none of them is something a player can open.
|
||||
static YtDlpPlay? parse(Map<String, Object?> j) {
|
||||
final picked = j['requested_formats'] is List
|
||||
? [for (final f in j['requested_formats'] as List) if (f is Map) Map<String, Object?>.from(f)]
|
||||
: [if (j['url'] is String) j];
|
||||
if (picked.isEmpty) return null;
|
||||
final urls = <String>[];
|
||||
for (final f in picked) {
|
||||
final url = _playUrl(f);
|
||||
if (url == null) return null;
|
||||
urls.add(url);
|
||||
}
|
||||
final first = picked.first;
|
||||
final headers = <String, String>{
|
||||
if (first['http_headers'] is Map)
|
||||
for (final e in (first['http_headers'] as Map).entries)
|
||||
if (e.key is String && e.value is String) e.key as String: e.value as String,
|
||||
};
|
||||
final cookie = cookieHeaderOf(first['cookies']);
|
||||
if (cookie.isNotEmpty) headers['Cookie'] = cookie;
|
||||
final subs = <YtDlpSubtitle>[];
|
||||
final requested = j['requested_subtitles'];
|
||||
if (requested is Map) {
|
||||
for (final e in requested.entries) {
|
||||
final lang = '${e.key}', v = e.value;
|
||||
if (lang == 'danmaku' || v is! Map || v['url'] is! String) continue;
|
||||
final ext = v['ext'] is String ? v['ext'] as String : '';
|
||||
if (ext == 'xml' || ext == 'json') continue;
|
||||
subs.add(YtDlpSubtitle(lang, v['url'] as String,
|
||||
name: v['name'] is String ? v['name'] as String : '', ext: ext));
|
||||
}
|
||||
}
|
||||
return YtDlpPlay(urls, headers: headers, subtitles: subs);
|
||||
}
|
||||
|
||||
static String? _playUrl(Map<String, Object?> f) {
|
||||
final protocol = f['protocol'] is String ? f['protocol'] as String : 'https';
|
||||
if (protocol.startsWith('http_dash')) {
|
||||
final manifest = f['manifest_url'];
|
||||
return manifest is String && manifest.startsWith('http') ? manifest : null;
|
||||
}
|
||||
final url = f['url'];
|
||||
if (url is! String || !(url.startsWith('http://') || url.startsWith('https://'))) return null;
|
||||
return const {'http', 'https', 'm3u8', 'm3u8_native'}.contains(protocol) ? url : null;
|
||||
}
|
||||
}
|
||||
|
||||
/// A Cookie header from yt-dlp's `cookies` field (`name=value; Domain=...;
|
||||
/// Path=/; name2=value2; ...`): the name=value pairs, attributes dropped.
|
||||
String cookieHeaderOf(Object? cookies) {
|
||||
if (cookies is! String || cookies.trim().isEmpty) return '';
|
||||
const attributes = {'domain', 'path', 'expires', 'max-age', 'secure', 'httponly', 'samesite', 'comment', 'version'};
|
||||
final pairs = <String>[];
|
||||
for (final part in cookies.split(';')) {
|
||||
final t = part.trim();
|
||||
final i = t.indexOf('=');
|
||||
final name = (i < 0 ? t : t.substring(0, i)).trim();
|
||||
if (i <= 0 || attributes.contains(name.toLowerCase())) continue;
|
||||
pairs.add('$name=${t.substring(i + 1).trim()}');
|
||||
}
|
||||
return pairs.join('; ');
|
||||
}
|
||||
|
||||
/// The last `ERROR:` line of yt-dlp's stderr without its prefixes
|
||||
/// (`ERROR: [twitter] 123: No video` -> `No video`), else the last line.
|
||||
String ytDlpErrorText(String stderr) {
|
||||
@@ -602,7 +708,11 @@ class YtDlp {
|
||||
final e = await err;
|
||||
_log.warning('resolve ${logUrl(url)}: exit $code after ${sw.elapsedMilliseconds} ms; stderr tail:\n'
|
||||
'${_tail(e)}');
|
||||
throw YtDlpException(ytDlpErrorText(e));
|
||||
final text = ytDlpErrorText(e);
|
||||
// A site that hides the video from guests: say which cookies went.
|
||||
final note = mayNeedSignIn(text) ? await cookieNote(url) : null;
|
||||
if (note != null) _log.info('resolve ${logUrl(url)}: $note');
|
||||
throw YtDlpException(text, cookies: note);
|
||||
}
|
||||
try {
|
||||
final j = jsonDecode(await out);
|
||||
@@ -616,6 +726,59 @@ class YtDlp {
|
||||
}
|
||||
}
|
||||
|
||||
/// Which cookies a run for [url] uses and what they hold for its site
|
||||
/// ([cookieSite]), in plain words (D11): none; the private file and how
|
||||
/// many of its cookies are for the site; a browser, and how many cookies
|
||||
/// for the site yt-dlp could read from it (one run of yt-dlp that writes
|
||||
/// the browser's cookies to a private file in [dir], counted and deleted
|
||||
/// at once; values are never logged).
|
||||
Future<String> cookieNote(String url) async {
|
||||
final site = cookieSite(Uri.tryParse(url)?.host ?? '');
|
||||
final source = cookies.read(count: false);
|
||||
switch (source) {
|
||||
case null:
|
||||
return 'no cookies were used';
|
||||
case CookieFile(:final path):
|
||||
try {
|
||||
final n = parseCookies(await File(path).readAsString()).where((c) => cookieSite(c.host) == site).length;
|
||||
return cookieNoteText(source, site, n);
|
||||
} catch (_) {
|
||||
return cookieNoteText(source, site, -1);
|
||||
}
|
||||
case CookieBrowser():
|
||||
return cookieNoteText(source, site, -1, read: await _browserCookies(source, site));
|
||||
}
|
||||
}
|
||||
|
||||
/// How many cookies for [site] yt-dlp reads from [browser], or why it
|
||||
/// cannot.
|
||||
Future<({int count, String? error})> _browserCookies(CookieBrowser browser, String site) async {
|
||||
final exe = await ensure();
|
||||
if (exe == null) return (count: -1, error: lastError ?? 'yt-dlp is not available');
|
||||
final jar = File(p.join(dir, '.cookie-check-$pid.txt'));
|
||||
try {
|
||||
// Private before yt-dlp writes into it (it keeps the mode).
|
||||
await jar.writeAsString('# Netscape HTTP Cookie File\n');
|
||||
if (!Platform.isWindows) await Process.run('chmod', ['600', jar.path]);
|
||||
// No URL: yt-dlp loads the browser's cookies, saves the jar and stops.
|
||||
final r = await Process.run(
|
||||
exe, ['--ignore-config', '--cookies-from-browser', browser.spec, '--cookies', jar.path])
|
||||
.timeout(const Duration(seconds: 30));
|
||||
final cookies = parseCookiesOrEmpty(await jar.readAsString());
|
||||
final err = '${r.stderr}';
|
||||
if (cookies.isEmpty && err.contains('ERROR:') && !err.contains('provide at least one URL')) {
|
||||
return (count: -1, error: ytDlpErrorText(err));
|
||||
}
|
||||
return (count: cookies.where((c) => cookieSite(c.host) == site).length, error: null);
|
||||
} on TimeoutException {
|
||||
return (count: -1, error: 'yt-dlp took longer than 30 s reading them');
|
||||
} catch (e) {
|
||||
return (count: -1, error: '$e');
|
||||
} finally {
|
||||
if (await jar.exists()) await jar.delete().catchError((_) => jar);
|
||||
}
|
||||
}
|
||||
|
||||
/// [args] for the log: the cookie source named, not its path or spec.
|
||||
static String _argsForLog(List<String> args) {
|
||||
final out = <String>[];
|
||||
@@ -640,8 +803,14 @@ class YtDlp {
|
||||
}
|
||||
|
||||
/// The command line [resolve] runs.
|
||||
List<String> resolveArgs(String url) =>
|
||||
['-J', '--no-playlist', '--no-warnings', '--ignore-config', ...cookies.args(), '--', url];
|
||||
List<String> resolveArgs(String url) => [
|
||||
'-J', '--no-playlist', '--no-warnings', '--ignore-config',
|
||||
// What a player opens ([YtDlpPlay]): yt-dlp's own pick at most
|
||||
// 1080p, and the page's subtitles but Bilibili's danmaku (an XML
|
||||
// comment feed, not subtitles).
|
||||
'-f', playFormat, '--write-subs', '--sub-langs', playSubLangs,
|
||||
...cookies.args(), '--', url,
|
||||
];
|
||||
|
||||
Future<void> close() => _changes.close();
|
||||
}
|
||||
|
||||
@@ -30,26 +30,93 @@ void main() {
|
||||
expect(again.map((c) => c.toLine()), cookies.map((c) => c.toLine()));
|
||||
});
|
||||
|
||||
test('kept sites become a private file yt-dlp and mpv are pointed at', () async {
|
||||
test('kept sites become a private file yt-dlp is pointed at', () async {
|
||||
final dir = await Directory.systemTemp.createTemp('cookies_test');
|
||||
addTearDown(() => dir.delete(recursive: true));
|
||||
final src = File('${dir.path}/export.json')..writeAsStringSync(_json);
|
||||
final y = YtDlp('${dir.path}/yt-dlp, a=b');
|
||||
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', 'U']);
|
||||
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '-f', playFormat, '--write-subs', '--sub-langs', playSubLangs, '--', 'U']);
|
||||
|
||||
final kept = await y.cookies.import(src.path, sites: {'bilibili.com'});
|
||||
expect([kept.count, ...kept.sites], [1, 'bilibili.com']);
|
||||
final path = y.cookies.filePath;
|
||||
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--cookies', path, '--', 'U']);
|
||||
expect(y.cookies.mpvRawOptions(), 'cookies=%${path.length}%$path');
|
||||
expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '-f', playFormat, '--write-subs', '--sub-langs', playSubLangs, '--cookies', path, '--', 'U']);
|
||||
if (!Platform.isWindows) expect(File(path).statSync().modeString(), 'rw-------');
|
||||
|
||||
await y.cookies.saveBrowser('firefox', 'work');
|
||||
expect(y.resolveArgs('U').sublist(4, 6), ['--cookies-from-browser', 'firefox:work']);
|
||||
expect(y.resolveArgs('U').sublist(9, 11), ['--cookies-from-browser', 'firefox:work']);
|
||||
expect(File(path).existsSync(), isFalse);
|
||||
expect(y.cookies.mpvRawOptions(), 'cookies-from-browser=%12%firefox:work');
|
||||
|
||||
await y.cookies.remove();
|
||||
expect(y.cookies.read(), isNull);
|
||||
});
|
||||
|
||||
// D11: a sign-in error says which cookies were used and what they hold
|
||||
// for the site.
|
||||
group('cookie notes', () {
|
||||
test('the texts', () {
|
||||
expect(cookieNoteText(null, 'bilibili.com', -1), 'no cookies were used');
|
||||
const file = CookieFile('/x', 3, ['youtube.com']);
|
||||
expect(cookieNoteText(file, 'bilibili.com', 0), 'used the cookies file, which has none for bilibili.com');
|
||||
expect(cookieNoteText(file, 'bilibili.com', 4),
|
||||
'used the cookies file (4 for bilibili.com: signed out or expired?)');
|
||||
const ff = CookieBrowser('firefox', 'work');
|
||||
expect(cookieNoteText(ff, 'bilibili.com', -1, read: (count: 0, error: null)),
|
||||
"used Firefox (work)'s cookies, which have none for bilibili.com: sign in to bilibili.com there");
|
||||
expect(cookieNoteText(ff, 'bilibili.com', -1, read: (count: 12, error: null)),
|
||||
"used Firefox (work)'s cookies (12 for bilibili.com: that account may not see it)");
|
||||
expect(cookieNoteText(const CookieBrowser('chrome'), 'x.com', -1, read: (count: -1, error: 'locked')),
|
||||
"used Chrome's cookies, but yt-dlp couldn't read them: locked");
|
||||
expect(withCookieNote(noFormatsText, 'no cookies were used'),
|
||||
'No video found: it may need signing in (no cookies were used)');
|
||||
expect(withCookieNote(needsCookiesText, 'x'), 'This site needs cookies (x) — Settings → Player → yt-dlp → Cookies');
|
||||
expect(withCookieNote(notFoundText, 'x'), notFoundText);
|
||||
expect(mayNeedSignIn('No video formats found!'), isTrue);
|
||||
expect(mayNeedSignIn('HTTP Error 404'), isFalse);
|
||||
});
|
||||
|
||||
test("a failed resolve names the file's cookies for the site; a browser's are counted by yt-dlp",
|
||||
() async {
|
||||
final dir = await Directory.systemTemp.createTemp('cookie_note');
|
||||
addTearDown(() => dir.delete(recursive: true));
|
||||
// A stand-in yt-dlp: a version, a resolve that finds no formats, and
|
||||
// the browser read (no URL) that writes the jar it is given.
|
||||
final fake = File('${dir.path}/yt-dlp');
|
||||
await fake.writeAsString(r'''#!/bin/sh
|
||||
case "$1" in --version) echo 2099.01.01; exit 0;; esac
|
||||
jar=""; prev=""
|
||||
for a in "$@"; do [ "$prev" = "--cookies" ] && jar="$a"; prev="$a"; done
|
||||
case "$*" in *-J*) echo "ERROR: [BiliBili] BV1: No video formats found!" >&2; exit 1;; esac
|
||||
printf '# Netscape HTTP Cookie File\n.bilibili.com\tTRUE\t/\tTRUE\t2000000000\tSESSDATA\tx\n.bilibili.com\tTRUE\t/\tFALSE\t2000000000\tbuvid3\ty\n.other.org\tTRUE\t/\tFALSE\t0\ta\tb\n' > "$jar"
|
||||
echo "Extracted 3 cookies from firefox" >&2
|
||||
echo "yt-dlp: error: You must provide at least one URL." >&2; exit 2
|
||||
''');
|
||||
await Process.run('chmod', ['+x', fake.path]);
|
||||
final y = YtDlp(dir.path, updateEvery: const Duration(days: 9999));
|
||||
await File('${dir.path}/last-check').writeAsString(DateTime.now().toIso8601String());
|
||||
const page = 'https://www.bilibili.com/video/BV1';
|
||||
|
||||
Future<YtDlpException> fail() async {
|
||||
try {
|
||||
await y.resolve(page);
|
||||
} on YtDlpException catch (e) {
|
||||
return e;
|
||||
}
|
||||
throw StateError('resolved');
|
||||
}
|
||||
|
||||
expect((await fail()).cookies, 'no cookies were used');
|
||||
await y.cookies.saveFile(parseCookies('.youtube.com\tTRUE\t/\tFALSE\t0\tSID\tv\n'));
|
||||
expect((await fail()).cookies, 'used the cookies file, which has none for bilibili.com');
|
||||
await y.cookies.saveBrowser('firefox');
|
||||
final e = await fail();
|
||||
expect(e.message, contains('No video formats found'));
|
||||
expect(e.cookies, "used Firefox's cookies (2 for bilibili.com: that account may not see it)");
|
||||
expect(dir.listSync().map((f) => f.path).where((p) => p.contains('cookie-check')), isEmpty,
|
||||
reason: 'the browser jar is deleted at once');
|
||||
// The probe carries it into the plain words.
|
||||
expect(LinkProbe.page(Uri.parse(page), ytDlpError: e.message, cookies: e.cookies).problem,
|
||||
"No video found: it may need signing in (used Firefox's cookies (2 for bilibili.com: that account may not see it))");
|
||||
}, skip: Platform.isWindows ? 'the stand-in yt-dlp is a shell script' : false);
|
||||
});
|
||||
}
|
||||
@@ -261,4 +261,46 @@ void main() {
|
||||
expect(pick(['avc1.640033', 'vp09.00.40.08', 'av01.0.08M.08']), 'v2');
|
||||
expect(pick(['avc1.640033', 'vp09.00.40.08']), 'v1');
|
||||
});
|
||||
|
||||
test('what a player opens: yt-dlp\'s pick, an EDL for video + audio, headers with the cookies, subtitles but danmaku', () {
|
||||
final split = YtDlpInfo.parse({
|
||||
'title': 'Page',
|
||||
'formats': <Object>[],
|
||||
'requested_formats': [
|
||||
{
|
||||
'format_id': 'v', 'url': 'https://cdn/v.m4s', 'protocol': 'https',
|
||||
'http_headers': {'Referer': 'https://www.bilibili.com/', 'User-Agent': 'UA'},
|
||||
'cookies': 'SESSDATA=abc; Domain=.bilibili.com; Path=/; Secure; Expires=1999999999; buvid3=xyz; Domain=.bilibili.com',
|
||||
},
|
||||
{'format_id': 'a', 'url': 'https://cdn/a.m4s', 'protocol': 'https', 'http_headers': {'Referer': 'other'}},
|
||||
],
|
||||
'requested_subtitles': {
|
||||
'zh-CN': {'url': 'https://subs/zh.vtt', 'ext': 'vtt', 'name': 'Chinese'},
|
||||
'danmaku': {'url': 'https://comment/1.xml', 'ext': 'xml'},
|
||||
},
|
||||
}).play!;
|
||||
expect(split.urls, ['https://cdn/v.m4s', 'https://cdn/a.m4s']);
|
||||
expect(split.mpvUrl, mpvEdl(['https://cdn/v.m4s', 'https://cdn/a.m4s']));
|
||||
expect(split.headers, {'Referer': 'https://www.bilibili.com/', 'User-Agent': 'UA', 'Cookie': 'SESSDATA=abc; buvid3=xyz'});
|
||||
expect([for (final s in split.subtitles) '${s.lang} ${s.name} ${s.url}'], ['zh-CN Chinese https://subs/zh.vtt']);
|
||||
|
||||
final single = YtDlpInfo.parse({
|
||||
'title': 'Tweet', 'url': 'https://video.twimg.com/v.mp4', 'protocol': 'https', 'ext': 'mp4',
|
||||
'http_headers': {'User-Agent': 'UA'},
|
||||
}).play!;
|
||||
expect(single.mpvUrl, 'https://video.twimg.com/v.mp4');
|
||||
expect(single.headers, {'User-Agent': 'UA'});
|
||||
|
||||
final hls = YtDlpInfo.parse({
|
||||
'title': 'Live', 'is_live': true,
|
||||
'requested_formats': [{'url': 'https://x/live.m3u8', 'protocol': 'm3u8_native'}],
|
||||
}).play!;
|
||||
expect(hls.mpvUrl, 'https://x/live.m3u8', reason: 'a live stream plays, it is only not relayed');
|
||||
expect(YtDlpInfo.parse({'title': 'DASH', 'requested_formats': [{'url': 'https://x/f', 'protocol': 'http_dash_segments', 'manifest_url': 'https://x/m.mpd'}]}).play?.mpvUrl,
|
||||
'https://x/m.mpd');
|
||||
expect(YtDlpInfo.parse({'title': 'none', 'formats': <Object>[]}).play, isNull);
|
||||
expect(YtDlpInfo.parse({'title': 'rtmp', 'url': 'rtmp://x/y', 'protocol': 'rtmp'}).play, isNull);
|
||||
expect(cookieHeaderOf(null), '');
|
||||
expect(cookieHeaderOf('a=1; Path=/; HttpOnly; b=2=3'), 'a=1; b=2=3');
|
||||
});
|
||||
}
|
||||
@@ -112,14 +112,41 @@ class ChannelRelayFeed extends RelayFeed {
|
||||
class _Pull {
|
||||
final int id, start, end;
|
||||
int pos;
|
||||
final buf = BytesBuilder(copy: false);
|
||||
// The slice being filled: its chunks so far (readers take them as they
|
||||
// come, see [RelayBuffer.read]).
|
||||
final parts = <Uint8List>[];
|
||||
int filled = 0;
|
||||
_Pull(this.id, this.start, this.end) : pos = start;
|
||||
|
||||
/// Bytes of the slice being filled from [from] (offset in the slice).
|
||||
Uint8List? partFrom(int from) {
|
||||
if (from >= filled) return null;
|
||||
final out = BytesBuilder(copy: false);
|
||||
var at = 0;
|
||||
for (final p in parts) {
|
||||
final end = at + p.length;
|
||||
if (end > from) out.add(at >= from ? p : Uint8List.sublistView(p, from - at));
|
||||
at = end;
|
||||
}
|
||||
return out.takeBytes();
|
||||
}
|
||||
|
||||
Uint8List takeSlice() {
|
||||
final out = BytesBuilder(copy: false);
|
||||
for (final p in parts) {
|
||||
out.add(p);
|
||||
}
|
||||
parts.clear();
|
||||
filled = 0;
|
||||
return out.takeBytes();
|
||||
}
|
||||
}
|
||||
|
||||
class _Viewer {
|
||||
int pos = 0;
|
||||
int open = 0; // requests in flight
|
||||
int newest = 0; // sequence of its newest request
|
||||
(int slice, int req)? blocking; // where its newest request waits
|
||||
DateTime lastActive = DateTime.now();
|
||||
}
|
||||
|
||||
@@ -134,9 +161,11 @@ class _Viewer {
|
||||
/// slowest and [ahead] past the furthest viewer. Evicted by distance from
|
||||
/// the viewers, never LRU.
|
||||
/// - Filled only while someone watches: a viewer blocked on a slice comes
|
||||
/// first (oldest wait first); then read-ahead.
|
||||
/// first (oldest wait first); then read-ahead. A blocked viewer reads the
|
||||
/// slice being filled as its bytes arrive, not once it is whole.
|
||||
/// - Only a viewer's newest request moves its position or asks the feed
|
||||
/// for anything; older ones still get bytes once they are buffered.
|
||||
/// for anything; older ones still get bytes once they are buffered, and
|
||||
/// stop holding the feed once superseded (a seek elsewhere).
|
||||
/// Everything lives in memory: dropping the buffer is the cleanup.
|
||||
class RelayBuffer implements TrackBuffer {
|
||||
/// Set by the relay when a media's tracks share the budget.
|
||||
@@ -166,6 +195,8 @@ class RelayBuffer implements TrackBuffer {
|
||||
final _slices = <int, Uint8List>{};
|
||||
final _waiters = <int, List<Completer<void>>>{};
|
||||
final _blocked = <int, int>{}; // slice -> wait start (us)
|
||||
// The driving requests waiting on each blocked slice.
|
||||
final _blockedBy = <int, Set<int>>{};
|
||||
final _viewers = <String, _Viewer>{};
|
||||
@override
|
||||
FeedClock clock = FeedClock();
|
||||
@@ -256,18 +287,19 @@ class RelayBuffer implements TrackBuffer {
|
||||
_endPull(pl);
|
||||
return;
|
||||
}
|
||||
pl.buf.add(Uint8List.sublistView(data, off, off + take));
|
||||
pl.parts.add(Uint8List.sublistView(data, off, off + take));
|
||||
pl.filled += take;
|
||||
off += take;
|
||||
pl.pos += take;
|
||||
if (pl.pos == _sliceEnd(s) + 1) {
|
||||
_slices[s] = pl.buf.takeBytes();
|
||||
for (final c in _waiters.remove(s) ?? const <Completer<void>>[]) {
|
||||
c.complete();
|
||||
}
|
||||
if (_shouldStop(pl, s)) {
|
||||
_endPull(pl);
|
||||
return;
|
||||
}
|
||||
final done = pl.pos == _sliceEnd(s) + 1;
|
||||
if (done) _slices[s] = pl.takeSlice();
|
||||
// Whole, or more of it: whoever waits on it reads on.
|
||||
for (final c in _waiters.remove(s) ?? const <Completer<void>>[]) {
|
||||
c.complete();
|
||||
}
|
||||
if (done && _shouldStop(pl, s)) {
|
||||
_endPull(pl);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -405,25 +437,54 @@ class RelayBuffer implements TrackBuffer {
|
||||
await _kick.future.timeout(const Duration(seconds: 5), onTimeout: () {});
|
||||
}
|
||||
|
||||
Future<bool> _waitSlice(int s, bool drives) async {
|
||||
/// The bytes of slice [s] from [from] (offset in the slice) as soon as
|
||||
/// there are any: the whole slice's rest, or what the pull filling it has
|
||||
/// so far. Null when they did not come within [waitLimit]. [drives]: the
|
||||
/// request is its viewer's newest ([driving] says while it still is), so
|
||||
/// it may move the feed.
|
||||
Future<Uint8List?> _waitBytes(_Viewer v, int s, int from, int req, bool Function() driving) async {
|
||||
final deadline = DateTime.now().add(waitLimit);
|
||||
while (!_slices.containsKey(s)) {
|
||||
if (_disposed || DateTime.now().isAfter(deadline)) return false;
|
||||
if (drives) {
|
||||
_blocked.putIfAbsent(s, () => DateTime.now().microsecondsSinceEpoch);
|
||||
// A pull nobody waits on gives way at once; one serving another
|
||||
// blocked viewer hands over at its next slice boundary.
|
||||
try {
|
||||
while (true) {
|
||||
final whole = _slices[s];
|
||||
if (whole != null) return from == 0 ? whole : Uint8List.sublistView(whole, from);
|
||||
final pl = _pull;
|
||||
final at = pl == null ? -1 : pl.pos ~/ slice;
|
||||
if (pl != null && at != s && !_blocked.containsKey(at)) _endPull(pl);
|
||||
_wake();
|
||||
if (pl != null && pl.pos ~/ slice == s && pl.start <= s * slice) {
|
||||
final part = pl.partFrom(from);
|
||||
if (part != null) return part;
|
||||
}
|
||||
if (_disposed || DateTime.now().isAfter(deadline)) return null;
|
||||
if (driving()) {
|
||||
_blocked.putIfAbsent(s, () => DateTime.now().microsecondsSinceEpoch);
|
||||
_blockedBy.putIfAbsent(s, () => {}).add(req);
|
||||
v.blocking = (s, req);
|
||||
// A pull nobody waits on gives way at once; one serving another
|
||||
// blocked viewer hands over at its next slice boundary.
|
||||
final at = pl == null ? -1 : pl.pos ~/ slice;
|
||||
if (pl != null && at != s && !_blocked.containsKey(at)) _endPull(pl);
|
||||
_wake();
|
||||
} else {
|
||||
_unblock(s, req);
|
||||
}
|
||||
final c = Completer<void>();
|
||||
_waiters.putIfAbsent(s, () => []).add(c);
|
||||
await c.future.timeout(const Duration(seconds: 2), onTimeout: () {});
|
||||
}
|
||||
final c = Completer<void>();
|
||||
_waiters.putIfAbsent(s, () => []).add(c);
|
||||
await c.future.timeout(const Duration(seconds: 2), onTimeout: () {});
|
||||
} finally {
|
||||
_unblock(s, req);
|
||||
}
|
||||
}
|
||||
|
||||
// [req] no longer waits on slice [s]; the slice stops counting as
|
||||
// blocked once nobody driving does.
|
||||
void _unblock(int s, int req) {
|
||||
final by = _blockedBy[s];
|
||||
if (by == null || !by.remove(req)) return;
|
||||
if (by.isEmpty) {
|
||||
_blockedBy.remove(s);
|
||||
_blocked.remove(s);
|
||||
_wake();
|
||||
}
|
||||
_blocked.remove(s);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- viewers
|
||||
@@ -444,6 +505,10 @@ class RelayBuffer implements TrackBuffer {
|
||||
final tail = start >= size - pin;
|
||||
final me = ++_reqSeq;
|
||||
if (!tail) {
|
||||
// A seek: the request it supersedes stops holding the feed at once.
|
||||
final was = v.blocking;
|
||||
if (was != null) _unblock(was.$1, was.$2);
|
||||
v.blocking = null;
|
||||
v.newest = me;
|
||||
v.pos = start;
|
||||
}
|
||||
@@ -454,16 +519,14 @@ class RelayBuffer implements TrackBuffer {
|
||||
var off = start;
|
||||
while (off <= end) {
|
||||
final s = off ~/ slice;
|
||||
if (!await _waitSlice(s, tail || v.newest == me)) {
|
||||
throw StateError('relay: bytes at $off did not arrive');
|
||||
}
|
||||
final data = _slices[s];
|
||||
if (data == null) continue; // evicted between wake and read
|
||||
final from = off - s * slice, to = min(data.length, end - s * slice + 1);
|
||||
yield Uint8List.sublistView(data, from, to);
|
||||
bytesToViewers += to - from;
|
||||
onSent?.call(to - from);
|
||||
off += to - from;
|
||||
final data = await _waitBytes(v, s, off - s * slice, me, () => tail || v.newest == me);
|
||||
if (data == null) throw StateError('relay: bytes at $off did not arrive');
|
||||
final n = min(data.length, end - off + 1);
|
||||
if (n <= 0) continue;
|
||||
yield n == data.length ? data : Uint8List.sublistView(data, 0, n);
|
||||
bytesToViewers += n;
|
||||
onSent?.call(n);
|
||||
off += n;
|
||||
v.lastActive = DateTime.now();
|
||||
if (!tail && v.newest == me) v.pos = off;
|
||||
_wake();
|
||||
|
||||
Reference in new issue
Block a user