diff --git a/README.md b/README.md index 7ba09a7..919187d 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,8 @@ then everyone (you included) uses Join. | `--advertise-host` | none | host put in file URLs instead of the one each viewer dialled | | `--[no-]yt-dlp` | on | read web pages (YouTube, X, …) with yt-dlp when the server fetches a link | | `--yt-dlp-dir` | next to the binary | writable folder for the server's yt-dlp (systemd: the unit's `StateDirectory`) | -| `--config` | none | YAML or JSON file with the same settings (`port`, `secret`, `name`, `mediaPath`, `advertiseHost`, `ytDlp`, `ytDlpDir`); replaces the flags | +| `--yt-dlp-cookies` | none | cookies.txt or cookie-extension JSON export for that yt-dlp (sites that refuse guests); copied into the yt-dlp folder at start, removed from it when not given. **An account login**: keep it private, use a throwaway account for YouTube | +| `--config` | none | YAML or JSON file with the same settings (`port`, `secret`, `name`, `mediaPath`, `advertiseHost`, `ytDlp`, `ytDlpDir`, `ytDlpCookies`); replaces the flags | It prints `READY port=... name=...` once listening and stops cleanly on Ctrl+C or SIGTERM. A dedicated server has no host. @@ -316,6 +317,14 @@ Web pages work in all three: the app and the dedicated server each download [yt-dlp](https://github.com/yt-dlp/yt-dlp) by themselves (checked against the release's checksums, updated once a day; Settings → Player shows its version and has "Update now"), so there is nothing to install. +Sites that refuse guests (Bilibili answers "HTTP Error 412", YouTube may +ask to sign in) need cookies: Settings → Player → yt-dlp → Cookies reads +them from a browser, or from a cookies.txt or a cookie extension's JSON +export (picked or dropped; tick the sites to keep). They stay on this +computer and are used by its yt-dlp only (your own playback, "through +me", and a server you host). Cookies log yt-dlp in as you on those +sites: keep the file private, and for YouTube consider a throwaway +account (Google may flag accounts used with yt-dlp). YouTube works in all three too. The last two never show the link to anyone; they can't pass on live streams or pages that only offer streaming playlists (use "Everyone fetches it" for those). Your upload has to beat the video's diff --git a/apps/desktop/lib/src/playback/playback_controller.dart b/apps/desktop/lib/src/playback/playback_controller.dart index bb72805..f700c45 100644 --- a/apps/desktop/lib/src/playback/playback_controller.dart +++ b/apps/desktop/lib/src/playback/playback_controller.dart @@ -16,7 +16,7 @@ import '../sync/state_diff.dart'; import '../sync/user_seek.dart'; import 'frame_watchdog.dart'; import 'media_marks.dart'; -import 'yt_dlp_app.dart' show ytdlFormat; +import 'yt_dlp_app.dart' show AppYtDlp, ytdlFormat; /// Owns one mpv [Player] (and its [VideoController]) per session: created /// by [init] on join, released by [dispose] on leave, swapped by @@ -65,21 +65,31 @@ class PlaybackController { void Function(String error)? onYtdlError; /// Points mpv's yt-dlp hook at [path] (the app's managed copy) for web - /// page links, at most 1080p. Once per player. + /// page links, at most 1080p, once per player; its cookies + /// (`ytdl-raw-options`) whenever they changed. Future useYtDlp(String path) async { final p = _player; - if (p == null || identical(p, _ytdlSetOn)) return; + if (p == null) return; try { final dynamic platform = p.platform; - await platform?.setProperty('script-opts', 'ytdl_hook-ytdl_path=$path'); - await platform?.setProperty('ytdl-format', ytdlFormat); - _ytdlSetOn = p; + if (!identical(p, _ytdlSetOn)) { + await platform?.setProperty('script-opts', 'ytdl_hook-ytdl_path=$path'); + await platform?.setProperty('ytdl-format', ytdlFormat); + _ytdlSetOn = p; + _ytdlRaw = ''; + } + final raw = await AppYtDlp.instance.mpvRawOptions(); + if (raw != _ytdlRaw && identical(p, _player)) { + await platform?.setProperty('ytdl-raw-options', raw); + _ytdlRaw = raw; + } } catch (e) { debugPrint('yt-dlp path not set: $e'); } } Player? _ytdlSetOn; + String _ytdlRaw = ''; /// Fingerprint of the picture on screen, installed by the video view /// (`ShownFrameProbe.sample`). Without it the watchdog stays idle. diff --git a/apps/desktop/lib/src/playback/yt_dlp_app.dart b/apps/desktop/lib/src/playback/yt_dlp_app.dart index 7d19144..151e1a1 100644 --- a/apps/desktop/lib/src/playback/yt_dlp_app.dart +++ b/apps/desktop/lib/src/playback/yt_dlp_app.dart @@ -28,6 +28,16 @@ class AppYtDlp { /// update check afterwards). void warmUp() => unawaited(ytDlp.then((y) => y.ensure()).catchError((_) => null)); + /// mpv's `ytdl-raw-options` for the cookies in the yt-dlp folder ('' for + /// none; see [YtDlpCookies]). + Future mpvRawOptions() async { + try { + return (await ytDlp).cookies.mpvRawOptions(); + } catch (_) { + return ''; + } + } + final _probes = )>{}; static const _probeTtl = Duration(minutes: 10); diff --git a/apps/desktop/lib/src/shell/settings_tab.dart b/apps/desktop/lib/src/shell/settings_tab.dart index 39c4745..77bd812 100644 --- a/apps/desktop/lib/src/shell/settings_tab.dart +++ b/apps/desktop/lib/src/shell/settings_tab.dart @@ -1,8 +1,13 @@ import 'dart:async'; +import 'dart:io'; + +import 'package:file_picker/file_picker.dart'; import 'package:flutter/material.dart'; +import 'package:fox_utils/fox_utils.dart' show DesktopDropX, DropTarget; import 'package:signals/signals_flutter.dart'; -import 'package:sync_relay_core/core.dart' show YtDlp; +import 'package:sync_relay_core/core.dart' + show CookieBrowser, CookieFile, NetscapeCookie, YtDlp, YtDlpCookies, cookieBrowsers, cookiesBySite, parseCookies, suggestedCookieSites; import '../controls/player_settings.dart'; import '../jellyfin/accounts.dart'; @@ -420,6 +425,8 @@ class _YtDlpSettingsState extends State { 'Downloaded and kept up to date by the app; checked once a day.', style: TextStyle(color: scheme.onSurfaceVariant, fontSize: 12), ), + const SizedBox(height: 12), + YtDlpCookieSettings(cookies: y.cookies), ], ); }, @@ -429,6 +436,271 @@ class _YtDlpSettingsState extends State { } } +/// yt-dlp's cookies (sites that refuse guests): from a browser, or a +/// cookies.txt / JSON export picked or dropped, filtered by site into the +/// private copy in the yt-dlp folder (see [YtDlpCookies]). +class YtDlpCookieSettings extends StatefulWidget { + final YtDlpCookies cookies; + const YtDlpCookieSettings({super.key, required this.cookies}); + + @override + State createState() => _YtDlpCookieSettingsState(); +} + +class _YtDlpCookieSettingsState extends State { + late var _source = widget.cookies.read(); + String? _error; + bool _dragging = false; + + void _reload([String? error]) { + if (!mounted) return; + setState(() { + _source = widget.cookies.read(); + _error = error; + }); + } + + String get _summary => switch (_source) { + CookieFile f => '${f.count} cookie${f.count == 1 ? '' : 's'} for ${f.sites.join(', ')}', + CookieBrowser b => 'From ${b.browser}${b.profile.isEmpty ? '' : ' (profile ${b.profile})'}', + _ => 'No cookies: yt-dlp visits sites as a guest', + }; + + Future _pickFile() async { + final List files; + try { + files = await FilePicker.pickFiles( + dialogTitle: 'Pick a cookies.txt or a JSON cookie export', + type: FileType.custom, + allowedExtensions: const ['txt', 'json'], + ); + } catch (_) { + _reload('file dialog unavailable — try again'); + return; + } + final path = files.firstOrNull?.path; + if (path != null) await _importFile(path); + } + + Future _importFile(String path) async { + final List all; + try { + all = parseCookies(await File(path).readAsString()); + } on FormatException catch (e) { + _reload("Can't read that file: ${e.message}"); + return; + } on FileSystemException catch (e) { + _reload("Can't read that file: ${e.message}"); + return; + } + if (!mounted) return; + final bySite = cookiesBySite(all); + final sites = await showDialog>( + context: context, + builder: (_) => _CookieSitesDialog(bySite: {for (final e in bySite.entries) e.key: e.value.length}), + ); + if (sites == null || sites.isEmpty) return; + try { + await widget.cookies.saveFile([for (final s in sites) ...?bySite[s]]); + _reload(); + } catch (e) { + _reload('Could not keep the cookies: $e'); + } + } + + Future _pickBrowser() async { + final current = _source; + final picked = await showDialog( + context: context, + builder: (_) => _CookieBrowserDialog(initial: current is CookieBrowser ? current : null), + ); + if (picked == null) return; + try { + await widget.cookies.saveBrowser(picked.browser, picked.profile); + _reload(); + } catch (e) { + _reload('Could not save: $e'); + } + } + + Future _remove() async { + try { + await widget.cookies.remove(); + _reload(); + } catch (e) { + _reload('Could not remove the cookies: $e'); + } + } + + @override + Widget build(BuildContext context) { + final scheme = Theme.of(context).colorScheme; + return DropTarget( + onDragEntered: (_) => setState(() => _dragging = true), + onDragExited: (_) => setState(() => _dragging = false), + onDragDone: (d) { + setState(() => _dragging = false); + final path = DesktopDropX.pathsOf(d.files).firstOrNull; + if (path != null) _importFile(path); + }, + child: AnimatedContainer( + duration: const Duration(milliseconds: 150), + padding: const EdgeInsets.all(8), + decoration: BoxDecoration( + borderRadius: BorderRadius.circular(12), + border: Border.all(color: _dragging ? scheme.primary : scheme.outlineVariant), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + ListTile( + contentPadding: EdgeInsets.zero, + leading: const Icon(Icons.cookie_outlined), + title: const Text('Cookies'), + subtitle: Text(_summary, key: const ValueKey('ytdlp-cookies-summary')), + ), + if (_error != null) + Text(_error!, key: const ValueKey('ytdlp-cookies-error'), style: TextStyle(color: scheme.error)), + Wrap( + spacing: 8, + runSpacing: 8, + children: [ + OutlinedButton.icon( + onPressed: _pickBrowser, + icon: const Icon(Icons.public), + label: const Text('From a browser…'), + ), + OutlinedButton.icon( + onPressed: _pickFile, + icon: const Icon(Icons.file_open_outlined), + label: const Text('From a file…'), + ), + if (_source != null) + TextButton( + key: const ValueKey('ytdlp-cookies-remove'), + onPressed: _remove, + child: const Text('Remove cookies'), + ), + ], + ), + const SizedBox(height: 8), + Text( + 'For sites that refuse guests (Bilibili, YouTube asking to sign in). Drop a cookies.txt or a ' + 'cookie-extension JSON export here, or pick one; only the sites you tick are kept, on this computer only. ' + 'Cookies log yt-dlp in as you on those sites. Keep the file private; for YouTube consider a ' + 'throwaway account (Google may flag accounts used with yt-dlp).', + style: TextStyle(color: scheme.onSurfaceVariant, fontSize: 12), + ), + ], + ), + ), + ); + } +} + +/// Which sites of an imported cookie file to keep; pops the ticked ones. +class _CookieSitesDialog extends StatefulWidget { + /// Site -> number of cookies. + final Map bySite; + const _CookieSitesDialog({required this.bySite}); + + @override + State<_CookieSitesDialog> createState() => _CookieSitesDialogState(); +} + +class _CookieSitesDialogState extends State<_CookieSitesDialog> { + late final _on = {for (final s in widget.bySite.keys) if (suggestedCookieSites.contains(s)) s}; + + @override + Widget build(BuildContext context) { + return AlertDialog( + title: const Text('Keep cookies for'), + content: SizedBox( + width: 380, + child: ListView( + shrinkWrap: true, + children: [ + for (final e in widget.bySite.entries) + CheckboxListTile( + dense: true, + value: _on.contains(e.key), + title: Text(e.key), + subtitle: Text('${e.value} cookie${e.value == 1 ? '' : 's'}'), + onChanged: (v) => setState(() => v == true ? _on.add(e.key) : _on.remove(e.key)), + ), + ], + ), + ), + actions: [ + TextButton(onPressed: () => Navigator.pop(context), child: const Text('Cancel')), + FilledButton( + onPressed: _on.isEmpty ? null : () => Navigator.pop(context, {..._on}), + child: const Text('Keep'), + ), + ], + ); + } +} + +/// Which browser (and profile) yt-dlp reads cookies from. +class _CookieBrowserDialog extends StatefulWidget { + final CookieBrowser? initial; + const _CookieBrowserDialog({this.initial}); + + @override + State<_CookieBrowserDialog> createState() => _CookieBrowserDialogState(); +} + +class _CookieBrowserDialogState extends State<_CookieBrowserDialog> { + late var _browser = widget.initial?.browser ?? cookieBrowsers.first; + late final _profile = TextEditingController(text: widget.initial?.profile ?? ''); + + @override + void dispose() { + _profile.dispose(); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + return AlertDialog( + title: const Text('Cookies from a browser'), + content: SizedBox( + width: 380, + child: Column( + mainAxisSize: MainAxisSize.min, + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + DropdownButtonFormField( + initialValue: _browser, + decoration: const InputDecoration(labelText: 'Browser'), + items: [for (final b in cookieBrowsers) DropdownMenuItem(value: b, child: Text(b))], + onChanged: (v) => setState(() => _browser = v ?? _browser), + ), + const SizedBox(height: 12), + TextField( + controller: _profile, + decoration: const InputDecoration(labelText: 'Profile (optional)', hintText: 'the default profile'), + ), + const SizedBox(height: 12), + Text( + 'yt-dlp reads them each time it opens a link (the browser may need to be closed on Windows).', + style: TextStyle(color: Theme.of(context).colorScheme.onSurfaceVariant, fontSize: 12), + ), + ], + ), + ), + actions: [ + TextButton(onPressed: () => Navigator.pop(context), child: const Text('Cancel')), + FilledButton( + onPressed: () => Navigator.pop(context, CookieBrowser(_browser, _profile.text.trim())), + child: const Text('Use'), + ), + ], + ); + } +} + /// AnimeFlow (the Schedule tab's source): connected as whom, with which /// rights, or why not. class AnimeFlowCard extends StatefulWidget { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fea24a3..47df71c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -43,7 +43,7 @@ size (`ServerIsolate.setFile`): the host's in-session File swap then sends a - **Dedicated server**: `bin/server.dart` compiled with `dart compile exe` and shipped next to the app as `watch-together-server`. It parses flags (`--port`, `--secret`, `--name`, `--media`, `--advertise-host`, - `--jellyfin `, `--[no-]yt-dlp`, `--yt-dlp-dir`, `--config`; default port 8080) into the same map and calls `serve`; + `--jellyfin `, `--[no-]yt-dlp`, `--yt-dlp-dir`, `--yt-dlp-cookies`, `--config`; default port 8080) into the same map and calls `serve`; SIGINT and SIGTERM stop it. A Flutter binary cannot run without a display, which is why the server is its own build. @@ -529,8 +529,30 @@ all three link modes. Nothing is bundled or installed by hand. `--yt-dlp-dir`, else `$STATE_DIRECTORY/yt-dlp` (systemd), else `yt-dlp` next to the binary if writable, else `~/.cache/watch-together/yt-dlp`. -- **Resolve** (`YtDlp.resolve`): `yt-dlp -J --no-playlist --no-warnings - --ignore-config`, 25 s timeout: title, uploader, duration, thumbnail, +- **Cookies** (`YtDlpCookies`, `cookies.dart`): a yt-dlp folder holds at + most one source, `cookies.txt` (a private Netscape copy, mode 0600, + written to a `.part` that is made private before anything goes into + it) or `cookies-browser` (`BROWSER[:PROFILE]`). Every resolve adds + `--cookies ` or `--cookies-from-browser ` (the update and + `--version` runs do not); yt-dlp may write the copy back. Imports + (`parseCookies`) take a Netscape cookies.txt or a cookie extension's + JSON array (`name`, `value`, `domain`, `path`, `expirationDate` / + `expires`, `hostOnly`, `secure`, `httpOnly`, `session`; host-only + cookies keep their bare host with `FALSE`, the rest get a leading '.'; + session cookies expire `0`), grouped by site (`cookieSite`: the + registrable domain by a short rule, with google.com filed under + youtube.com and bilibili's CDNs under bilibili.com). The app: Settings + → Player → yt-dlp → Cookies (a browser and profile, or a file picked or + dropped, then the sites to keep, bilibili.com and youtube.com ticked + when present; "Remove cookies"); its hosted server shares the folder, + so it uses them too. mpv's hook gets the same as `ytdl-raw-options` + (`cookies=%N%` / `cookies-from-browser=%N%`, length-quoted), + re-set before each external link when it changed. The dedicated + server: `--yt-dlp-cookies ` / `ytDlpCookies` converted into its + folder at start; without it the copy and browser setting are removed. + Values are never logged and never leave the machine. +- **Resolve** (`YtDlp.resolve`, `resolveArgs`): `yt-dlp -J --no-playlist --no-warnings + --ignore-config` plus the cookies, 25 s timeout: title, uploader, duration, thumbnail, live or not, and what a relay reads (`pickTracks`), each format with its URL, `http_headers` and `http_chunk_size`. Only plain http(s) formats without DRM count: diff --git a/packages/core/lib/core.dart b/packages/core/lib/core.dart index 21fa406..01bed06 100644 --- a/packages/core/lib/core.dart +++ b/packages/core/lib/core.dart @@ -7,6 +7,7 @@ export 'src/protocol.dart'; export 'src/relay.dart'; export 'src/link_probe.dart'; export 'src/yt_dlp.dart'; +export 'src/cookies.dart'; export 'src/media_source.dart'; export 'src/sync_engine.dart'; export 'src/jellyfin/client.dart'; diff --git a/packages/core/lib/src/config.dart b/packages/core/lib/src/config.dart index 8101d7f..d1b2395 100644 --- a/packages/core/lib/src/config.dart +++ b/packages/core/lib/src/config.dart @@ -111,6 +111,10 @@ class AppConfig { /// Where that yt-dlp lives (writable); empty: a `yt-dlp` folder next to /// the server binary, else `~/.cache/watch-together/yt-dlp`. final String ytDlpDir; + /// Dedicated server: a cookies.txt or browser-extension JSON export + /// copied (converted) into [ytDlpDir] at start, so yt-dlp is logged in + /// (see `YtDlpCookies`); empty: no cookies. + final String ytDlpCookies; const AppConfig({ required this.port, @@ -123,6 +127,7 @@ class AppConfig { this.jellyfin, this.ytDlp = true, this.ytDlpDir = '', + this.ytDlpCookies = '', }); factory AppConfig.defaults() => @@ -141,6 +146,7 @@ class AppConfig { jellyfin: JellyfinCredential.fromMap(map['jellyfin']), ytDlp: (map['ytDlp'] as bool?) ?? true, ytDlpDir: (map['ytDlpDir'] as String?) ?? '', + ytDlpCookies: (map['ytDlpCookies'] as String?) ?? '', ); } on ConfigException { rethrow; @@ -160,6 +166,7 @@ class AppConfig { if (jellyfin != null) 'jellyfin': jellyfin!.toMap(), 'ytDlp': ytDlp, 'ytDlpDir': ytDlpDir, + 'ytDlpCookies': ytDlpCookies, }; static Future loadFile(String path) async => diff --git a/packages/core/lib/src/cookies.dart b/packages/core/lib/src/cookies.dart new file mode 100644 index 0000000..578045f --- /dev/null +++ b/packages/core/lib/src/cookies.dart @@ -0,0 +1,297 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as p; + +// Cookies for yt-dlp: some sites refuse guests (Bilibili answers HTTP 412, +// YouTube may ask to sign in). A machine's yt-dlp folder holds at most one +// source: a private Netscape cookies.txt (mode 0600) or the name of a +// browser to read them from. They only ever go to this machine's yt-dlp; +// values are never logged or sent anywhere. + +/// One cookie, as a line of a Netscape cookies.txt. +class NetscapeCookie { + /// With a leading '.' when [includeSubdomains]. + final String domain; + final bool includeSubdomains; + final String path; + final bool secure; + /// Unix seconds; 0 for a session cookie. + final int expires; + final String name; + final String value; + final bool httpOnly; + const NetscapeCookie({ + required this.domain, + this.includeSubdomains = true, + this.path = '/', + this.secure = false, + this.expires = 0, + required this.name, + required this.value, + this.httpOnly = false, + }); + + /// The host without a leading '.', lowercase. + String get host => (domain.startsWith('.') ? domain.substring(1) : domain).toLowerCase(); + + String toLine() => [ + '${httpOnly ? '#HttpOnly_' : ''}$domain', + includeSubdomains ? 'TRUE' : 'FALSE', + path, + secure ? 'TRUE' : 'FALSE', + '$expires', + name, + value, + ].join('\t'); +} + +/// Reads a Netscape cookies.txt or the JSON array browser extensions +/// export (Cookie-Editor, "Get cookies.txt" style: objects with name, +/// value, domain, path, expirationDate or expires, httpOnly, secure, +/// hostOnly, session; also `{"cookies": [...]}`). Throws [FormatException] +/// without quoting the input (it holds logins). +List parseCookies(String text) { + final t = text.trimLeft(); + if (t.startsWith('[') || t.startsWith('{')) return _fromJson(t); + final out = []; + for (var line in const LineSplitter().convert(text)) { + var httpOnly = false; + if (line.startsWith('#HttpOnly_')) { + httpOnly = true; + line = line.substring('#HttpOnly_'.length); + } else if (line.startsWith('#') || line.trim().isEmpty) { + continue; + } + final f = line.split('\t'); + if (f.length < 7) continue; + out.add(NetscapeCookie( + domain: f[0], + includeSubdomains: f[1].toUpperCase() == 'TRUE', + path: f[2], + secure: f[3].toUpperCase() == 'TRUE', + expires: int.tryParse(f[4]) ?? double.tryParse(f[4])?.toInt() ?? 0, + name: f[5], + value: f.sublist(6).join('\t'), + httpOnly: httpOnly, + )); + } + if (out.isEmpty) throw const FormatException('no cookies found (expected a cookies.txt or a JSON export)'); + return out; +} + +List _fromJson(String text) { + Object? root; + try { + root = jsonDecode(text); + } on FormatException { + throw const FormatException('not valid JSON'); + } + if (root is Map && root['cookies'] is List) root = root['cookies']; + if (root is! List) throw const FormatException('expected a JSON array of cookies'); + final out = []; + for (final c in root) { + if (c is! Map) continue; + final name = c['name'], value = c['value'], domain = c['domain']; + if (name is! String || value is! String || domain is! String || domain.isEmpty) continue; + final hostOnly = c['hostOnly'] == true; + final exp = c['expirationDate'] ?? c['expires'] ?? c['expiry']; + final session = c['session'] == true || exp is! num || exp <= 0; + final path = c['path']; + out.add(NetscapeCookie( + domain: hostOnly ? (domain.startsWith('.') ? domain.substring(1) : domain) : (domain.startsWith('.') ? domain : '.$domain'), + includeSubdomains: !hostOnly, + path: path is String && path.isNotEmpty ? path : '/', + secure: c['secure'] == true, + expires: session ? 0 : exp.toInt(), + name: name, + value: value, + httpOnly: c['httpOnly'] == true, + )); + } + if (out.isEmpty) throw const FormatException('no cookies found in the JSON'); + return out; +} + +/// [cookies] as a Netscape cookies.txt (what yt-dlp's `--cookies` reads). +String netscapeCookiesText(Iterable cookies) => [ + '# Netscape HTTP Cookie File', + '# Written by Starsync for yt-dlp. Holds logins: keep it private.', + '', + for (final c in cookies) c.toLine(), + '', + ].join('\n'); + +// Second-level labels under which a country code needs a third label +// (bbc.co.uk, example.com.cn). +const _secondLevel = {'co', 'com', 'net', 'org', 'ac', 'gov', 'edu', 'ne', 'or', 'go'}; + +// Domains one login spans, filed under one site. +const _siteAlias = { + 'google.com': 'youtube.com', + 'youtu.be': 'youtube.com', + 'hdslb.com': 'bilibili.com', + 'bilivideo.com': 'bilibili.com', + 'biliapi.net': 'bilibili.com', + 'b23.tv': 'bilibili.com', + 'x.com': 'twitter.com', +}; + +/// The registrable domain of [host] (`www.bilibili.com` -> `bilibili.com`, +/// `a.bbc.co.uk` -> `bbc.co.uk`), by a short rule rather than the full +/// public suffix list. +String registrableDomain(String host) { + final labels = host.toLowerCase().replaceFirst(RegExp(r'^\.+'), '').split('.').where((l) => l.isNotEmpty).toList(); + if (labels.length <= 2) return labels.join('.'); + final n = labels.length; + final keep = labels[n - 1].length == 2 && _secondLevel.contains(labels[n - 2]) ? 3 : 2; + return labels.sublist(n - keep).join('.'); +} + +/// The site a cookie of [host] belongs to: its registrable domain, with the +/// domains one login spans folded together (google.com under youtube.com). +String cookieSite(String host) { + final d = registrableDomain(host); + return _siteAlias[d] ?? d; +} + +/// [cookies] grouped by [cookieSite], sites sorted by name. +Map> cookiesBySite(Iterable cookies) { + final m = >{}; + for (final c in cookies) { + m.putIfAbsent(cookieSite(c.host), () => []).add(c); + } + return {for (final k in m.keys.toList()..sort()) k: m[k]!}; +} + +/// Sites ticked by default after an import: those yt-dlp is usually given +/// cookies for. +const suggestedCookieSites = {'bilibili.com', 'youtube.com'}; + +/// Browsers `--cookies-from-browser` reads. +const cookieBrowsers = ['firefox', 'chrome', 'chromium', 'brave', 'edge', 'opera', 'vivaldi']; + +/// Where a machine's yt-dlp gets cookies: a private file or a browser. +sealed class CookieSource { + const CookieSource(); +} + +/// [path] is the private copy in the yt-dlp folder. +class CookieFile extends CookieSource { + final String path; + final int count; + /// [cookieSite]s in it, sorted. + final List sites; + const CookieFile(this.path, this.count, this.sites); +} + +class CookieBrowser extends CookieSource { + final String browser; + /// Empty: the browser's default profile. + final String profile; + const CookieBrowser(this.browser, [this.profile = '']); + /// `BROWSER[:PROFILE]`, as `--cookies-from-browser` takes it. + String get spec => profile.isEmpty ? browser : '$browser:$profile'; +} + +/// The cookie setting kept in a yt-dlp folder ([dir]): `cookies.txt` +/// (mode 0600) or `cookies-browser` (the `BROWSER[:PROFILE]` spec), never +/// both. +class YtDlpCookies { + final String dir; + const YtDlpCookies(this.dir); + + String get filePath => p.join(dir, 'cookies.txt'); + String get _browserPath => p.join(dir, 'cookies-browser'); + + /// The current source, or null for none. Without [count] a file is not + /// parsed ([CookieFile.count] -1, no sites). + CookieSource? read({bool count = true}) { + try { + final b = File(_browserPath); + if (b.existsSync()) { + final spec = b.readAsStringSync().trim(); + if (spec.isNotEmpty) { + final i = spec.indexOf(':'); + return i < 0 ? CookieBrowser(spec) : CookieBrowser(spec.substring(0, i), spec.substring(i + 1)); + } + } + final f = File(filePath); + if (f.existsSync()) { + if (!count) return CookieFile(filePath, -1, const []); + final cookies = parseCookies(f.readAsStringSync()); + return CookieFile(filePath, cookies.length, cookiesBySite(cookies).keys.toList()); + } + } catch (_) {} + return null; + } + + /// yt-dlp arguments for [source] (default: [read]). + List args([CookieSource? source]) => switch (source ?? read(count: false)) { + CookieFile(:final path) => ['--cookies', path], + CookieBrowser b => ['--cookies-from-browser', b.spec], + null => const [], + }; + + /// The same as mpv's `ytdl-raw-options` value ('' for none); each value + /// length-quoted (`%N%`) so commas and '=' in paths survive. + String mpvRawOptions([CookieSource? source]) { + String q(String v) => '%${utf8.encode(v).length}%$v'; + return switch (source ?? read(count: false)) { + CookieFile(:final path) => 'cookies=${q(path)}', + CookieBrowser b => 'cookies-from-browser=${q(b.spec)}', + null => '', + }; + } + + /// Keeps [cookies] as the private copy, replacing any browser setting. + Future saveFile(List cookies) async { + await Directory(dir).create(recursive: true); + final part = File('$filePath.part$pid'); + try { + // Private before anything is written into it. + await part.writeAsString(''); + if (!Platform.isWindows) { + final r = await Process.run('chmod', ['600', part.path]); + if (r.exitCode != 0) throw FileSystemException('chmod failed', part.path); + } + await part.writeAsString(netscapeCookiesText(cookies), flush: true); + final target = File(filePath); + if (Platform.isWindows && await target.exists()) await target.delete(); + await part.rename(filePath); + } finally { + if (await part.exists()) await part.delete().catchError((_) => part); + } + await _delete(_browserPath); + return CookieFile(filePath, cookies.length, cookiesBySite(cookies).keys.toList()); + } + + /// Converts the cookies.txt or JSON export at [source] into the private + /// copy, keeping only [sites] ([cookieSite]s; null: all). + Future import(String source, {Set? sites}) async { + final all = parseCookies(await File(source).readAsString()); + final kept = sites == null ? all : [for (final c in all) if (sites.contains(cookieSite(c.host))) c]; + if (kept.isEmpty) throw const FormatException('no cookies for the chosen sites'); + return saveFile(kept); + } + + /// Reads cookies from [browser] (one of [cookieBrowsers]) from now on, + /// replacing any file. + Future saveBrowser(String browser, [String profile = '']) async { + if (!cookieBrowsers.contains(browser)) throw ArgumentError.value(browser, 'browser'); + await Directory(dir).create(recursive: true); + await File(_browserPath).writeAsString(CookieBrowser(browser, profile.trim()).spec); + await _delete(filePath); + } + + /// No cookies any more. + Future remove() async { + await _delete(filePath); + await _delete(_browserPath); + } + + static Future _delete(String path) async { + final f = File(path); + if (await f.exists()) await f.delete(); + } +} diff --git a/packages/core/lib/src/yt_dlp.dart b/packages/core/lib/src/yt_dlp.dart index 631d70f..cc461a2 100644 --- a/packages/core/lib/src/yt_dlp.dart +++ b/packages/core/lib/src/yt_dlp.dart @@ -7,6 +7,7 @@ import 'dart:typed_data'; import 'package:crypto/crypto.dart'; import 'package:path/path.dart' as p; +import 'cookies.dart'; import 'logging.dart'; // yt-dlp: turns a web page (YouTube, Twitter/X, ...) into media URLs. The @@ -284,6 +285,9 @@ class YtDlp { Future? _ensuring; Future? _updating; + /// Cookies every resolve sends (see [YtDlpCookies]), kept in [dir]. + late final cookies = YtDlpCookies(dir); + String get managedPath => p.join(dir, Platform.isWindows ? 'yt-dlp.exe' : 'yt-dlp'); String get _stampPath => p.join(dir, 'last-check'); @@ -482,12 +486,12 @@ class YtDlp { } } - /// Asks yt-dlp what [url] is. Throws [YtDlpException] with yt-dlp's own - /// reason on failure. + /// Asks yt-dlp what [url] is, with [cookies] if any. Throws + /// [YtDlpException] with yt-dlp's own reason on failure. Future resolve(String url, {Duration timeout = const Duration(seconds: 25)}) async { final exe = await ensure(); if (exe == null) throw YtDlpException(lastError ?? 'yt-dlp is not available'); - final proc = await Process.start(exe, ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', url]); + final proc = await Process.start(exe, resolveArgs(url)); final out = proc.stdout.transform(utf8.decoder).join(); final err = proc.stderr.transform(utf8.decoder).join(); final int code; @@ -507,6 +511,10 @@ class YtDlp { } } + /// The command line [resolve] runs. + List resolveArgs(String url) => + ['-J', '--no-playlist', '--no-warnings', '--ignore-config', ...cookies.args(), '--', url]; + Future close() => _changes.close(); } diff --git a/packages/core/test/cookies_test.dart b/packages/core/test/cookies_test.dart new file mode 100644 index 0000000..8ed65a4 --- /dev/null +++ b/packages/core/test/cookies_test.dart @@ -0,0 +1,55 @@ +// yt-dlp cookies: the JSON export -> Netscape conversion, the per-site +// filter, and what yt-dlp and mpv are given. Synthetic cookies only. +library; + +import 'dart:io'; + +import 'package:sync_relay_core/core.dart'; +import 'package:test/test.dart'; + +const _json = '''[ + {"domain": ".bilibili.com", "name": "SESSDATA", "value": "s1", "path": "/", "expirationDate": 1893456000.5, + "hostOnly": false, "httpOnly": true, "secure": true, "session": false}, + {"domain": "www.youtube.com", "name": "PREF", "value": "a=b", "hostOnly": true, "session": true}, + {"domain": "accounts.google.com", "name": "SID", "value": "g", "expires": 1893456000}, + {"domain": ".example.co.uk", "name": "x", "value": "y", "path": "/p"} +]'''; + +void main() { + test('JSON export converts to Netscape lines, grouped by site', () { + final cookies = parseCookies(_json); + expect(cookies.map((c) => c.toLine()), [ + '#HttpOnly_.bilibili.com\tTRUE\t/\tTRUE\t1893456000\tSESSDATA\ts1', + 'www.youtube.com\tFALSE\t/\tFALSE\t0\tPREF\ta=b', + '.accounts.google.com\tTRUE\t/\tFALSE\t1893456000\tSID\tg', + '.example.co.uk\tTRUE\t/p\tFALSE\t0\tx\ty', + ]); + expect(cookiesBySite(cookies).keys, ['bilibili.com', 'example.co.uk', 'youtube.com']); + // Netscape text reads back the same. + final again = parseCookies(netscapeCookiesText(cookies)); + expect(again.map((c) => c.toLine()), cookies.map((c) => c.toLine())); + }); + + test('kept sites become a private file yt-dlp and mpv are pointed at', () async { + final dir = await Directory.systemTemp.createTemp('cookies_test'); + addTearDown(() => dir.delete(recursive: true)); + final src = File('${dir.path}/export.json')..writeAsStringSync(_json); + final y = YtDlp('${dir.path}/yt-dlp, a=b'); + expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', 'U']); + + final kept = await y.cookies.import(src.path, sites: {'bilibili.com'}); + expect([kept.count, ...kept.sites], [1, 'bilibili.com']); + final path = y.cookies.filePath; + expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--cookies', path, '--', 'U']); + expect(y.cookies.mpvRawOptions(), 'cookies=%${path.length}%$path'); + if (!Platform.isWindows) expect(File(path).statSync().modeString(), 'rw-------'); + + await y.cookies.saveBrowser('firefox', 'work'); + expect(y.resolveArgs('U').sublist(4, 6), ['--cookies-from-browser', 'firefox:work']); + expect(File(path).existsSync(), isFalse); + expect(y.cookies.mpvRawOptions(), 'cookies-from-browser=%12%firefox:work'); + + await y.cookies.remove(); + expect(y.cookies.read(), isNull); + }); +} diff --git a/packages/server/bin/server.dart b/packages/server/bin/server.dart index e1915b4..cc427a8 100644 --- a/packages/server/bin/server.dart +++ b/packages/server/bin/server.dart @@ -25,6 +25,10 @@ Future main(List args) async { 'downloaded on first use.') ..addOption('yt-dlp-dir', defaultsTo: '', help: 'Writable folder for that yt-dlp (default: yt-dlp next to this binary).') + ..addOption('yt-dlp-cookies', + defaultsTo: '', + help: 'cookies.txt or browser-extension JSON export to log that yt-dlp in (sites that refuse guests). ' + 'An account login: keep it private. Copied into --yt-dlp-dir at start.') ..addOption('config', help: 'YAML or JSON config file (keys as in AppConfig); overrides the other flags.') ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help.'); final ArgResults parsed; @@ -60,6 +64,7 @@ Future main(List args) async { advertiseHost: '${parsed['advertise-host']}', ytDlp: parsed['yt-dlp'] as bool, ytDlpDir: '${parsed['yt-dlp-dir']}', + ytDlpCookies: '${parsed['yt-dlp-cookies']}', ); } final jellyfinPath = parsed['jellyfin'] as String?; @@ -76,8 +81,10 @@ Future main(List args) async { jellyfin: await JellyfinCredential.loadFile(jellyfinPath), ytDlp: c.ytDlp, ytDlpDir: c.ytDlpDir, + ytDlpCookies: c.ytDlpCookies, ); } + await _setUpCookies(config); } on ConfigException catch (e) { stderr.writeln(e.message); exitCode = 64; @@ -116,3 +123,23 @@ Future main(List args) async { await s.cancel(); } } + +/// The private cookie copy in the server's yt-dlp folder: converted from +/// [AppConfig.ytDlpCookies], or removed when none is given (so dropping the +/// flag logs yt-dlp out). +Future _setUpCookies(AppConfig config) async { + if (!config.ytDlp) return; + final cookies = YtDlpCookies(serverYtDlpDir(config.ytDlpDir)); + try { + if (config.ytDlpCookies.isEmpty) { + await cookies.remove(); + return; + } + final kept = await cookies.import(config.ytDlpCookies); + stdout.writeln('yt-dlp cookies: ${kept.count} for ${kept.sites.join(', ')}'); + } on FormatException catch (e) { + throw ConfigException('--yt-dlp-cookies ${config.ytDlpCookies}: ${e.message}'); + } on FileSystemException catch (e) { + throw ConfigException('--yt-dlp-cookies: ${e.message} (${e.path})'); + } +}