yt-dlp cookies: from a browser or a file (cookies.txt or extension JSON), per site, private; server --yt-dlp-cookies

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/core/lib/core.dart

@@ -7,6 +7,7 @@ export 'src/protocol.dart';
 export 'src/relay.dart';
 export 'src/link_probe.dart';
 export 'src/yt_dlp.dart';
+export 'src/cookies.dart';
 export 'src/media_source.dart';
 export 'src/sync_engine.dart';
 export 'src/jellyfin/client.dart';

packages/core/lib/src/config.dart

@@ -111,6 +111,10 @@ class AppConfig {
   /// Where that yt-dlp lives (writable); empty: a `yt-dlp` folder next to
   /// the server binary, else `~/.cache/watch-together/yt-dlp`.
   final String ytDlpDir;
+  /// Dedicated server: a cookies.txt or browser-extension JSON export
+  /// copied (converted) into [ytDlpDir] at start, so yt-dlp is logged in
+  /// (see `YtDlpCookies`); empty: no cookies.
+  final String ytDlpCookies;
 
   const AppConfig({
     required this.port,
@@ -123,6 +127,7 @@ class AppConfig {
     this.jellyfin,
     this.ytDlp = true,
     this.ytDlpDir = '',
+    this.ytDlpCookies = '',
   });
 
   factory AppConfig.defaults() =>
@@ -141,6 +146,7 @@ class AppConfig {
         jellyfin: JellyfinCredential.fromMap(map['jellyfin']),
         ytDlp: (map['ytDlp'] as bool?) ?? true,
         ytDlpDir: (map['ytDlpDir'] as String?) ?? '',
+        ytDlpCookies: (map['ytDlpCookies'] as String?) ?? '',
       );
     } on ConfigException {
       rethrow;
@@ -160,6 +166,7 @@ class AppConfig {
         if (jellyfin != null) 'jellyfin': jellyfin!.toMap(),
         'ytDlp': ytDlp,
         'ytDlpDir': ytDlpDir,
+        'ytDlpCookies': ytDlpCookies,
       };
 
   static Future<AppConfig> loadFile(String path) async =>

packages/core/lib/src/cookies.dart

@@ -0,0 +1,297 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+
+// Cookies for yt-dlp: some sites refuse guests (Bilibili answers HTTP 412,
+// YouTube may ask to sign in). A machine's yt-dlp folder holds at most one
+// source: a private Netscape cookies.txt (mode 0600) or the name of a
+// browser to read them from. They only ever go to this machine's yt-dlp;
+// values are never logged or sent anywhere.
+
+/// One cookie, as a line of a Netscape cookies.txt.
+class NetscapeCookie {
+  /// With a leading '.' when [includeSubdomains].
+  final String domain;
+  final bool includeSubdomains;
+  final String path;
+  final bool secure;
+  /// Unix seconds; 0 for a session cookie.
+  final int expires;
+  final String name;
+  final String value;
+  final bool httpOnly;
+  const NetscapeCookie({
+    required this.domain,
+    this.includeSubdomains = true,
+    this.path = '/',
+    this.secure = false,
+    this.expires = 0,
+    required this.name,
+    required this.value,
+    this.httpOnly = false,
+  });
+
+  /// The host without a leading '.', lowercase.
+  String get host => (domain.startsWith('.') ? domain.substring(1) : domain).toLowerCase();
+
+  String toLine() => [
+        '${httpOnly ? '#HttpOnly_' : ''}$domain',
+        includeSubdomains ? 'TRUE' : 'FALSE',
+        path,
+        secure ? 'TRUE' : 'FALSE',
+        '$expires',
+        name,
+        value,
+      ].join('\t');
+}
+
+/// Reads a Netscape cookies.txt or the JSON array browser extensions
+/// export (Cookie-Editor, "Get cookies.txt" style: objects with name,
+/// value, domain, path, expirationDate or expires, httpOnly, secure,
+/// hostOnly, session; also `{"cookies": [...]}`). Throws [FormatException]
+/// without quoting the input (it holds logins).
+List<NetscapeCookie> parseCookies(String text) {
+  final t = text.trimLeft();
+  if (t.startsWith('[') || t.startsWith('{')) return _fromJson(t);
+  final out = <NetscapeCookie>[];
+  for (var line in const LineSplitter().convert(text)) {
+    var httpOnly = false;
+    if (line.startsWith('#HttpOnly_')) {
+      httpOnly = true;
+      line = line.substring('#HttpOnly_'.length);
+    } else if (line.startsWith('#') || line.trim().isEmpty) {
+      continue;
+    }
+    final f = line.split('\t');
+    if (f.length < 7) continue;
+    out.add(NetscapeCookie(
+      domain: f[0],
+      includeSubdomains: f[1].toUpperCase() == 'TRUE',
+      path: f[2],
+      secure: f[3].toUpperCase() == 'TRUE',
+      expires: int.tryParse(f[4]) ?? double.tryParse(f[4])?.toInt() ?? 0,
+      name: f[5],
+      value: f.sublist(6).join('\t'),
+      httpOnly: httpOnly,
+    ));
+  }
+  if (out.isEmpty) throw const FormatException('no cookies found (expected a cookies.txt or a JSON export)');
+  return out;
+}
+
+List<NetscapeCookie> _fromJson(String text) {
+  Object? root;
+  try {
+    root = jsonDecode(text);
+  } on FormatException {
+    throw const FormatException('not valid JSON');
+  }
+  if (root is Map && root['cookies'] is List) root = root['cookies'];
+  if (root is! List) throw const FormatException('expected a JSON array of cookies');
+  final out = <NetscapeCookie>[];
+  for (final c in root) {
+    if (c is! Map) continue;
+    final name = c['name'], value = c['value'], domain = c['domain'];
+    if (name is! String || value is! String || domain is! String || domain.isEmpty) continue;
+    final hostOnly = c['hostOnly'] == true;
+    final exp = c['expirationDate'] ?? c['expires'] ?? c['expiry'];
+    final session = c['session'] == true || exp is! num || exp <= 0;
+    final path = c['path'];
+    out.add(NetscapeCookie(
+      domain: hostOnly ? (domain.startsWith('.') ? domain.substring(1) : domain) : (domain.startsWith('.') ? domain : '.$domain'),
+      includeSubdomains: !hostOnly,
+      path: path is String && path.isNotEmpty ? path : '/',
+      secure: c['secure'] == true,
+      expires: session ? 0 : exp.toInt(),
+      name: name,
+      value: value,
+      httpOnly: c['httpOnly'] == true,
+    ));
+  }
+  if (out.isEmpty) throw const FormatException('no cookies found in the JSON');
+  return out;
+}
+
+/// [cookies] as a Netscape cookies.txt (what yt-dlp's `--cookies` reads).
+String netscapeCookiesText(Iterable<NetscapeCookie> cookies) => [
+      '# Netscape HTTP Cookie File',
+      '# Written by Starsync for yt-dlp. Holds logins: keep it private.',
+      '',
+      for (final c in cookies) c.toLine(),
+      '',
+    ].join('\n');
+
+// Second-level labels under which a country code needs a third label
+// (bbc.co.uk, example.com.cn).
+const _secondLevel = {'co', 'com', 'net', 'org', 'ac', 'gov', 'edu', 'ne', 'or', 'go'};
+
+// Domains one login spans, filed under one site.
+const _siteAlias = {
+  'google.com': 'youtube.com',
+  'youtu.be': 'youtube.com',
+  'hdslb.com': 'bilibili.com',
+  'bilivideo.com': 'bilibili.com',
+  'biliapi.net': 'bilibili.com',
+  'b23.tv': 'bilibili.com',
+  'x.com': 'twitter.com',
+};
+
+/// The registrable domain of [host] (`www.bilibili.com` -> `bilibili.com`,
+/// `a.bbc.co.uk` -> `bbc.co.uk`), by a short rule rather than the full
+/// public suffix list.
+String registrableDomain(String host) {
+  final labels = host.toLowerCase().replaceFirst(RegExp(r'^\.+'), '').split('.').where((l) => l.isNotEmpty).toList();
+  if (labels.length <= 2) return labels.join('.');
+  final n = labels.length;
+  final keep = labels[n - 1].length == 2 && _secondLevel.contains(labels[n - 2]) ? 3 : 2;
+  return labels.sublist(n - keep).join('.');
+}
+
+/// The site a cookie of [host] belongs to: its registrable domain, with the
+/// domains one login spans folded together (google.com under youtube.com).
+String cookieSite(String host) {
+  final d = registrableDomain(host);
+  return _siteAlias[d] ?? d;
+}
+
+/// [cookies] grouped by [cookieSite], sites sorted by name.
+Map<String, List<NetscapeCookie>> cookiesBySite(Iterable<NetscapeCookie> cookies) {
+  final m = <String, List<NetscapeCookie>>{};
+  for (final c in cookies) {
+    m.putIfAbsent(cookieSite(c.host), () => []).add(c);
+  }
+  return {for (final k in m.keys.toList()..sort()) k: m[k]!};
+}
+
+/// Sites ticked by default after an import: those yt-dlp is usually given
+/// cookies for.
+const suggestedCookieSites = {'bilibili.com', 'youtube.com'};
+
+/// Browsers `--cookies-from-browser` reads.
+const cookieBrowsers = ['firefox', 'chrome', 'chromium', 'brave', 'edge', 'opera', 'vivaldi'];
+
+/// Where a machine's yt-dlp gets cookies: a private file or a browser.
+sealed class CookieSource {
+  const CookieSource();
+}
+
+/// [path] is the private copy in the yt-dlp folder.
+class CookieFile extends CookieSource {
+  final String path;
+  final int count;
+  /// [cookieSite]s in it, sorted.
+  final List<String> sites;
+  const CookieFile(this.path, this.count, this.sites);
+}
+
+class CookieBrowser extends CookieSource {
+  final String browser;
+  /// Empty: the browser's default profile.
+  final String profile;
+  const CookieBrowser(this.browser, [this.profile = '']);
+  /// `BROWSER[:PROFILE]`, as `--cookies-from-browser` takes it.
+  String get spec => profile.isEmpty ? browser : '$browser:$profile';
+}
+
+/// The cookie setting kept in a yt-dlp folder ([dir]): `cookies.txt`
+/// (mode 0600) or `cookies-browser` (the `BROWSER[:PROFILE]` spec), never
+/// both.
+class YtDlpCookies {
+  final String dir;
+  const YtDlpCookies(this.dir);
+
+  String get filePath => p.join(dir, 'cookies.txt');
+  String get _browserPath => p.join(dir, 'cookies-browser');
+
+  /// The current source, or null for none. Without [count] a file is not
+  /// parsed ([CookieFile.count] -1, no sites).
+  CookieSource? read({bool count = true}) {
+    try {
+      final b = File(_browserPath);
+      if (b.existsSync()) {
+        final spec = b.readAsStringSync().trim();
+        if (spec.isNotEmpty) {
+          final i = spec.indexOf(':');
+          return i < 0 ? CookieBrowser(spec) : CookieBrowser(spec.substring(0, i), spec.substring(i + 1));
+        }
+      }
+      final f = File(filePath);
+      if (f.existsSync()) {
+        if (!count) return CookieFile(filePath, -1, const []);
+        final cookies = parseCookies(f.readAsStringSync());
+        return CookieFile(filePath, cookies.length, cookiesBySite(cookies).keys.toList());
+      }
+    } catch (_) {}
+    return null;
+  }
+
+  /// yt-dlp arguments for [source] (default: [read]).
+  List<String> args([CookieSource? source]) => switch (source ?? read(count: false)) {
+        CookieFile(:final path) => ['--cookies', path],
+        CookieBrowser b => ['--cookies-from-browser', b.spec],
+        null => const [],
+      };
+
+  /// The same as mpv's `ytdl-raw-options` value ('' for none); each value
+  /// length-quoted (`%N%`) so commas and '=' in paths survive.
+  String mpvRawOptions([CookieSource? source]) {
+    String q(String v) => '%${utf8.encode(v).length}%$v';
+    return switch (source ?? read(count: false)) {
+      CookieFile(:final path) => 'cookies=${q(path)}',
+      CookieBrowser b => 'cookies-from-browser=${q(b.spec)}',
+      null => '',
+    };
+  }
+
+  /// Keeps [cookies] as the private copy, replacing any browser setting.
+  Future<CookieFile> saveFile(List<NetscapeCookie> cookies) async {
+    await Directory(dir).create(recursive: true);
+    final part = File('$filePath.part$pid');
+    try {
+      // Private before anything is written into it.
+      await part.writeAsString('');
+      if (!Platform.isWindows) {
+        final r = await Process.run('chmod', ['600', part.path]);
+        if (r.exitCode != 0) throw FileSystemException('chmod failed', part.path);
+      }
+      await part.writeAsString(netscapeCookiesText(cookies), flush: true);
+      final target = File(filePath);
+      if (Platform.isWindows && await target.exists()) await target.delete();
+      await part.rename(filePath);
+    } finally {
+      if (await part.exists()) await part.delete().catchError((_) => part);
+    }
+    await _delete(_browserPath);
+    return CookieFile(filePath, cookies.length, cookiesBySite(cookies).keys.toList());
+  }
+
+  /// Converts the cookies.txt or JSON export at [source] into the private
+  /// copy, keeping only [sites] ([cookieSite]s; null: all).
+  Future<CookieFile> import(String source, {Set<String>? sites}) async {
+    final all = parseCookies(await File(source).readAsString());
+    final kept = sites == null ? all : [for (final c in all) if (sites.contains(cookieSite(c.host))) c];
+    if (kept.isEmpty) throw const FormatException('no cookies for the chosen sites');
+    return saveFile(kept);
+  }
+
+  /// Reads cookies from [browser] (one of [cookieBrowsers]) from now on,
+  /// replacing any file.
+  Future<void> saveBrowser(String browser, [String profile = '']) async {
+    if (!cookieBrowsers.contains(browser)) throw ArgumentError.value(browser, 'browser');
+    await Directory(dir).create(recursive: true);
+    await File(_browserPath).writeAsString(CookieBrowser(browser, profile.trim()).spec);
+    await _delete(filePath);
+  }
+
+  /// No cookies any more.
+  Future<void> remove() async {
+    await _delete(filePath);
+    await _delete(_browserPath);
+  }
+
+  static Future<void> _delete(String path) async {
+    final f = File(path);
+    if (await f.exists()) await f.delete();
+  }
+}

packages/core/lib/src/yt_dlp.dart

@@ -7,6 +7,7 @@ import 'dart:typed_data';
 import 'package:crypto/crypto.dart';
 import 'package:path/path.dart' as p;
 
+import 'cookies.dart';
 import 'logging.dart';
 
 // yt-dlp: turns a web page (YouTube, Twitter/X, ...) into media URLs. The
@@ -284,6 +285,9 @@ class YtDlp {
   Future<String?>? _ensuring;
   Future<void>? _updating;
 
+  /// Cookies every resolve sends (see [YtDlpCookies]), kept in [dir].
+  late final cookies = YtDlpCookies(dir);
+
   String get managedPath => p.join(dir, Platform.isWindows ? 'yt-dlp.exe' : 'yt-dlp');
   String get _stampPath => p.join(dir, 'last-check');
 
@@ -482,12 +486,12 @@ class YtDlp {
     }
   }
 
-  /// Asks yt-dlp what [url] is. Throws [YtDlpException] with yt-dlp's own
-  /// reason on failure.
+  /// Asks yt-dlp what [url] is, with [cookies] if any. Throws
+  /// [YtDlpException] with yt-dlp's own reason on failure.
   Future<YtDlpInfo> resolve(String url, {Duration timeout = const Duration(seconds: 25)}) async {
     final exe = await ensure();
     if (exe == null) throw YtDlpException(lastError ?? 'yt-dlp is not available');
-    final proc = await Process.start(exe, ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', url]);
+    final proc = await Process.start(exe, resolveArgs(url));
     final out = proc.stdout.transform(utf8.decoder).join();
     final err = proc.stderr.transform(utf8.decoder).join();
     final int code;
@@ -507,6 +511,10 @@ class YtDlp {
     }
   }
 
+  /// The command line [resolve] runs.
+  List<String> resolveArgs(String url) =>
+      ['-J', '--no-playlist', '--no-warnings', '--ignore-config', ...cookies.args(), '--', url];
+
   Future<void> close() => _changes.close();
 }
 

packages/core/test/cookies_test.dart

@@ -0,0 +1,55 @@
+// yt-dlp cookies: the JSON export -> Netscape conversion, the per-site
+// filter, and what yt-dlp and mpv are given. Synthetic cookies only.
+library;
+
+import 'dart:io';
+
+import 'package:sync_relay_core/core.dart';
+import 'package:test/test.dart';
+
+const _json = '''[
+  {"domain": ".bilibili.com", "name": "SESSDATA", "value": "s1", "path": "/", "expirationDate": 1893456000.5,
+   "hostOnly": false, "httpOnly": true, "secure": true, "session": false},
+  {"domain": "www.youtube.com", "name": "PREF", "value": "a=b", "hostOnly": true, "session": true},
+  {"domain": "accounts.google.com", "name": "SID", "value": "g", "expires": 1893456000},
+  {"domain": ".example.co.uk", "name": "x", "value": "y", "path": "/p"}
+]''';
+
+void main() {
+  test('JSON export converts to Netscape lines, grouped by site', () {
+    final cookies = parseCookies(_json);
+    expect(cookies.map((c) => c.toLine()), [
+      '#HttpOnly_.bilibili.com\tTRUE\t/\tTRUE\t1893456000\tSESSDATA\ts1',
+      'www.youtube.com\tFALSE\t/\tFALSE\t0\tPREF\ta=b',
+      '.accounts.google.com\tTRUE\t/\tFALSE\t1893456000\tSID\tg',
+      '.example.co.uk\tTRUE\t/p\tFALSE\t0\tx\ty',
+    ]);
+    expect(cookiesBySite(cookies).keys, ['bilibili.com', 'example.co.uk', 'youtube.com']);
+    // Netscape text reads back the same.
+    final again = parseCookies(netscapeCookiesText(cookies));
+    expect(again.map((c) => c.toLine()), cookies.map((c) => c.toLine()));
+  });
+
+  test('kept sites become a private file yt-dlp and mpv are pointed at', () async {
+    final dir = await Directory.systemTemp.createTemp('cookies_test');
+    addTearDown(() => dir.delete(recursive: true));
+    final src = File('${dir.path}/export.json')..writeAsStringSync(_json);
+    final y = YtDlp('${dir.path}/yt-dlp, a=b');
+    expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', 'U']);
+
+    final kept = await y.cookies.import(src.path, sites: {'bilibili.com'});
+    expect([kept.count, ...kept.sites], [1, 'bilibili.com']);
+    final path = y.cookies.filePath;
+    expect(y.resolveArgs('U'), ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--cookies', path, '--', 'U']);
+    expect(y.cookies.mpvRawOptions(), 'cookies=%${path.length}%$path');
+    if (!Platform.isWindows) expect(File(path).statSync().modeString(), 'rw-------');
+
+    await y.cookies.saveBrowser('firefox', 'work');
+    expect(y.resolveArgs('U').sublist(4, 6), ['--cookies-from-browser', 'firefox:work']);
+    expect(File(path).existsSync(), isFalse);
+    expect(y.cookies.mpvRawOptions(), 'cookies-from-browser=%12%firefox:work');
+
+    await y.cookies.remove();
+    expect(y.cookies.read(), isNull);
+  });
+}

packages/server/bin/server.dart

@@ -25,6 +25,10 @@ Future<void> main(List<String> args) async {
             'downloaded on first use.')
     ..addOption('yt-dlp-dir',
         defaultsTo: '', help: 'Writable folder for that yt-dlp (default: yt-dlp next to this binary).')
+    ..addOption('yt-dlp-cookies',
+        defaultsTo: '',
+        help: 'cookies.txt or browser-extension JSON export to log that yt-dlp in (sites that refuse guests). '
+            'An account login: keep it private. Copied into --yt-dlp-dir at start.')
     ..addOption('config', help: 'YAML or JSON config file (keys as in AppConfig); overrides the other flags.')
     ..addFlag('help', abbr: 'h', negatable: false, help: 'Show this help.');
   final ArgResults parsed;
@@ -60,6 +64,7 @@ Future<void> main(List<String> args) async {
         advertiseHost: '${parsed['advertise-host']}',
         ytDlp: parsed['yt-dlp'] as bool,
         ytDlpDir: '${parsed['yt-dlp-dir']}',
+        ytDlpCookies: '${parsed['yt-dlp-cookies']}',
       );
     }
     final jellyfinPath = parsed['jellyfin'] as String?;
@@ -76,8 +81,10 @@ Future<void> main(List<String> args) async {
         jellyfin: await JellyfinCredential.loadFile(jellyfinPath),
         ytDlp: c.ytDlp,
         ytDlpDir: c.ytDlpDir,
+        ytDlpCookies: c.ytDlpCookies,
       );
     }
+    await _setUpCookies(config);
   } on ConfigException catch (e) {
     stderr.writeln(e.message);
     exitCode = 64;
@@ -116,3 +123,23 @@ Future<void> main(List<String> args) async {
     await s.cancel();
   }
 }
+
+/// The private cookie copy in the server's yt-dlp folder: converted from
+/// [AppConfig.ytDlpCookies], or removed when none is given (so dropping the
+/// flag logs yt-dlp out).
+Future<void> _setUpCookies(AppConfig config) async {
+  if (!config.ytDlp) return;
+  final cookies = YtDlpCookies(serverYtDlpDir(config.ytDlpDir));
+  try {
+    if (config.ytDlpCookies.isEmpty) {
+      await cookies.remove();
+      return;
+    }
+    final kept = await cookies.import(config.ytDlpCookies);
+    stdout.writeln('yt-dlp cookies: ${kept.count} for ${kept.sites.join(', ')}');
+  } on FormatException catch (e) {
+    throw ConfigException('--yt-dlp-cookies ${config.ytDlpCookies}: ${e.message}');
+  } on FileSystemException catch (e) {
+    throw ConfigException('--yt-dlp-cookies: ${e.message} (${e.path})');
+  }
+}