yt-dlp for page links in every Link mode; app icon and Linux launcher entry
yt-dlp: a managed, checksum-verified, daily-updated yt-dlp in the app and the dedicated server (--[no-]yt-dlp, --yt-dlp-dir). Pasted links are probed: direct files stay links, pages are resolved for title, uploader, duration and thumbnail. Everyone fetches it: mpv's ytdl hook on the managed copy. I pass it on / the server fetches it: the best single-file format, re-resolved once when its URL expires. Relay names never split the /r/ path; viewers of server-fetched pages wait for the relay to go live. No wire change. Icon: two stars orbiting a play button (assets/icon/icon.svg, PNGs, the Windows .ico). GTK app id moe.senkochan.watch_together; release builds install their .desktop file and icons into ~/.local/share so Wayland shows the icon; the window icon is set for X11. README shows the icon. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
foxyzandClaude Opus 5.5 committed
1 parent
9ddd85de90
commit
745717a4a0
40 files changed
+1444
-123
No files matched your search
@@ -5,6 +5,8 @@ export 'src/clock_sync.dart';
|
||||
export 'src/config.dart';
|
||||
export 'src/protocol.dart';
|
||||
export 'src/relay.dart';
|
||||
export 'src/link_probe.dart';
|
||||
export 'src/yt_dlp.dart';
|
||||
export 'src/media_source.dart';
|
||||
export 'src/sync_engine.dart';
|
||||
export 'src/jellyfin/client.dart';
|
||||
|
||||
@@ -105,6 +105,12 @@ class AppConfig {
|
||||
final String hostKey;
|
||||
/// Optional login for relaying Jellyfin media to guests; null = no relay.
|
||||
final JellyfinCredential? jellyfin;
|
||||
/// "The server fetches it" reads web pages (YouTube, Twitter/X, ...)
|
||||
/// through its own yt-dlp, downloaded on first use.
|
||||
final bool ytDlp;
|
||||
/// Where that yt-dlp lives (writable); empty: a `yt-dlp` folder next to
|
||||
/// the server binary, else `~/.cache/watch-together/yt-dlp`.
|
||||
final String ytDlpDir;
|
||||
|
||||
const AppConfig({
|
||||
required this.port,
|
||||
@@ -115,6 +121,8 @@ class AppConfig {
|
||||
this.advertiseHost = '',
|
||||
this.hostKey = '',
|
||||
this.jellyfin,
|
||||
this.ytDlp = true,
|
||||
this.ytDlpDir = '',
|
||||
});
|
||||
|
||||
factory AppConfig.defaults() =>
|
||||
@@ -131,6 +139,8 @@ class AppConfig {
|
||||
advertiseHost: (map['advertiseHost'] as String?) ?? '',
|
||||
hostKey: (map['hostKey'] as String?) ?? '',
|
||||
jellyfin: JellyfinCredential.fromMap(map['jellyfin']),
|
||||
ytDlp: (map['ytDlp'] as bool?) ?? true,
|
||||
ytDlpDir: (map['ytDlpDir'] as String?) ?? '',
|
||||
);
|
||||
} on ConfigException {
|
||||
rethrow;
|
||||
@@ -148,6 +158,8 @@ class AppConfig {
|
||||
'advertiseHost': advertiseHost,
|
||||
'hostKey': hostKey,
|
||||
if (jellyfin != null) 'jellyfin': jellyfin!.toMap(),
|
||||
'ytDlp': ytDlp,
|
||||
'ytDlpDir': ytDlpDir,
|
||||
};
|
||||
|
||||
static Future<AppConfig> loadFile(String path) async =>
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'relay.dart';
|
||||
import 'yt_dlp.dart';
|
||||
|
||||
// What a pasted link is: a media file (range-readable as is) or a web page
|
||||
// that yt-dlp turns into one. Used where the bytes are fetched: the
|
||||
// putter's app (preview, "I pass it on") and the server ("the server
|
||||
// fetches it").
|
||||
|
||||
const _mediaExtensions = {
|
||||
'mkv', 'mp4', 'm4v', 'webm', 'mov', 'avi', 'wmv', 'flv', 'ts', 'm2ts', 'mpg', 'mpeg', 'ogv', //
|
||||
'mp3', 'm4a', 'flac', 'opus', 'ogg', 'wav',
|
||||
};
|
||||
|
||||
const _fileTypes = {'application/mp4', 'application/ogg', 'application/x-matroska'};
|
||||
const _genericTypes = {'', 'application/octet-stream', 'binary/octet-stream', 'application/binary'};
|
||||
|
||||
/// True when a `Range: bytes=0-0` answer ([status], [contentType]'s mime
|
||||
/// type) means [url] is a media file: a video/audio type, or a generic one
|
||||
/// on a media file name. Pages, HLS playlists and errors are not.
|
||||
bool looksLikeMediaFile({required int status, required String contentType, required Uri url}) {
|
||||
if (status != 200 && status != 206) return false;
|
||||
final type = contentType.toLowerCase();
|
||||
if (type.contains('mpegurl')) return false;
|
||||
if (type.startsWith('video/') || type.startsWith('audio/') || _fileTypes.contains(type)) return true;
|
||||
return _genericTypes.contains(type) && hasMediaExtension(url);
|
||||
}
|
||||
|
||||
/// [url]'s file name ends in a media extension (`.mkv`, `.mp4`, ...).
|
||||
bool hasMediaExtension(Uri url) {
|
||||
final last = url.pathSegments.where((s) => s.isNotEmpty).lastOrNull ?? '';
|
||||
final dot = last.lastIndexOf('.');
|
||||
return dot > 0 && _mediaExtensions.contains(last.substring(dot + 1).toLowerCase());
|
||||
}
|
||||
|
||||
/// A page's `og:title`, else its `<title>`, entities decoded; null if
|
||||
/// neither.
|
||||
String? htmlTitle(String html) {
|
||||
for (final tag in RegExp(r'<meta\b[^>]*>', caseSensitive: false).allMatches(html)) {
|
||||
final t = tag.group(0)!;
|
||||
if (!RegExp(r'''(?:property|name)\s*=\s*["']og:title["']''', caseSensitive: false).hasMatch(t)) continue;
|
||||
final c = RegExp('content\\s*=\\s*(?:"([^"]*)"|\'([^\']*)\')', caseSensitive: false).firstMatch(t);
|
||||
final v = _unescape(c?.group(1) ?? c?.group(2) ?? '');
|
||||
if (v.isNotEmpty) return v;
|
||||
}
|
||||
final m = RegExp(r'<title[^>]*>([\s\S]*?)</title>', caseSensitive: false).firstMatch(html);
|
||||
final v = _unescape(m?.group(1) ?? '');
|
||||
return v.isEmpty ? null : v;
|
||||
}
|
||||
|
||||
String _unescape(String s) => s
|
||||
.replaceAllMapped(RegExp(r'&#(x?)([0-9a-fA-F]+);'), (m) {
|
||||
final n = int.tryParse(m.group(2)!, radix: m.group(1)!.isEmpty ? 10 : 16);
|
||||
return n == null ? m.group(0)! : String.fromCharCode(n);
|
||||
})
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll(''', "'")
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll(' ', ' ')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll(RegExp(r'\s+'), ' ')
|
||||
.trim();
|
||||
|
||||
/// The text for a page yt-dlp cannot read.
|
||||
String notAVideoText(String? ytDlpError) =>
|
||||
ytDlpError == null ? 'Not a video' : "Not a video (and yt-dlp couldn't read it: $ytDlpError)";
|
||||
|
||||
/// The text for a page with only split or HLS streams.
|
||||
const noSingleFileText = 'This page has no single-file video to pass on (only separate video and audio '
|
||||
"streams). Choose 'Everyone fetches it' instead.";
|
||||
|
||||
class LinkProbeException implements Exception {
|
||||
final String message;
|
||||
const LinkProbeException(this.message);
|
||||
@override
|
||||
String toString() => message;
|
||||
}
|
||||
|
||||
/// What [probeLink] found.
|
||||
class LinkProbe {
|
||||
final Uri url;
|
||||
final bool isPage;
|
||||
/// A file's byte size; -1 when unknown or a page.
|
||||
final int size;
|
||||
/// yt-dlp's answer for a page it could read.
|
||||
final YtDlpInfo? info;
|
||||
/// A page's title from its HTML when yt-dlp could not read it.
|
||||
final String? pageTitle;
|
||||
/// Why yt-dlp could not read the page.
|
||||
final String? ytDlpError;
|
||||
const LinkProbe.file(this.url, this.size)
|
||||
: isPage = false,
|
||||
info = null,
|
||||
pageTitle = null,
|
||||
ytDlpError = null;
|
||||
const LinkProbe.page(this.url, {this.info, this.pageTitle, this.ytDlpError})
|
||||
: isPage = true,
|
||||
size = -1;
|
||||
|
||||
/// The name to show: yt-dlp's title, the page's, else the host; null
|
||||
/// for a file (named by its file name).
|
||||
String? get title {
|
||||
if (!isPage) return null;
|
||||
final t = info?.title ?? pageTitle ?? '';
|
||||
return t.isNotEmpty ? t : url.host;
|
||||
}
|
||||
|
||||
/// Why a relay cannot read this link, or null when it can.
|
||||
String? get relayProblem => !isPage
|
||||
? null
|
||||
: info == null
|
||||
? notAVideoText(ytDlpError)
|
||||
: info!.single == null
|
||||
? noSingleFileText
|
||||
: null;
|
||||
}
|
||||
|
||||
const _probeLimit = 512 * 1024;
|
||||
|
||||
/// Finds out what [url] is: a `Range: bytes=0-0` GET; a media file is
|
||||
/// done, anything else goes to yt-dlp ([ytDlp] null: no yt-dlp), and when
|
||||
/// yt-dlp cannot read it, the page's own title. Throws
|
||||
/// [LinkProbeException] when the link cannot be reached at all.
|
||||
Future<LinkProbe> probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async {
|
||||
final c = client ?? (HttpClient()..connectionTimeout = const Duration(seconds: 10));
|
||||
try {
|
||||
String? body;
|
||||
Object? getError;
|
||||
try {
|
||||
final req = await c.getUrl(url).timeout(const Duration(seconds: 15));
|
||||
req.headers
|
||||
..set(HttpHeaders.rangeHeader, 'bytes=0-0')
|
||||
..set(HttpHeaders.acceptEncodingHeader, 'identity');
|
||||
final res = await req.close().timeout(const Duration(seconds: 15));
|
||||
final type = res.headers.contentType?.mimeType ?? '';
|
||||
if (looksLikeMediaFile(status: res.statusCode, contentType: type, url: url)) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
final total = res.headers.value(HttpHeaders.contentRangeHeader)?.split('/').last;
|
||||
final n = int.tryParse(total ?? '') ?? (res.statusCode == 200 ? res.contentLength : -1);
|
||||
return LinkProbe.file(url, n > 0 ? n : -1);
|
||||
}
|
||||
body = type.contains('html') ? await _readText(res) : null;
|
||||
if (body == null) await res.drain<void>().catchError((_) {});
|
||||
} catch (e) {
|
||||
getError = e;
|
||||
}
|
||||
String? ytErr;
|
||||
if (ytDlp != null) {
|
||||
try {
|
||||
return LinkProbe.page(url, info: await ytDlp.resolve(url.toString()));
|
||||
} catch (e) {
|
||||
ytErr = '$e';
|
||||
}
|
||||
}
|
||||
if (getError != null) {
|
||||
throw LinkProbeException('cannot open the link: $getError${ytErr == null ? '' : ' (yt-dlp: $ytErr)'}');
|
||||
}
|
||||
// A server honouring the Range sent one byte of the page: get it all.
|
||||
if (body != null && body.length < 2) {
|
||||
try {
|
||||
final req = await c.getUrl(url).timeout(const Duration(seconds: 15));
|
||||
body = await _readText(await req.close().timeout(const Duration(seconds: 15)));
|
||||
} catch (_) {}
|
||||
}
|
||||
return LinkProbe.page(url, pageTitle: body == null ? null : htmlTitle(body), ytDlpError: ytErr);
|
||||
} finally {
|
||||
if (client == null) c.close(force: true);
|
||||
}
|
||||
}
|
||||
|
||||
Future<String> _readText(HttpClientResponse res) async {
|
||||
final out = BytesBuilder(copy: false);
|
||||
try {
|
||||
await for (final chunk in res.timeout(const Duration(seconds: 15))) {
|
||||
out.add(chunk);
|
||||
if (out.length >= _probeLimit) break;
|
||||
}
|
||||
} catch (_) {}
|
||||
return utf8.decode(out.takeBytes(), allowMalformed: true);
|
||||
}
|
||||
|
||||
/// A page's single-file format (see [YtDlpInfo.single]), range-read with
|
||||
/// yt-dlp's headers. Format URLs expire: on 403/410 the page is resolved
|
||||
/// again once and the read goes on from where it stopped.
|
||||
class PageRelayBytes implements RelayBytes {
|
||||
final Uri page;
|
||||
final YtDlp ytDlp;
|
||||
YtDlpFormat? _format;
|
||||
HttpRelayBytes? _inner;
|
||||
int? _size;
|
||||
var _gen = 0;
|
||||
Future<void>? _refreshing;
|
||||
PageRelayBytes(this.page, this.ytDlp, {YtDlpFormat? format}) : _format = format;
|
||||
|
||||
Future<YtDlpFormat> _resolve() async {
|
||||
final f = (await ytDlp.resolve(page.toString())).single;
|
||||
if (f == null) throw const LinkProbeException(noSingleFileText);
|
||||
return f;
|
||||
}
|
||||
|
||||
Future<HttpRelayBytes> _open() async =>
|
||||
_inner ??= () {
|
||||
final f = _format!;
|
||||
return HttpRelayBytes(f.url, headers: f.headers);
|
||||
}();
|
||||
|
||||
static bool _expired(Object e) => e is UpstreamHttpException && (e.status == 403 || e.status == 410);
|
||||
|
||||
Future<void> _refresh(int gen) {
|
||||
if (gen != _gen) return Future.value();
|
||||
return _refreshing ??= () async {
|
||||
try {
|
||||
final f = await _resolve();
|
||||
final old = _inner;
|
||||
_format = f;
|
||||
_inner = null;
|
||||
_gen++;
|
||||
await old?.close();
|
||||
} finally {
|
||||
_refreshing = null;
|
||||
}
|
||||
}();
|
||||
}
|
||||
|
||||
@override
|
||||
Future<int> size() async {
|
||||
if (_size != null) return _size!;
|
||||
_format ??= await _resolve();
|
||||
final gen = _gen;
|
||||
try {
|
||||
return _size = await (await _open()).size();
|
||||
} catch (e) {
|
||||
if (!_expired(e)) rethrow;
|
||||
await _refresh(gen);
|
||||
return _size = await (await _open()).size();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Stream<List<int>> read(int start, int end) async* {
|
||||
var pos = start;
|
||||
var retried = false;
|
||||
while (true) {
|
||||
final gen = _gen;
|
||||
final inner = await _open();
|
||||
try {
|
||||
await for (final chunk in inner.read(pos, end)) {
|
||||
pos += chunk.length;
|
||||
yield chunk;
|
||||
}
|
||||
return;
|
||||
} catch (e) {
|
||||
if (retried || !_expired(e)) rethrow;
|
||||
retried = true;
|
||||
await _refresh(gen);
|
||||
final n = await (await _open()).size();
|
||||
if (_size != null && n != _size) throw const LinkProbeException('the video changed on the page');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> close() async => _inner?.close();
|
||||
}
|
||||
|
||||
/// A link of either kind: probed on first [size] (or given [probe]), then
|
||||
/// read as a file ([HttpRelayBytes]) or a page ([PageRelayBytes]). Throws
|
||||
/// [LinkProbeException] with the reason when it cannot be relayed.
|
||||
class LinkRelayBytes implements RelayBytes {
|
||||
final Uri url;
|
||||
final YtDlp? ytDlp;
|
||||
LinkProbe? _probe;
|
||||
RelayBytes? _inner;
|
||||
LinkRelayBytes(this.url, {this.ytDlp, LinkProbe? probe}) : _probe = probe;
|
||||
|
||||
/// The probe, after [size].
|
||||
LinkProbe? get probe => _probe;
|
||||
|
||||
@override
|
||||
Future<int> size() async {
|
||||
if (_inner == null) {
|
||||
final probe = _probe ??= await probeLink(url, ytDlp: ytDlp);
|
||||
final problem = probe.relayProblem;
|
||||
if (problem != null) throw LinkProbeException(problem);
|
||||
_inner = probe.isPage ? PageRelayBytes(url, ytDlp!, format: probe.info!.single) : HttpRelayBytes(url);
|
||||
}
|
||||
return _inner!.size();
|
||||
}
|
||||
|
||||
@override
|
||||
Stream<List<int>> read(int start, int end) {
|
||||
final inner = _inner;
|
||||
if (inner == null) return Stream.error(StateError('size() first'));
|
||||
return inner.read(start, end);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> close() async => _inner?.close();
|
||||
}
|
||||
@@ -27,6 +27,11 @@ import 'logging.dart';
|
||||
|
||||
final _log = createLogger('relay');
|
||||
|
||||
/// The `<name>` segment of a `/r/` URL: the media name URL-encoded, its
|
||||
/// slashes made `_` first (a page title can hold a URL, and an encoded
|
||||
/// slash still splits the path). The server ignores the segment.
|
||||
String relayUrlName(String name) => Uri.encodeComponent(name.replaceAll(RegExp(r'[/\\]'), '_'));
|
||||
|
||||
/// Binary relay frame: 4-byte big-endian pull id, then payload.
|
||||
Uint8List encodeRelayFrame(int id, List<int> payload) {
|
||||
final out = Uint8List(4 + payload.length);
|
||||
@@ -66,6 +71,12 @@ class FileRelayBytes implements RelayBytes {
|
||||
Future<void> close() async {}
|
||||
}
|
||||
|
||||
/// A non-2xx answer from an [HttpRelayBytes] upstream.
|
||||
class UpstreamHttpException extends HttpException {
|
||||
final int status;
|
||||
UpstreamHttpException(this.status, Uri url) : super('upstream HTTP $status', uri: url);
|
||||
}
|
||||
|
||||
/// An HTTP(S) resource with Range support, e.g. a link or a Jellyfin
|
||||
/// stream; [headers] (auth) never leave this machine.
|
||||
class HttpRelayBytes implements RelayBytes {
|
||||
@@ -85,7 +96,7 @@ class HttpRelayBytes implements RelayBytes {
|
||||
final res = await req.close().timeout(const Duration(seconds: 20));
|
||||
if (res.statusCode != 200 && res.statusCode != 206) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
throw HttpException('upstream HTTP ${res.statusCode}', uri: url);
|
||||
throw UpstreamHttpException(res.statusCode, url);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:ffi' show Abi;
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:path/path.dart' as p;
|
||||
|
||||
import 'logging.dart';
|
||||
|
||||
// yt-dlp: turns a web page (YouTube, Twitter/X, ...) into media URLs. The
|
||||
// app and the server each keep their own copy, the official standalone
|
||||
// build downloaded from GitHub and checked against the release's
|
||||
// SHA2-256SUMS, refreshed at most once a day. Nothing is installed by hand.
|
||||
|
||||
final _log = createLogger('yt-dlp');
|
||||
|
||||
class YtDlpException implements Exception {
|
||||
final String message;
|
||||
const YtDlpException(this.message);
|
||||
@override
|
||||
String toString() => message;
|
||||
}
|
||||
|
||||
/// One playable format of a resolved page: a URL and the headers yt-dlp
|
||||
/// says to send with it.
|
||||
class YtDlpFormat {
|
||||
final String id;
|
||||
final Uri url;
|
||||
final Map<String, String> headers;
|
||||
final int? height;
|
||||
final String ext;
|
||||
const YtDlpFormat({required this.id, required this.url, this.headers = const {}, this.height, this.ext = ''});
|
||||
}
|
||||
|
||||
/// What `yt-dlp -J` says about a page.
|
||||
class YtDlpInfo {
|
||||
final String title;
|
||||
final String uploader;
|
||||
final String extractor;
|
||||
final String webpageUrl;
|
||||
/// Seconds; null when unknown (a live stream).
|
||||
final double? duration;
|
||||
final String? thumbnail;
|
||||
/// The best format holding video and audio in one http(s) file (what a
|
||||
/// relay can range-read); null when the page only has split or HLS
|
||||
/// streams.
|
||||
final YtDlpFormat? single;
|
||||
const YtDlpInfo({
|
||||
required this.title,
|
||||
this.uploader = '',
|
||||
this.extractor = '',
|
||||
this.webpageUrl = '',
|
||||
this.duration,
|
||||
this.thumbnail,
|
||||
this.single,
|
||||
});
|
||||
|
||||
/// Parses `yt-dlp -J` output; a playlist answer uses its first entry.
|
||||
static YtDlpInfo parse(Map<String, Object?> j) {
|
||||
final entries = j['entries'];
|
||||
if (j['_type'] == 'playlist' && entries is List) {
|
||||
final first = entries.whereType<Map>().firstOrNull;
|
||||
if (first == null) throw const YtDlpException('the page has no videos');
|
||||
return parse(Map<String, Object?>.from(first));
|
||||
}
|
||||
String str(String k) => j[k] is String ? (j[k] as String).trim() : '';
|
||||
final formats = j['formats'] is List ? (j['formats'] as List) : [if (j['url'] is String) j];
|
||||
return YtDlpInfo(
|
||||
title: str('title').isNotEmpty ? str('title') : str('fulltitle'),
|
||||
uploader: str('uploader').isNotEmpty ? str('uploader') : str('channel'),
|
||||
extractor: str('extractor_key').isNotEmpty ? str('extractor_key') : str('extractor'),
|
||||
webpageUrl: str('webpage_url'),
|
||||
duration: j['duration'] is num ? (j['duration'] as num).toDouble() : null,
|
||||
thumbnail: str('thumbnail').isEmpty ? null : str('thumbnail'),
|
||||
single: pickSingleFile(formats),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const _singleFileExts = {'mp4', 'webm', 'mov', 'm4v', 'mkv'};
|
||||
|
||||
/// The best single-file format of [formats] (yt-dlp format maps): video
|
||||
/// and audio both present (or not known to be absent), plain http(s), a
|
||||
/// video container; the tallest up to [maxHeight], else the shortest above
|
||||
/// it; mp4/webm and higher bitrate break ties.
|
||||
YtDlpFormat? pickSingleFile(List<Object?> formats, {int maxHeight = 1080}) {
|
||||
final ok = <(Map, int)>[];
|
||||
for (final f in formats) {
|
||||
if (f is! Map) continue;
|
||||
final url = f['url'];
|
||||
final protocol = f['protocol'];
|
||||
if (url is! String || (protocol != 'http' && protocol != 'https')) continue;
|
||||
if (f['vcodec'] == 'none' || f['acodec'] == 'none') continue;
|
||||
if (!_singleFileExts.contains(f['ext'])) continue;
|
||||
final uri = Uri.tryParse(url);
|
||||
if (uri == null || !(uri.isScheme('http') || uri.isScheme('https'))) continue;
|
||||
ok.add((f, f['height'] is num ? (f['height'] as num).toInt() : 0));
|
||||
}
|
||||
if (ok.isEmpty) return null;
|
||||
double tbr(Map f) => f['tbr'] is num ? (f['tbr'] as num).toDouble() : 0;
|
||||
int rank((Map, int) a, (Map, int) b) {
|
||||
final (fa, ha) = a;
|
||||
final (fb, hb) = b;
|
||||
final ina = ha <= maxHeight, inb = hb <= maxHeight;
|
||||
if (ina != inb) return ina ? -1 : 1;
|
||||
if (ha != hb) return ina ? hb.compareTo(ha) : ha.compareTo(hb);
|
||||
final ma = fa['ext'] == 'mp4' || fa['ext'] == 'webm', mb = fb['ext'] == 'mp4' || fb['ext'] == 'webm';
|
||||
if (ma != mb) return ma ? -1 : 1;
|
||||
return tbr(fb).compareTo(tbr(fa));
|
||||
}
|
||||
|
||||
ok.sort(rank);
|
||||
final (f, h) = ok.first;
|
||||
final headers = f['http_headers'];
|
||||
return YtDlpFormat(
|
||||
id: '${f['format_id'] ?? ''}',
|
||||
url: Uri.parse(f['url'] as String),
|
||||
headers: headers is Map ? {for (final e in headers.entries) '${e.key}': '${e.value}'} : const {},
|
||||
height: h == 0 ? null : h,
|
||||
ext: '${f['ext']}',
|
||||
);
|
||||
}
|
||||
|
||||
/// The official standalone build for this platform, or null if there is
|
||||
/// none.
|
||||
String? ytDlpAssetName(Abi abi, {bool musl = false}) => switch (abi) {
|
||||
Abi.linuxX64 => musl ? 'yt-dlp_musllinux' : 'yt-dlp_linux',
|
||||
Abi.linuxArm64 => musl ? 'yt-dlp_musllinux_aarch64' : 'yt-dlp_linux_aarch64',
|
||||
Abi.windowsX64 => 'yt-dlp.exe',
|
||||
Abi.windowsArm64 => 'yt-dlp_arm64.exe',
|
||||
Abi.windowsIA32 => 'yt-dlp_x86.exe',
|
||||
Abi.macosX64 || Abi.macosArm64 => 'yt-dlp_macos',
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// `SHA2-256SUMS` lines (`<hex> <name>`) as name -> lowercase hex.
|
||||
Map<String, String> parseSha256Sums(String text) => {
|
||||
for (final line in const LineSplitter().convert(text))
|
||||
if (RegExp(r'^([0-9a-fA-F]{64})\s+\*?(\S+)\s*$').firstMatch(line) case final m?)
|
||||
m.group(2)!: m.group(1)!.toLowerCase(),
|
||||
};
|
||||
|
||||
/// The last `ERROR:` line of yt-dlp's stderr without its prefixes
|
||||
/// (`ERROR: [twitter] 123: No video` -> `No video`), else the last line.
|
||||
String ytDlpErrorText(String stderr) {
|
||||
final lines = const LineSplitter().convert(stderr).map((l) => l.trim()).where((l) => l.isNotEmpty).toList();
|
||||
if (lines.isEmpty) return 'yt-dlp failed';
|
||||
final err = lines.lastWhere((l) => l.startsWith('ERROR:'), orElse: () => lines.last);
|
||||
return err.replaceFirst(RegExp(r'^ERROR:\s*'), '').replaceFirst(RegExp(r'^\[[^\]]+\]\s*[^:\s]*:\s*'), '');
|
||||
}
|
||||
|
||||
/// Finds, downloads, updates and runs yt-dlp. The managed copy lives in
|
||||
/// [dir] (writable); without it a `yt-dlp` on PATH is the fallback when
|
||||
/// the download fails. Every failure is non-fatal: [lastError] says why.
|
||||
class YtDlp {
|
||||
final String dir;
|
||||
final bool enabled;
|
||||
/// How often [ensure] looks for a newer release in the background.
|
||||
final Duration updateEvery;
|
||||
final HttpClient Function() _http;
|
||||
static const releases = 'https://github.com/yt-dlp/yt-dlp/releases';
|
||||
|
||||
YtDlp(this.dir, {this.enabled = true, this.updateEvery = const Duration(days: 1), HttpClient Function()? http})
|
||||
: _http = http ?? HttpClient.new;
|
||||
|
||||
String? _path;
|
||||
/// The binary in use, once found.
|
||||
String? get path => _path;
|
||||
/// `yt-dlp --version` of [path].
|
||||
String? version;
|
||||
String? lastError;
|
||||
/// A download or update is running.
|
||||
bool busy = false;
|
||||
final _changes = StreamController<void>.broadcast();
|
||||
/// Fires when [version], [lastError] or [busy] change.
|
||||
Stream<void> get changes => _changes.stream;
|
||||
Future<String?>? _ensuring;
|
||||
Future<void>? _updating;
|
||||
|
||||
String get managedPath => p.join(dir, Platform.isWindows ? 'yt-dlp.exe' : 'yt-dlp');
|
||||
String get _stampPath => p.join(dir, 'last-check');
|
||||
|
||||
void _changed() {
|
||||
if (!_changes.isClosed) _changes.add(null);
|
||||
}
|
||||
|
||||
void _fail(String message) {
|
||||
lastError = message;
|
||||
_log.warning(message);
|
||||
_changed();
|
||||
}
|
||||
|
||||
/// The binary to run, downloading it first if needed; null (see
|
||||
/// [lastError]) when there is none. A newer release is looked for in the
|
||||
/// background at most every [updateEvery].
|
||||
Future<String?> ensure() {
|
||||
final known = _path;
|
||||
if (known != null && (known == 'yt-dlp' || File(known).existsSync())) {
|
||||
unawaited(update());
|
||||
return Future.value(known);
|
||||
}
|
||||
return _ensuring ??= _ensure().whenComplete(() => _ensuring = null);
|
||||
}
|
||||
|
||||
Future<String?> _ensure() async {
|
||||
if (!enabled) {
|
||||
lastError = 'yt-dlp is turned off';
|
||||
return null;
|
||||
}
|
||||
if (await File(managedPath).exists()) {
|
||||
final v = await _versionOf(managedPath);
|
||||
if (v != null) {
|
||||
_use(managedPath, v);
|
||||
unawaited(update());
|
||||
return managedPath;
|
||||
}
|
||||
}
|
||||
try {
|
||||
await _download(await _latestTag());
|
||||
return managedPath;
|
||||
} catch (e) {
|
||||
final onPath = await _versionOf('yt-dlp');
|
||||
if (onPath != null) {
|
||||
_use('yt-dlp', onPath);
|
||||
_fail('could not download yt-dlp ($e); using the one on PATH');
|
||||
return 'yt-dlp';
|
||||
}
|
||||
_fail('could not download yt-dlp: $e');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
void _use(String path, String v) {
|
||||
_path = path;
|
||||
version = v;
|
||||
_changed();
|
||||
}
|
||||
|
||||
Future<String?> _versionOf(String exe) async {
|
||||
try {
|
||||
final r = await Process.run(exe, ['--version']).timeout(const Duration(seconds: 20));
|
||||
final v = '${r.stdout}'.trim();
|
||||
return r.exitCode == 0 && v.isNotEmpty ? v : null;
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// Replaces the managed copy when a newer release is out. Without
|
||||
/// [force], at most every [updateEvery] (remembered across restarts).
|
||||
Future<void> update({bool force = false}) => _updating ??= _update(force).whenComplete(() => _updating = null);
|
||||
|
||||
Future<void> _update(bool force) async {
|
||||
if (!enabled) return;
|
||||
try {
|
||||
if (!force) {
|
||||
final stamp = File(_stampPath);
|
||||
if (await stamp.exists()) {
|
||||
final at = DateTime.tryParse((await stamp.readAsString()).trim());
|
||||
if (at != null && DateTime.now().difference(at) < updateEvery) return;
|
||||
}
|
||||
}
|
||||
final tag = await _latestTag();
|
||||
final managed = await File(managedPath).exists() ? await _versionOf(managedPath) : null;
|
||||
if (managed != tag) {
|
||||
await _download(tag);
|
||||
} else {
|
||||
_use(managedPath, managed!);
|
||||
await _stamp();
|
||||
}
|
||||
if (lastError != null) {
|
||||
lastError = null;
|
||||
_changed();
|
||||
}
|
||||
} catch (e) {
|
||||
_fail('yt-dlp update failed: $e');
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _stamp() async {
|
||||
try {
|
||||
await File(_stampPath).writeAsString(DateTime.now().toIso8601String());
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
/// The tag `releases/latest` redirects to.
|
||||
Future<String> _latestTag() async {
|
||||
final client = _http();
|
||||
try {
|
||||
final req = await client.getUrl(Uri.parse('$releases/latest')).timeout(const Duration(seconds: 20));
|
||||
req.followRedirects = false;
|
||||
final res = await req.close().timeout(const Duration(seconds: 20));
|
||||
await res.drain<void>().catchError((_) {});
|
||||
final loc = res.headers.value(HttpHeaders.locationHeader) ?? '';
|
||||
final tag = Uri.tryParse(loc)?.pathSegments.lastOrNull ?? '';
|
||||
if (!res.isRedirect || tag.isEmpty || tag == 'latest') {
|
||||
throw YtDlpException('no latest release (HTTP ${res.statusCode})');
|
||||
}
|
||||
return tag;
|
||||
} finally {
|
||||
client.close(force: true);
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _download(String tag) async {
|
||||
final asset = ytDlpAssetName(Abi.current(), musl: _isMusl());
|
||||
if (asset == null) throw YtDlpException('no yt-dlp build for ${Abi.current()}');
|
||||
busy = true;
|
||||
_changed();
|
||||
final client = _http();
|
||||
final part = File('$managedPath.part$pid');
|
||||
try {
|
||||
await Directory(dir).create(recursive: true);
|
||||
final sums = parseSha256Sums(utf8.decode(await _get(client, '$releases/download/$tag/SHA2-256SUMS')));
|
||||
final want = sums[asset];
|
||||
if (want == null) throw YtDlpException('$asset is not in the release checksums');
|
||||
_log.info('downloading yt-dlp $tag ($asset)');
|
||||
final req = await client.getUrl(Uri.parse('$releases/download/$tag/$asset')).timeout(const Duration(seconds: 30));
|
||||
final res = await req.close().timeout(const Duration(seconds: 30));
|
||||
if (res.statusCode != 200) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
throw YtDlpException('download HTTP ${res.statusCode}');
|
||||
}
|
||||
final sink = part.openWrite();
|
||||
final hashOut = _DigestSink();
|
||||
final hasher = sha256.startChunkedConversion(hashOut);
|
||||
await for (final chunk in res.timeout(const Duration(seconds: 60))) {
|
||||
hasher.add(chunk);
|
||||
sink.add(chunk);
|
||||
}
|
||||
hasher.close();
|
||||
await sink.close();
|
||||
if (hashOut.value.toString() != want) throw const YtDlpException('checksum mismatch');
|
||||
if (!Platform.isWindows) {
|
||||
final r = await Process.run('chmod', ['755', part.path]);
|
||||
if (r.exitCode != 0) throw YtDlpException('chmod failed: ${r.stderr}');
|
||||
}
|
||||
final v = await _versionOf(part.path);
|
||||
if (v == null) throw const YtDlpException('the downloaded yt-dlp does not run');
|
||||
final target = File(managedPath);
|
||||
// Windows cannot rename onto an existing file.
|
||||
if (Platform.isWindows && await target.exists()) await target.delete();
|
||||
await part.rename(managedPath);
|
||||
_use(managedPath, v);
|
||||
await _stamp();
|
||||
_log.info('yt-dlp $v ready at $managedPath');
|
||||
} finally {
|
||||
client.close(force: true);
|
||||
if (await part.exists()) await part.delete().catchError((_) => part);
|
||||
busy = false;
|
||||
_changed();
|
||||
}
|
||||
}
|
||||
|
||||
Future<List<int>> _get(HttpClient client, String url) async {
|
||||
final req = await client.getUrl(Uri.parse(url)).timeout(const Duration(seconds: 20));
|
||||
final res = await req.close().timeout(const Duration(seconds: 20));
|
||||
if (res.statusCode != 200) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
throw YtDlpException('$url: HTTP ${res.statusCode}');
|
||||
}
|
||||
final out = BytesBuilder(copy: false);
|
||||
await for (final c in res.timeout(const Duration(seconds: 20))) {
|
||||
out.add(c);
|
||||
}
|
||||
return out.takeBytes();
|
||||
}
|
||||
|
||||
static bool _isMusl() {
|
||||
if (!Platform.isLinux) return false;
|
||||
try {
|
||||
return Directory('/lib').listSync().any((e) => p.basename(e.path).startsWith('ld-musl-'));
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// Asks yt-dlp what [url] is. Throws [YtDlpException] with yt-dlp's own
|
||||
/// reason on failure.
|
||||
Future<YtDlpInfo> resolve(String url, {Duration timeout = const Duration(seconds: 25)}) async {
|
||||
final exe = await ensure();
|
||||
if (exe == null) throw YtDlpException(lastError ?? 'yt-dlp is not available');
|
||||
final proc = await Process.start(exe, ['-J', '--no-playlist', '--no-warnings', '--ignore-config', '--', url]);
|
||||
final out = proc.stdout.transform(utf8.decoder).join();
|
||||
final err = proc.stderr.transform(utf8.decoder).join();
|
||||
final int code;
|
||||
try {
|
||||
code = await proc.exitCode.timeout(timeout);
|
||||
} on TimeoutException {
|
||||
proc.kill();
|
||||
throw YtDlpException('yt-dlp took longer than ${timeout.inSeconds} s');
|
||||
}
|
||||
if (code != 0) throw YtDlpException(ytDlpErrorText(await err));
|
||||
try {
|
||||
final j = jsonDecode(await out);
|
||||
if (j is! Map) throw const FormatException('not an object');
|
||||
return YtDlpInfo.parse(Map<String, Object?>.from(j));
|
||||
} on FormatException catch (e) {
|
||||
throw YtDlpException('unreadable yt-dlp output: ${e.message}');
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> close() => _changes.close();
|
||||
}
|
||||
|
||||
class _DigestSink implements Sink<Digest> {
|
||||
late Digest value;
|
||||
@override
|
||||
void add(Digest data) => value = data;
|
||||
@override
|
||||
void close() {}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
// yt-dlp output parsing, the single-file pick and the link classifier.
|
||||
// The live test (real download from GitHub, a real tweet) runs only with
|
||||
// YTDLP_LIVE=1.
|
||||
library;
|
||||
|
||||
import 'dart:ffi' show Abi;
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:sync_relay_core/core.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
Map<String, Object?> _f(String id, String protocol, Object? vcodec, Object? acodec, int? height,
|
||||
{String ext = 'mp4', num tbr = 0}) =>
|
||||
{
|
||||
'format_id': id, 'protocol': protocol, 'vcodec': vcodec, 'acodec': acodec, 'height': height, //
|
||||
'ext': ext, 'tbr': tbr, 'url': 'https://cdn.example/$id', 'http_headers': {'User-Agent': 'UA'},
|
||||
};
|
||||
|
||||
void main() {
|
||||
group('single-file pick', () {
|
||||
test('a tweet: the tallest progressive mp4 (codecs unknown), not the split HLS', () {
|
||||
final f = pickSingleFile([
|
||||
_f('hls-audio', 'm3u8_native', 'none', null, null),
|
||||
_f('http-256', 'https', null, null, 270, tbr: 256),
|
||||
_f('http-2176', 'https', null, null, 598, tbr: 2176),
|
||||
_f('http-832', 'https', null, null, 360, tbr: 832),
|
||||
_f('hls-430', 'm3u8_native', 'avc1', 'none', 598),
|
||||
])!;
|
||||
expect((f.id, f.height), ('http-2176', 598));
|
||||
expect(f.headers, {'User-Agent': 'UA'});
|
||||
});
|
||||
|
||||
test('split or HLS only: none', () {
|
||||
expect(
|
||||
pickSingleFile([
|
||||
_f('140', 'https', 'none', 'mp4a', null, ext: 'm4a'),
|
||||
_f('137', 'https', 'avc1', 'none', 1080),
|
||||
_f('96', 'm3u8_native', 'avc1', 'mp4a', 1080),
|
||||
]),
|
||||
isNull);
|
||||
});
|
||||
|
||||
test('up to 1080p, else the shortest above; mp4/webm win ties', () {
|
||||
expect(
|
||||
pickSingleFile([
|
||||
_f('4k', 'https', 'vp9', 'opus', 2160, ext: 'webm'),
|
||||
_f('720', 'https', 'avc1', 'mp4a', 720),
|
||||
_f('1080mkv', 'https', 'avc1', 'mp4a', 1080, ext: 'mkv'),
|
||||
_f('1080', 'https', 'avc1', 'mp4a', 1080),
|
||||
])!.id,
|
||||
'1080');
|
||||
expect(
|
||||
pickSingleFile([
|
||||
_f('4k', 'https', 'vp9', 'opus', 2160, ext: 'webm'),
|
||||
_f('1440', 'https', 'vp9', 'opus', 1440, ext: 'webm'),
|
||||
])!.id,
|
||||
'1440');
|
||||
});
|
||||
|
||||
test('info: metadata, a playlist answer uses its first entry', () {
|
||||
final info = YtDlpInfo.parse({
|
||||
'_type': 'playlist',
|
||||
'entries': [
|
||||
{
|
||||
'title': 'a girl. - https://t.co/x', 'uploader': 'a girl.', 'duration': 6.3, //
|
||||
'thumbnail': 'https://pbs/x.jpg', 'extractor_key': 'Twitter',
|
||||
'formats': [_f('http-2176', 'https', null, null, 598)],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect((info.title, info.uploader, info.duration, info.thumbnail, info.extractor, info.single?.id),
|
||||
('a girl. - https://t.co/x', 'a girl.', 6.3, 'https://pbs/x.jpg', 'Twitter', 'http-2176'));
|
||||
});
|
||||
});
|
||||
|
||||
test('link classifier: media types and media file names are files; pages, HLS, errors are not', () {
|
||||
bool file(int status, String type, String url) =>
|
||||
looksLikeMediaFile(status: status, contentType: type, url: Uri.parse(url));
|
||||
expect(file(206, 'video/mp4', 'https://a/watch'), isTrue);
|
||||
expect(file(200, 'audio/mpeg', 'https://a/x'), isTrue);
|
||||
expect(file(206, 'application/octet-stream', 'https://a/dir/ep%201.MKV?sig=1'), isTrue);
|
||||
expect(file(206, 'application/octet-stream', 'https://a/download'), isFalse);
|
||||
expect(file(200, 'text/html', 'https://x.com/u/status/1/video/1'), isFalse);
|
||||
expect(file(200, 'text/html', 'https://a/fake.mp4'), isFalse);
|
||||
expect(file(200, 'application/vnd.apple.mpegurl', 'https://a/index.m3u8'), isFalse);
|
||||
expect(file(403, 'video/mp4', 'https://a/x.mp4'), isFalse);
|
||||
});
|
||||
|
||||
test('a page title in a /r/ URL stays one path segment', () {
|
||||
expect(relayUrlName('a girl. - https://t.co/x'), 'a%20girl.%20-%20https%3A__t.co_x');
|
||||
});
|
||||
|
||||
test('page title: og:title first (any attribute order), else <title>, entities decoded', () {
|
||||
expect(htmlTitle('<title>Plain</title><meta content="Tom & Jerry '24" property="og:title">'),
|
||||
"Tom & Jerry '24");
|
||||
expect(htmlTitle("<head><TITLE>\n A "page"\n</TITLE></head>"), 'A "page"');
|
||||
expect(htmlTitle('<p>nothing</p>'), isNull);
|
||||
});
|
||||
|
||||
test('release files: checksums, asset per platform, error text', () {
|
||||
final sums = parseSha256Sums('${'a' * 64} yt-dlp\n${'B' * 64} yt-dlp.exe\ngarbage\n');
|
||||
expect(sums, {'yt-dlp': 'a' * 64, 'yt-dlp.exe': 'b' * 64});
|
||||
expect(ytDlpAssetName(Abi.linuxX64), 'yt-dlp_linux');
|
||||
expect(ytDlpAssetName(Abi.linuxArm64, musl: true), 'yt-dlp_musllinux_aarch64');
|
||||
expect(ytDlpAssetName(Abi.windowsX64), 'yt-dlp.exe');
|
||||
expect(ytDlpAssetName(Abi.androidArm), isNull);
|
||||
expect(ytDlpErrorText('WARNING: x\nERROR: [twitter] 123: No video could be found in this tweet\n'),
|
||||
'No video could be found in this tweet');
|
||||
expect(ytDlpErrorText('ERROR: Unsupported URL: https://example.com/'), 'Unsupported URL: https://example.com/');
|
||||
});
|
||||
|
||||
test('live: download yt-dlp, resolve a tweet, relay its single file', () async {
|
||||
final dir = await Directory.systemTemp.createTemp('ytdlp_live');
|
||||
addTearDown(() => dir.delete(recursive: true));
|
||||
final y = YtDlp(dir.path);
|
||||
expect(await y.ensure(), y.managedPath, reason: '${y.lastError}');
|
||||
expect(y.version, isNotNull);
|
||||
final url = Uri.parse('https://x.com/velcrotea/status/2108421180701188522/video/1');
|
||||
final probe = await probeLink(url, ytDlp: y);
|
||||
expect(probe.isPage, isTrue);
|
||||
expect(probe.title, startsWith('a girl.'));
|
||||
expect(probe.relayProblem, isNull);
|
||||
final bytes = LinkRelayBytes(url, ytDlp: y, probe: probe);
|
||||
final size = await bytes.size();
|
||||
var n = 0;
|
||||
await for (final c in bytes.read(0, size - 1)) {
|
||||
n += c.length;
|
||||
}
|
||||
await bytes.close();
|
||||
expect(n, size);
|
||||
}, timeout: const Timeout(Duration(minutes: 3)), skip: Platform.environment['YTDLP_LIVE'] == '1' ? false : 'YTDLP_LIVE not set');
|
||||
}
|
||||
Reference in new issue
Block a user