forked from Foxyz/sync_replay_project
Sessions leave redacted logs; viewers wait for a member relay's source before opening it
Logs: one redaction pass (query strings to names, member tokens,
passwords, cookies, auth headers) for every line; daily files with size
and age caps in the app (hosted server included) and the dedicated
server (--log-dir); swaps, relay channels, per-member /r /m /j outcomes,
server fetches, mpv load errors, yt-dlp runs. Settings > About: Copy
diagnostics, Open logs folder.
Bilibili through someone else: the viewer opened /r/ before the source
dialled in; the server holds that request and mpv gives up after 5 s
("could not be opened"). Viewers now open a member relay once it is
live, the putter starts its source before its own player, and track
sizes are read in parallel.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
foxyzandClaude Opus 5.5 committed
1 parent
572104f49e
commit
4ab37c55cf
25 files changed
+1124
-108
No files matched your search
@@ -4,6 +4,7 @@ import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'constants.dart';
|
||||
import 'logging.dart';
|
||||
import 'relay.dart';
|
||||
import 'relay_pieces.dart';
|
||||
import 'yt_dlp.dart';
|
||||
@@ -170,6 +171,8 @@ class LinkProbe {
|
||||
: null;
|
||||
}
|
||||
|
||||
final _log = createLogger('link');
|
||||
|
||||
const _probeLimit = 512 * 1024;
|
||||
|
||||
/// Finds out what [url] is: a `Range: bytes=0-0` GET; a media file is
|
||||
@@ -177,6 +180,17 @@ const _probeLimit = 512 * 1024;
|
||||
/// yt-dlp cannot read it, the page's own title. Throws
|
||||
/// [LinkProbeException] when the link cannot be reached at all.
|
||||
Future<LinkProbe> probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async {
|
||||
final p = await _probeLink(url, ytDlp: ytDlp, client: client);
|
||||
_log.info('probe ${logUrl(url)}: ${p.isPage ? 'page' : 'file of ${p.size} bytes'}'
|
||||
'${p.status == 0 ? '' : ', HTTP ${p.status}'}'
|
||||
'${p.info == null ? '' : ', yt-dlp read it'}'
|
||||
'${p.ytDlpError == null ? '' : ', yt-dlp: ${p.ytDlpError}'}'
|
||||
'${p.problem == null ? '' : '; problem: ${p.problem}'}'
|
||||
'${p.relayProblem == null ? '' : '; relay: ${p.relayProblem}'}');
|
||||
return p;
|
||||
}
|
||||
|
||||
Future<LinkProbe> _probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async {
|
||||
final c = client ?? (HttpClient()..connectionTimeout = const Duration(seconds: 10));
|
||||
try {
|
||||
String? body;
|
||||
@@ -210,6 +224,7 @@ Future<LinkProbe> probeLink(Uri url, {YtDlp? ytDlp, HttpClient? client}) async {
|
||||
}
|
||||
}
|
||||
if (getError != null) {
|
||||
_log.warning('probe ${logUrl(url)}: cannot open it: $getError${ytErr == null ? '' : ' (yt-dlp: $ytErr)'}');
|
||||
throw LinkProbeException('cannot open the link: $getError${ytErr == null ? '' : ' (yt-dlp: $ytErr)'}');
|
||||
}
|
||||
// A server honouring the Range sent one byte of the page: get it all.
|
||||
@@ -300,6 +315,7 @@ class PageRelayMedia implements RelayMedia {
|
||||
Future<void> _refresh(int gen) {
|
||||
if (gen != _gen) return Future.value();
|
||||
return _refreshing ??= () async {
|
||||
_log.info('relay of ${logUrl(page)}: a format URL expired (403/410), resolving the page again');
|
||||
try {
|
||||
const changed = LinkProbeException('the video changed on the page');
|
||||
final info = await _info();
|
||||
|
||||
@@ -1,19 +1,367 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:logging/logging.dart';
|
||||
|
||||
// One log for the app (its hosted server's records included) and one for
|
||||
// the dedicated server: every record goes through [redactLog] once, then
|
||||
// to the console and, once [startLogFiles] ran, to a rotating file
|
||||
// (ARCHITECTURE "Logging").
|
||||
|
||||
Logger createLogger(String name) => Logger(name);
|
||||
|
||||
void initLogging({Level level = Level.INFO}) {
|
||||
var _listening = false;
|
||||
var _console = true;
|
||||
LogFiles? _files;
|
||||
// Lines logged before the files are open (the app finds its folder only
|
||||
// after Flutter is up): written first once they are.
|
||||
final _early = <String>[];
|
||||
const _earlyMax = 2000;
|
||||
|
||||
/// The open log files, once [startLogFiles] ran.
|
||||
LogFiles? get logFiles => _files;
|
||||
|
||||
/// Routes every log record to the console ([console]) and, after
|
||||
/// [startLogFiles], to the files. Safe to call again (only [level] and
|
||||
/// [console] change).
|
||||
void initLogging({Level level = Level.INFO, bool console = true}) {
|
||||
Logger.root.level = level;
|
||||
Logger.root.onRecord.listen((r) {
|
||||
// ignore: avoid_print
|
||||
print('${r.time.toIso8601String()} ${r.level.name} ${r.loggerName}: ${r.message}');
|
||||
if (r.error != null) {
|
||||
// ignore: avoid_print
|
||||
print(' error: ${r.error}');
|
||||
}
|
||||
if (r.stackTrace != null) {
|
||||
// ignore: avoid_print
|
||||
print('${r.stackTrace}');
|
||||
}
|
||||
});
|
||||
_console = console;
|
||||
if (_listening) return;
|
||||
_listening = true;
|
||||
Logger.root.onRecord.listen((r) =>
|
||||
writeLogLine(formatLogLine(r.time, r.level.name, r.loggerName, r.message, error: r.error, stack: r.stackTrace)));
|
||||
}
|
||||
|
||||
/// Writes the files in [dir] from now on: `<prefix>-YYYY-MM-DD.log`,
|
||||
/// rotated by day and [LogFiles.maxFileBytes], pruned by age and total
|
||||
/// size. Lines logged before go in first. A folder that cannot be written
|
||||
/// leaves logging on the console only (and says so there).
|
||||
LogFiles? startLogFiles(String dir, {String prefix = 'app'}) {
|
||||
if (_files != null) return _files;
|
||||
try {
|
||||
final f = LogFiles(dir, prefix: prefix)..open();
|
||||
_files = f;
|
||||
for (final l in _early) {
|
||||
f.write(l);
|
||||
}
|
||||
_early.clear();
|
||||
return f;
|
||||
} catch (e) {
|
||||
// ignore: avoid_print
|
||||
print('log files off: cannot write $dir ($e)');
|
||||
_early.clear();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// One finished, redacted line to the console and the file.
|
||||
void writeLogLine(String line) {
|
||||
if (_console) {
|
||||
// ignore: avoid_print
|
||||
print(line);
|
||||
}
|
||||
final f = _files;
|
||||
if (f != null) {
|
||||
f.write(line);
|
||||
} else if (_early.length < _earlyMax) {
|
||||
_early.add(line);
|
||||
}
|
||||
}
|
||||
|
||||
/// `2026-10-12 18:44:27.202 WARNING relay: message`, then the error and
|
||||
/// stack trace on their own lines; all of it through [redactLog].
|
||||
String formatLogLine(DateTime time, String level, String logger, String message, {Object? error, Object? stack}) {
|
||||
final b = StringBuffer('${logTimestamp(time)} $level $logger: $message');
|
||||
if (error != null) b.write('\n error: $error');
|
||||
final s = '${stack ?? ''}'.trimRight();
|
||||
if (s.isNotEmpty) b.write('\n$s');
|
||||
return redactLog(b.toString());
|
||||
}
|
||||
|
||||
String _two(int n) => n.toString().padLeft(2, '0');
|
||||
|
||||
/// Local time as the log writes it (`2026-10-12 18:44:27.202`).
|
||||
String logTimestamp(DateTime t) => '${t.year}-${_two(t.month)}-${_two(t.day)} '
|
||||
'${_two(t.hour)}:${_two(t.minute)}:${_two(t.second)}.${t.millisecond.toString().padLeft(3, '0')}';
|
||||
|
||||
/// The time a log line starts with, or null for a continuation line.
|
||||
DateTime? logLineTime(String line) {
|
||||
final m = _lineTime.firstMatch(line);
|
||||
if (m == null) return null;
|
||||
return DateTime.tryParse('${m[1]}T${m[2]}');
|
||||
}
|
||||
|
||||
final _lineTime = RegExp(r'^(\d{4}-\d{2}-\d{2}) (\d{2}:\d{2}:\d{2}\.\d{3}) ');
|
||||
|
||||
/// A record from another isolate (the hosted server's, `LEVEL logger:
|
||||
/// message`) logged here under [tag]: `server.relay` at its own level.
|
||||
void relogLine(String tag, String line) {
|
||||
final m = RegExp(r'^([A-Z]+) ([^:\s]+): ').firstMatch(line);
|
||||
final level = Level.LEVELS.where((l) => l.name == m?[1]).firstOrNull;
|
||||
if (m == null || level == null) {
|
||||
Logger(tag).info(line);
|
||||
return;
|
||||
}
|
||||
Logger('$tag.${m[2]}').log(level, line.substring(m.end));
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- redaction
|
||||
|
||||
// Names whose value is a secret wherever they appear as `name=value`,
|
||||
// `"name":"value"` or `Name="value"` (Jellyfin's MediaBrowser header).
|
||||
const _secretNames = r'(?:x-emby-token|x-mediabrowser-token|x-emby-authorization|authorization|set-cookie|cookie|'
|
||||
r'api_?key|access_?token|refresh_?token|id_?token|auth_?token|token|password|passwd|secret|host_?key|'
|
||||
r'sessdata|bili_jct|client_?secret)';
|
||||
final _jsonSecret = RegExp('"($_secretNames)"\\s*:\\s*"(?:[^"\\\\]|\\\\.)*"', caseSensitive: false);
|
||||
final _quotedSecret = RegExp('\\b($_secretNames)\\s*=\\s*"[^"]*"', caseSensitive: false);
|
||||
// Header-like `Name: value` runs to the end of the line (a cookie
|
||||
// header's `;`-separated pairs, MediaBrowser's comma list).
|
||||
final _headerSecret = RegExp(
|
||||
'\\b(authorization|cookie|set-cookie|x-emby-authorization|x-emby-token|x-mediabrowser-token)\\s*:\\s*[^\\n]*',
|
||||
caseSensitive: false);
|
||||
final _pairSecret = RegExp('\\b($_secretNames)\\s*=\\s*[^&\\s"\',;)}\\]]+', caseSensitive: false);
|
||||
final _bearer = RegExp(r'\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}', caseSensitive: false);
|
||||
final _url = RegExp(r'''\b(?:https?|wss?)://[^\s"'<>]+''', caseSensitive: false);
|
||||
// A member token in a server path: /r/<sid>/<token>/..., /m, /j, /a, /i.
|
||||
final _pathToken = RegExp(r'(/[rmjai]/[^/\s?#]+/)[0-9a-fA-F]{32}\b');
|
||||
final _userInfo = RegExp(r'^([a-z]+://)[^/@\s]+@', caseSensitive: false);
|
||||
|
||||
/// [s] with nothing secret left: every URL's query string down to its
|
||||
/// parameter names (signed CDN links, `api_key`), user:password@ in URLs,
|
||||
/// member tokens in `/r/ /m/ /j/ /a/ /i/` paths, and the values of
|
||||
/// passwords, tokens, API keys, cookies and auth headers. Applied to
|
||||
/// every log line ([formatLogLine]).
|
||||
String redactLog(String s) {
|
||||
if (s.isEmpty) return s;
|
||||
var out = s.replaceAllMapped(_url, (m) => _redactUrl(m[0]!));
|
||||
out = out.replaceAllMapped(_pathToken, (m) => '${m[1]}***');
|
||||
out = out.replaceAllMapped(_jsonSecret, (m) => '"${m[1]}":"***"');
|
||||
out = out.replaceAllMapped(_headerSecret, (m) => '${m[1]}: ***');
|
||||
out = out.replaceAllMapped(_quotedSecret, (m) => '${m[1]}="***"');
|
||||
out = out.replaceAllMapped(_pairSecret, (m) => '${m[1]}=***');
|
||||
out = out.replaceAllMapped(_bearer, (m) => '${m[1]} ***');
|
||||
return out;
|
||||
}
|
||||
|
||||
String _redactUrl(String url) {
|
||||
var u = url.replaceFirstMapped(_userInfo, (m) => '${m[1]}***@');
|
||||
final hash = u.indexOf('#');
|
||||
if (hash >= 0) u = u.substring(0, hash);
|
||||
final q = u.indexOf('?');
|
||||
if (q < 0) return u;
|
||||
final names = u
|
||||
.substring(q + 1)
|
||||
.split('&')
|
||||
.map((p) => p.split('=').first)
|
||||
.where((n) => n.isNotEmpty)
|
||||
.map((n) => n.length > 40 ? '${n.substring(0, 40)}…' : n)
|
||||
.toList();
|
||||
return '${u.substring(0, q)}?${names.isEmpty ? '' : '${names.join('&')}=…'}';
|
||||
}
|
||||
|
||||
/// [url] for a log line: scheme, host and path, the query cut to its
|
||||
/// parameter names (see [redactLog]).
|
||||
String logUrl(Object? url) => redactLog('$url');
|
||||
|
||||
// --------------------------------------------------------------- flood control
|
||||
|
||||
/// At most [burst] lines per key in each [window]; the next line let
|
||||
/// through says how many were dropped. For floods (ffmpeg retrying a
|
||||
/// refused request, a viewer's range reads).
|
||||
class LogThrottle {
|
||||
final int burst;
|
||||
final Duration window;
|
||||
final DateTime Function() _now;
|
||||
final _keys = <String, ({DateTime start, int count, int dropped})>{};
|
||||
LogThrottle({this.burst = 5, this.window = const Duration(seconds: 30), DateTime Function()? now})
|
||||
: _now = now ?? DateTime.now;
|
||||
|
||||
/// Null to drop the line; else what to put before it: '' or
|
||||
/// `(12 more like this dropped) `.
|
||||
String? allow(String key) {
|
||||
final now = _now();
|
||||
final k = _keys[key];
|
||||
if (k == null || now.difference(k.start) >= window) {
|
||||
final dropped = k?.dropped ?? 0;
|
||||
_keys.remove(key);
|
||||
_keys[key] = (start: now, count: 1, dropped: 0);
|
||||
if (_keys.length > 500) _keys.remove(_keys.keys.first);
|
||||
return dropped > 0 ? '($dropped more like this dropped) ' : '';
|
||||
}
|
||||
if (k.count < burst) {
|
||||
_keys[key] = (start: k.start, count: k.count + 1, dropped: k.dropped);
|
||||
return '';
|
||||
}
|
||||
_keys[key] = (start: k.start, count: k.count, dropped: k.dropped + 1);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ files
|
||||
|
||||
/// Rotating log files in [dir]: `<prefix>-YYYY-MM-DD.log` (then `.1.log`,
|
||||
/// `.2.log` ... once one passes [maxFileBytes]); files older than
|
||||
/// [keepDays] and the oldest beyond [maxTotalBytes] are deleted whenever a
|
||||
/// file is opened.
|
||||
class LogFiles {
|
||||
final String dir;
|
||||
final String prefix;
|
||||
final int maxFileBytes;
|
||||
final int maxTotalBytes;
|
||||
final int keepDays;
|
||||
final DateTime Function() _now;
|
||||
LogFiles(this.dir,
|
||||
{this.prefix = 'app',
|
||||
this.maxFileBytes = 10 * 1024 * 1024,
|
||||
this.maxTotalBytes = 100 * 1024 * 1024,
|
||||
this.keepDays = 7,
|
||||
DateTime Function()? now})
|
||||
: _now = now ?? DateTime.now;
|
||||
|
||||
RandomAccessFile? _out;
|
||||
String? _day;
|
||||
var _size = 0;
|
||||
String? _path;
|
||||
|
||||
/// The file being written.
|
||||
String? get currentPath => _path;
|
||||
|
||||
String _dayOf(DateTime t) => '${t.year}-${_two(t.month)}-${_two(t.day)}';
|
||||
|
||||
String _dayOfFile(File f) => f.uri.pathSegments.last.substring(prefix.length + 1, prefix.length + 11);
|
||||
|
||||
/// Opens today's file (creating [dir]); throws when it cannot.
|
||||
void open() {
|
||||
Directory(dir).createSync(recursive: true);
|
||||
_roll(_dayOf(_now()));
|
||||
}
|
||||
|
||||
void _roll(String day) {
|
||||
_out?.closeSync();
|
||||
_out = null;
|
||||
_day = day;
|
||||
for (var i = 0;; i++) {
|
||||
final path = '$dir${Platform.pathSeparator}$prefix-$day${i == 0 ? '' : '.$i'}.log';
|
||||
final f = File(path);
|
||||
final size = f.existsSync() ? f.lengthSync() : 0;
|
||||
if (size < maxFileBytes) {
|
||||
_out = f.openSync(mode: FileMode.append);
|
||||
_size = size;
|
||||
_path = path;
|
||||
break;
|
||||
}
|
||||
}
|
||||
prune();
|
||||
}
|
||||
|
||||
/// Appends [line]; a new day or a full file moves on to the next file.
|
||||
void write(String line) {
|
||||
try {
|
||||
final day = _dayOf(_now());
|
||||
if (_out == null || day != _day || _size >= maxFileBytes) _roll(day);
|
||||
final bytes = utf8.encode('$line\n');
|
||||
_out!.writeFromSync(bytes);
|
||||
_size += bytes.length;
|
||||
} catch (_) {
|
||||
// A full disk or a removed folder must not take the app down.
|
||||
}
|
||||
}
|
||||
|
||||
/// This prefix's files, oldest first.
|
||||
List<File> files() {
|
||||
final d = Directory(dir);
|
||||
if (!d.existsSync()) return const [];
|
||||
final name = RegExp('^${RegExp.escape(prefix)}-(\\d{4}-\\d{2}-\\d{2})(?:\\.(\\d+))?\\.log\$');
|
||||
final found = <(String, int, File)>[];
|
||||
for (final e in d.listSync()) {
|
||||
if (e is! File) continue;
|
||||
final m = name.firstMatch(e.uri.pathSegments.last);
|
||||
if (m != null) found.add((m[1]!, int.tryParse(m[2] ?? '') ?? 0, e));
|
||||
}
|
||||
found.sort((a, b) => a.$1 != b.$1 ? a.$1.compareTo(b.$1) : a.$2.compareTo(b.$2));
|
||||
return [for (final f in found) f.$3];
|
||||
}
|
||||
|
||||
/// Deletes files older than [keepDays], then the oldest while all of
|
||||
/// them pass [maxTotalBytes]; never the one being written.
|
||||
void prune() {
|
||||
final cutoff = _dayOf(_now().subtract(Duration(days: keepDays)));
|
||||
final kept = <File>[];
|
||||
for (final f in files()) {
|
||||
if (_dayOfFile(f).compareTo(cutoff) < 0 && f.path != _path) {
|
||||
_delete(f);
|
||||
} else {
|
||||
kept.add(f);
|
||||
}
|
||||
}
|
||||
var total = kept.fold<int>(0, (a, f) => a + _len(f));
|
||||
for (final f in kept) {
|
||||
if (total <= maxTotalBytes) break;
|
||||
if (f.path == _path) continue;
|
||||
total -= _len(f);
|
||||
_delete(f);
|
||||
}
|
||||
}
|
||||
|
||||
static int _len(File f) {
|
||||
try {
|
||||
return f.lengthSync();
|
||||
} catch (_) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void _delete(File f) {
|
||||
try {
|
||||
f.deleteSync();
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
/// The lines of the last [since] (a record's continuation lines with
|
||||
/// it), oldest first; at most about [maxBytes], the newest kept.
|
||||
String recent(Duration since, {int maxBytes = 4 * 1024 * 1024}) {
|
||||
final from = _now().subtract(since);
|
||||
final fromDay = _dayOf(from);
|
||||
final out = <String>[];
|
||||
var size = 0;
|
||||
var done = false;
|
||||
for (final f in files().reversed) {
|
||||
if (done || _dayOfFile(f).compareTo(fromDay) < 0) break;
|
||||
List<String> lines;
|
||||
try {
|
||||
lines = const LineSplitter().convert(utf8.decode(f.readAsBytesSync(), allowMalformed: true));
|
||||
} catch (_) {
|
||||
continue;
|
||||
}
|
||||
// Newest first; a record's continuation lines come before its head.
|
||||
final block = <String>[];
|
||||
for (final l in lines.reversed) {
|
||||
block.add(l);
|
||||
final t = logLineTime(l);
|
||||
if (t == null) continue;
|
||||
if (t.isBefore(from)) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
for (final b in block) {
|
||||
size += b.length + 1;
|
||||
out.add(b);
|
||||
}
|
||||
block.clear();
|
||||
if (size >= maxBytes) {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out.reversed.join('\n');
|
||||
}
|
||||
|
||||
void close() {
|
||||
try {
|
||||
_out?.closeSync();
|
||||
} catch (_) {}
|
||||
_out = null;
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,8 @@ import 'relay_pieces.dart';
|
||||
// the tracks; a cancelled pull's unacked bytes are written off.
|
||||
|
||||
final _log = createLogger('relay');
|
||||
final _upstreamThrottle = LogThrottle(burst: 3);
|
||||
final _readThrottle = LogThrottle(burst: 3);
|
||||
|
||||
/// The `<name>` segment of a `/r/` URL: the media name URL-encoded, its
|
||||
/// slashes made `_` first (a page title can hold a URL, and an encoded
|
||||
@@ -104,6 +106,15 @@ class UpstreamHttpException extends HttpException {
|
||||
UpstreamHttpException(this.status, Uri url) : super('upstream HTTP $status', uri: url);
|
||||
}
|
||||
|
||||
/// Logs an upstream answer that is not the media (status, what was
|
||||
/// asked, the names of the headers sent), a few per host and status at a
|
||||
/// time.
|
||||
void logUpstreamRefusal(int status, Uri url, String what, Iterable<String> headerNames) {
|
||||
final pass = _upstreamThrottle.allow('${url.host} $status');
|
||||
if (pass == null) return;
|
||||
_log.warning('${pass}upstream $status for $what of ${logUrl(url)} (sent headers: ${headerNames.join(', ')})');
|
||||
}
|
||||
|
||||
/// An HTTP(S) resource with Range support, e.g. a link or a Jellyfin
|
||||
/// stream; [headers] (auth) never leave this machine. A read is one GET
|
||||
/// per [chunkSize] bytes, back to back (see
|
||||
@@ -127,6 +138,7 @@ class HttpRelayBytes implements RelayBytes {
|
||||
final res = await req.close().timeout(const Duration(seconds: 20));
|
||||
if (res.statusCode != 200 && res.statusCode != 206) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
logUpstreamRefusal(res.statusCode, url, range, headers.keys);
|
||||
throw UpstreamHttpException(res.statusCode, url);
|
||||
}
|
||||
return res;
|
||||
@@ -144,6 +156,7 @@ class HttpRelayBytes implements RelayBytes {
|
||||
final total = res.headers.value(HttpHeaders.contentRangeHeader)?.split('/').last;
|
||||
final n = int.tryParse(total ?? '') ?? (res.statusCode == 200 ? res.contentLength : -1);
|
||||
if (n <= 0) throw HttpException('upstream size unknown', uri: url);
|
||||
_log.info('upstream ${logUrl(url)}: ${res.statusCode} $type, $n bytes (headers: ${headers.keys.join(', ')})');
|
||||
return _size = n;
|
||||
}
|
||||
|
||||
@@ -354,8 +367,19 @@ class RelaySource {
|
||||
/// Opens the media (a page is resolved) and starts serving; throws
|
||||
/// when it cannot be read.
|
||||
Future<void> start() async {
|
||||
final tracks = await media.open();
|
||||
final infos = [for (final t in tracks) await relayTrackInfo(t)];
|
||||
final sw = Stopwatch()..start();
|
||||
final List<RelayTrack> tracks;
|
||||
final List<Map<String, Object?>> infos;
|
||||
try {
|
||||
tracks = await media.open();
|
||||
// The tracks' first upstream reads at once: viewers wait on them.
|
||||
infos = await Future.wait([for (final t in tracks) relayTrackInfo(t)]);
|
||||
} catch (e) {
|
||||
_log.warning('relay source ${logUrl(url)}: cannot open the media after ${sw.elapsedMilliseconds} ms: $e');
|
||||
rethrow;
|
||||
}
|
||||
_log.info('relay source ${logUrl(url)}: media opened in ${sw.elapsedMilliseconds} ms '
|
||||
'(${infos.map((i) => i['size'] != null ? '${i['kind'] == '' ? 'file' : i['kind']} ${i['size']} bytes' : '${i['kind']} ${(i['pieces'] as List?)?.length} pieces').join(' + ')})');
|
||||
_tracks = tracks;
|
||||
unawaited(_loop({
|
||||
't': 'info',
|
||||
@@ -365,6 +389,7 @@ class RelaySource {
|
||||
|
||||
Future<void> _loop(Map<String, Object?> info) async {
|
||||
var attempt = 0;
|
||||
final where = logUrl(url);
|
||||
while (!_closed) {
|
||||
try {
|
||||
final ws = await WebSocket.connect(url.toString()).timeout(const Duration(seconds: 15));
|
||||
@@ -373,17 +398,21 @@ class RelaySource {
|
||||
break;
|
||||
}
|
||||
_ws = ws;
|
||||
_log.info('relay channel up: $where${attempt > 0 ? ' (after $attempt retries)' : ''}');
|
||||
attempt = 0;
|
||||
ws.add(jsonEncode(info));
|
||||
await for (final m in ws) {
|
||||
if (m is String) _onText(ws, m);
|
||||
}
|
||||
if (ws.closeCode == relayCloseNotSource) {
|
||||
_log.info('relay channel: not the source any more, stopping');
|
||||
_log.info('relay channel: not the source any more, stopping ($where, sent $bytesSent bytes)');
|
||||
_closed = true;
|
||||
} else if (!_closed) {
|
||||
_log.warning('relay channel down: $where, code ${ws.closeCode} ${ws.closeReason ?? ''}; reconnecting');
|
||||
}
|
||||
} catch (e) {
|
||||
_log.fine('relay channel: $e');
|
||||
final pass = attempt < 3 || attempt % 10 == 0 ? '' : null;
|
||||
if (pass != null) _log.warning('relay channel: cannot reach $where (attempt ${attempt + 1}): $e');
|
||||
}
|
||||
_ws = null;
|
||||
_abandonAll();
|
||||
@@ -477,7 +506,8 @@ class RelaySource {
|
||||
if (!p.cancelled) ws.add(jsonEncode({'t': 'end', 'id': p.id}));
|
||||
} catch (e) {
|
||||
if (!p.cancelled) {
|
||||
_log.warning('relay read ${p.id} failed: $e');
|
||||
final pass = _readThrottle.allow('$e');
|
||||
if (pass != null) _log.warning('${pass}relay read ${p.id} failed: $e');
|
||||
try {
|
||||
ws.add(jsonEncode({'t': 'fail', 'id': p.id, 'reason': '$e'}));
|
||||
} catch (_) {}
|
||||
|
||||
@@ -343,6 +343,7 @@ Future<UpstreamPieces> fetchHlsPlaylist(HttpClient client, Uri url, Map<String,
|
||||
final res = await req.close().timeout(const Duration(seconds: 20));
|
||||
if (res.statusCode != 200) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
logUpstreamRefusal(res.statusCode, url, 'the playlist', headers.keys);
|
||||
throw UpstreamHttpException(res.statusCode, url);
|
||||
}
|
||||
final text = await res.transform(utf8.decoder).join().timeout(const Duration(seconds: 30));
|
||||
@@ -361,6 +362,7 @@ Stream<List<int>> readPiece(HttpClient client, PieceSource s, Map<String, String
|
||||
final res = await req.close().timeout(const Duration(seconds: 20));
|
||||
if (res.statusCode != 200 && res.statusCode != 206) {
|
||||
await res.drain<void>().catchError((_) {});
|
||||
logUpstreamRefusal(res.statusCode, s.url, 'a piece', headers.keys);
|
||||
throw UpstreamHttpException(res.statusCode, s.url);
|
||||
}
|
||||
if (len == null) {
|
||||
|
||||
@@ -51,6 +51,11 @@ class YtDlpFormat {
|
||||
this.protocol = 'https',
|
||||
this.fragments});
|
||||
|
||||
/// For the log: no URL query, header names only.
|
||||
String describe() => '$id $protocol $ext${height == null ? '' : ' ${height}p'} @${url.host}'
|
||||
' headers=[${headers.keys.join(',')}]${chunkSize == null ? '' : ' chunk=$chunkSize'}'
|
||||
'${fragments == null ? '' : ' fragments=${fragments!.length}'}';
|
||||
|
||||
/// Relayed as pieces (see relay_pieces.dart), not as a byte stream.
|
||||
bool get isPieces => _piecesProtocols.contains(protocol);
|
||||
|
||||
@@ -128,6 +133,17 @@ class YtDlpInfo {
|
||||
this.formats = const {},
|
||||
});
|
||||
|
||||
/// One line for the log: extractor, duration, live, and each picked
|
||||
/// track's format (id, protocol, container, height, CDN host, the names
|
||||
/// of the headers it is fetched with).
|
||||
String describe() {
|
||||
final t = tracks.isEmpty
|
||||
? 'nothing a relay can read'
|
||||
: tracks.map((t) => '${t.kind.isEmpty ? 'file' : t.kind} ${t.format.describe()}').join(' + ');
|
||||
return '$extractor${isLive ? ' live' : ''}, ${duration?.toStringAsFixed(0) ?? '?'} s, '
|
||||
'${formats.length} relayable formats; tracks: $t';
|
||||
}
|
||||
|
||||
/// Parses `yt-dlp -J` output; a playlist answer uses its first entry.
|
||||
static YtDlpInfo parse(Map<String, Object?> j) {
|
||||
final entries = j['entries'];
|
||||
@@ -568,7 +584,10 @@ class YtDlp {
|
||||
Future<YtDlpInfo> resolve(String url, {Duration timeout = const Duration(seconds: 25)}) async {
|
||||
final exe = await ensure();
|
||||
if (exe == null) throw YtDlpException(lastError ?? 'yt-dlp is not available');
|
||||
final proc = await Process.start(exe, resolveArgs(url));
|
||||
final args = resolveArgs(url);
|
||||
final sw = Stopwatch()..start();
|
||||
_log.info('resolve ${logUrl(url)} (yt-dlp $version, ${_argsForLog(args)})');
|
||||
final proc = await Process.start(exe, args);
|
||||
final out = proc.stdout.transform(utf8.decoder).join();
|
||||
final err = proc.stderr.transform(utf8.decoder).join();
|
||||
final int code;
|
||||
@@ -576,18 +595,50 @@ class YtDlp {
|
||||
code = await proc.exitCode.timeout(timeout);
|
||||
} on TimeoutException {
|
||||
proc.kill();
|
||||
_log.warning('resolve ${logUrl(url)}: no answer in ${timeout.inSeconds} s, killed');
|
||||
throw YtDlpException('yt-dlp took longer than ${timeout.inSeconds} s');
|
||||
}
|
||||
if (code != 0) throw YtDlpException(ytDlpErrorText(await err));
|
||||
if (code != 0) {
|
||||
final e = await err;
|
||||
_log.warning('resolve ${logUrl(url)}: exit $code after ${sw.elapsedMilliseconds} ms; stderr tail:\n'
|
||||
'${_tail(e)}');
|
||||
throw YtDlpException(ytDlpErrorText(e));
|
||||
}
|
||||
try {
|
||||
final j = jsonDecode(await out);
|
||||
if (j is! Map) throw const FormatException('not an object');
|
||||
return YtDlpInfo.parse(Map<String, Object?>.from(j));
|
||||
final info = YtDlpInfo.parse(Map<String, Object?>.from(j));
|
||||
_log.info('resolved ${logUrl(url)} in ${sw.elapsedMilliseconds} ms: ${info.describe()}');
|
||||
return info;
|
||||
} on FormatException catch (e) {
|
||||
_log.warning('resolve ${logUrl(url)}: unreadable output (${e.message})');
|
||||
throw YtDlpException('unreadable yt-dlp output: ${e.message}');
|
||||
}
|
||||
}
|
||||
|
||||
/// [args] for the log: the cookie source named, not its path or spec.
|
||||
static String _argsForLog(List<String> args) {
|
||||
final out = <String>[];
|
||||
for (var i = 0; i < args.length; i++) {
|
||||
final a = args[i];
|
||||
if ((a == '--cookies' || a == '--cookies-from-browser') && i + 1 < args.length) {
|
||||
out.add(a == '--cookies' ? '--cookies <file>' : '--cookies-from-browser <browser>');
|
||||
i++;
|
||||
} else if (a == '--') {
|
||||
break;
|
||||
} else {
|
||||
out.add(a);
|
||||
}
|
||||
}
|
||||
return out.join(' ');
|
||||
}
|
||||
|
||||
/// The last lines of yt-dlp's stderr, indented.
|
||||
static String _tail(String stderr, {int lines = 8}) {
|
||||
final all = const LineSplitter().convert(stderr).where((l) => l.trim().isNotEmpty).toList();
|
||||
return all.skip(all.length > lines ? all.length - lines : 0).map((l) => ' $l').join('\n');
|
||||
}
|
||||
|
||||
/// The command line [resolve] runs.
|
||||
List<String> resolveArgs(String url) =>
|
||||
['-J', '--no-playlist', '--no-warnings', '--ignore-config', ...cookies.args(), '--', url];
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:sync_relay_core/core.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
void main() {
|
||||
group('redactLog', () {
|
||||
const tok = '50989dda73c45a4710751f8aff4a1982';
|
||||
|
||||
test('signed URLs keep scheme, host, path and parameter names only', () {
|
||||
final out = redactLog('GET https://upos-hz-mirrorakam.akamaized.net/a/b-30080.m4s?deadline=1791826934&upsig=0f61b79b&hdnts=exp~acl -> 403');
|
||||
expect(out, 'GET https://upos-hz-mirrorakam.akamaized.net/a/b-30080.m4s?deadline&upsig&hdnts=… -> 403');
|
||||
expect(redactLog('https://jf.example/Videos/1/stream?static=true&api_key=abc123'),
|
||||
'https://jf.example/Videos/1/stream?static&api_key=***');
|
||||
expect(redactLog('http://user:pw@host/x#frag'), 'http://***@host/x');
|
||||
});
|
||||
|
||||
test('member tokens in server paths', () {
|
||||
expect(redactLog('open http://1.2.3.4:8080/r/default/$tok/2/0/name'), 'open http://1.2.3.4:8080/r/default/***/2/0/name');
|
||||
expect(redactLog('ws://h:1/r/default/$tok/source?q=7'), 'ws://h:1/r/default/***/source?q=…');
|
||||
expect(redactLog('/m/default/$tok/file.mkv range=bytes=0-'), '/m/default/***/file.mkv range=bytes=0-');
|
||||
expect(redactLog('edl://!no_clip;%160%http://h/r/default/$tok/2/0/n;!new_stream;!no_clip;%160%http://h/r/default/$tok/2/1/n'),
|
||||
isNot(contains(tok)));
|
||||
});
|
||||
|
||||
test('secret values by name: pairs, JSON, quoted, headers, bearer', () {
|
||||
expect(redactLog('api_key=abc123&x=1'), 'api_key=***&x=1');
|
||||
expect(redactLog('{"token":"$tok","name":"bob","secret":"hunter2"}'), '{"token":"***","name":"bob","secret":"***"}');
|
||||
expect(redactLog('MediaBrowser Client="WT", Device="pc", Token="deadbeef"'), contains('Token="***"'));
|
||||
expect(redactLog('headers: {Cookie: SESSDATA=abc; bili_jct=x}'), 'headers: {Cookie: ***');
|
||||
expect(redactLog('Authorization: Bearer abcdefgh12345678'), 'Authorization: ***');
|
||||
expect(redactLog('sent Bearer abcdefgh12345678 to it'), 'sent Bearer *** to it');
|
||||
expect(redactLog('password=pw1 hostKey=k1 X-Emby-Token=t1'), 'password=*** hostKey=*** X-Emby-Token=***');
|
||||
});
|
||||
|
||||
test('leaves ordinary text alone', () {
|
||||
const plain = 'yt-dlp cookies: 5 for youtube.com; relay pull 3 failed: upstream HTTP 403; tokens: 2 members';
|
||||
expect(redactLog(plain), plain);
|
||||
});
|
||||
|
||||
test('formatLogLine redacts the error and stack too', () {
|
||||
final line = formatLogLine(DateTime(2026, 1, 2, 3, 4, 5, 6), 'WARNING', 'relay', 'read failed',
|
||||
error: 'HttpException: upstream HTTP 403, uri = https://cdn/x.m4s?sig=abc');
|
||||
expect(line, '2026-01-02 03:04:05.006 WARNING relay: read failed\n error: HttpException: upstream HTTP 403, uri = https://cdn/x.m4s?sig=…');
|
||||
expect(logLineTime(line), DateTime(2026, 1, 2, 3, 4, 5, 6));
|
||||
});
|
||||
});
|
||||
|
||||
test('LogThrottle lets a burst through, then counts what it dropped', () {
|
||||
var now = DateTime(2026);
|
||||
final t = LogThrottle(burst: 2, window: const Duration(seconds: 10), now: () => now);
|
||||
expect([t.allow('a'), t.allow('a'), t.allow('a'), t.allow('b')], ['', '', null, '']);
|
||||
now = now.add(const Duration(seconds: 11));
|
||||
expect(t.allow('a'), '(1 more like this dropped) ');
|
||||
});
|
||||
|
||||
group('LogFiles', () {
|
||||
late Directory dir;
|
||||
setUp(() async => dir = await Directory.systemTemp.createTemp('logfiles'));
|
||||
tearDown(() async => dir.delete(recursive: true));
|
||||
|
||||
test('rolls by size and day, prunes by age and total, reads recent lines', () {
|
||||
var now = DateTime(2026, 10, 12, 10);
|
||||
File('${dir.path}/app-2026-10-01.log').writeAsStringSync('old\n');
|
||||
File('${dir.path}/server-2026-10-01.log').writeAsStringSync('not ours\n');
|
||||
final f = LogFiles(dir.path, maxFileBytes: 100, maxTotalBytes: 1000, now: () => now)..open();
|
||||
expect(File('${dir.path}/app-2026-10-01.log').existsSync(), isFalse);
|
||||
expect(File('${dir.path}/server-2026-10-01.log').existsSync(), isTrue);
|
||||
String line(String m) => formatLogLine(now, 'INFO', 't', m);
|
||||
f.write(line('a' * 60));
|
||||
f.write(line('b'));
|
||||
expect(f.currentPath, endsWith('app-2026-10-12.log'));
|
||||
f.write(line('b2'));
|
||||
expect(f.currentPath, endsWith('app-2026-10-12.1.log'));
|
||||
now = DateTime(2026, 10, 13, 9);
|
||||
f.write(line('c'));
|
||||
f.write(' continuation of c');
|
||||
expect(f.currentPath, endsWith('app-2026-10-13.log'));
|
||||
final recent = f.recent(const Duration(hours: 2));
|
||||
expect(recent, '${line('c')}\n continuation of c');
|
||||
expect(f.recent(const Duration(days: 2)).split('\n').first, contains('a' * 60));
|
||||
f.close();
|
||||
// Total cap: the oldest go first.
|
||||
final small = LogFiles(dir.path, maxFileBytes: 100, maxTotalBytes: 150, now: () => now)..open();
|
||||
expect(small.files().map((x) => x.uri.pathSegments.last), ['app-2026-10-12.1.log', 'app-2026-10-13.log']);
|
||||
small.close();
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user