Protocol 15: one shared session log on the server, chat never logged, Mark a problem

Every app streams its redacted diagnostic lines (server time, batched
every 10 s, gzip, buffered while offline) to POST /l/; the server merges
them with its own into session-YYYY-MM-DD.log in time order, tagged by
who. Opt-out in Settings > About; Mark a problem (Ctrl+Shift+B, the
player menus, About) writes a marker with an optional note. Copy
diagnostics can include the server's session log. Chat text never
reaches any log.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
foxyzandClaude Opus 5.5 committed 2026-10-12 19:41:06 +03:00
1 parent 4ab37c55cf
commit 7487b4deb1
29 files changed
+1380 -47

No files matched your search

+5 -2
View File
@@ -31,7 +31,8 @@ Future<void> main(List<String> args) async {
'An account login: keep it private. Copied into --yt-dlp-dir at start.')
..addOption('config', help: 'YAML or JSON config file (keys as in AppConfig); overrides the other flags.')
..addOption('log-dir',
help: 'Folder for the rotating log files (server-YYYY-MM-DD.log, kept 7 days). '
help: 'Folder for the rotating log files (server-YYYY-MM-DD.log, and session-YYYY-MM-DD.log with '
'every member\'s shared diagnostics merged in; kept 7 days). '
'Default: logs next to --config, else logs next to this binary (systemd: \$STATE_DIRECTORY/logs).')
..addFlag('upgrade',
negatable: false,
@@ -117,7 +118,9 @@ Future<void> main(List<String> args) async {
}
final events = ReceivePort('server events');
final done = serve(config.toMap(), events: events.sendPort);
// The session log (every member's diagnostics and the server's records
// merged) goes next to the server's own.
final done = serve({...config.toMap(), if (files != null) 'logDir': files.dir}, events: events.sendPort);
final signals = <StreamSubscription<ProcessSignal>>[];
await for (final msg in events) {
if (msg is! Map) continue;
+1
View File
@@ -7,3 +7,4 @@ export 'src/jellyfin_relay.dart';
export 'src/relay_buffer.dart';
export 'src/relay_routes.dart';
export 'src/relay_stats.dart';
export 'src/session_log.dart';
+5 -2
View File
@@ -19,7 +19,8 @@ import 'sync_server.dart';
/// swap; a following `swap` with its name and size puts it up) and answers
/// `{'ok':true,'name':n,'size':s}` or `{'ok':false}`. Without [events] a
/// start failure throws. `'loopbackOnly': true` in [config] (next to the
/// [AppConfig] keys) binds 127.0.0.1 only: the app's solo session.
/// [AppConfig] keys) binds 127.0.0.1 only: the app's solo session;
/// `'logDir'` is the folder of the session log ([SyncServer.logDir]).
Future<void> serve(Map<String, Object?> config, {SendPort? events}) async {
StreamSubscription<pkg_logging.LogRecord>? logs;
if (events != null) {
@@ -34,7 +35,9 @@ Future<void> serve(Map<String, Object?> config, {SendPort? events}) async {
}
final SyncServer server;
try {
server = SyncServer(AppConfig.fromMap(config), loopbackOnly: config['loopbackOnly'] == true);
final logDir = config['logDir'];
server = SyncServer(AppConfig.fromMap(config),
loopbackOnly: config['loopbackOnly'] == true, logDir: logDir is String && logDir.isNotEmpty ? logDir : null);
await server.start();
} catch (e) {
await logs?.cancel();
+288
View File
@@ -0,0 +1,288 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:logging/logging.dart' as pkg_logging;
import 'package:relic/relic.dart';
import 'package:sync_relay_core/core.dart';
import 'server_hub.dart';
import 'session.dart';
// The session log (ARCHITECTURE "Logging"): every member's diagnostics
// (`POST /l/...`) and the server's own records, merged in server-clock
// order into one file per day, `session-YYYY-MM-DD.log`, in the server's
// log folder.
class _Entry {
final int t;
final int seq;
final String text;
const _Entry(this.t, this.seq, this.text);
}
/// The merged log. Lines wait [window] (members send every 10 s) and are
/// written in time order; a line later than that (a reconnect's backlog)
/// still goes in with its own time, a little out of order.
class SessionLog {
final LogFiles files;
final Duration window;
final DateTime Function() _now;
final _pending = <_Entry>[];
var _seq = 0;
StreamSubscription<pkg_logging.LogRecord>? _own;
/// Per member token: the version and share state already noted.
final _noted = <String, ({String version, bool share})>{};
SessionLog(String dir,
{this.window = const Duration(seconds: 15),
List<String> shareWith = const ['app', 'server'],
DateTime Function()? now,
int maxFileBytes = 10 * 1024 * 1024,
int maxTotalBytes = 100 * 1024 * 1024})
: _now = now ?? DateTime.now,
files = LogFiles(dir,
prefix: sessionLogPrefix,
shareWith: shareWith,
maxFileBytes: maxFileBytes,
maxTotalBytes: maxTotalBytes,
now: now) {
files.open();
}
/// Takes this isolate's own log records as the server's lines.
void captureOwn() {
_own ??= pkg_logging.Logger.root.onRecord.listen((r) {
final b = StringBuffer(r.message);
if (r.error != null) b.write('\n error: ${r.error}');
final s = '${r.stackTrace ?? ''}'.trimRight();
if (s.isNotEmpty) b.write('\n$s');
add(r.time.millisecondsSinceEpoch, 'server', r.level.name, r.loggerName, '$b');
});
}
static final _control = RegExp(r'[\x00-\x08\x0b-\x1f\x7f]');
static final _loggerName = RegExp(r'^[A-Za-z0-9_.\-]{1,40}$');
static const _levels = {'FINEST', 'FINER', 'FINE', 'CONFIG', 'INFO', 'WARNING', 'SEVERE', 'SHOUT'};
static const _quiet = {'FINEST', 'FINER', 'FINE', 'CONFIG', 'INFO'};
static String _clean(String s, int max) {
var out = s.replaceAll('\r', '').replaceAll(_control, ' ');
if (out.length > max) out = '${out.substring(0, max)}…';
return out;
}
static String _two(int n) => n.toString().padLeft(2, '0');
static String _clock(DateTime t) =>
'${_two(t.hour)}:${_two(t.minute)}:${_two(t.second)}.${t.millisecond.toString().padLeft(3, '0')}';
/// One record at [t] (epoch ms, server clock) from [who] (`name (id)`
/// or `server`): `HH:MM:SS.mmm [who] logger: message`, the level before
/// the logger from WARNING up, continuation lines indented. Redacted
/// again here.
void add(int t, String who, String level, String logger, String message) {
final lvl = _levels.contains(level) ? level : 'INFO';
final name = _loggerName.hasMatch(logger) ? logger : 'app';
final body = redactLog(_clean(message, 8192)).split('\n');
final head = '[${_clean(who, 80).replaceAll('\n', ' ')}] ${_quiet.contains(lvl) ? '' : '$lvl '}$name: ${body.first}';
_push(t, [head, for (final l in body.skip(1)) ' $l'].join('\n'));
}
/// A problem mark by [name] ([who]) at [t]:
/// `=== PROBLEM marked by <name>: <note> ===`.
void mark(int t, String who, String name, String note) {
final n = redactLog(_clean(note, RelayConstants.maxProblemNoteLength).replaceAll('\n', ' ')).trim();
_push(t, '[${_clean(who, 80).replaceAll('\n', ' ')}] === PROBLEM marked by $name${n.isEmpty ? '' : ': $n'} ===');
}
/// A line of the server's about a member, now.
void note(String message) => add(_now().millisecondsSinceEpoch, 'server', 'INFO', 'diag', message);
void _push(int t, String text) {
final now = _now().millisecondsSinceEpoch;
// A clock far ahead lands now; older than the files keep is dropped.
if (t > now + 60000) t = now;
if (t < now - files.keepDays * 86400000) return;
_pending.add(_Entry(t, _seq++, text));
// A flood must not grow the wait without bound: write early.
if (_pending.length > 20000) flush(all: true);
}
/// Writes the lines older than [window] ([all]: every one), in order.
/// Called every second.
void flush({bool all = false}) {
if (_pending.isEmpty) return;
final until = all ? null : _now().millisecondsSinceEpoch - window.inMilliseconds;
_pending.sort(_order);
var n = 0;
for (final e in _pending) {
if (until != null && e.t > until) break;
final at = DateTime.fromMillisecondsSinceEpoch(e.t);
files.write('${_clock(at)} ${e.text}', at: at);
n++;
}
_pending.removeRange(0, n);
}
static int _order(_Entry a, _Entry b) => a.t != b.t ? a.t.compareTo(b.t) : a.seq.compareTo(b.seq);
/// A batch from the member [token] ([who] for the log, [name] for a
/// mark): its lines, a note the first time (or when its version or
/// sharing changes), how many it dropped, its marks.
void takeBatch(String token, String who, String name, DiagBatch batch) {
final version = _clean(batch.version, 100).replaceAll('\n', ' ');
final before = _noted[token];
if (before == null || before.version != version || before.share != batch.share) {
_noted[token] = (version: version, share: batch.share);
final what = version.isEmpty ? '' : ' ($version)';
if (!batch.share) {
note("$who doesn't share diagnostics$what");
} else if (before == null || !before.share) {
note('$who shares diagnostics$what');
} else {
note('$who now runs $version');
}
}
if (batch.share) {
for (final l in batch.lines) {
add(l.t, who, l.level, l.logger, l.message);
}
if (batch.dropped > 0) {
final t = batch.lines.isEmpty ? _now().millisecondsSinceEpoch : batch.lines.first.t;
add(t, who, 'WARNING', 'diag', '${batch.dropped} lines dropped (the app could not send them in time)');
}
}
for (final m in batch.marks) {
mark(m.t, who, name, m.note);
}
}
/// The lines of the last [since], written and waiting, oldest first; at
/// most about [maxBytes], the newest kept.
String recent(Duration since, {int maxBytes = 8 * 1024 * 1024}) {
final from = _now().subtract(since);
final fromDay = _day(from);
final out = <String>[];
final head = RegExp(r'^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) ');
for (final f in files.files()) {
final name = f.uri.pathSegments.last;
final day = name.substring(sessionLogPrefix.length + 1, sessionLogPrefix.length + 11);
if (day.compareTo(fromDay) < 0) continue;
final date = DateTime.tryParse(day);
if (date == null) continue;
List<String> lines;
try {
lines = const LineSplitter().convert(utf8.decode(f.readAsBytesSync(), allowMalformed: true));
} catch (_) {
continue;
}
var keep = false;
for (final l in lines) {
final m = head.firstMatch(l);
if (m != null) {
final t = DateTime(date.year, date.month, date.day, int.parse(m[1]!), int.parse(m[2]!), int.parse(m[3]!),
int.parse(m[4]!));
keep = !t.isBefore(from);
}
if (keep) out.add(l);
}
}
final fromMs = from.millisecondsSinceEpoch;
final waiting = [..._pending]..sort(_order);
for (final e in waiting) {
if (e.t >= fromMs) out.add('${_clock(DateTime.fromMillisecondsSinceEpoch(e.t))} ${e.text}');
}
var size = 0, start = out.length;
while (start > 0 && size + out[start - 1].length + 1 <= maxBytes) {
size += out[--start].length + 1;
}
return out.sublist(start).join('\n');
}
static String _day(DateTime t) => '${t.year}-${_two(t.month)}-${_two(t.day)}';
Future<void> close() async {
await _own?.cancel();
_own = null;
flush(all: true);
files.close();
}
}
class _Rate {
var start = DateTime.now();
var posts = 0;
var bytes = 0;
}
/// `POST /l/<sid>/<token>`: a member's diagnostics batch ([DiagBatch],
/// gzip) into the [sessionLog]; `GET /l/<sid>/<token>?hours=N`: the session log of
/// the last N hours (gzip text), for any member. Without one (a
/// server with no log folder, the solo session) batches are taken and
/// dropped, and the GET is 404.
void mountSessionLogRoutes(RelicApp app, ServerHub hub, SessionLog? Function() sessionLog) {
Response status(int code, [String? why]) => Response(code, body: Body.fromString(why ?? 'status $code'));
final rates = <String, _Rate>{};
ViewerToken? member(Request req) {
final session = hub.sessions.get(req.pathParameters.raw[#sid] ?? '');
final token = req.pathParameters.raw[#token] ?? '';
if (session == null || session.gate(token) != MediaGate.ok) return null;
return session.tokens[token];
}
app.post('/l/:sid/:token', (final req) async {
final viewer = member(req);
if (viewer == null) return status(401);
final body = BytesBuilder(copy: false);
try {
await for (final c in req.read(maxLength: RelayConstants.diagBatchBytes)) {
body.add(c);
}
} catch (_) {
return status(413);
}
final DiagBatch batch;
try {
batch = DiagBatch.decode(body.takeBytes());
} on FormatException catch (e) {
return status(e.message == 'too large' ? 413 : 400, 'bad batch');
}
final now = DateTime.now();
if (rates.length > 500) rates.removeWhere((k, _) => !hub.session.tokens.containsKey(k));
final rate = rates.putIfAbsent(viewer.token, _Rate.new);
if (now.difference(rate.start) >= const Duration(minutes: 1)) {
rate
..start = now
..posts = 0
..bytes = 0;
}
final size = batch.lines.fold<int>(0, (a, l) => a + l.size);
if (rate.posts >= RelayConstants.diagPostsPerMin || rate.bytes + size > RelayConstants.diagBytesPerMin) {
return Response(429,
body: Body.fromString('too many'), headers: Headers.build((h) => h['retry-after'] = ['30']));
}
rate
..posts += 1
..bytes += size;
final log = sessionLog();
if (log == null) return Response.noContent();
log.takeBatch(viewer.token, hub.who(viewer.token), viewer.name, batch);
return Response.noContent();
});
app.get('/l/:sid/:token', (final req) {
if (member(req) == null) return status(401);
final log = sessionLog();
if (log == null) return status(404, 'this server keeps no session log');
final hours = (int.tryParse(req.url.queryParameters['hours'] ?? '') ?? 6).clamp(1, 24 * 7);
final text = log.recent(Duration(hours: hours));
return Response.ok(
body: Body.fromData(Uint8List.fromList(gzip.encode(utf8.encode(text))),
mimeType: MimeType('application', 'gzip')),
headers: Headers.build((h) => h['cache-control'] = ['no-store']),
);
});
}
+22 -1
View File
@@ -11,6 +11,7 @@ import 'media_endpoint.dart';
import 'relay_routes.dart';
import 'server_hub.dart';
import 'session.dart';
import 'session_log.dart';
/// [configured], else systemd's `$STATE_DIRECTORY/yt-dlp`, else `yt-dlp`
/// next to the server binary when that is writable (not under `dart run`),
@@ -56,8 +57,16 @@ class SyncServer {
/// IPv4 interface.
final bool loopbackOnly;
/// The folder of the session log (`session-YYYY-MM-DD.log`, every
/// member's diagnostics and the server's records merged); null: none.
final String? logDir;
SessionLog? _sessionLog;
/// The session log while running with a [logDir].
SessionLog? get sessionLog => _sessionLog;
/// [ytDlp] replaces the server's own (tests).
SyncServer(this.config, {YtDlp? ytDlp, this.loopbackOnly = false}) {
SyncServer(this.config, {YtDlp? ytDlp, this.loopbackOnly = false, this.logDir}) {
hub = ServerHub(sessions, config.secret,
hostKey: config.hostKey,
jellyfin: config.jellyfin,
@@ -103,6 +112,7 @@ class SyncServer {
mountRelayRoutes(_app, sessions, relayAt: hub.relayAt, sourceOf: hub.sourceOf, onLog: logRequest);
mountAvatarRoute(_app, hub);
mountArtRoute(_app, hub);
mountSessionLogRoutes(_app, hub, () => _sessionLog);
mountJellyfinRelayRoute(
_app, sessions, () => hub.jellyfin, () => _relayClient ??= HttpClient(),
next: () => hub.nextJellyfinRef, onLog: logRequest);
@@ -139,6 +149,14 @@ class SyncServer {
if (config.secret.isEmpty) {
_log.warning('no password set: open server, anyone who can reach it can join');
}
final dir = logDir;
if (dir != null && _sessionLog == null) {
try {
_sessionLog = SessionLog(dir)..captureOwn();
} catch (e) {
_log.warning('no session log: cannot write $dir ($e)');
}
}
await _resolveJellyfinServerId();
_mountRoutes();
_server = await _app.serve(
@@ -158,6 +176,7 @@ class SyncServer {
hub.sampleRates();
hub.broadcastState();
hub.broadcastFeed();
_sessionLog?.flush();
});
_log.info('server "${config.name}" listening on $boundPort');
}
@@ -193,5 +212,7 @@ class SyncServer {
await _app.close();
_server = null;
_log.info('server "${config.name}" stopped');
await _sessionLog?.close();
_sessionLog = null;
}
}
+130
View File
@@ -0,0 +1,130 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:logging/logging.dart';
import 'package:sync_relay_core/core.dart';
import 'package:sync_relay_server/server.dart';
import 'package:test/test.dart';
import 'package:web_socket_channel/web_socket_channel.dart';
void main() {
late Directory dir;
setUp(() async => dir = await Directory.systemTemp.createTemp('session_log'));
tearDown(() async => dir.delete(recursive: true));
String read() => File('${dir.path}/session-2026-10-12.log').readAsStringSync();
test('merges lines of every side in server-clock order after the window', () async {
var now = DateTime(2026, 10, 12, 18, 0, 30);
final log = SessionLog(dir.path, window: const Duration(seconds: 15), now: () => now);
int at(int s, [int ms = 0]) => DateTime(2026, 10, 12, 18, 0, s, ms).millisecondsSinceEpoch;
log.add(at(12), 'Bob (m2)', 'INFO', 'relay', 'second');
log.add(at(10, 5), 'server', 'WARNING', 'server_hub', 'first\n error: boom');
log.add(at(20), 'Alice (m1)', 'INFO', 'mpv', 'too new');
log.add(at(14), 'Alice (m1)', 'INFO', 'media', 'open http://h:1/r/default/50989dda73c45a4710751f8aff4a1982/2/0/x?sig=abc');
log.mark(at(13), 'Alice (m1)', 'Alice', 'stutter\nhere');
log.flush();
expect(read().trimRight().split('\n'), [
'18:00:10.005 [server] WARNING server_hub: first',
' error: boom',
'18:00:12.000 [Bob (m2)] relay: second',
'18:00:13.000 [Alice (m1)] === PROBLEM marked by Alice: stutter here ===',
'18:00:14.000 [Alice (m1)] media: open http://h:1/r/default/***/2/0/x?sig=…',
]);
expect(log.recent(const Duration(hours: 1)), endsWith('[Alice (m1)] mpv: too new'), reason: 'waiting lines too');
now = DateTime(2026, 10, 12, 18, 1);
await log.close();
expect(read(), contains('18:00:20.000 [Alice (m1)] mpv: too new'));
});
test('takeBatch: notes the version once, opt-out once, dropped lines', () async {
final now = DateTime(2026, 10, 12, 18);
final log = SessionLog(dir.path, now: () => now);
final t = now.millisecondsSinceEpoch;
log.takeBatch('tok', 'Bob (m2)', 'Bob',
DiagBatch(version: 'WT 1.0', lines: [DiagLine(t: t, level: 'INFO', logger: 'app', message: 'hi')], dropped: 3));
log.takeBatch('tok', 'Bob (m2)', 'Bob', DiagBatch(version: 'WT 1.0', lines: [DiagLine(t: t, level: 'INFO', logger: 'app', message: 'again')]));
log.takeBatch('tok2', 'Cleo (m3)', 'Cleo', const DiagBatch(version: 'WT 1.0', share: false, lines: [DiagLine(t: 0, level: 'INFO', logger: 'app', message: 'never')]));
log.takeBatch('tok2', 'Cleo (m3)', 'Cleo', const DiagBatch(version: 'WT 1.0', share: false));
await log.close();
final text = read();
expect('shares diagnostics'.allMatches(text).length, 1);
expect(text, contains('[server] diag: Bob (m2) shares diagnostics (WT 1.0)'));
expect(text, contains("[server] diag: Cleo (m3) doesn't share diagnostics (WT 1.0)"));
expect("doesn't share".allMatches(text).length, 1);
expect(text, contains('[Bob (m2)] WARNING diag: 3 lines dropped'));
expect(text, isNot(contains('never')));
});
group('live', () {
late SyncServer server;
setUp(() async {
server = SyncServer(const AppConfig(port: 0, secret: 's', name: 't'), logDir: dir.path);
await server.start();
});
tearDown(() async => server.stop());
Future<(WebSocketChannel, String)> join(String name) async {
final ch = WebSocketChannel.connect(Uri.parse('ws://127.0.0.1:${server.boundPort}/sync'));
final msgs = StreamIterator(ch.stream);
ch.sink.add(jsonEncode(HelloMessage(
proto: RelayConstants.protoVersion,
secret: 's',
user: HelloUser(name: name),
media: const HelloMedia(name: '', size: 0, duration: 0))
.toJson()));
while (await msgs.moveNext()) {
final m = jsonDecode('${msgs.current}') as Map;
if (m['t'] == 'welcome') {
unawaited(() async {
while (await msgs.moveNext()) {}
}());
return (ch, m['token'] as String);
}
}
throw StateError('no welcome');
}
DiagUploader uploader(String token, {String version = 'WT 1.0'}) => DiagUploader(
target: () => (base: Uri.parse('http://127.0.0.1:${server.boundPort}'), token: token, offsetMs: 0),
version: version);
test('members send lines and marks; chat never lands; any member reads it back', () async {
final (alice, aliceTok) = await join('Alice');
final (bob, bobTok) = await join('Bob');
final a = uploader(aliceTok)..start();
final b = uploader(bobTok)
..share = false
..start();
Logger('relay').info('alice line password=hunter2');
alice.sink.add(jsonEncode(const ChatMessage(text: 'secret chat words').toJson()));
await Future<void>.delayed(const Duration(milliseconds: 200));
expect(await a.flush(), isTrue);
expect(await b.flush(), isTrue);
expect(await a.markProblem('audio drifts'), isTrue);
final text = await a.fetchSessionLog(hours: 1);
expect(text, isNotNull);
expect(text, contains('[Alice (m1)] relay: alice line password=***'));
expect(text, contains('=== PROBLEM marked by Alice: audio drifts ==='));
expect(text, contains("Bob (m2) doesn't share diagnostics"));
expect(text, contains('[server] server_hub: join Alice'));
expect(text, isNot(contains('secret chat words')));
// Bob reads it too (friends); a stranger does not.
expect(await b.fetchSessionLog(), contains('PROBLEM'));
expect(await uploader('0' * 32).fetchSessionLog(), isNull);
final bad = await HttpClient().postUrl(Uri.parse('http://127.0.0.1:${server.boundPort}/l/default/$aliceTok'));
bad.add(utf8.encode('not gzip'));
expect((await bad.close()).statusCode, 400);
await a.stop();
await b.stop();
await alice.sink.close();
await bob.sink.close();
await server.stop();
final files = dir.listSync().map((e) => e.uri.pathSegments.last).where((n) => n.startsWith('session-'));
final all = [for (final n in files) File('${dir.path}/$n').readAsStringSync()].join();
expect(all, contains('PROBLEM'));
expect(all, isNot(contains('secret chat words')));
});
});
}
+2 -2
View File
@@ -64,8 +64,8 @@ void main() {
});
group('core welcome + proto', () {
test('proto 14: avatars by hash, profile intent', () {
expect(RelayConstants.protoVersion, 14);
test('proto 15: avatars by hash, profile intent', () {
expect(RelayConstants.protoVersion, 15);
const v = MemberView(id: 'm1', name: 'a', ready: false, inSet: false, buffered: 0, stale: false);
expect(v.toJson().containsKey('avatarHash'), isFalse);
expect(v.toJson().containsKey('avatar'), isFalse);