forked from Foxyz/sync_replay_project
Protocol 15: one shared session log on the server, chat never logged, Mark a problem
Every app streams its redacted diagnostic lines (server time, batched every 10 s, gzip, buffered while offline) to POST /l/; the server merges them with its own into session-YYYY-MM-DD.log in time order, tagged by who. Opt-out in Settings > About; Mark a problem (Ctrl+Shift+B, the player menus, About) writes a marker with an optional note. Copy diagnostics can include the server's session log. Chat text never reaches any log. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
foxyzandClaude Opus 5.5 committed
1 parent
4ab37c55cf
commit
7487b4deb1
29 files changed
+1380
-47
No files matched your search
@@ -1,6 +1,7 @@
|
||||
export 'src/constants.dart';
|
||||
export 'src/errors.dart';
|
||||
export 'src/logging.dart';
|
||||
export 'src/diag_share.dart';
|
||||
export 'src/clock_sync.dart';
|
||||
export 'src/config.dart';
|
||||
export 'src/protocol.dart';
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
abstract class RelayConstants {
|
||||
static const int protoVersion = 14;
|
||||
static const int protoVersion = 15;
|
||||
|
||||
// Relay (`/r/...`): a member's bytes through the server's buffer.
|
||||
static const int relaySliceBytes = 1 << 20;
|
||||
@@ -70,4 +70,20 @@ abstract class RelayConstants {
|
||||
static const int maxQueueItems = 100;
|
||||
static const int maxQueueTextLength = 200;
|
||||
static const int queuePlayTimeoutSecs = 30;
|
||||
|
||||
/// Diagnostics sharing (`/l/...`, ARCHITECTURE "Logging"): a member's
|
||||
/// batch carries at most [diagBatchBytes] of lines (gzip on the wire, at
|
||||
/// most [diagBatchBytes] compressed and [diagDecodedBytes] unpacked); it
|
||||
/// keeps up to [diagBufferBytes] while it cannot send (oldest dropped)
|
||||
/// and sends every [diagIntervalSecs]. The server takes at most
|
||||
/// [diagPostsPerMin] posts and [diagBytesPerMin] unpacked bytes per member
|
||||
/// and minute (429 beyond). A "Mark a problem" note is cut to
|
||||
/// [maxProblemNoteLength] characters.
|
||||
static const int diagBatchBytes = 256 * 1024;
|
||||
static const int diagDecodedBytes = 1 << 20;
|
||||
static const int diagBufferBytes = 2 << 20;
|
||||
static const int diagIntervalSecs = 10;
|
||||
static const int diagPostsPerMin = 30;
|
||||
static const int diagBytesPerMin = 4 << 20;
|
||||
static const int maxProblemNoteLength = 200;
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
import 'dart:async';
|
||||
import 'dart:collection';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:logging/logging.dart';
|
||||
|
||||
import 'constants.dart';
|
||||
import 'logging.dart';
|
||||
|
||||
// Diagnostics sharing (ARCHITECTURE "Logging", PROTOCOL "Diagnostics"):
|
||||
// each app sends its log lines to the server, which merges them with its
|
||||
// own into one session log. The batch format both sides use, and the
|
||||
// app's sender.
|
||||
|
||||
/// One log record of a member: [t] in epoch ms (the app's clock while it
|
||||
/// waits, the server's on the wire), the level name, the logger and the
|
||||
/// redacted message (an error and stack trace on the lines below).
|
||||
class DiagLine {
|
||||
final int t;
|
||||
final String level;
|
||||
final String logger;
|
||||
final String message;
|
||||
const DiagLine({required this.t, required this.level, required this.logger, required this.message});
|
||||
|
||||
/// Roughly its bytes in a batch.
|
||||
int get size => message.length + logger.length + level.length + 32;
|
||||
|
||||
Map<String, Object?> toJson() => {'t': t, 'l': level, 'n': logger, 'm': message};
|
||||
|
||||
static DiagLine? fromJson(Object? j) {
|
||||
if (j is! Map) return null;
|
||||
final t = j['t'], l = j['l'], n = j['n'], m = j['m'];
|
||||
if (t is! num || !t.isFinite || m is! String) return null;
|
||||
return DiagLine(t: t.toInt(), level: l is String ? l : 'INFO', logger: n is String ? n : 'app', message: m);
|
||||
}
|
||||
}
|
||||
|
||||
/// A "Mark a problem" from a member: when (epoch ms, server clock) and
|
||||
/// its optional note.
|
||||
class DiagMark {
|
||||
final int t;
|
||||
final String note;
|
||||
const DiagMark({required this.t, this.note = ''});
|
||||
|
||||
Map<String, Object?> toJson() => {'t': t, if (note.isNotEmpty) 'note': note};
|
||||
|
||||
static DiagMark? fromJson(Object? j) {
|
||||
if (j is! Map) return null;
|
||||
final t = j['t'], note = j['note'];
|
||||
if (t is! num || !t.isFinite) return null;
|
||||
return DiagMark(t: t.toInt(), note: note is String ? note : '');
|
||||
}
|
||||
}
|
||||
|
||||
/// The body of `POST /l/<sid>/<token>` (gzip of its JSON): the app's
|
||||
/// [version], whether it [share]s its lines (false: none, the server notes
|
||||
/// it once), the [lines], how many it [dropped] since the last batch (its
|
||||
/// buffer was full) and any problem [marks].
|
||||
class DiagBatch {
|
||||
final String version;
|
||||
final bool share;
|
||||
final List<DiagLine> lines;
|
||||
final int dropped;
|
||||
final List<DiagMark> marks;
|
||||
const DiagBatch({this.version = '', this.share = true, this.lines = const [], this.dropped = 0, this.marks = const []});
|
||||
|
||||
Map<String, Object?> toJson() => {
|
||||
if (version.isNotEmpty) 'v': version,
|
||||
if (!share) 'share': false,
|
||||
if (lines.isNotEmpty) 'lines': [for (final l in lines) l.toJson()],
|
||||
if (dropped > 0) 'dropped': dropped,
|
||||
if (marks.isNotEmpty) 'marks': [for (final m in marks) m.toJson()],
|
||||
};
|
||||
|
||||
List<int> encode() => gzip.encode(utf8.encode(jsonEncode(toJson())));
|
||||
|
||||
factory DiagBatch.fromJson(Map<String, Object?> j) {
|
||||
final lines = j['lines'], marks = j['marks'], dropped = j['dropped'], v = j['v'];
|
||||
return DiagBatch(
|
||||
version: v is String ? v : '',
|
||||
share: j['share'] != false,
|
||||
lines: lines is List ? lines.map(DiagLine.fromJson).whereType<DiagLine>().toList() : const [],
|
||||
dropped: dropped is num && dropped.isFinite && dropped > 0 ? dropped.toInt() : 0,
|
||||
marks: marks is List ? marks.map(DiagMark.fromJson).whereType<DiagMark>().toList() : const [],
|
||||
);
|
||||
}
|
||||
|
||||
/// Unpacks a body; throws [FormatException] when it is not gzip of a
|
||||
/// batch or unpacks to more than [maxBytes].
|
||||
static DiagBatch decode(List<int> body, {int maxBytes = RelayConstants.diagDecodedBytes}) {
|
||||
final text = utf8.decode(gunzipCapped(body, maxBytes), allowMalformed: true);
|
||||
final j = jsonDecode(text);
|
||||
if (j is! Map) throw const FormatException('not a batch');
|
||||
return DiagBatch.fromJson(Map<String, Object?>.from(j));
|
||||
}
|
||||
}
|
||||
|
||||
/// [body] gunzipped, or [FormatException] once it passes [maxBytes] (a
|
||||
/// small body must not unpack to gigabytes).
|
||||
List<int> gunzipCapped(List<int> body, int maxBytes) {
|
||||
final out = BytesBuilder(copy: false);
|
||||
final sink = _CappedSink(out, maxBytes);
|
||||
try {
|
||||
gzip.decoder.startChunkedConversion(sink)
|
||||
..add(body)
|
||||
..close();
|
||||
} on FormatException {
|
||||
rethrow;
|
||||
} catch (e) {
|
||||
throw FormatException('not gzip: $e');
|
||||
}
|
||||
return out.takeBytes();
|
||||
}
|
||||
|
||||
class _CappedSink implements Sink<List<int>> {
|
||||
final BytesBuilder out;
|
||||
final int max;
|
||||
_CappedSink(this.out, this.max);
|
||||
|
||||
@override
|
||||
void add(List<int> data) {
|
||||
if (out.length + data.length > max) throw const FormatException('too large');
|
||||
out.add(data);
|
||||
}
|
||||
|
||||
@override
|
||||
void close() {}
|
||||
}
|
||||
|
||||
/// Where the app's lines go: the server's base URL (`http://host:port`),
|
||||
/// this member's token and the server's clock minus the local one (ms).
|
||||
typedef DiagTarget = ({Uri base, String token, double offsetMs});
|
||||
|
||||
/// The app's side: keeps its log records (not the hosted server's, which
|
||||
/// the server logs itself) while in a session, and sends them to the
|
||||
/// server every [RelayConstants.diagIntervalSecs] ([tick]) and on [stop].
|
||||
/// [target] is null while there is nowhere to send (reconnecting, no
|
||||
/// clock sample yet): lines wait, up to [bufferBytes], the oldest dropped
|
||||
/// and counted. With [share] off nothing is kept or sent but one batch
|
||||
/// saying so per connection; problem marks ([markProblem]) go either way.
|
||||
class DiagUploader {
|
||||
final DiagTarget? Function() target;
|
||||
/// The app and its version, e.g. `Watch Together 1.4.0 on linux`.
|
||||
final String version;
|
||||
final int bufferBytes;
|
||||
final int batchBytes;
|
||||
final Duration interval;
|
||||
final DateTime Function() _now;
|
||||
DiagUploader({
|
||||
required this.target,
|
||||
required this.version,
|
||||
this.bufferBytes = RelayConstants.diagBufferBytes,
|
||||
this.batchBytes = RelayConstants.diagBatchBytes,
|
||||
this.interval = const Duration(seconds: RelayConstants.diagIntervalSecs),
|
||||
DateTime Function()? now,
|
||||
}) : _now = now ?? DateTime.now;
|
||||
|
||||
/// Loggers never sent: the hosted server's records (`server.*`, the
|
||||
/// server logs them itself) and the local copy of a mark ([markLogger]).
|
||||
static bool sends(String logger) => logger != 'server' && !logger.startsWith('server.') && logger != markLogger;
|
||||
|
||||
/// The local log's copy of a problem mark.
|
||||
static const markLogger = 'problem';
|
||||
static final _markLog = Logger(markLogger);
|
||||
|
||||
final _lines = Queue<DiagLine>();
|
||||
var _size = 0;
|
||||
var _dropped = 0;
|
||||
final _marks = <DiagMark>[];
|
||||
StreamSubscription<LogRecord>? _sub;
|
||||
HttpClient? _http;
|
||||
var _busy = false;
|
||||
var _again = false;
|
||||
DateTime? _last;
|
||||
/// The token the server was told "doesn't share" for.
|
||||
String? _toldOff;
|
||||
var _share = true;
|
||||
|
||||
bool get running => _sub != null;
|
||||
|
||||
/// Lines waiting to go.
|
||||
int get pending => _lines.length;
|
||||
|
||||
/// Share the log lines (Settings); off drops what waits.
|
||||
bool get share => _share;
|
||||
set share(bool on) {
|
||||
if (on == _share) return;
|
||||
_share = on;
|
||||
if (!on) _clear();
|
||||
_toldOff = null;
|
||||
}
|
||||
|
||||
/// Starts keeping the log's records (a session starts).
|
||||
void start() {
|
||||
_sub ??= Logger.root.onRecord.listen(_onRecord);
|
||||
_last = _now();
|
||||
}
|
||||
|
||||
void _onRecord(LogRecord r) {
|
||||
if (!_share || !sends(r.loggerName)) return;
|
||||
final b = StringBuffer(r.message);
|
||||
if (r.error != null) b.write('\n error: ${r.error}');
|
||||
final s = '${r.stackTrace ?? ''}'.trimRight();
|
||||
if (s.isNotEmpty) b.write('\n$s');
|
||||
add(DiagLine(t: r.time.millisecondsSinceEpoch, level: r.level.name, logger: r.loggerName, message: redactLog('$b')));
|
||||
}
|
||||
|
||||
/// Keeps [line] (local clock) for the next batch.
|
||||
void add(DiagLine line) {
|
||||
_lines.add(line);
|
||||
_size += line.size;
|
||||
_trim();
|
||||
}
|
||||
|
||||
void _trim() {
|
||||
while (_size > bufferBytes && _lines.isNotEmpty) {
|
||||
_size -= _lines.removeFirst().size;
|
||||
_dropped++;
|
||||
}
|
||||
}
|
||||
|
||||
void _clear() {
|
||||
_lines.clear();
|
||||
_size = 0;
|
||||
_dropped = 0;
|
||||
}
|
||||
|
||||
/// "Mark a problem": a marker line in the server's session log with
|
||||
/// [note] (cut to [RelayConstants.maxProblemNoteLength], one line), sent
|
||||
/// now with the lines waiting. Also written to the local log. True once
|
||||
/// the server has it; false when it waits (not connected).
|
||||
Future<bool> markProblem(String note) {
|
||||
var clean = note.replaceAll(RegExp(r'[\x00-\x1f\x7f]+'), ' ').trim();
|
||||
if (clean.length > RelayConstants.maxProblemNoteLength) {
|
||||
clean = clean.substring(0, RelayConstants.maxProblemNoteLength);
|
||||
}
|
||||
_markLog.warning('=== PROBLEM marked${clean.isEmpty ? '' : ': $clean'} ===');
|
||||
_marks.add(DiagMark(t: _now().millisecondsSinceEpoch, note: redactLog(clean)));
|
||||
return flush();
|
||||
}
|
||||
|
||||
/// Once a second (the app's ticker): sends when [interval] has passed.
|
||||
void tick() {
|
||||
final last = _last;
|
||||
if (last != null && _now().difference(last) < interval) return;
|
||||
unawaited(flush());
|
||||
}
|
||||
|
||||
/// Sends what waits: true when the server took all of it (or there was
|
||||
/// nothing to send).
|
||||
Future<bool> flush() async {
|
||||
if (_busy) {
|
||||
// Once the one under way is done (a mark goes now, not next tick).
|
||||
_again = true;
|
||||
return false;
|
||||
}
|
||||
final tg = target();
|
||||
if (tg == null) return false;
|
||||
_busy = true;
|
||||
_last = _now();
|
||||
try {
|
||||
final offset = tg.offsetMs.round();
|
||||
List<DiagMark> takeMarks() {
|
||||
final m = [for (final x in _marks) DiagMark(t: x.t + offset, note: x.note)];
|
||||
_marks.clear();
|
||||
return m;
|
||||
}
|
||||
|
||||
if (!_share) {
|
||||
if (_toldOff == tg.token && _marks.isEmpty) return true;
|
||||
final marks = takeMarks();
|
||||
final ok = await _post(tg, DiagBatch(version: version, share: false, marks: marks));
|
||||
if (ok) {
|
||||
_toldOff = tg.token;
|
||||
} else {
|
||||
_marks.insertAll(0, [for (final m in marks) DiagMark(t: m.t - offset, note: m.note)]);
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
// A few batches at most per flush; the rest goes next time.
|
||||
for (var i = 0; i < 8 && (_lines.isNotEmpty || _marks.isNotEmpty || _dropped > 0); i++) {
|
||||
final taken = <DiagLine>[];
|
||||
var bytes = 0;
|
||||
while (_lines.isNotEmpty && (taken.isEmpty || bytes + _lines.first.size <= batchBytes)) {
|
||||
final l = _lines.removeFirst();
|
||||
_size -= l.size;
|
||||
bytes += l.size;
|
||||
taken.add(l);
|
||||
}
|
||||
final dropped = _dropped;
|
||||
_dropped = 0;
|
||||
final marks = takeMarks();
|
||||
final batch = DiagBatch(
|
||||
version: version,
|
||||
lines: [for (final l in taken) DiagLine(t: l.t + offset, level: l.level, logger: l.logger, message: l.message)],
|
||||
dropped: dropped,
|
||||
marks: marks,
|
||||
);
|
||||
if (!await _post(tg, batch)) {
|
||||
// Back in front, in order; the cap applies again.
|
||||
for (final l in taken.reversed) {
|
||||
_lines.addFirst(l);
|
||||
_size += l.size;
|
||||
}
|
||||
_dropped += dropped;
|
||||
_marks.insertAll(0, [for (final m in marks) DiagMark(t: m.t - offset, note: m.note)]);
|
||||
_trim();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return _lines.isEmpty;
|
||||
} finally {
|
||||
_busy = false;
|
||||
if (_again) {
|
||||
_again = false;
|
||||
if (_sub != null) unawaited(flush());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static Uri _httpUri(Uri base, String path, [Map<String, String>? query]) => base.replace(
|
||||
scheme: base.scheme == 'wss' || base.scheme == 'https' ? 'https' : 'http',
|
||||
path: path,
|
||||
queryParameters: query,
|
||||
fragment: null);
|
||||
|
||||
/// The server's `http://host:port` for a `/sync` socket URL.
|
||||
static Uri baseOf(Uri wsUri) => _httpUri(wsUri, '/');
|
||||
|
||||
Future<bool> _post(DiagTarget tg, DiagBatch batch) async {
|
||||
try {
|
||||
final http = _http ??= HttpClient()..connectionTimeout = const Duration(seconds: 5);
|
||||
final req = await http.postUrl(_httpUri(tg.base, '/l/default/${tg.token}')).timeout(const Duration(seconds: 5));
|
||||
final body = batch.encode();
|
||||
req.headers.contentType = ContentType('application', 'gzip');
|
||||
req.contentLength = body.length;
|
||||
req.add(body);
|
||||
final res = await req.close().timeout(const Duration(seconds: 10));
|
||||
await res.drain<void>();
|
||||
// 413: this batch can never go (should not happen); drop it.
|
||||
return res.statusCode == 204 || res.statusCode == 200 || res.statusCode == 413;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// The server's session log of the last [hours] (every member and the
|
||||
/// server, merged), or null when it cannot be had (not connected, a
|
||||
/// server without one, an older server).
|
||||
Future<String?> fetchSessionLog({int hours = 6}) async {
|
||||
final tg = target();
|
||||
if (tg == null) return null;
|
||||
try {
|
||||
final http = _http ??= HttpClient()..connectionTimeout = const Duration(seconds: 5);
|
||||
final req = await http
|
||||
.getUrl(_httpUri(tg.base, '/l/default/${tg.token}', {'hours': '$hours'}))
|
||||
.timeout(const Duration(seconds: 5));
|
||||
final res = await req.close().timeout(const Duration(seconds: 10));
|
||||
final bytes = <int>[];
|
||||
await for (final c in res.timeout(const Duration(seconds: 30))) {
|
||||
bytes.addAll(c);
|
||||
}
|
||||
if (res.statusCode != 200) return null;
|
||||
return utf8.decode(gunzipCapped(bytes, 64 << 20), allowMalformed: true);
|
||||
} catch (_) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// The session ends: what waits goes now (best effort, [timeout]), then
|
||||
/// nothing is kept until the next [start].
|
||||
Future<void> stop({Duration timeout = const Duration(milliseconds: 1500)}) async {
|
||||
await _sub?.cancel();
|
||||
_sub = null;
|
||||
if ((_lines.isNotEmpty || _marks.isNotEmpty || _dropped > 0) && target() != null) {
|
||||
try {
|
||||
await flush().timeout(timeout);
|
||||
} catch (_) {}
|
||||
}
|
||||
_clear();
|
||||
_marks.clear();
|
||||
_toldOff = null;
|
||||
_last = null;
|
||||
_http?.close(force: true);
|
||||
_http = null;
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,10 @@ import 'package:logging/logging.dart';
|
||||
|
||||
Logger createLogger(String name) => Logger(name);
|
||||
|
||||
/// The server's combined log of every member and itself
|
||||
/// (`session-YYYY-MM-DD.log`, ARCHITECTURE "Logging").
|
||||
const sessionLogPrefix = 'session';
|
||||
|
||||
var _listening = false;
|
||||
var _console = true;
|
||||
LogFiles? _files;
|
||||
@@ -35,12 +39,13 @@ void initLogging({Level level = Level.INFO, bool console = true}) {
|
||||
|
||||
/// Writes the files in [dir] from now on: `<prefix>-YYYY-MM-DD.log`,
|
||||
/// rotated by day and [LogFiles.maxFileBytes], pruned by age and total
|
||||
/// size. Lines logged before go in first. A folder that cannot be written
|
||||
/// leaves logging on the console only (and says so there).
|
||||
/// size (shared with the session log, [sessionLogPrefix], in the same
|
||||
/// folder). Lines logged before go in first. A folder that cannot be
|
||||
/// written leaves logging on the console only (and says so there).
|
||||
LogFiles? startLogFiles(String dir, {String prefix = 'app'}) {
|
||||
if (_files != null) return _files;
|
||||
try {
|
||||
final f = LogFiles(dir, prefix: prefix)..open();
|
||||
final f = LogFiles(dir, prefix: prefix, shareWith: const [sessionLogPrefix])..open();
|
||||
_files = f;
|
||||
for (final l in _early) {
|
||||
f.write(l);
|
||||
@@ -123,13 +128,13 @@ final _headerSecret = RegExp(
|
||||
final _pairSecret = RegExp('\\b($_secretNames)\\s*=\\s*[^&\\s"\',;)}\\]]+', caseSensitive: false);
|
||||
final _bearer = RegExp(r'\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}', caseSensitive: false);
|
||||
final _url = RegExp(r'''\b(?:https?|wss?)://[^\s"'<>]+''', caseSensitive: false);
|
||||
// A member token in a server path: /r/<sid>/<token>/..., /m, /j, /a, /i.
|
||||
final _pathToken = RegExp(r'(/[rmjai]/[^/\s?#]+/)[0-9a-fA-F]{32}\b');
|
||||
// A member token in a server path: /r/<sid>/<token>/..., /m, /j, /a, /i, /l.
|
||||
final _pathToken = RegExp(r'(/[rmjail]/[^/\s?#]+/)[0-9a-fA-F]{32}\b');
|
||||
final _userInfo = RegExp(r'^([a-z]+://)[^/@\s]+@', caseSensitive: false);
|
||||
|
||||
/// [s] with nothing secret left: every URL's query string down to its
|
||||
/// parameter names (signed CDN links, `api_key`), user:password@ in URLs,
|
||||
/// member tokens in `/r/ /m/ /j/ /a/ /i/` paths, and the values of
|
||||
/// member tokens in `/r/ /m/ /j/ /a/ /i/ /l/` paths, and the values of
|
||||
/// passwords, tokens, API keys, cookies and auth headers. Applied to
|
||||
/// every log line ([formatLogLine]).
|
||||
String redactLog(String s) {
|
||||
@@ -203,16 +208,21 @@ class LogThrottle {
|
||||
/// Rotating log files in [dir]: `<prefix>-YYYY-MM-DD.log` (then `.1.log`,
|
||||
/// `.2.log` ... once one passes [maxFileBytes]); files older than
|
||||
/// [keepDays] and the oldest beyond [maxTotalBytes] are deleted whenever a
|
||||
/// file is opened.
|
||||
/// file is opened. [shareWith]: other prefixes in [dir] that count toward
|
||||
/// the same [maxTotalBytes] (the app's or server's log and the session
|
||||
/// log); the oldest of all of them go first, never another prefix's file
|
||||
/// of today (its writer may have it open).
|
||||
class LogFiles {
|
||||
final String dir;
|
||||
final String prefix;
|
||||
final List<String> shareWith;
|
||||
final int maxFileBytes;
|
||||
final int maxTotalBytes;
|
||||
final int keepDays;
|
||||
final DateTime Function() _now;
|
||||
LogFiles(this.dir,
|
||||
{this.prefix = 'app',
|
||||
this.shareWith = const [],
|
||||
this.maxFileBytes = 10 * 1024 * 1024,
|
||||
this.maxTotalBytes = 100 * 1024 * 1024,
|
||||
this.keepDays = 7,
|
||||
@@ -229,7 +239,10 @@ class LogFiles {
|
||||
|
||||
String _dayOf(DateTime t) => '${t.year}-${_two(t.month)}-${_two(t.day)}';
|
||||
|
||||
String _dayOfFile(File f) => f.uri.pathSegments.last.substring(prefix.length + 1, prefix.length + 11);
|
||||
String _dayOfFile(File f) => _fileDay(prefix, f);
|
||||
|
||||
static String _fileDay(String prefix, File f) =>
|
||||
f.uri.pathSegments.last.substring(prefix.length + 1, prefix.length + 11);
|
||||
|
||||
/// Opens today's file (creating [dir]); throws when it cannot.
|
||||
void open() {
|
||||
@@ -256,9 +269,10 @@ class LogFiles {
|
||||
}
|
||||
|
||||
/// Appends [line]; a new day or a full file moves on to the next file.
|
||||
void write(String line) {
|
||||
/// [at]: the line's own time picks the day's file (else now).
|
||||
void write(String line, {DateTime? at}) {
|
||||
try {
|
||||
final day = _dayOf(_now());
|
||||
final day = _dayOf(at ?? _now());
|
||||
if (_out == null || day != _day || _size >= maxFileBytes) _roll(day);
|
||||
final bytes = utf8.encode('$line\n');
|
||||
_out!.writeFromSync(bytes);
|
||||
@@ -269,7 +283,10 @@ class LogFiles {
|
||||
}
|
||||
|
||||
/// This prefix's files, oldest first.
|
||||
List<File> files() {
|
||||
List<File> files() => filesOf(prefix);
|
||||
|
||||
/// The files of [prefix] in [dir], oldest first.
|
||||
List<File> filesOf(String prefix) {
|
||||
final d = Directory(dir);
|
||||
if (!d.existsSync()) return const [];
|
||||
final name = RegExp('^${RegExp.escape(prefix)}-(\\d{4}-\\d{2}-\\d{2})(?:\\.(\\d+))?\\.log\$');
|
||||
@@ -295,10 +312,18 @@ class LogFiles {
|
||||
kept.add(f);
|
||||
}
|
||||
}
|
||||
var total = kept.fold<int>(0, (a, f) => a + _len(f));
|
||||
for (final f in kept) {
|
||||
final today = _dayOf(_now());
|
||||
// Every file under the budget, oldest day first: (day, prefix, file).
|
||||
final all = [
|
||||
for (final f in kept) (_dayOfFile(f), true, f),
|
||||
for (final p in shareWith)
|
||||
if (p != prefix)
|
||||
for (final f in filesOf(p)) (_fileDay(p, f), false, f),
|
||||
]..sort((a, b) => a.$1.compareTo(b.$1));
|
||||
var total = all.fold<int>(0, (a, f) => a + _len(f.$3));
|
||||
for (final (day, own, f) in all) {
|
||||
if (total <= maxTotalBytes) break;
|
||||
if (f.path == _path) continue;
|
||||
if (f.path == _path || (!own && day == today)) continue;
|
||||
total -= _len(f);
|
||||
_delete(f);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:sync_relay_core/core.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
void main() {
|
||||
test('a batch survives gzip both ways; a bomb or junk is refused', () {
|
||||
const b = DiagBatch(
|
||||
version: 'WT 1.0',
|
||||
lines: [DiagLine(t: 5, level: 'WARNING', logger: 'mpv', message: 'x\n error: y')],
|
||||
dropped: 2,
|
||||
marks: [DiagMark(t: 7, note: 'lag')]);
|
||||
final back = DiagBatch.decode(b.encode());
|
||||
expect(back.toJson(), b.toJson());
|
||||
expect(DiagBatch.decode(const DiagBatch(share: false).encode()).share, isFalse);
|
||||
expect(() => DiagBatch.decode(gzip.encode(List.filled(2 << 20, 32)), maxBytes: 1 << 20), throwsFormatException);
|
||||
expect(() => DiagBatch.decode(utf8.encode('plain')), throwsFormatException);
|
||||
});
|
||||
|
||||
group('DiagUploader', () {
|
||||
late HttpServer http;
|
||||
final got = <DiagBatch>[];
|
||||
var accept = true;
|
||||
setUp(() async {
|
||||
got.clear();
|
||||
accept = true;
|
||||
http = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
|
||||
http.listen((req) async {
|
||||
final body = await req.fold<List<int>>([], (a, c) => a..addAll(c));
|
||||
if (accept) got.add(DiagBatch.decode(body));
|
||||
req.response.statusCode = accept ? 204 : 503;
|
||||
await req.response.close();
|
||||
});
|
||||
});
|
||||
tearDown(() => http.close(force: true));
|
||||
|
||||
DiagUploader make({bool connected = true, int buffer = 1 << 20}) => DiagUploader(
|
||||
target: () => connected ? (base: Uri.parse('http://127.0.0.1:${http.port}'), token: 't', offsetMs: 1000) : null,
|
||||
version: 'WT 1.0',
|
||||
bufferBytes: buffer);
|
||||
|
||||
test('sends its records on the server clock, not the hosted server\'s; keeps them while it cannot', () async {
|
||||
final u = make()..start();
|
||||
Logger('relay').info('hello api_key=abc');
|
||||
Logger('server.relay').info('the server logs this itself');
|
||||
await Future<void>.delayed(Duration.zero);
|
||||
accept = false;
|
||||
expect(await u.flush(), isFalse);
|
||||
expect(u.pending, 1);
|
||||
accept = true;
|
||||
expect(await u.flush(), isTrue);
|
||||
final line = got.single.lines.single;
|
||||
expect(line.message, 'hello api_key=***');
|
||||
expect(line.logger, 'relay');
|
||||
expect(got.single.version, 'WT 1.0');
|
||||
await u.stop();
|
||||
});
|
||||
|
||||
test('a full buffer drops the oldest and says how many', () async {
|
||||
final u = make(buffer: 400);
|
||||
for (var i = 0; i < 10; i++) {
|
||||
u.add(DiagLine(t: i, level: 'INFO', logger: 'a', message: 'line $i ${'x' * 40}'));
|
||||
}
|
||||
expect(await u.flush(), isTrue);
|
||||
expect(got.single.dropped, greaterThan(0));
|
||||
expect(got.single.lines.last.message, startsWith('line 9'));
|
||||
expect(got.single.lines.first.t, greaterThanOrEqualTo(1000), reason: 'offset to the server clock');
|
||||
});
|
||||
|
||||
test('opted out: no lines, one "does not share" per connection; marks still go', () async {
|
||||
final u = make()
|
||||
..share = false
|
||||
..start();
|
||||
Logger('relay').info('private');
|
||||
expect(await u.flush(), isTrue);
|
||||
expect(await u.flush(), isTrue);
|
||||
expect(got.length, 1);
|
||||
expect(got.single.share, isFalse);
|
||||
expect(got.single.lines, isEmpty);
|
||||
expect(await u.markProblem('it\nfroze'), isTrue);
|
||||
expect(got.last.marks.single.note, 'it froze');
|
||||
await u.stop();
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -19,6 +19,7 @@ void main() {
|
||||
expect(redactLog('open http://1.2.3.4:8080/r/default/$tok/2/0/name'), 'open http://1.2.3.4:8080/r/default/***/2/0/name');
|
||||
expect(redactLog('ws://h:1/r/default/$tok/source?q=7'), 'ws://h:1/r/default/***/source?q=…');
|
||||
expect(redactLog('/m/default/$tok/file.mkv range=bytes=0-'), '/m/default/***/file.mkv range=bytes=0-');
|
||||
expect(redactLog('POST /l/default/$tok'), 'POST /l/default/***');
|
||||
expect(redactLog('edl://!no_clip;%160%http://h/r/default/$tok/2/0/n;!new_stream;!no_clip;%160%http://h/r/default/$tok/2/1/n'),
|
||||
isNot(contains(tok)));
|
||||
});
|
||||
@@ -85,5 +86,16 @@ void main() {
|
||||
expect(small.files().map((x) => x.uri.pathSegments.last), ['app-2026-10-12.1.log', 'app-2026-10-13.log']);
|
||||
small.close();
|
||||
});
|
||||
|
||||
test('the session log shares the total budget, never touching another writer\'s today', () {
|
||||
final now = DateTime(2026, 10, 12, 10);
|
||||
File('${dir.path}/session-2026-10-10.log').writeAsStringSync('s' * 100);
|
||||
File('${dir.path}/session-2026-10-12.log').writeAsStringSync('s' * 100);
|
||||
File('${dir.path}/app-2026-10-11.log').writeAsStringSync('a' * 100);
|
||||
final f = LogFiles(dir.path, maxTotalBytes: 150, shareWith: const ['session'], now: () => now)..open();
|
||||
final left = dir.listSync().map((e) => e.uri.pathSegments.last).toSet();
|
||||
expect(left, {'session-2026-10-12.log', 'app-2026-10-12.log'});
|
||||
f.close();
|
||||
});
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user