Relay: viewers keep the file size (headers within 4 s, connection: close), retries back off, 412 re-resolves

Two server behaviours left ffmpeg (8.1/9.0) without the file size, after
which every seek failed with "size -38" until a reload:
- dart:io holds response headers until body bytes exist: a stalled
  source meant no headers within mpv's 5 s timeout. Relay byte responses
  are now written on the raw connection, headers with the first bytes or
  after 4 s, and a swap ends waiting reads with a plain close.
- every response closed its socket without saying connection: close, so
  ffmpeg 8.x reused a dead connection for its next seek (pol's X clips).
  The server now says it whenever the request asks.
A failed pull waits 250 ms doubling to 8 s instead of re-pulling at once
(the ~130 ms storm that drew Bilibili's 412/403); 412 counts as an expired
URL and re-resolves the page, at most once per 60 s. The app reloads a
player once per media when ffmpeg reports the lost size anyway. P37: a
read that waits past ffmpeg's retry budget still ends the file early.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/core/lib/src/constants.dart

@@ -31,6 +31,13 @@ abstract class RelayConstants {
   /// A viewer request waits this long for its bytes (source away) before
   /// the server gives up on it.
   static const int relayWaitSecs = 60;
+  /// Failed pulls in a row back off: 250 ms doubling up to this, back to
+  /// none once bytes come again.
+  static const int relayRetryMaxMs = 8000;
+  /// A relay viewer's response headers wait at most this long for its
+  /// first bytes, under mpv's 5 s `network-timeout`: a header that never
+  /// comes in time costs ffmpeg the file's size for good.
+  static const int relayHeaderWaitMs = 4000;
   /// The warm relay (Up next's first relayed item, before it goes up)
   /// keeps this much ahead of and behind its viewers' preloads, besides
   /// the pins: about what a preload player reads ahead (48 MiB), never the

packages/core/lib/src/link_probe.dart

@@ -282,9 +282,12 @@ Future<String> _readText(HttpClientResponse res) async {
 /// audio, or segmented playlists). A byte track is range-read with
 /// yt-dlp's headers in GETs of the format's `http_chunk_size`; a pieces
 /// track's playlist (or DASH fragments) is read on first use, then each
-/// piece is one GET. Format URLs expire: on 403/410 from any track the page
-/// is resolved again once for all tracks (same format ids, same sizes and
-/// piece counts) and the read goes on from where it stopped.
+/// piece is one GET. Format URLs expire: on 403/410/412 from any track the
+/// page is resolved again once for all tracks (same format ids, same sizes
+/// and piece counts) and the read goes on from where it stopped. Another
+/// expiry within [refreshGap] of that resolve fails the read instead (the
+/// server backs off and asks again), so a site refusing the fresh URLs too
+/// is not resolved again and again.
 class PageRelayMedia implements RelayMedia {
   final Uri page;
   final YtDlp ytDlp;
@@ -296,7 +299,10 @@ class PageRelayMedia implements RelayMedia {
   final _http = HttpClient();
   var _gen = 0;
   Future<void>? _refreshing;
-  PageRelayMedia(this.page, this.ytDlp, {List<YtDlpTrack>? tracks}) : _picked = tracks;
+  DateTime? _refreshedAt;
+  final Duration refreshGap;
+  PageRelayMedia(this.page, this.ytDlp, {List<YtDlpTrack>? tracks, this.refreshGap = const Duration(seconds: 60)})
+      : _picked = tracks;
 
   Future<YtDlpInfo> _info() async {
     final info = await ytDlp.resolve(page.toString());
@@ -335,14 +341,22 @@ class PageRelayMedia implements RelayMedia {
     return fetchHlsPlaylist(_http, f.url, f.headers);
   }
 
-  static bool _expired(Object e) => e is UpstreamHttpException && (e.status == 403 || e.status == 410);
+  /// A format URL the site no longer takes: 403, 410, or Bilibili's 412.
+  static bool _expired(Object e) =>
+      e is UpstreamHttpException && (e.status == 403 || e.status == 410 || e.status == 412);
 
   /// Resolves the page again: the same format ids with the same sizes and
   /// piece shapes, else the video changed.
-  Future<void> _refresh(int gen) {
+  Future<void> _refresh(int gen, Object cause) {
     if (gen != _gen) return Future.value();
+    final last = _refreshedAt;
+    if (_refreshing == null && last != null && DateTime.now().difference(last) < refreshGap) {
+      // Resolved a moment ago and refused again: not once more.
+      return Future.error(cause);
+    }
     return _refreshing ??= () async {
-      _log.info('relay of ${logUrl(page)}: a format URL expired (403/410), resolving the page again');
+      _refreshedAt = DateTime.now();
+      _log.info('relay of ${logUrl(page)}: a format URL expired (HTTP ${cause is UpstreamHttpException ? cause.status : '?'}), resolving the page again');
       try {
         const changed = LinkProbeException('the video changed on the page');
         final info = await _info();
@@ -384,7 +398,7 @@ class PageRelayMedia implements RelayMedia {
       return _sizes[i] = await _open(i).size();
     } catch (e) {
       if (!_expired(e)) rethrow;
-      await _refresh(gen);
+      await _refresh(gen, e);
       return _sizes[i] = await _open(i).size();
     }
   }
@@ -397,7 +411,7 @@ class PageRelayMedia implements RelayMedia {
       return _pieces[i] = await _loadPieces(_picked![i].format);
     } catch (e) {
       if (!_expired(e)) rethrow;
-      await _refresh(gen);
+      await _refresh(gen, e);
       return _pieces[i] ??= await _loadPieces(_picked![i].format);
     }
   }
@@ -416,7 +430,7 @@ class PageRelayMedia implements RelayMedia {
       } catch (e) {
         if (retried || !_expired(e)) rethrow;
         retried = true;
-        await _refresh(gen);
+        await _refresh(gen, e);
       }
     }
   }
@@ -435,7 +449,7 @@ class PageRelayMedia implements RelayMedia {
       } catch (e) {
         if (retried || !_expired(e)) rethrow;
         retried = true;
-        await _refresh(gen);
+        await _refresh(gen, e);
       }
     }
   }

packages/core/test/page_relay_test.dart

@@ -0,0 +1,107 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:sync_relay_core/core.dart';
+import 'package:test/test.dart';
+
+// A page relayed by its formats: an expired format URL (403/410/412) is
+// resolved again once, and not again and again when the site keeps
+// refusing.
+
+/// yt-dlp that resolves the page to the origin's current format URL.
+class _CountingYtDlp extends YtDlp {
+  final Uri Function() url;
+  var resolves = 0;
+  _CountingYtDlp(super.dir, this.url);
+
+  @override
+  Future<YtDlpInfo> resolve(String page, {Duration timeout = const Duration(seconds: 25)}) async {
+    resolves++;
+    final f = YtDlpFormat(id: 'v', url: url());
+    return YtDlpInfo(title: 'page', tracks: [(kind: '', format: f)], formats: {'v': f});
+  }
+}
+
+void main() {
+  late HttpServer origin;
+  late Directory tmp;
+  // Only `/v<gen>.mp4` of the current generation is served; older ones get
+  // [refusal] (a format URL that expired).
+  var gen = 1;
+  var refusal = 412;
+  final bytes = Uint8List.fromList(List.generate(100000, (i) => i % 251));
+
+  setUp(() async {
+    gen = 1;
+    refusal = 412;
+    tmp = await Directory.systemTemp.createTemp('page-relay');
+    origin = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
+    origin.listen((req) async {
+      if (req.uri.path != '/v$gen.mp4') {
+        req.response.statusCode = refusal;
+        await req.response.close();
+        return;
+      }
+      final m = RegExp(r'bytes=(\d+)-(\d+)').firstMatch(req.headers.value('range') ?? '')!;
+      final start = int.parse(m.group(1)!), end = int.parse(m.group(2)!);
+      req.response
+        ..statusCode = 206
+        ..headers.contentType = ContentType('video', 'mp4')
+        ..headers.set('content-range', 'bytes $start-$end/${bytes.length}')
+        ..add(bytes.sublist(start, end + 1));
+      await req.response.close();
+    });
+  });
+
+  tearDown(() async {
+    await origin.close(force: true);
+    await tmp.delete(recursive: true);
+  });
+
+  Uri current() => Uri.parse('http://127.0.0.1:${origin.port}/v$gen.mp4');
+
+  Future<List<int>> read(RelayTrack t, int start, int end) async =>
+      [for (final c in await t.bytes!.read(start, end).toList()) ...c];
+
+  for (final status in [403, 410, 412]) {
+    test('HTTP $status: the page is resolved again once and the read goes on', () async {
+      refusal = status;
+      final y = _CountingYtDlp(tmp.path, current);
+      final media = PageRelayMedia(Uri.parse('https://example.com/watch'), y);
+      addTearDown(media.close);
+      final t = (await media.open()).single;
+      expect(await t.bytes!.size(), bytes.length);
+      expect(await read(t, 0, 999), bytes.sublist(0, 1000));
+      gen = 2; // the first URL expired
+      expect(await read(t, 5000, 5999), bytes.sublist(5000, 6000));
+      expect(y.resolves, 2);
+    });
+  }
+
+  test('refused again right after a resolve: the read fails, no resolve storm', () async {
+    final y = _CountingYtDlp(tmp.path, current);
+    final media = PageRelayMedia(Uri.parse('https://example.com/watch'), y);
+    addTearDown(media.close);
+    final t = (await media.open()).single;
+    await t.bytes!.size();
+    gen = 2;
+    expect(await read(t, 0, 9), bytes.sublist(0, 10));
+    gen = 3; // the fresh URL is refused too
+    for (var i = 0; i < 5; i++) {
+      await expectLater(read(t, 0, 9), throwsA(isA<UpstreamHttpException>().having((e) => e.status, 'status', 412)));
+    }
+    expect(y.resolves, 2);
+  });
+
+  test('another expiry after the gap resolves again', () async {
+    final y = _CountingYtDlp(tmp.path, current);
+    final media = PageRelayMedia(Uri.parse('https://example.com/watch'), y, refreshGap: Duration.zero);
+    addTearDown(media.close);
+    final t = (await media.open()).single;
+    await t.bytes!.size();
+    for (gen = 2; gen <= 4; gen++) {
+      expect(await read(t, 100, 199), bytes.sublist(100, 200));
+    }
+    expect(y.resolves, 4);
+  });
+}