using Godot; using System; // A thin 3D panel showing Godot Control UI rendered into a SubViewport (the project's ONE // SubViewport screen, see AGENTS.md), plus the touch-first hybrid cursor that drives it. // Input-agnostic: callers feed per-hand rays (origin + forward dir), trigger and thumbstick; // the screen projects them onto its plane and injects InputEventMouseMotion/MouseButton into // the SubViewport via Viewport.PushInput(event, in_local_coords: true). // // Cursor rules per hand (panel-local frame: X right, Y up, +Z = front normal, origin = center): // TOUCH — the stylus tip (origin + dir * TipLength) is within HoverDepth in front of the plane // (or pushed up to PushThroughDepth behind it) and over the panel: the tip's // projection is the cursor. Crossing ContactDepth from the front presses, backing out // past ReleaseDepth releases (hysteresis; entering from the side/behind never presses). // LASER — otherwise, the ray hits the panel from the front: the hit is the cursor and the // trigger (edge) clicks; holding the trigger drags (sliders). // One Godot mouse exists per viewport, so one hand owns it: the hand holding a press keeps it, // else a touching hand beats a laser hand, else the previous owner is preferred. // SWIPE vs TAP: a touch landing on a vertically scrollable ScrollContainer (not on a slider or // scrollbar inside it) holds its press back. Travel beyond SwipeThreshold before release = a // swipe: the content follows the tip (and flings on release), no click is ever sent. Released // sooner = a tap: press + release are sent together at the contact point. Thumbstick Y = wheel. public partial class MenuScreen : Node3D { public const float TipLength = 0.12f; public const float HoverDepth = 0.06f; public const float ContactDepth = 0.006f; public const float ReleaseDepth = 0.018f; public const float PushThroughDepth = 0.08f; public const float TouchEdgeMargin = 0.015f; public const float ScrollDeadzone = 0.3f; public const float ScrollNotchesPerSecond = 14f; public const float SwipeThreshold = 16f; // px (~6 mm) of tip travel before a touch is a swipe public const float FlingDecay = 5f; // 1/s public const float FlingMinSpeed = 30f; // px/s public enum PointerMode { None, Touch, Laser } public struct PointerInput { public bool Valid; public Vector3 Origin; public Vector3 Direction; public bool Trigger; public Vector2 Stick; } private struct HandState { public PointerMode Mode; public Vector3 Local; public Vector2 Pixel; public float Depth; public float RayDistance; public bool Armed; public bool TriggerPrev; public bool WantDown; } public Vector2 SizeMeters { get; } public Vector2I SizePixels { get; } public SubViewport Viewport { get; private set; } public StaticBody3D Body { get; private set; } // Fired when a hand presses down on the UI (touch contact or trigger) — for haptics. public event Action PointerPressed; private readonly HandState[] _hands = new HandState[2]; private readonly MeshInstance3D[] _dots = new MeshInstance3D[2]; private readonly StandardMaterial3D[] _dotMaterials = new StandardMaterial3D[2]; private int _owner = -1; private bool _down; private bool _mouseInside; private Vector2 _lastSent = new(-1, -1); private float _scrollAccum; // Touch held back for swipe-vs-tap (only while _down). private bool _pending; private bool _swiping; private ScrollContainer _swipeTarget; private Vector2 _pressPixel; private float _swipeLastY; private float _swipeScroll; private float _swipeVelocity; private ScrollContainer _flingTarget; private float _flingVelocity; private StandardMaterial3D _screenMaterial; public MenuScreen() : this(new Vector2(0.36f, 0.24f), new Vector2I(960, 640)) { } public MenuScreen(Vector2 sizeMeters, Vector2I sizePixels) { SizeMeters = sizeMeters; SizePixels = sizePixels; Build(); } public override void _Ready() { // ViewportTexture is fetched once the viewport is inside the tree (docs: may return // incorrect data before the scene root is ready). _screenMaterial.AlbedoTexture = Viewport.GetTexture(); } public PointerMode GetMode(int hand) => _hands[hand].Mode; public bool IsPressing(int hand) => _owner == hand && _down; public bool IsSwiping => _down && _pending && _swiping; public Vector2 GetPixel(int hand) => _hands[hand].Pixel; public bool TryGetLaserDistance(int hand, out float distance) { distance = _hands[hand].RayDistance; return Visible && _hands[hand].Mode == PointerMode.Laser; } public Vector2 LocalToPixel(Vector3 local) => new( (local.X / SizeMeters.X + 0.5f) * SizePixels.X, (0.5f - local.Y / SizeMeters.Y) * SizePixels.Y); public Vector3 PixelToLocal(Vector2 pixel) => new( (pixel.X / SizePixels.X - 0.5f) * SizeMeters.X, (0.5f - pixel.Y / SizePixels.Y) * SizeMeters.Y, 0f); public Vector3 PixelToGlobal(Vector2 pixel) => GlobalTransform * PixelToLocal(pixel); public void SetContent(Control content) { content.SetAnchorsAndOffsetsPreset(Control.LayoutPreset.FullRect); Viewport.AddChild(content); } public void SetActive(bool active) { if (!active) ReleaseAll(); Visible = active; Viewport.RenderTargetUpdateMode = active ? SubViewport.UpdateMode.Always : SubViewport.UpdateMode.Disabled; foreach (Node child in Body.GetChildren()) if (child is CollisionShape3D shape) shape.SetDeferred(CollisionShape3D.PropertyName.Disabled, !active); } public void ReleaseAll() { if (_owner >= 0 && _down && !_pending) SendButton(_lastSent, false); _down = false; _pending = false; _flingTarget = null; _owner = -1; if (_mouseInside) { Viewport.NotifyMouseExited(); _mouseInside = false; } _scrollAccum = 0f; for (int i = 0; i < 2; i++) { // TriggerPrev survives: a trigger still held across a reset must not re-click. _hands[i] = new HandState { TriggerPrev = _hands[i].TriggerPrev }; if (_dots[i] != null) _dots[i].Visible = false; } } private void Build() { Viewport = new SubViewport { Name = "MenuViewport", Size = SizePixels, Disable3D = true, TransparentBg = false, GuiEmbedSubwindows = true, HandleInputLocally = true, Msaa2D = Godot.Viewport.Msaa.Msaa4X, RenderTargetUpdateMode = SubViewport.UpdateMode.Disabled, }; AddChild(Viewport); // Screen quad with explicit UVs so the pixel mapping above is ours, not QuadMesh's. float hx = SizeMeters.X * 0.5f, hy = SizeMeters.Y * 0.5f; var st = new SurfaceTool(); st.Begin(Mesh.PrimitiveType.Triangles); Vector3[] corners = { new(-hx, hy, 0), new(hx, hy, 0), new(hx, -hy, 0), new(-hx, -hy, 0) }; Vector2[] uvs = { new(0, 0), new(1, 0), new(1, 1), new(0, 1) }; int[] order = { 0, 1, 2, 0, 2, 3 }; foreach (int i in order) { st.SetNormal(Vector3.Back); st.SetUV(uvs[i]); st.AddVertex(corners[i]); } _screenMaterial = new StandardMaterial3D { ShadingMode = BaseMaterial3D.ShadingModeEnum.Unshaded, TextureFilter = BaseMaterial3D.TextureFilterEnum.Linear, CullMode = BaseMaterial3D.CullModeEnum.Disabled, }; MeshInstance3D screen = new() { Name = "Screen", Mesh = st.Commit(), CastShadow = GeometryInstance3D.ShadowCastingSetting.Off, }; screen.SetSurfaceOverrideMaterial(0, _screenMaterial); AddChild(screen); // Thin backing frame (hides the mirrored back side, gives the panel an edge). StandardMaterial3D frameMaterial = new() { AlbedoColor = new Color(0.05f, 0.06f, 0.08f) }; MeshInstance3D frame = new() { Name = "Frame", Mesh = new BoxMesh { Size = new Vector3(SizeMeters.X + 0.012f, SizeMeters.Y + 0.012f, 0.008f) }, Position = new Vector3(0, 0, -0.0045f), CastShadow = GeometryInstance3D.ShadowCastingSetting.Off, }; frame.SetSurfaceOverrideMaterial(0, frameMaterial); AddChild(frame); // UI layer 2, mask 0: lasers/raycasts see it, props pass through it. Body = new StaticBody3D { Name = "PanelBody", CollisionLayer = 2, CollisionMask = 0 }; Body.AddChild(new CollisionShape3D { Shape = new BoxShape3D { Size = new Vector3(SizeMeters.X, SizeMeters.Y, 0.02f) }, Position = new Vector3(0, 0, -0.005f), }); AddChild(Body); for (int i = 0; i < 2; i++) { _dotMaterials[i] = new StandardMaterial3D { ShadingMode = BaseMaterial3D.ShadingModeEnum.Unshaded, AlbedoColor = Colors.White, NoDepthTest = true, RenderPriority = 1, }; _dots[i] = new MeshInstance3D { Name = $"Cursor{i}", Mesh = new SphereMesh { Radius = 0.004f, Height = 0.008f, RadialSegments = 12, Rings = 6 }, CastShadow = GeometryInstance3D.ShadowCastingSetting.Off, Visible = false, }; _dots[i].SetSurfaceOverrideMaterial(0, _dotMaterials[i]); AddChild(_dots[i]); } } // Call once per frame with both hands' input (index 0 = left, 1 = right). public void UpdatePointers(PointerInput[] inputs, double delta) { if (!Visible) { // Keep trigger edges current while hidden so a held trigger can't click on summon. for (int hand = 0; hand < 2 && hand < inputs.Length; hand++) _hands[hand].TriggerPrev = inputs[hand].Trigger; return; } for (int hand = 0; hand < 2; hand++) Classify(hand, hand < inputs.Length ? inputs[hand] : default); int owner = PickOwner(); if (owner != _owner) { if (_owner >= 0 && _down) { if (!_pending) SendButton(_lastSent, false); _down = false; _pending = false; } _owner = owner; _scrollAccum = 0f; } if (_owner < 0) { if (_mouseInside) { Viewport.NotifyMouseExited(); _mouseInside = false; } } else { HandState h = _hands[_owner]; if (!_mouseInside) { Viewport.NotifyMouseEntered(); _mouseInside = true; } if (h.Pixel != _lastSent) SendMotion(h.Pixel); int current = _owner; if (h.WantDown != _down) { _down = h.WantDown; if (_down) { _flingTarget = null; if (h.Mode == PointerMode.Touch && FindSwipeTarget() is ScrollContainer target) BeginPending(target, h.Pixel); else SendButton(h.Pixel, true); PointerPressed?.Invoke(current, h.Mode); } else if (_pending) EndPending(); else SendButton(h.Pixel, false); } else if (_down && _pending) UpdateSwipe(h.Pixel, (float)delta); // A UI handler may have reset us (ReleaseAll) during the injected event. if (_owner == current && current < inputs.Length) Scroll(inputs[current].Stick, h.Pixel, (float)delta); } Fling((float)delta); UpdateDots(); } // The scroll container a touch at the current mouse position would swipe: walk up from the // hovered control; a Range (slider, scrollbar) on the way keeps the normal immediate press. private ScrollContainer FindSwipeTarget() { for (Node n = Viewport.GuiGetHoveredControl(); n is Control c; n = n.GetParent()) { if (c is Godot.Range) return null; if (c is ScrollContainer sc) { VScrollBar bar = sc.GetVScrollBar(); if (bar.MaxValue - bar.Page > 0.5) return sc; } } return null; } private void BeginPending(ScrollContainer target, Vector2 pixel) { _pending = true; _swiping = false; _swipeTarget = target; _pressPixel = pixel; _swipeVelocity = 0f; } private void UpdateSwipe(Vector2 pixel, float delta) { if (!IsInstanceValid(_swipeTarget)) return; if (!_swiping) { if (pixel.DistanceTo(_pressPixel) < SwipeThreshold) return; _swiping = true; _swipeLastY = pixel.Y; _swipeScroll = _swipeTarget.ScrollVertical; return; } // Content follows the tip: dragging up reveals what is below. float dy = pixel.Y - _swipeLastY; _swipeLastY = pixel.Y; _swipeScroll -= dy; if (!SetScroll(_swipeTarget)) _swipeScroll = _swipeTarget.ScrollVertical; // at an end: no stored overscroll if (delta > 0f) _swipeVelocity = Mathf.Lerp(_swipeVelocity, -dy / delta, 1f - Mathf.Exp(-delta * 30f)); } private void EndPending() { _pending = false; if (_swiping) { _flingTarget = _swipeTarget; _flingVelocity = _swipeVelocity; _swipeScroll = _flingTarget.ScrollVertical; } else { // A tap: the whole click at the contact point (buttons fire on release anyway). Vector2 back = _hands[_owner].Pixel; SendMotion(_pressPixel); SendButton(_pressPixel, true); SendButton(_pressPixel, false); if (back != _pressPixel) SendMotion(back); } _swiping = false; _swipeTarget = null; } private void Fling(float delta) { if (_flingTarget == null) return; if (!IsInstanceValid(_flingTarget) || Mathf.Abs(_flingVelocity) < FlingMinSpeed) { _flingTarget = null; return; } _swipeScroll += _flingVelocity * delta; if (!SetScroll(_flingTarget)) _flingTarget = null; // hit an end _flingVelocity *= Mathf.Exp(-FlingDecay * delta); } // Applies _swipeScroll; false when the container clamped it (an end was reached). private bool SetScroll(ScrollContainer target) { int want = Mathf.RoundToInt(_swipeScroll); target.ScrollVertical = want; return target.ScrollVertical == want; } private void Classify(int hand, PointerInput input) { HandState h = _hands[hand]; PointerMode mode = PointerMode.None; bool wasOwnerDown = _owner == hand && _down; if (input.Valid && input.Direction.LengthSquared() > 1e-6f) { Transform3D inv = GlobalTransform.AffineInverse(); Vector3 from = inv * input.Origin; Vector3 dir = (inv.Basis * input.Direction).Normalized(); Vector3 tip = from + dir * TipLength; float hx = SizeMeters.X * 0.5f, hy = SizeMeters.Y * 0.5f; bool tipOver = Mathf.Abs(tip.X) <= hx + TouchEdgeMargin && Mathf.Abs(tip.Y) <= hy + TouchEdgeMargin; if (from.Z > 0f && tipOver && tip.Z <= HoverDepth && tip.Z >= -PushThroughDepth) { mode = PointerMode.Touch; h.Local = new Vector3(Mathf.Clamp(tip.X, -hx, hx), Mathf.Clamp(tip.Y, -hy, hy), 0f); h.Depth = tip.Z; h.RayDistance = 0f; } else if (from.Z > 0f && dir.Z < -1e-4f) { float t = -from.Z / dir.Z; Vector3 hit = from + dir * t; if (Mathf.Abs(hit.X) <= hx && Mathf.Abs(hit.Y) <= hy) { mode = PointerMode.Laser; h.Local = hit; h.Depth = float.MaxValue; h.RayDistance = t; } } } h.Mode = mode; if (mode != PointerMode.None) { Vector2 px = LocalToPixel(h.Local); h.Pixel = new Vector2( Mathf.Clamp(px.X, 0f, SizePixels.X - 1), Mathf.Clamp(px.Y, 0f, SizePixels.Y - 1)); } switch (mode) { case PointerMode.Touch: h.WantDown = wasOwnerDown ? h.Depth <= ReleaseDepth : h.Armed && h.Depth <= ContactDepth; break; case PointerMode.Laser: h.WantDown = wasOwnerDown ? input.Trigger : input.Trigger && !h.TriggerPrev; break; default: h.WantDown = false; break; } h.Armed = mode == PointerMode.Touch && h.Depth > ContactDepth; h.TriggerPrev = input.Trigger; _hands[hand] = h; } private int PickOwner() { if (_owner >= 0 && _down && _hands[_owner].Mode != PointerMode.None) return _owner; int best = -1; for (int hand = 0; hand < 2; hand++) { if (_hands[hand].Mode != PointerMode.Touch) continue; if (best < 0 || _hands[hand].Depth < _hands[best].Depth) best = hand; } if (best >= 0) return best; if (_owner >= 0 && _hands[_owner].Mode == PointerMode.Laser) return _owner; for (int hand = 0; hand < 2; hand++) if (_hands[hand].Mode == PointerMode.Laser) return hand; return -1; } private void Scroll(Vector2 stick, Vector2 pixel, float delta) { if (Mathf.Abs(stick.Y) < ScrollDeadzone) { _scrollAccum = 0f; return; } _scrollAccum += stick.Y * ScrollNotchesPerSecond * delta; while (Mathf.Abs(_scrollAccum) >= 1f) { bool up = _scrollAccum > 0f; _scrollAccum -= up ? 1f : -1f; SendWheel(pixel, up ? MouseButton.WheelUp : MouseButton.WheelDown); } } public void SendWheel(Vector2 pixel, MouseButton wheel) { foreach (bool pressed in new[] { true, false }) { Viewport.PushInput(new InputEventMouseButton { Position = pixel, GlobalPosition = pixel, ButtonIndex = wheel, Pressed = pressed, Factor = 1f, ButtonMask = _down && !_pending ? MouseButtonMask.Left : 0, }, true); } } private void SendMotion(Vector2 pixel) { Vector2 relative = _lastSent.X < 0 ? Vector2.Zero : pixel - _lastSent; Viewport.PushInput(new InputEventMouseMotion { Position = pixel, GlobalPosition = pixel, Relative = relative, ButtonMask = _down && !_pending ? MouseButtonMask.Left : 0, }, true); _lastSent = pixel; } private void SendButton(Vector2 pixel, bool pressed) { Viewport.PushInput(new InputEventMouseButton { Position = pixel, GlobalPosition = pixel, ButtonIndex = MouseButton.Left, Pressed = pressed, ButtonMask = pressed ? MouseButtonMask.Left : 0, }, true); _lastSent = pixel; } private void UpdateDots() { for (int hand = 0; hand < 2; hand++) { HandState h = _hands[hand]; MeshInstance3D dot = _dots[hand]; dot.Visible = h.Mode != PointerMode.None; if (!dot.Visible) continue; bool pressing = _owner == hand && _down; float scale; Color color; if (h.Mode == PointerMode.Touch) { // Shrinks as the tip approaches the surface: a proximity cue for touch. float k = Mathf.Clamp(h.Depth / HoverDepth, 0f, 1f); scale = Mathf.Lerp(0.6f, 1.6f, k); color = pressing ? new Color(0.5f, 1f, 0.6f) : new Color(0.55f, 0.85f, 1f); } else { scale = 1f; color = pressing ? new Color(1f, 0.85f, 0.4f) : new Color(1f, 0.55f, 0.15f); } if (_owner != hand) color.A = 0.5f; _dotMaterials[hand].AlbedoColor = color; _dotMaterials[hand].Transparency = color.A < 1f ? BaseMaterial3D.TransparencyEnum.Alpha : BaseMaterial3D.TransparencyEnum.Disabled; dot.Position = new Vector3(h.Local.X, h.Local.Y, 0.002f); dot.Scale = Vector3.One * scale; } } }