feat(mobile): adapt touch mouse behavior

Signed-off-by: luckfox-eng29 <eng29@luckfox.com>

ui/src/components/Video/VideoElement.tsx

@@ -13,9 +13,10 @@ export const VideoElement = forwardRef<HTMLVideoElement, VideoElementProps>(
   ({ onPlaying, style, className }, ref) => {
     const setVirtualKeyboardEnabled = useHidStore(state => state.setVirtualKeyboardEnabled);
     const isVirtualKeyboardEnabled = useHidStore(state => state.isVirtualKeyboardEnabled);
+    const allowTapToOpenVirtualKeyboard = useHidStore(state => state.allowTapToOpenVirtualKeyboard);
 
     const handleClick = () => {
-      if (isMobile && !isVirtualKeyboardEnabled) {
+      if (isMobile && allowTapToOpenVirtualKeyboard && !isVirtualKeyboardEnabled) {
         setVirtualKeyboardEnabled(true);
       }
     };
@@ -46,4 +47,4 @@ export const VideoElement = forwardRef<HTMLVideoElement, VideoElementProps>(
   }
 );
 
-VideoElement.displayName = "VideoElement";
+VideoElement.displayName = "VideoElement";

ui/src/components/VideoOverlay.tsx

@@ -278,9 +278,10 @@ export function HDMIErrorOverlay({ show, hdmiState }: HDMIErrorOverlayProps) {
 
   const setVirtualKeyboardEnabled = useHidStore(state => state.setVirtualKeyboardEnabled);
   const isVirtualKeyboardEnabled = useHidStore(state => state.isVirtualKeyboardEnabled);
+  const allowTapToOpenVirtualKeyboard = useHidStore(state => state.allowTapToOpenVirtualKeyboard);
 
   const handleClick = () => {
-    if (isMobile && !isVirtualKeyboardEnabled) {
+    if (isMobile && allowTapToOpenVirtualKeyboard && !isVirtualKeyboardEnabled) {
       setVirtualKeyboardEnabled(true);
     }
   };

ui/src/hooks/stores.ts

@@ -600,6 +600,8 @@ export interface HidState {
 
   isVirtualKeyboardEnabled: boolean;
   setVirtualKeyboardEnabled: (enabled: boolean) => void;
+  allowTapToOpenVirtualKeyboard: boolean;
+  setAllowTapToOpenVirtualKeyboard: (enabled: boolean) => void;
 
   isPasteModeEnabled: boolean;
   setPasteModeEnabled: (enabled: boolean) => void;
@@ -665,6 +667,8 @@ export const useHidStore = create<HidState>((set, get) => ({
 
   isVirtualKeyboardEnabled: false,
   setVirtualKeyboardEnabled: enabled => set({ isVirtualKeyboardEnabled: enabled }),
+  allowTapToOpenVirtualKeyboard: true,
+  setAllowTapToOpenVirtualKeyboard: enabled => set({ allowTapToOpenVirtualKeyboard: enabled }),
 
   isPasteModeEnabled: false,
   setPasteModeEnabled: enabled => set({ isPasteModeEnabled: enabled }),

ui/src/layout/core/desktop/DesktopMobile.tsx

@@ -1,5 +1,5 @@
 import React, { useEffect, useRef, useState } from "react";
-import { BsMouseFill, BsLockFill, BsUnlockFill } from "react-icons/bs";
+import { BsKeyboardFill, BsLockFill, BsUnlockFill } from "react-icons/bs";
 import { useReactAt } from "i18n-auto-extractor/react";
 
 import VirtualKeyboard from "@components/VirtualKeyboard";
@@ -41,7 +41,47 @@ import VirtualMediaSource from "@/layout/components_side/VirtualMediaSource";
 import { useJsonRpc } from "@/hooks/useJsonRpc";
 import OcrOverlay from "@components/OcrOverlay";
 
+const GestureIcon = ({ className = "h-4 w-4" }: { className?: string }) => (
+  <svg viewBox="0 0 24 24" className={className} fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+    <circle cx="10" cy="10" r="6.5" />
+    <path d="M14.8 14.8L21 21" />
+    <path d="M10 7.5V12.5" />
+    <path d="M7.5 10H12.5" />
+  </svg>
+);
+
+const ResetViewIcon = ({ className = "h-4 w-4" }: { className?: string }) => (
+  <svg viewBox="0 0 24 24" className={className} fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+    <path d="M19 12A7 7 0 1 1 12 5" />
+    <path d="M12 2L12 6L16 6" />
+  </svg>
+);
+
+const MouseStickIcon = ({ className = "h-4 w-4" }: { className?: string }) => (
+  <svg viewBox="0 0 24 24" className={className} fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+    <rect x="7" y="3" width="10" height="18" rx="5" />
+    <path d="M12 3V8" />
+    <circle cx="12" cy="13" r="1.5" fill="currentColor" stroke="none" />
+  </svg>
+);
+
+const FourWayMoveIcon = ({ className = "h-5 w-5" }: { className?: string }) => (
+  <svg viewBox="0 0 24 24" className={className} fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round" aria-hidden="true">
+    <path d="M12 3V21" />
+    <path d="M3 12H21" />
+    <path d="M12 3L9 6" />
+    <path d="M12 3L15 6" />
+    <path d="M12 21L9 18" />
+    <path d="M12 21L15 18" />
+    <path d="M3 12L6 9" />
+    <path d="M3 12L6 15" />
+    <path d="M21 12L18 9" />
+    <path d="M21 12L18 15" />
+  </svg>
+);
+
 export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number }) {
+  const joystickSpeedLevels = [1.4, 1.05, 0.7];
   const { $at } = useReactAt();
   const { isDark } = useTheme();
   const videoElm = useRef<HTMLVideoElement>(null);
@@ -57,50 +97,119 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
   const videoStream = useVideoStream(videoElm as React.RefObject<HTMLVideoElement>, audioElm as React.RefObject<HTMLAudioElement>);
   const pointerLock = usePointerLock(videoElm as React.RefObject<HTMLVideoElement>);
   useFullscreen(videoElm as React.RefObject<HTMLVideoElement>, pointerLock, isFullscreen);
-  const touchZoom = useTouchZoom(zoomContainerRef as React.RefObject<HTMLDivElement>);
+  const [isTouchGestureEnabled, setIsTouchGestureEnabled] = useState(true);
+  const touchZoom = useTouchZoom(zoomContainerRef as React.RefObject<HTMLDivElement>, isTouchGestureEnabled);
   const { handleGlobalPaste } = usePasteHandler(pasteCaptureRef as React.RefObject<HTMLTextAreaElement>);
   const keyboardEvents = useKeyboardEvents(pasteCaptureRef as React.RefObject<HTMLTextAreaElement>, isReinitializingGadget);
   const [showVirtualMouseButtons, setShowVirtualMouseButtons] = useState(false);
+  const [showVirtualJoystick, setShowVirtualJoystick] = useState(false);
+  const [joystickVector, setJoystickVector] = useState({ x: 0, y: 0 });
+  const [joystickSensitivity, setJoystickSensitivity] = useState(1);
+  const [joystickPos, setJoystickPos] = useState({ x: 16, y: 24 });
   const [lockedButtons, setLockedButtons] = useState(0);
   const mouseEvents = useMouseEvents(videoElm as React.RefObject<HTMLVideoElement>, pointerLock, touchZoom, showVirtualMouseButtons, lockedButtons);
   const overlays = useVideoOverlays(videoStream, pointerLock, videoEffects);
 
   const forceHttp = useSettingsStore(state => state.forceHttp);
+  const mouseMode = useSettingsStore(state => state.mouseMode);
   const mouseX = useMouseStore(state => state.mouseX);
   const mouseY = useMouseStore(state => state.mouseY);
+  const allowTapToOpenVirtualKeyboard = useHidStore(state => state.allowTapToOpenVirtualKeyboard);
+  const setAllowTapToOpenVirtualKeyboard = useHidStore(state => state.setAllowTapToOpenVirtualKeyboard);
   const [send] = useJsonRpc();
   const [leftBtnPos, setLeftBtnPos] = useState({ x: 40, y: 40 });
   const [rightBtnPos, setRightBtnPos] = useState({ x: 120, y: 40 });
-  const [leftLockPos, setLeftLockPos] = useState({ x: 40, y: 110 });
-  const [rightLockPos, setRightLockPos] = useState({ x: 120, y: 110 });
+  const [wheelPos, setWheelPos] = useState({ x: 184, y: 140 });
 
-  const [draggingBtn, setDraggingBtn] = useState<"left" | "right" | "leftLock" | "rightLock" | null>(null);
+  const [draggingBtn, setDraggingBtn] = useState<"left" | "right" | "wheel" | null>(null);
   const dragOffset = useRef({ x: 0, y: 0 });
+  const joystickAreaRef = useRef<HTMLDivElement>(null);
+  const joystickPointerIdRef = useRef<number | null>(null);
+  const joystickVectorRef = useRef({ x: 0, y: 0 });
+  const joystickFrameRef = useRef<number | null>(null);
+  const joystickLastTsRef = useRef<number | null>(null);
+  const joystickMovePointerIdRef = useRef<number | null>(null);
+  const joystickMoveHoldTimerRef = useRef<number | null>(null);
+  const joystickMoveEnabledRef = useRef(false);
   
   const activeButtonsRef = useRef(0);
   
   useEffect(() => {
     if (isFullscreen) {
       setShowVirtualMouseButtons(false);
+      setShowVirtualJoystick(false);
     }
   }, [isFullscreen]);
 
+  useEffect(() => {
+    joystickVectorRef.current = joystickVector;
+  }, [joystickVector]);
+
+  useEffect(() => {
+    if (!showVirtualJoystick) {
+      joystickPointerIdRef.current = null;
+      joystickMovePointerIdRef.current = null;
+      if (joystickMoveHoldTimerRef.current !== null) {
+        window.clearTimeout(joystickMoveHoldTimerRef.current);
+        joystickMoveHoldTimerRef.current = null;
+      }
+      joystickMoveEnabledRef.current = false;
+      joystickLastTsRef.current = null;
+      setJoystickVector({ x: 0, y: 0 });
+      if (joystickFrameRef.current !== null) {
+        cancelAnimationFrame(joystickFrameRef.current);
+        joystickFrameRef.current = null;
+      }
+      return;
+    }
+
+    const tick = (timestamp: number) => {
+      const prevTs = joystickLastTsRef.current ?? timestamp;
+      joystickLastTsRef.current = timestamp;
+      const frameScale = Math.min(2, Math.max(0.5, (timestamp - prevTs) / 16.67));
+      const vector = joystickVectorRef.current;
+      const container = containerRef.current;
+      const containerWidth = container?.clientWidth ?? 1280;
+      // Reduce sensitivity on small screens to avoid over-shooting.
+      const resolutionFactor = Math.max(0.35, Math.min(1, containerWidth / 1280));
+      const speed = 12 * resolutionFactor * joystickSensitivity;
+      const dx = Math.round(vector.x * speed * frameScale);
+      const dy = Math.round(vector.y * speed * frameScale);
+      if (dx !== 0 || dy !== 0) {
+        mouseEvents.sendVirtualRelativeMovement(dx, dy, 0);
+      }
+      joystickFrameRef.current = requestAnimationFrame(tick);
+    };
+
+    joystickFrameRef.current = requestAnimationFrame(tick);
+    return () => {
+      if (joystickFrameRef.current !== null) {
+        cancelAnimationFrame(joystickFrameRef.current);
+        joystickFrameRef.current = null;
+      }
+      joystickLastTsRef.current = null;
+    };
+  }, [showVirtualJoystick, mouseEvents, joystickSensitivity]);
+
   const updateButtons = (mask: number, isDown: boolean) => {
     if (isReinitializingGadget) return;
     
     let newButtons = activeButtonsRef.current;
     if (isDown) {
         newButtons |= mask;
+    } else if (lockedButtons & mask) {
+        // Keep pressed while lock is enabled.
+        newButtons |= mask;
     } else {
         newButtons &= ~mask;
     }
-    
-    if (!isDown && (lockedButtons & mask)) {
-        setLockedButtons(prev => prev & ~mask);
-    }
 
     activeButtonsRef.current = newButtons;
-    send("absMouseReport", { x: mouseX, y: mouseY, buttons: newButtons });
+    if (mouseMode === "relative") {
+      mouseEvents.sendVirtualRelativeMovement(0, 0, newButtons);
+    } else {
+      send("absMouseReport", { x: mouseX, y: mouseY, buttons: newButtons });
+    }
   };
   
   const toggleLock = (mask: number) => {
@@ -117,10 +226,14 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
      }
      
      activeButtonsRef.current = newButtons;
-     send("absMouseReport", { x: mouseX, y: mouseY, buttons: newButtons });
+     if (mouseMode === "relative") {
+       mouseEvents.sendVirtualRelativeMovement(0, 0, newButtons);
+     } else {
+       send("absMouseReport", { x: mouseX, y: mouseY, buttons: newButtons });
+     }
   };
 
-  const handlePointerDown = (e: React.PointerEvent<HTMLDivElement>, type: "left" | "right" | "leftLock" | "rightLock") => {
+  const handlePointerDown = (e: React.PointerEvent<HTMLDivElement>, type: "left" | "right" | "wheel") => {
     const target = e.currentTarget;
     const rect = target.getBoundingClientRect();
     dragOffset.current = {
@@ -138,26 +251,121 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
     const containerRect = container.getBoundingClientRect();
     const x = e.clientX - containerRect.left - dragOffset.current.x;
     const y = e.clientY - containerRect.top - dragOffset.current.y;
-    const clampedX = Math.max(0, Math.min(containerRect.width - 56, x));
-    const clampedY = Math.max(0, Math.min(containerRect.height - 56, y));
+    const dragWidth = draggingBtn === "wheel" ? 32 : 56;
+    const dragHeight = draggingBtn === "wheel" ? 68 : 56;
+    const clampedX = Math.max(0, Math.min(containerRect.width - dragWidth, x));
+    const clampedY = Math.max(0, Math.min(containerRect.height - dragHeight, y));
     if (draggingBtn === "left") {
       setLeftBtnPos({ x: clampedX, y: clampedY });
     } else if (draggingBtn === "right") {
       setRightBtnPos({ x: clampedX, y: clampedY });
-    } else if (draggingBtn === "leftLock") {
-      setLeftLockPos({ x: clampedX, y: clampedY });
-    } else if (draggingBtn === "rightLock") {
-      setRightLockPos({ x: clampedX, y: clampedY });
+    } else if (draggingBtn === "wheel") {
+      setWheelPos({ x: clampedX, y: clampedY });
     }
   };
 
-  const handlePointerUp = (e: React.PointerEvent<HTMLDivElement>, type: "left" | "right" | "leftLock" | "rightLock") => {
+  const handlePointerUp = (e: React.PointerEvent<HTMLDivElement>, type: "left" | "right" | "wheel") => {
     const wasDragging = draggingBtn === type;
     setDraggingBtn(null);
     (e.currentTarget as HTMLDivElement).releasePointerCapture(e.pointerId);
     if (!wasDragging) return;
   };
 
+  const updateJoystickVector = (clientX: number, clientY: number) => {
+    const joystickElm = joystickAreaRef.current;
+    if (!joystickElm) return;
+    const rect = joystickElm.getBoundingClientRect();
+    const centerX = rect.left + rect.width / 2;
+    const centerY = rect.top + rect.height / 2;
+    const rawX = clientX - centerX;
+    const rawY = clientY - centerY;
+    const maxRadius = 32;
+    const length = Math.hypot(rawX, rawY);
+    if (!length || length <= maxRadius) {
+      setJoystickVector({ x: rawX / maxRadius, y: rawY / maxRadius });
+      return;
+    }
+    const scale = maxRadius / length;
+    setJoystickVector({ x: (rawX * scale) / maxRadius, y: (rawY * scale) / maxRadius });
+  };
+
+  const handleJoystickPointerDown = (e: React.PointerEvent<HTMLDivElement>) => {
+    if (joystickMovePointerIdRef.current !== null) return;
+    e.preventDefault();
+    joystickPointerIdRef.current = e.pointerId;
+    e.currentTarget.setPointerCapture(e.pointerId);
+    updateJoystickVector(e.clientX, e.clientY);
+  };
+
+  const handleJoystickPointerMove = (e: React.PointerEvent<HTMLDivElement>) => {
+    if (joystickPointerIdRef.current !== e.pointerId) return;
+    e.preventDefault();
+    updateJoystickVector(e.clientX, e.clientY);
+  };
+
+  const handleJoystickPointerUp = (e: React.PointerEvent<HTMLDivElement>) => {
+    if (joystickPointerIdRef.current !== e.pointerId) return;
+    joystickPointerIdRef.current = null;
+    e.currentTarget.releasePointerCapture(e.pointerId);
+    setJoystickVector({ x: 0, y: 0 });
+  };
+
+  const handleJoystickMoveStart = (e: React.PointerEvent<HTMLDivElement>) => {
+    joystickMovePointerIdRef.current = e.pointerId;
+    joystickMoveEnabledRef.current = false;
+    if (joystickMoveHoldTimerRef.current !== null) {
+      window.clearTimeout(joystickMoveHoldTimerRef.current);
+    }
+    joystickMoveHoldTimerRef.current = window.setTimeout(() => {
+      joystickMoveEnabledRef.current = true;
+    }, 350);
+    e.currentTarget.setPointerCapture(e.pointerId);
+    e.preventDefault();
+    e.stopPropagation();
+  };
+
+  const handleJoystickMove = (e: React.PointerEvent<HTMLDivElement>) => {
+    if (joystickMovePointerIdRef.current !== e.pointerId) return;
+    if (!joystickMoveEnabledRef.current) return;
+    const container = containerRef.current;
+    if (!container) return;
+    const containerRect = container.getBoundingClientRect();
+    const joystickSize = 80;
+    const nextLeft = e.clientX - containerRect.left - joystickSize / 2;
+    const nextTop = e.clientY - containerRect.top - joystickSize / 2;
+    const clampedLeft = Math.max(0, Math.min(containerRect.width - joystickSize, nextLeft));
+    const clampedTop = Math.max(0, Math.min(containerRect.height - joystickSize, nextTop));
+    const nextBottom = containerRect.height - joystickSize - clampedTop;
+    setJoystickPos({ x: clampedLeft, y: nextBottom });
+    e.preventDefault();
+    e.stopPropagation();
+  };
+
+  const handleJoystickMoveEnd = (e: React.PointerEvent<HTMLDivElement>) => {
+    if (joystickMovePointerIdRef.current !== e.pointerId) return;
+    if (joystickMoveHoldTimerRef.current !== null) {
+      window.clearTimeout(joystickMoveHoldTimerRef.current);
+      joystickMoveHoldTimerRef.current = null;
+    }
+    joystickMoveEnabledRef.current = false;
+    joystickMovePointerIdRef.current = null;
+    e.currentTarget.releasePointerCapture(e.pointerId);
+    e.preventDefault();
+    e.stopPropagation();
+  };
+
+  const isMouseControlEnabled = showVirtualJoystick || showVirtualMouseButtons;
+  const joystickSpeedIndex = joystickSpeedLevels.reduce((bestIndex, value, index, arr) => {
+    const bestDistance = Math.abs(arr[bestIndex] - joystickSensitivity);
+    const currentDistance = Math.abs(value - joystickSensitivity);
+    return currentDistance < bestDistance ? index : bestIndex;
+  }, 0);
+  const toggleMouseControl = () => {
+    const nextEnabled = !isMouseControlEnabled;
+    setShowVirtualJoystick(nextEnabled);
+    setShowVirtualMouseButtons(nextEnabled);
+  };
+
   useEffect(() => {
     const keyboardCleanup = keyboardEvents.setupKeyboardEvents();
     const videoCleanup = videoStream.setupVideoEventListeners();
@@ -303,107 +511,292 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
                     className="pointer-events-none absolute inset-0"
                     onPointerMove={handlePointerMove}
                   >
-                    <div
-                      className={cx(
-                        "pointer-events-auto absolute right-3 top-3 flex h-8 w-8 items-center justify-center rounded-full text-white text-xs",
-                        isDark ? "bg-gray-500/70" : "bg-black/30",
-                      )}
-                      style={{
-                        touchAction: "none",
-                      }}
-                      onClick={() => setShowVirtualMouseButtons(prev => !prev)}
-                    >
-                      <BsMouseFill className="h-4 w-4" />
+                    <div className="pointer-events-auto absolute right-3 top-3 grid grid-cols-[auto_auto_auto] grid-rows-2 gap-1.5">
+                      <div
+                        className={cx(
+                          "flex h-8 w-8 items-center justify-center rounded-full text-white",
+                          isTouchGestureEnabled
+                            ? "bg-green-600/80"
+                            : "bg-gray-500/70",
+                        )}
+                        style={{
+                          touchAction: "none",
+                        }}
+                        onClick={() => setIsTouchGestureEnabled(prev => !prev)}
+                      >
+                        <GestureIcon />
+                      </div>
+                      <div
+                        className={cx(
+                          "flex h-8 w-8 items-center justify-center rounded-full text-white",
+                          isMouseControlEnabled
+                            ? "bg-green-600/80"
+                            : "bg-gray-500/70",
+                        )}
+                        style={{
+                          touchAction: "none",
+                        }}
+                        onClick={toggleMouseControl}
+                      >
+                        <MouseStickIcon />
+                      </div>
+                      <div
+                        className={cx(
+                          "flex h-8 w-8 items-center justify-center rounded-full text-white",
+                          allowTapToOpenVirtualKeyboard
+                            ? "bg-green-600/80"
+                            : "bg-gray-500/70",
+                        )}
+                        style={{
+                          touchAction: "none",
+                        }}
+                        onClick={() => setAllowTapToOpenVirtualKeyboard(!allowTapToOpenVirtualKeyboard)}
+                      >
+                        <BsKeyboardFill className="h-4 w-4" />
+                      </div>
+                      <div
+                        className={cx(
+                          "col-span-3 flex h-8 items-center justify-center rounded-full text-white",
+                          isDark ? "bg-gray-500/70" : "bg-black/30",
+                        )}
+                        style={{
+                          touchAction: "none",
+                        }}
+                        onClick={() => touchZoom.resetTransform()}
+                      >
+                        <ResetViewIcon />
+                      </div>
                     </div>
                     {showVirtualMouseButtons && (
                       <>
                         <div
                           className={cx(
-                            "pointer-events-auto absolute flex h-14 w-14 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100",
-                            isDark ? "bg-gray-500/70" : "bg-black/30",
+                            "pointer-events-auto absolute flex h-14 w-14 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100 relative",
+                            (lockedButtons & 1) ? "bg-green-600/80" : (isDark ? "bg-gray-500/70" : "bg-black/30"),
                           )}
                           style={{
                             left: leftBtnPos.x,
                             top: leftBtnPos.y,
                             touchAction: "none",
                           }}
-                          onPointerDown={e => {
-                            handlePointerDown(e, "left");
+                          onPointerDown={() => {
                             updateButtons(1, true);
                           }}
-                          onPointerUp={e => {
-                            handlePointerUp(e, "left");
+                          onPointerUp={() => {
                             updateButtons(1, false);
                           }}
                         >
                           L
+                          <div
+                            className="absolute -left-1 -top-1 flex h-5 w-5 items-center justify-center rounded-full bg-black/45 text-white"
+                            onPointerDown={e => {
+                              e.stopPropagation();
+                              handlePointerDown(e, "left");
+                            }}
+                            onPointerMove={e => {
+                              e.stopPropagation();
+                              handlePointerMove(e);
+                            }}
+                            onPointerUp={e => {
+                              e.stopPropagation();
+                              handlePointerUp(e, "left");
+                            }}
+                          >
+                            <FourWayMoveIcon className="h-3 w-3" />
+                          </div>
+                          <div
+                            className={cx(
+                              "absolute -right-1 -top-1 flex h-5 w-5 items-center justify-center rounded-full text-white",
+                              (lockedButtons & 1) ? "bg-green-600/90" : "bg-black/45",
+                            )}
+                            onPointerDown={e => {
+                              e.stopPropagation();
+                            }}
+                            onClick={e => {
+                              e.stopPropagation();
+                              toggleLock(1);
+                            }}
+                          >
+                            {(lockedButtons & 1) ? <BsLockFill className="h-3 w-3" /> : <BsUnlockFill className="h-3 w-3" />}
+                          </div>
                         </div>
 
                         <div
                           className={cx(
-                            "pointer-events-auto absolute flex h-14 w-14 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100",
-                            (lockedButtons & 1) ? "bg-green-600/80" : (isDark ? "bg-gray-500/70" : "bg-black/30"),
-                          )}
-                          style={{
-                            left: leftLockPos.x,
-                            top: leftLockPos.y,
-                            touchAction: "none",
-                          }}
-                          onPointerDown={e => {
-                            handlePointerDown(e, "leftLock");
-                          }}
-                          onPointerUp={e => {
-                            handlePointerUp(e, "leftLock");
-                          }}
-                          onClick={() => toggleLock(1)}
-                        >
-                          {(lockedButtons & 1) ? <BsLockFill /> : <BsUnlockFill />} L
-                        </div>
-
-                        <div
-                          className={cx(
-                            "pointer-events-auto absolute flex h-14 w-14 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100",
-                            isDark ? "bg-gray-500/70" : "bg-black/30",
+                            "pointer-events-auto absolute flex h-14 w-14 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100 relative",
+                            (lockedButtons & 2) ? "bg-green-600/80" : (isDark ? "bg-gray-500/70" : "bg-black/30"),
                           )}
                           style={{
                             left: rightBtnPos.x,
                             top: rightBtnPos.y,
                             touchAction: "none",
                           }}
-                          onPointerDown={e => {
-                            handlePointerDown(e, "right");
+                          onPointerDown={() => {
                             updateButtons(2, true);
                           }}
-                          onPointerUp={e => {
-                            handlePointerUp(e, "right");
+                          onPointerUp={() => {
                             updateButtons(2, false);
                           }}
                         >
                           R
+                          <div
+                            className="absolute -left-1 -top-1 flex h-5 w-5 items-center justify-center rounded-full bg-black/45 text-white"
+                            onPointerDown={e => {
+                              e.stopPropagation();
+                              handlePointerDown(e, "right");
+                            }}
+                            onPointerMove={e => {
+                              e.stopPropagation();
+                              handlePointerMove(e);
+                            }}
+                            onPointerUp={e => {
+                              e.stopPropagation();
+                              handlePointerUp(e, "right");
+                            }}
+                          >
+                            <FourWayMoveIcon className="h-3 w-3" />
+                          </div>
+                          <div
+                            className={cx(
+                              "absolute -right-1 -top-1 flex h-5 w-5 items-center justify-center rounded-full text-white",
+                              (lockedButtons & 2) ? "bg-green-600/90" : "bg-black/45",
+                            )}
+                            onPointerDown={e => {
+                              e.stopPropagation();
+                            }}
+                            onClick={e => {
+                              e.stopPropagation();
+                              toggleLock(2);
+                            }}
+                          >
+                            {(lockedButtons & 2) ? <BsLockFill className="h-3 w-3" /> : <BsUnlockFill className="h-3 w-3" />}
+                          </div>
                         </div>
 
                         <div
-                          className={cx(
-                            "pointer-events-auto absolute flex h-14 w-14 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100",
-                            (lockedButtons & 2) ? "bg-green-600/80" : (isDark ? "bg-gray-500/70" : "bg-black/30"),
-                          )}
+                          className="pointer-events-auto absolute"
                           style={{
-                            left: rightLockPos.x,
-                            top: rightLockPos.y,
+                            left: wheelPos.x,
+                            top: wheelPos.y,
                             touchAction: "none",
                           }}
-                          onPointerDown={e => {
-                            handlePointerDown(e, "rightLock");
-                          }}
-                          onPointerUp={e => {
-                            handlePointerUp(e, "rightLock");
-                          }}
-                          onClick={() => toggleLock(2)}
                         >
-                          {(lockedButtons & 2) ? <BsLockFill /> : <BsUnlockFill />} R
+                          <div
+                            className={cx(
+                              "flex h-8 w-8 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100",
+                              isDark ? "bg-gray-500/70" : "bg-black/30",
+                            )}
+                            onClick={() => send("wheelReport", { wheelY: 1 })}
+                          >
+                            ▲
+                          </div>
+                          <div
+                            className="mt-1 flex h-5 w-8 items-center justify-center rounded-full bg-black/45 text-white"
+                            onPointerDown={e => {
+                              e.stopPropagation();
+                              handlePointerDown(e, "wheel");
+                            }}
+                            onPointerMove={e => {
+                              e.stopPropagation();
+                              handlePointerMove(e);
+                            }}
+                            onPointerUp={e => {
+                              e.stopPropagation();
+                              handlePointerUp(e, "wheel");
+                            }}
+                          >
+                            <FourWayMoveIcon className="h-3 w-3" />
+                          </div>
+                          <div
+                            className={cx(
+                              "mt-1 flex h-8 w-8 items-center justify-center rounded-full text-white text-xs active:scale-90 transition-transform duration-100",
+                              isDark ? "bg-gray-500/70" : "bg-black/30",
+                            )}
+                            onClick={() => send("wheelReport", { wheelY: -1 })}
+                          >
+                            ▼
+                          </div>
                         </div>
                       </>
                     )}
+                    {showVirtualJoystick && (
+                      <div
+                        className="pointer-events-auto absolute"
+                        style={{ left: joystickPos.x, bottom: joystickPos.y }}
+                      >
+                        <div className="absolute -top-7 left-0 flex items-center gap-1">
+                          <div
+                            className={cx(
+                              "flex h-6 w-6 items-center justify-center rounded-full text-white transition-transform duration-100 active:scale-95",
+                              isDark ? "bg-gray-500/70" : "bg-black/30",
+                            )}
+                            style={{ touchAction: "none" }}
+                            onPointerDown={handleJoystickMoveStart}
+                            onPointerMove={handleJoystickMove}
+                            onPointerUp={handleJoystickMoveEnd}
+                            onPointerCancel={handleJoystickMoveEnd}
+                          >
+                            <FourWayMoveIcon className="h-4 w-4" />
+                          </div>
+                        </div>
+                        <div className="absolute right-[-36px] top-0 flex h-20 items-center">
+                          <div
+                            className={cx(
+                              "relative flex h-16 w-3 flex-col justify-between rounded-full py-1",
+                              isDark ? "bg-white/25" : "bg-black/20",
+                            )}
+                            style={{ touchAction: "none" }}
+                          >
+                            {joystickSpeedLevels.map((level, index) => (
+                              <button
+                                key={level}
+                                type="button"
+                                className={cx(
+                                  "relative z-10 h-3 w-3 rounded-full border",
+                                  joystickSpeedIndex === index
+                                    ? "border-blue-300 bg-blue-400"
+                                    : (isDark ? "border-white/60 bg-white/40" : "border-black/40 bg-black/20"),
+                                )}
+                                onClick={() => setJoystickSensitivity(level)}
+                                aria-label={`Set joystick speed ${level.toFixed(2)}`}
+                              />
+                            ))}
+                            <div
+                              className="pointer-events-none absolute left-full top-1/2 ml-[1px] -translate-y-1/2 border-y-[4px] border-l-[6px] border-y-transparent border-l-blue-400"
+                              style={{
+                                top: `${(joystickSpeedIndex / (joystickSpeedLevels.length - 1)) * 100}%`,
+                              }}
+                            />
+                          </div>
+                          <div className="ml-1 flex h-16 flex-col justify-between text-[8px] text-white/80">
+                            <span>Fast</span>
+                            <span>Slow</span>
+                          </div>
+                        </div>
+                        <div
+                          ref={joystickAreaRef}
+                          className={cx(
+                            "flex h-20 w-20 items-center justify-center rounded-full border",
+                            isDark ? "border-white/40 bg-black/20" : "border-black/30 bg-white/20",
+                          )}
+                          style={{ touchAction: "none" }}
+                          onPointerDown={handleJoystickPointerDown}
+                          onPointerMove={handleJoystickPointerMove}
+                          onPointerUp={handleJoystickPointerUp}
+                          onPointerCancel={handleJoystickPointerUp}
+                        >
+                          <div
+                            className={cx(
+                              "h-9 w-9 rounded-full",
+                              isDark ? "bg-white/70" : "bg-black/50",
+                            )}
+                            style={{
+                              transform: `translate(${joystickVector.x * 24}px, ${joystickVector.y * 24}px)`,
+                            }}
+                          />
+                        </div>
+                      </div>
+                    )}
                   </div>
                 </div>
                 <VirtualKeyboard />

ui/src/layout/core/desktop/hooks/useMouseEvents.ts

@@ -25,12 +25,13 @@ export const useMouseEvents = (
   const { setMousePosition, setMouseMove } = useMouseStore();
   const { width: videoWidth, height: videoHeight } = useVideoStore();
   const isReinitializingGadget = useHidStore(state => state.isReinitializingGadget);
+  const touchDragActiveRef = useRef(false);
 
   const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
 
   const sendRelMouseMovement = useCallback(
-    (x: number, y: number, buttons: number) => {
-      if (settings.mouseMode !== "relative") return;
+    (x: number, y: number, buttons: number, force = false) => {
+      if (!force && settings.mouseMode !== "relative") return;
       // Don't send mouse events while reinitializing gadget
       if (isReinitializingGadget) return;
       send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
@@ -50,6 +51,13 @@ export const useMouseEvents = (
     [send, setMousePosition, settings.mouseMode, isReinitializingGadget],
   );
 
+  const sendVirtualRelativeMovement = useCallback(
+    (x: number, y: number, buttons = 0) => {
+      sendRelMouseMovement(x, y, buttons, true);
+    },
+    [sendRelMouseMovement],
+  );
+
   const relMouseMoveHandler = useCallback(
     (e: MouseEvent) => {
       const pt = (e as unknown as PointerEvent).pointerType as unknown as string;
@@ -76,11 +84,17 @@ export const useMouseEvents = (
   const absMouseMoveHandler = useCallback(
     (e: MouseEvent) => {
       const pt = (e as unknown as PointerEvent).pointerType as unknown as string;
+      const pointerEvent = e as unknown as PointerEvent;
+      const eventType = pointerEvent.type;
       if (pt === "touch") {
         if (touchZoom) {
-            const touchCount = touchZoom.activeTouchPointers.current.size;
-            const eventType = (e as unknown as PointerEvent).type;
-            if (touchCount >= 2 && eventType !== "pointerup") return;
+          const touchCount = touchZoom.activeTouchPointers.current.size;
+          if (touchCount >= 2) {
+            if (eventType === "pointerup" || eventType === "pointercancel") {
+              touchDragActiveRef.current = false;
+            }
+            return;
+          }
         }
       }
 
@@ -135,28 +149,20 @@ export const useMouseEvents = (
       let buttons = e.buttons;
 
       if (pt === "touch") {
-        const touchCount = touchZoom ? touchZoom.activeTouchPointers.current.size : 1;
-        const pointerEvent = e as unknown as PointerEvent;
-        const eventType = pointerEvent.type;
-
-        if (eventType === "pointerup") {
-          if (touchCount >= 2 || disableTouchClick) {
-            buttons = 0;
-          } else {
-            buttons = 1;
-          }
-        } else {
+        if (eventType === "pointerdown") {
+          touchDragActiveRef.current = !disableTouchClick;
+        }
+        if (eventType === "pointerup" || eventType === "pointercancel") {
           buttons = 0;
+          touchDragActiveRef.current = false;
+        } else {
+          buttons = touchDragActiveRef.current ? 1 : 0;
         }
       }
 
       buttons |= externalButtons;
 
       sendAbsMouseMovement(x, y, buttons);
-
-      if (pt === "touch" && buttons !== externalButtons && (e as unknown as PointerEvent).type === "pointerup") {
-        sendAbsMouseMovement(x, y, externalButtons);
-      }
     },
     [settings.mouseMode, videoElm, videoWidth, videoHeight, sendAbsMouseMovement, touchZoom, disableTouchClick, externalButtons],
   );
@@ -216,8 +222,10 @@ export const useMouseEvents = (
     };
 
     videoElmRefValue.addEventListener("mousemove", eventHandler, { signal });
+    videoElmRefValue.addEventListener("pointermove", eventHandler, { signal });
     videoElmRefValue.addEventListener("pointerdown", eventHandler, { signal });
     videoElmRefValue.addEventListener("pointerup", eventHandler, { signal });
+    videoElmRefValue.addEventListener("pointercancel", eventHandler, { signal });
     videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
       signal,
       passive: true,
@@ -255,5 +263,6 @@ export const useMouseEvents = (
 
   return {
     setupMouseEvents,
+    sendVirtualRelativeMovement,
   };
 };

ui/src/layout/core/desktop/hooks/useTouchZoom.ts

@@ -1,7 +1,8 @@
 import { useEffect, useRef, useState } from "react";
 
 export const useTouchZoom = (
-  containerRef: React.RefObject<HTMLDivElement>
+  containerRef: React.RefObject<HTMLDivElement>,
+  gestureEnabled = true,
 ) => {
   const [mobileScale, setMobileScale] = useState(1);
   const [mobileTx, setMobileTx] = useState(0);
@@ -17,67 +18,83 @@ export const useTouchZoom = (
     if (!el) return;
     const abortController = new AbortController();
     const signal = abortController.signal;
+    const isPointInVideo = (x: number, y: number) => {
+      const video = el.querySelector("video") as HTMLVideoElement | null;
+      if (!video) return false;
+      const vRect = video.getBoundingClientRect();
+      return x >= vRect.left && x <= vRect.right && y >= vRect.top && y <= vRect.bottom;
+    };
 
     const onPointerDown = (e: PointerEvent) => {
       if (e.pointerType !== "touch") return;
       activeTouchPointers.current.set(e.pointerId, { x: e.clientX, y: e.clientY });
+      if (!gestureEnabled) return;
+      let shouldHandleLocalGesture = false;
+
       if (activeTouchPointers.current.size === 1) {
         const now = Date.now();
-        let isInVideo = false;
-        const video = el.querySelector("video") as HTMLVideoElement | null;
-        if (video) {
-          const vRect = video.getBoundingClientRect();
-          if (
-            e.clientX >= vRect.left &&
-            e.clientX <= vRect.right &&
-            e.clientY >= vRect.top &&
-            e.clientY <= vRect.bottom
-          ) {
-            isInVideo = true;
-          }
-        }
+        const isInVideo = isPointInVideo(e.clientX, e.clientY);
         if (!isInVideo) {
           if (now - lastTapAt.current < 300) {
             setMobileScale(1);
             setMobileTx(0);
             setMobileTy(0);
           }
+          shouldHandleLocalGesture = true;
         }
         lastTapAt.current = now;
-        lastPanPoint.current = { x: e.clientX, y: e.clientY };
+        if (mobileScale > 1) {
+          lastPanPoint.current = { x: e.clientX, y: e.clientY };
+          shouldHandleLocalGesture = true;
+        } else {
+          lastPanPoint.current = null;
+        }
       } else if (activeTouchPointers.current.size === 2) {
         const pts = Array.from(activeTouchPointers.current.values());
         const d = Math.hypot(pts[0].x - pts[1].x, pts[0].y - pts[1].y);
         initialPinchDistance.current = d;
         initialPinchScale.current = mobileScale;
+        shouldHandleLocalGesture = true;
+      }
+
+      if (shouldHandleLocalGesture) {
+        e.preventDefault();
+        e.stopPropagation();
       }
-      e.preventDefault();
-      e.stopPropagation();
     };
 
     const onPointerMove = (e: PointerEvent) => {
       if (e.pointerType !== "touch") return;
       const prev = activeTouchPointers.current.get(e.pointerId);
       activeTouchPointers.current.set(e.pointerId, { x: e.clientX, y: e.clientY });
+      if (!gestureEnabled) return;
       const pts = Array.from(activeTouchPointers.current.values());
+      let shouldHandleLocalGesture = false;
+
       if (pts.length === 2 && initialPinchDistance.current) {
         const d = Math.hypot(pts[0].x - pts[1].x, pts[0].y - pts[1].y);
         const factor = d / initialPinchDistance.current;
         const next = Math.max(1, Math.min(4, initialPinchScale.current * factor));
         setMobileScale(next);
-      } else if (pts.length === 1 && lastPanPoint.current && prev) {
+        shouldHandleLocalGesture = true;
+      } else if (pts.length === 1 && mobileScale > 1 && lastPanPoint.current && prev) {
         const dx = e.clientX - lastPanPoint.current.x;
         const dy = e.clientY - lastPanPoint.current.y;
         lastPanPoint.current = { x: e.clientX, y: e.clientY };
         setMobileTx(v => v + dx);
         setMobileTy(v => v + dy);
+        shouldHandleLocalGesture = true;
+      }
+
+      if (shouldHandleLocalGesture) {
+        e.preventDefault();
+        e.stopPropagation();
       }
-      e.preventDefault();
-      e.stopPropagation();
     };
 
     const onPointerUp = (e: PointerEvent) => {
       if (e.pointerType !== "touch") return;
+      const wasHandlingLocalGesture = gestureEnabled && (initialPinchDistance.current !== null || mobileScale > 1);
       activeTouchPointers.current.delete(e.pointerId);
       if (activeTouchPointers.current.size < 2) {
         initialPinchDistance.current = null;
@@ -85,8 +102,11 @@ export const useTouchZoom = (
       if (activeTouchPointers.current.size === 0) {
         lastPanPoint.current = null;
       }
-      e.preventDefault();
-      e.stopPropagation();
+
+      if (wasHandlingLocalGesture) {
+        e.preventDefault();
+        e.stopPropagation();
+      }
     };
 
     el.addEventListener("pointerdown", onPointerDown, { signal });
@@ -95,7 +115,15 @@ export const useTouchZoom = (
     el.addEventListener("pointercancel", onPointerUp, { signal });
 
     return () => abortController.abort();
-  }, [mobileScale, containerRef]);
+  }, [mobileScale, containerRef, gestureEnabled]);
+
+  const resetTransform = () => {
+    initialPinchDistance.current = null;
+    lastPanPoint.current = null;
+    setMobileScale(1);
+    setMobileTx(0);
+    setMobileTy(0);
+  };
 
   useEffect(() => {
     const container = containerRef.current;
@@ -115,5 +143,6 @@ export const useTouchZoom = (
     mobileTy,
     activeTouchPointers,
     lastPanPoint,
+    resetTransform,
   };
 };