clone/kvm
1
0
Fork 0
mirror of https://github.com/luckfox-eng29/kvm.git synced 2026-01-18 11:38:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: hid rpc channel (#755)

Browse Source 
* feat: use hidRpcChannel to save bandwidth

* chore: simplify handshake of hid rpc

* add logs

* chore: add timeout when writing to hid endpoints

* fix issues

* chore: show hid rpc version

* refactor hidrpc marshal / unmarshal

* add queues for keyboard / mouse event

* chore: change logging level of JSONRPC send event to trace

* minor changes related to logging

* fix: nil check

* chore: add comments and remove unused code

* add useMouse

* chore: log msg data only when debug or trace mode

* chore: make tslint happy

* chore: unlock keyboardStateLock before calling onKeysDownChange

* chore: remove keyPressReportApiAvailable

* chore: change version handle

* chore: clean up unused functions

* remove comments
This commit is contained in:
Aveline
2025-09-04 22:27:56 +02:00
committed by GitHub
parent e8ef82e582
commit bcc307b147
21 changed files with 1292 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ui/src/components/InfoBar.tsx
View File

@@ -10,10 +10,12 @@ import {
VideoState
} from "@/hooks/stores";
import { keys, modifiers } from "@/keyboardMappings";
import { useHidRpc } from "@/hooks/useHidRpc";
export default function InfoBar() {
const { keysDownState } = useHidStore();
const { mouseX, mouseY, mouseMove } = useMouseStore();
const { rpcHidStatus } = useHidRpc();
const videoClientSize = useVideoStore(
(state: VideoState) => `${Math.round(state.clientWidth)}x${Math.round(state.clientHeight)}`,
@@ -46,7 +48,7 @@ export default function InfoBar() {
const modifierNames = Object.entries(modifiers).filter(([_, mask]) => (activeModifierMask & mask) !== 0).map(([name, _]) => name);
const keyNames = Object.entries(keys).filter(([_, value]) => keysDown.includes(value)).map(([name, _]) => name);
return [...modifierNames,...keyNames].join(", ");
return [...modifierNames, ...keyNames].join(", ");
}, [keysDownState, showPressedKeys]);
return (
@@ -100,6 +102,12 @@ export default function InfoBar() {
<span className="text-xs">{hdmiState}</span>
</div>
)}
{debugMode && (
<div className="flex w-[156px] items-center gap-x-1">
<span className="text-xs font-semibold">HidRPC State:</span>
<span className="text-xs">{rpcHidStatus}</span>
</div>
)}
{showPressedKeys && (
<div className="flex items-center gap-x-1">

173
ui/src/components/WebRTCVideo.tsx
View File

@@ -7,15 +7,14 @@ import MacroBar from "@/components/MacroBar";
import InfoBar from "@components/InfoBar";
import notifications from "@/notifications";
import useKeyboard from "@/hooks/useKeyboard";
import { useJsonRpc } from "@/hooks/useJsonRpc";
import { cx } from "@/cva.config";
import { keys } from "@/keyboardMappings";
import {
useMouseStore,
useRTCStore,
useSettingsStore,
useVideoStore,
} from "@/hooks/stores";
import useMouse from "@/hooks/useMouse";
import {
HDMIErrorOverlay,
@@ -31,10 +30,18 @@ export default function WebRTCVideo() {
const [isPlaying, setIsPlaying] = useState(false);
const [isPointerLockActive, setIsPointerLockActive] = useState(false);
const [isKeyboardLockActive, setIsKeyboardLockActive] = useState(false);
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
// Store hooks
const settings = useSettingsStore();
const { handleKeyPress, resetKeyboardState } = useKeyboard();
const { setMousePosition, setMouseMove } = useMouseStore();
const {
getRelMouseMoveHandler,
getAbsMouseMoveHandler,
getMouseWheelHandler,
resetMousePosition,
} = useMouse();
const {
setClientSize: setVideoClientSize,
setSize: setVideoSize,
@@ -55,15 +62,9 @@ export default function WebRTCVideo() {
const hdmiError = ["no_lock", "no_signal", "out_of_range"].includes(hdmiState);
const isVideoLoading = !isPlaying;
// Mouse wheel states
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
// Misc states and hooks
const { send } = useJsonRpc();
// Video-related
const handleResize = useCallback(
( { width, height }: { width: number | undefined; height: number | undefined }) => {
({ width, height }: { width: number | undefined; height: number | undefined }) => {
if (!videoElm.current) return;
// Do something with width and height, e.g.:
setVideoClientSize(width || 0, height || 0);
@@ -99,7 +100,6 @@ export default function WebRTCVideo() {
);
// Pointer lock and keyboard lock related
const isPointerLockPossible = window.location.protocol === "https:" || window.location.hostname === "localhost";
const isFullscreenEnabled = document.fullscreenEnabled;
const checkNavigatorPermissions = useCallback(async (permissionName: string) => {
@@ -211,129 +211,29 @@ export default function WebRTCVideo() {
}
};
document.addEventListener("fullscreenchange ", handleFullscreenChange);
document.addEventListener("fullscreenchange", handleFullscreenChange);
}, [releaseKeyboardLock]);
// Mouse-related
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
send("relMouseReport", { dx: calcDelta(x), dy: calcDelta(y), buttons });
setMouseMove({ x, y, buttons });
},
[send, setMouseMove, settings.mouseMode],
const absMouseMoveHandler = useMemo(
() => getAbsMouseMoveHandler({
videoClientWidth,
videoClientHeight,
videoWidth,
videoHeight,
}),
[getAbsMouseMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight],
);
const relMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (settings.mouseMode !== "relative") return;
if (isPointerLockActive === false && isPointerLockPossible) return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[isPointerLockActive, isPointerLockPossible, sendRelMouseMovement, settings.mouseMode],
const relMouseMoveHandler = useMemo(
() => getRelMouseMoveHandler(),
[getRelMouseMoveHandler],
);
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (settings.mouseMode !== "absolute") return;
send("absMouseReport", { x, y, buttons });
// We set that for the debug info bar
setMousePosition(x, y);
},
[send, setMousePosition, settings.mouseMode],
const mouseWheelHandler = useMemo(
() => getMouseWheelHandler(),
[getMouseWheelHandler],
);
const absMouseMoveHandler = useCallback(
(e: MouseEvent) => {
if (!videoClientWidth || !videoClientHeight) return;
if (settings.mouseMode !== "absolute") return;
// Get the aspect ratios of the video element and the video stream
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
let offsetX = 0;
let offsetY = 0;
if (videoElementAspectRatio > videoStreamAspectRatio) {
// Pillarboxing: black bars on the left and right
effectiveWidth = videoClientHeight * videoStreamAspectRatio;
offsetX = (videoClientWidth - effectiveWidth) / 2;
} else if (videoElementAspectRatio < videoStreamAspectRatio) {
// Letterboxing: black bars on the top and bottom
effectiveHeight = videoClientWidth / videoStreamAspectRatio;
offsetY = (videoClientHeight - effectiveHeight) / 2;
}
// Clamp mouse position within the effective video boundaries
const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
// Map clamped mouse position to the video stream's coordinate system
const relativeX = (clampedX - offsetX) / effectiveWidth;
const relativeY = (clampedY - offsetY) / effectiveHeight;
// Convert to HID absolute coordinate system (0-32767 range)
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
},
[settings.mouseMode, videoClientWidth, videoClientHeight, videoWidth, videoHeight, sendAbsMouseMovement],
);
const mouseWheelHandler = useCallback(
(e: WheelEvent) => {
if (settings.scrollThrottling && blockWheelEvent) {
return;
}
// Determine if the wheel event is an accel scroll value
const isAccel = Math.abs(e.deltaY) >= 100;
// Calculate the accel scroll value
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
// Apply clamping (i.e. min and max mouse wheel hardware value)
const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
// Invert the clamped scroll value to match expected behavior
const invertedScrollValue = -clampedScrollValue;
send("wheelReport", { wheelY: invertedScrollValue });
// Apply blocking delay based of throttling settings
if (settings.scrollThrottling && !blockWheelEvent) {
setBlockWheelEvent(true);
setTimeout(() => setBlockWheelEvent(false), settings.scrollThrottling);
}
},
[send, blockWheelEvent, settings],
);
const resetMousePosition = useCallback(() => {
sendAbsMouseMovement(0, 0, 0);
}, [sendAbsMouseMovement]);
const keyDownHandler = useCallback(
(e: KeyboardEvent) => {
e.preventDefault();
@@ -357,7 +257,7 @@ export default function WebRTCVideo() {
}
console.debug(`Key down: ${hidKey}`);
handleKeyPress(hidKey, true);
if (!isKeyboardLockActive && hidKey === keys.MetaLeft) {
// If the left meta key was just pressed and we're not keyboard locked
// we'll never see the keyup event because the browser is going to lose
@@ -488,14 +388,16 @@ export default function WebRTCVideo() {
function setMouseModeEventListeners() {
const videoElmRefValue = videoElm.current;
if (!videoElmRefValue) return;
const isRelativeMouseMode = (settings.mouseMode === "relative");
const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler;
const abortController = new AbortController();
const signal = abortController.signal;
videoElmRefValue.addEventListener("mousemove", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("pointerup", isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler, { signal });
videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal });
videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal });
videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal });
videoElmRefValue.addEventListener("wheel", mouseWheelHandler, {
signal,
passive: true,
@@ -523,7 +425,16 @@ export default function WebRTCVideo() {
abortController.abort();
};
},
[absMouseMoveHandler, isPointerLockActive, isPointerLockPossible, mouseWheelHandler, relMouseMoveHandler, requestPointerLock, resetMousePosition, settings.mouseMode],
[
isPointerLockActive,
isPointerLockPossible,
requestPointerLock,
absMouseMoveHandler,
relMouseMoveHandler,
mouseWheelHandler,
resetMousePosition,
settings.mouseMode,
],
);
const containerRef = useRef<HTMLDivElement>(null);

302
ui/src/hooks/hidRpc.ts Normal file
View File

@@ -0,0 +1,302 @@
import { KeyboardLedState, KeysDownState } from "./stores";
export const HID_RPC_MESSAGE_TYPES = {
Handshake: 0x01,
KeyboardReport: 0x02,
PointerReport: 0x03,
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
}
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
export const HID_RPC_VERSION = 0x01;
const withinUint8Range = (value: number) => {
return value >= 0 && value <= 255;
};
const fromInt32toUint8 = (n: number) => {
if (n !== n >> 0) {
throw new Error(`Number ${n} is not within the int32 range`);
}
return new Uint8Array([
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
]);
};
const fromInt8ToUint8 = (n: number) => {
if (n < -128 || n > 127) {
throw new Error(`Number ${n} is not within the int8 range`);
}
return (n >> 0) & 0xFF;
};
const keyboardLedStateMasks = {
num_lock: 1 << 0,
caps_lock: 1 << 1,
scroll_lock: 1 << 2,
compose: 1 << 3,
kana: 1 << 4,
shift: 1 << 6,
}
export class RpcMessage {
messageType: HidRpcMessageType;
constructor(messageType: HidRpcMessageType) {
this.messageType = messageType;
}
marshal(): Uint8Array {
throw new Error("Not implemented");
}
public static unmarshal(_data: Uint8Array): RpcMessage | undefined {
throw new Error("Not implemented");
}
}
export class HandshakeMessage extends RpcMessage {
version: number;
constructor(version: number) {
super(HID_RPC_MESSAGE_TYPES.Handshake);
this.version = version;
}
marshal(): Uint8Array {
return new Uint8Array([this.messageType, this.version]);
}
public static unmarshal(data: Uint8Array): HandshakeMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid handshake message length: ${data.length}`);
}
return new HandshakeMessage(data[0]);
}
}
export class KeypressReportMessage extends RpcMessage {
private _key = 0;
private _press = false;
get key(): number {
return this._key;
}
set key(value: number) {
if (!withinUint8Range(value)) {
throw new Error(`Key ${value} is not within the uint8 range`);
}
this._key = value;
}
get press(): boolean {
return this._press;
}
set press(value: boolean) {
this._press = value;
}
constructor(key: number, press: boolean) {
super(HID_RPC_MESSAGE_TYPES.KeypressReport);
this.key = key;
this.press = press;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.key,
this.press ? 1 : 0,
]);
}
public static unmarshal(data: Uint8Array): KeypressReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keypress report message length: ${data.length}`);
}
return new KeypressReportMessage(data[0], data[1] === 1);
}
}
export class KeyboardReportMessage extends RpcMessage {
private _keys: number[] = [];
private _modifier = 0;
get keys(): number[] {
return this._keys;
}
set keys(value: number[]) {
value.forEach((k) => {
if (!withinUint8Range(k)) {
throw new Error(`Key ${k} is not within the uint8 range`);
}
});
this._keys = value;
}
get modifier(): number {
return this._modifier;
}
set modifier(value: number) {
if (!withinUint8Range(value)) {
throw new Error(`Modifier ${value} is not within the uint8 range`);
}
this._modifier = value;
}
constructor(keys: number[], modifier: number) {
super(HID_RPC_MESSAGE_TYPES.KeyboardReport);
this.keys = keys;
this.modifier = modifier;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.modifier,
...this.keys,
]);
}
public static unmarshal(data: Uint8Array): KeyboardReportMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard report message length: ${data.length}`);
}
return new KeyboardReportMessage(Array.from(data.slice(1)), data[0]);
}
}
export class KeyboardLedStateMessage extends RpcMessage {
keyboardLedState: KeyboardLedState;
constructor(keyboardLedState: KeyboardLedState) {
super(HID_RPC_MESSAGE_TYPES.KeyboardLedState);
this.keyboardLedState = keyboardLedState;
}
public static unmarshal(data: Uint8Array): KeyboardLedStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard led state message length: ${data.length}`);
}
const s = data[0];
const state = {
num_lock: (s & keyboardLedStateMasks.num_lock) !== 0,
caps_lock: (s & keyboardLedStateMasks.caps_lock) !== 0,
scroll_lock: (s & keyboardLedStateMasks.scroll_lock) !== 0,
compose: (s & keyboardLedStateMasks.compose) !== 0,
kana: (s & keyboardLedStateMasks.kana) !== 0,
shift: (s & keyboardLedStateMasks.shift) !== 0,
} as KeyboardLedState;
return new KeyboardLedStateMessage(state);
}
}
export class KeysDownStateMessage extends RpcMessage {
keysDownState: KeysDownState;
constructor(keysDownState: KeysDownState) {
super(HID_RPC_MESSAGE_TYPES.KeysDownState);
this.keysDownState = keysDownState;
}
public static unmarshal(data: Uint8Array): KeysDownStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keys down state message length: ${data.length}`);
}
return new KeysDownStateMessage({
modifier: data[0],
keys: Array.from(data.slice(1))
});
}
}
export class PointerReportMessage extends RpcMessage {
x: number;
y: number;
buttons: number;
constructor(x: number, y: number, buttons: number) {
super(HID_RPC_MESSAGE_TYPES.PointerReport);
this.x = x;
this.y = y;
this.buttons = buttons;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
...fromInt32toUint8(this.x),
...fromInt32toUint8(this.y),
this.buttons,
]);
}
}
export class MouseReportMessage extends RpcMessage {
dx: number;
dy: number;
buttons: number;
constructor(dx: number, dy: number, buttons: number) {
super(HID_RPC_MESSAGE_TYPES.MouseReport);
this.dx = dx;
this.dy = dy;
this.buttons = buttons;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
fromInt8ToUint8(this.dx),
fromInt8ToUint8(this.dy),
this.buttons,
]);
}
}
export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage,
[HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
[HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
}
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {
if (data.length < 1) {
throw new Error(`Invalid HID RPC message length: ${data.length}`);
}
const payload = data.slice(1);
const messageType = data[0];
if (!(messageType in messageRegistry)) {
throw new Error(`Unknown HID RPC message type: ${messageType}`);
}
return messageRegistry[messageType].unmarshal(payload);
};

18
ui/src/hooks/stores.ts
View File

@@ -105,6 +105,12 @@ export interface RTCState {
setRpcDataChannel: (channel: RTCDataChannel) => void;
rpcDataChannel: RTCDataChannel | null;
rpcHidProtocolVersion: number | null;
setRpcHidProtocolVersion: (version: number) => void;
rpcHidChannel: RTCDataChannel | null;
setRpcHidChannel: (channel: RTCDataChannel) => void;
peerConnectionState: RTCPeerConnectionState | null;
setPeerConnectionState: (state: RTCPeerConnectionState) => void;
@@ -151,6 +157,12 @@ export const useRTCStore = create<RTCState>(set => ({
rpcDataChannel: null,
setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
rpcHidProtocolVersion: null,
setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }),
rpcHidChannel: null,
setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }),
transceiver: null,
setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }),
@@ -449,9 +461,6 @@ export interface HidState {
keysDownState: KeysDownState;
setKeysDownState: (state: KeysDownState) => void;
keyPressReportApiAvailable: boolean;
setkeyPressReportApiAvailable: (available: boolean) => void;
isVirtualKeyboardEnabled: boolean;
setVirtualKeyboardEnabled: (enabled: boolean) => void;
@@ -469,9 +478,6 @@ export const useHidStore = create<HidState>(set => ({
keysDownState: { modifier: 0, keys: [0,0,0,0,0,0] } as KeysDownState,
setKeysDownState: (state: KeysDownState): void => set({ keysDownState: state }),
keyPressReportApiAvailable: true,
setkeyPressReportApiAvailable: (available: boolean) => set({ keyPressReportApiAvailable: available }),
isVirtualKeyboardEnabled: false,
setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),

150
ui/src/hooks/useHidRpc.ts Normal file
View File

@@ -0,0 +1,150 @@
import { useCallback, useEffect, useMemo } from "react";
import { useRTCStore } from "@/hooks/stores";
import {
HID_RPC_VERSION,
HandshakeMessage,
KeyboardReportMessage,
KeypressReportMessage,
MouseReportMessage,
PointerReportMessage,
RpcMessage,
unmarshalHidRpcMessage,
} from "./hidRpc";
export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore();
const rpcHidReady = useMemo(() => {
return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null;
}, [rpcHidChannel, rpcHidProtocolVersion]);
const rpcHidStatus = useMemo(() => {
if (!rpcHidChannel) return "N/A";
if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState;
if (!rpcHidProtocolVersion) return "handshaking";
return `ready (v${rpcHidProtocolVersion})`;
}, [rpcHidChannel, rpcHidProtocolVersion]);
const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => {
if (rpcHidChannel?.readyState !== "open") return;
if (!rpcHidReady && !ignoreHandshakeState) return;
let data: Uint8Array | undefined;
try {
data = message.marshal();
} catch (e) {
console.error("Failed to send HID RPC message", e);
}
if (!data) return;
rpcHidChannel?.send(data as unknown as ArrayBuffer);
}, [rpcHidChannel, rpcHidReady]);
const reportKeyboardEvent = useCallback(
(keys: number[], modifier: number) => {
sendMessage(new KeyboardReportMessage(keys, modifier));
}, [sendMessage],
);
const reportKeypressEvent = useCallback(
(key: number, press: boolean) => {
sendMessage(new KeypressReportMessage(key, press));
},
[sendMessage],
);
const reportAbsMouseEvent = useCallback(
(x: number, y: number, buttons: number) => {
sendMessage(new PointerReportMessage(x, y, buttons));
},
[sendMessage],
);
const reportRelMouseEvent = useCallback(
(dx: number, dy: number, buttons: number) => {
sendMessage(new MouseReportMessage(dx, dy, buttons));
},
[sendMessage],
);
const sendHandshake = useCallback(() => {
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
sendMessage(new HandshakeMessage(HID_RPC_VERSION), true);
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]);
const handleHandshake = useCallback((message: HandshakeMessage) => {
if (!message.version) {
console.error("Received handshake message without version", message);
return;
}
if (message.version > HID_RPC_VERSION) {
// we assume that the UI is always using the latest version of the HID RPC protocol
// so we can't support this
// TODO: use capabilities to determine rather than version number
console.error("Server is using a newer HID RPC version than the client", message);
return;
}
setRpcHidProtocolVersion(message.version);
}, [setRpcHidProtocolVersion]);
useEffect(() => {
if (!rpcHidChannel) return;
// send handshake message
sendHandshake();
const messageHandler = (e: MessageEvent) => {
if (typeof e.data === "string") {
console.warn("Received string data in HID RPC message handler", e.data);
return;
}
const message = unmarshalHidRpcMessage(new Uint8Array(e.data));
if (!message) {
console.warn("Received invalid HID RPC message", e.data);
return;
}
console.debug("Received HID RPC message", message);
switch (message.constructor) {
case HandshakeMessage:
handleHandshake(message as HandshakeMessage);
break;
default:
// not all events are handled here, the rest are handled by the onHidRpcMessage callback
break;
}
onHidRpcMessage?.(message);
};
rpcHidChannel.addEventListener("message", messageHandler);
return () => {
rpcHidChannel.removeEventListener("message", messageHandler);
};
},
[
rpcHidChannel,
onHidRpcMessage,
setRpcHidProtocolVersion,
sendHandshake,
handleHandshake,
],
);
return {
reportKeyboardEvent,
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,
};
}

108
ui/src/hooks/useKeyboard.ts
View File

@@ -1,13 +1,15 @@
import { useCallback } from "react";
import { KeysDownState, useHidStore, useRTCStore, hidKeyBufferSize, hidErrorRollOver } from "@/hooks/stores";
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
export default function useKeyboard() {
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { keysDownState, setKeysDownState } = useHidStore();
const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
// being tracked on the browser/client-side. When adding the keyPressReport API to the
@@ -19,7 +21,24 @@ export default function useKeyboard() {
// dynamically set when the device responds to the first key press event or reports its
// keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
const { keyPressReportApiAvailable, setkeyPressReportApiAvailable} = useHidStore();
// HidRPC is a binary format for exchanging keyboard and mouse events
const {
reportKeyboardEvent: sendKeyboardEventHidRpc,
reportKeypressEvent: sendKeypressEventHidRpc,
rpcHidReady,
} = useHidRpc((message) => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
break;
case KeyboardLedStateMessage:
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
break;
default:
break;
}
});
// sendKeyboardEvent is used to send the full keyboard state to the device for macro handling
// and resetting keyboard state. It sends the keys currently pressed and the modifier state.
@@ -27,63 +46,28 @@ export default function useKeyboard() {
// or just accept the state if it does not support (returning no result)
const sendKeyboardEvent = useCallback(
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
sendKeyboardEventHidRpc(state.keys, state.modifier);
return;
}
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
} else {
// If the device supports keyPressReport API, it will (also) return the keysDownState when we send
// the keyboardReport
const keysDownState = resp.result as KeysDownState;
if (keysDownState) {
setKeysDownState(keysDownState); // treat the response as the canonical state
setkeyPressReportApiAvailable(true); // if they returned a keysDownState, we ALSO know they also support keyPressReport
} else {
// older devices versions do not return the keyDownState
// so we just pretend they accepted what we sent
setKeysDownState(state);
setkeyPressReportApiAvailable(false); // we ALSO know they do not support keyPressReport
}
}
});
},
[rpcDataChannel?.readyState, send, setKeysDownState, setkeyPressReportApiAvailable],
);
// sendKeypressEvent is used to send a single key press/release event to the device.
// It sends the key and whether it is pressed or released.
// Older device version will not understand this request and will respond with
// an error with code -32601, which means that the RPC method name was not recognized.
// In that case we will switch to local key handling and update the keysDownState
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
const sendKeypressEvent = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
console.debug(`Send keypressEvent key: ${key}, press: ${press}`);
send("keypressReport", { key, press }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
// -32601 means the method is not supported because the device is running an older version
if (resp.error.code === -32601) {
console.error("Legacy device does not support keypressReport API, switching to local key down state handling", resp.error);
setkeyPressReportApiAvailable(false);
} else {
console.error(`Failed to send key ${key} press: ${press}`, resp.error);
}
} else {
const keysDownState = resp.result as KeysDownState;
if (keysDownState) {
setKeysDownState(keysDownState);
// we don't need to set keyPressReportApiAvailable here, because it's already true or we never landed here
}
}
});
},
[rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState],
[
rpcDataChannel?.readyState,
rpcHidReady,
send,
sendKeyboardEventHidRpc,
],
);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
@@ -135,12 +119,17 @@ export default function useKeyboard() {
// It then sends the full keyboard state to the device.
const handleKeyPress = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open") return;
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
if (keyPressReportApiAvailable) {
if (rpcHidReady) {
// if the keyPress api is available, we can just send the key press event
sendKeypressEvent(key, press);
// sendKeypressEvent is used to send a single key press/release event to the device.
// It sends the key and whether it is pressed or released.
// Older device version doesn't support this API, so we will switch to local key handling
// In that case we will switch to local key handling and update the keysDownState
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
sendKeypressEventHidRpc(key, press);
} else {
// if the keyPress api is not available, we need to handle the key locally
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
@@ -152,7 +141,14 @@ export default function useKeyboard() {
}
}
},
[keyPressReportApiAvailable, keysDownState, resetKeyboardState, rpcDataChannel?.readyState, sendKeyboardEvent, sendKeypressEvent],
[
rpcHidReady,
keysDownState,
resetKeyboardState,
rpcDataChannel?.readyState,
sendKeyboardEvent,
sendKeypressEventHidRpc,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists

172
ui/src/hooks/useMouse.ts Normal file
View File

@@ -0,0 +1,172 @@
import { useCallback, useState } from "react";
import { useJsonRpc } from "./useJsonRpc";
import { useHidRpc } from "./useHidRpc";
import { useMouseStore, useSettingsStore } from "./stores";
const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos);
export interface AbsMouseMoveHandlerProps {
videoClientWidth: number;
videoClientHeight: number;
videoWidth: number;
videoHeight: number;
}
export default function useMouse() {
// states
const { setMousePosition, setMouseMove } = useMouseStore();
const [blockWheelEvent, setBlockWheelEvent] = useState(false);
const { mouseMode, scrollThrottling } = useSettingsStore();
// RPC hooks
const { send } = useJsonRpc();
const { reportAbsMouseEvent, reportRelMouseEvent, rpcHidReady } = useHidRpc();
// Mouse-related
const sendRelMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (mouseMode !== "relative") return;
// if we ignore the event, double-click will not work
// if (x === 0 && y === 0 && buttons === 0) return;
const dx = calcDelta(x);
const dy = calcDelta(y);
if (rpcHidReady) {
reportRelMouseEvent(dx, dy, buttons);
} else {
// kept for backward compatibility
send("relMouseReport", { dx, dy, buttons });
}
setMouseMove({ x, y, buttons });
},
[
send,
reportRelMouseEvent,
setMouseMove,
mouseMode,
rpcHidReady,
],
);
const getRelMouseMoveHandler = useCallback(
() => (e: MouseEvent) => {
if (mouseMode !== "relative") return;
// Send mouse movement
const { buttons } = e;
sendRelMouseMovement(e.movementX, e.movementY, buttons);
},
[sendRelMouseMovement, mouseMode],
);
const sendAbsMouseMovement = useCallback(
(x: number, y: number, buttons: number) => {
if (mouseMode !== "absolute") return;
if (rpcHidReady) {
reportAbsMouseEvent(x, y, buttons);
} else {
// kept for backward compatibility
send("absMouseReport", { x, y, buttons });
}
// We set that for the debug info bar
setMousePosition(x, y);
},
[
send,
reportAbsMouseEvent,
setMousePosition,
mouseMode,
rpcHidReady,
],
);
const getAbsMouseMoveHandler = useCallback(
({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => (e: MouseEvent) => {
if (!videoClientWidth || !videoClientHeight) return;
if (mouseMode !== "absolute") return;
// Get the aspect ratios of the video element and the video stream
const videoElementAspectRatio = videoClientWidth / videoClientHeight;
const videoStreamAspectRatio = videoWidth / videoHeight;
// Calculate the effective video display area
let effectiveWidth = videoClientWidth;
let effectiveHeight = videoClientHeight;
let offsetX = 0;
let offsetY = 0;
if (videoElementAspectRatio > videoStreamAspectRatio) {
// Pillarboxing: black bars on the left and right
effectiveWidth = videoClientHeight * videoStreamAspectRatio;
offsetX = (videoClientWidth - effectiveWidth) / 2;
} else if (videoElementAspectRatio < videoStreamAspectRatio) {
// Letterboxing: black bars on the top and bottom
effectiveHeight = videoClientWidth / videoStreamAspectRatio;
offsetY = (videoClientHeight - effectiveHeight) / 2;
}
// Clamp mouse position within the effective video boundaries
const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth);
const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight);
// Map clamped mouse position to the video stream's coordinate system
const relativeX = (clampedX - offsetX) / effectiveWidth;
const relativeY = (clampedY - offsetY) / effectiveHeight;
// Convert to HID absolute coordinate system (0-32767 range)
const x = Math.round(relativeX * 32767);
const y = Math.round(relativeY * 32767);
// Send mouse movement
const { buttons } = e;
sendAbsMouseMovement(x, y, buttons);
}, [mouseMode, sendAbsMouseMovement],
);
const getMouseWheelHandler = useCallback(
() => (e: WheelEvent) => {
if (scrollThrottling && blockWheelEvent) {
return;
}
// Determine if the wheel event is an accel scroll value
const isAccel = Math.abs(e.deltaY) >= 100;
// Calculate the accel scroll value
const accelScrollValue = e.deltaY / 100;
// Calculate the no accel scroll value
const noAccelScrollValue = Math.sign(e.deltaY);
// Get scroll value
const scrollValue = isAccel ? accelScrollValue : noAccelScrollValue;
// Apply clamping (i.e. min and max mouse wheel hardware value)
const clampedScrollValue = Math.max(-127, Math.min(127, scrollValue));
// Invert the clamped scroll value to match expected behavior
const invertedScrollValue = -clampedScrollValue;
send("wheelReport", { wheelY: invertedScrollValue });
// Apply blocking delay based of throttling settings
if (scrollThrottling && !blockWheelEvent) {
setBlockWheelEvent(true);
setTimeout(() => setBlockWheelEvent(false), scrollThrottling);
}
},
[send, blockWheelEvent, scrollThrottling],
);
const resetMousePosition = useCallback(() => {
sendAbsMouseMovement(0, 0, 0);
}, [sendAbsMouseMovement]);
return {
getRelMouseMoveHandler,
getAbsMouseMoveHandler,
getMouseWheelHandler,
resetMousePosition,
};
}

16
ui/src/routes/devices.$id.tsx
View File

@@ -134,7 +134,8 @@ export default function KvmIdRoute() {
setRpcDataChannel,
isTurnServerInUse, setTurnServerInUse,
rpcDataChannel,
setTransceiver
setTransceiver,
setRpcHidChannel,
} = useRTCStore();
const location = useLocation();
@@ -481,6 +482,12 @@ export default function KvmIdRoute() {
setRpcDataChannel(rpcDataChannel);
};
const rpcHidChannel = pc.createDataChannel("hidrpc");
rpcHidChannel.binaryType = "arraybuffer";
rpcHidChannel.onopen = () => {
setRpcHidChannel(rpcHidChannel);
};
setPeerConnection(pc);
}, [
cleanupAndStopReconnecting,
@@ -491,6 +498,7 @@ export default function KvmIdRoute() {
setPeerConnection,
setPeerConnectionState,
setRpcDataChannel,
setRpcHidChannel,
setTransceiver,
]);
@@ -574,7 +582,6 @@ export default function KvmIdRoute() {
const {
keyboardLedState, setKeyboardLedState,
keysDownState, setKeysDownState, setUsbState,
setkeyPressReportApiAvailable
} = useHidStore();
const [hasUpdated, setHasUpdated] = useState(false);
@@ -612,7 +619,6 @@ export default function KvmIdRoute() {
const downState = resp.params as KeysDownState;
console.debug("Setting key down state:", downState);
setKeysDownState(downState);
setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport
}
if (resp.method === "otaState") {
@@ -689,7 +695,6 @@ export default function KvmIdRoute() {
if (resp.error.code === -32601) {
// if we don't support key down state, we know key press is also not available
console.warn("Failed to get key down state, switching to old-school", resp.error);
setkeyPressReportApiAvailable(false);
} else {
console.error("Failed to get key down state", resp.error);
}
@@ -697,11 +702,10 @@ export default function KvmIdRoute() {
const downState = resp.result as KeysDownState;
console.debug("Keyboard key down state", downState);
setKeysDownState(downState);
setkeyPressReportApiAvailable(true); // if they returned a keyDownState, we know they also support keyPressReport
}
setNeedKeyDownState(false);
});
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setkeyPressReportApiAvailable, setKeysDownState]);
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
// When the update is successful, we need to refresh the client javascript and show a success modal
useEffect(() => {