clone/kvm
1
0
Fork 0
mirror of https://github.com/luckfox-eng29/kvm.git synced 2026-01-18 03:28:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: release keyPress automatically (#796)

Browse Source 
* feat: release keyPress automatically

* send keepalive when pressing the key

* remove logging

* clean up logging

* chore: use unreliable channel to send keepalive events

* chore: use ordered unreliable channel for pointer events

* chore: adjust auto release key interval

* chore: update logging for kbdAutoReleaseLock

* chore: update comment for KEEPALIVE_INTERVAL

* fix: should cancelAutorelease when pressed is true

* fix: handshake won't happen if webrtc reconnects

* chore: add trace log for writeWithTimeout

* chore: add timeout for KeypressReport

* chore: use the proper key to send release command

* refactor: simplify HID RPC keyboard input handling and improve key state management

- Updated `handleHidRPCKeyboardInput` to return errors directly instead of keys down state.
- Refactored `rpcKeyboardReport` and `rpcKeypressReport` to return errors instead of states.
- Introduced a queue for managing key down state updates in the `Session` struct to prevent input handling stalls.
- Adjusted the `UpdateKeysDown` method to handle state changes more efficiently.
- Removed unnecessary logging and commented-out code for clarity.

* refactor: enhance keyboard auto-release functionality and key state management

* fix: correct Windows default auto-repeat delay comment from 1ms to 1s

* refactor: send keypress as early as possible

* refactor: replace console.warn with console.info for HID RPC channel events

* refactor: remove unused NewKeypressKeepAliveMessage function from HID RPC

* fix: handle error in key release process and log warnings

* fix: log warning on keypress report failure

* fix: update auto-release keyboard interval to 225

* refactor: enhance keep-alive handling and jitter compensation in HID RPC

- Implemented staleness guard to ignore outdated keep-alive packets.
- Added jitter compensation logic to adjust timer extensions based on packet arrival times.
- Introduced new methods for managing keep-alive state and reset functionality in the Session struct.
- Updated auto-release delay mechanism to use dynamic durations based on keep-alive timing.
- Adjusted keep-alive interval in the UI to improve responsiveness.

* gofmt

* clean up code

* chore: use dynamic duration for scheduleAutoRelease

* Use harcoded timer reset value for now

* fix: prevent nil pointer dereference when stopping timers in Close method

* refactor: remove nil check for kbdAutoReleaseTimers in DelayAutoReleaseWithDuration

* refactor: optimize dependencies in useHidRpc hooks

* refactor: streamline keep-alive timer management in useKeyboard hook

* refactor: clarify comments in useKeyboard hook for resetKeyboardState function

* refactor: reduce keysDownStateQueueSize

* refactor: close and reset keysDownStateQueue in newSession function

* chore: resolve conflicts

* resolve conflicts

---------

Co-authored-by: Adam Shiervani <adam.shiervani@gmail.com>
This commit is contained in:
Aveline
2025-09-18 13:35:47 +02:00
committed by GitHub
parent 72e3013337
commit afb146d78c
18 changed files with 767 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
ui/src/components/WebRTCVideo.tsx
View File

@@ -190,7 +190,7 @@ export default function WebRTCVideo() {
if (!isFullscreenEnabled || !videoElm.current) return;
// per https://wicg.github.io/keyboard-lock/#system-key-press-handler
// If keyboard lock is activated after fullscreen is already in effect, then the user my
// If keyboard lock is activated after fullscreen is already in effect, then the user my
// see multiple messages about how to exit fullscreen. For this reason, we recommend that
// developers call lock() before they enter fullscreen:
await requestKeyboardLock();
@@ -237,6 +237,7 @@ export default function WebRTCVideo() {
const keyDownHandler = useCallback(
(e: KeyboardEvent) => {
e.preventDefault();
if (e.repeat) return;
const code = getAdjustedKeyCode(e);
const hidKey = keys[code];

2
ui/src/components/popovers/PasteModal.tsx
View File

@@ -40,7 +40,7 @@ export default function PasteModal() {
const delayClassName = useMemo(() => debugMode ? "" : "hidden", [debugMode]);
const { setKeyboardLayout } = useSettingsStore();
const { selectedKeyboard } = useKeyboardLayout();
const { selectedKeyboard } = useKeyboardLayout();
useEffect(() => {
send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => {

12
ui/src/hooks/hidRpc.ts
View File

@@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = {
PointerReport: 0x03,
WheelReport: 0x04,
KeypressReport: 0x05,
KeypressKeepAliveReport: 0x09,
MouseReport: 0x06,
KeyboardMacroReport: 0x07,
CancelKeyboardMacroReport: 0x08,
@@ -409,6 +410,16 @@ export class MouseReportMessage extends RpcMessage {
}
}
export class KeypressKeepAliveMessage extends RpcMessage {
constructor() {
super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport);
}
marshal(): Uint8Array {
return new Uint8Array([this.messageType]);
}
}
export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage,
[HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage,
@@ -418,6 +429,7 @@ export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage,
[HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage,
[HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage,
}
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {

16
ui/src/hooks/stores.ts
View File

@@ -109,11 +109,17 @@ export interface RTCState {
setHidRpcDisabled: (disabled: boolean) => void;
rpcHidProtocolVersion: number | null;
setRpcHidProtocolVersion: (version: number) => void;
setRpcHidProtocolVersion: (version: number | null) => void;
rpcHidChannel: RTCDataChannel | null;
setRpcHidChannel: (channel: RTCDataChannel) => void;
rpcHidUnreliableChannel: RTCDataChannel | null;
setRpcHidUnreliableChannel: (channel: RTCDataChannel) => void;
rpcHidUnreliableNonOrderedChannel: RTCDataChannel | null;
setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => void;
peerConnectionState: RTCPeerConnectionState | null;
setPeerConnectionState: (state: RTCPeerConnectionState) => void;
@@ -164,11 +170,17 @@ export const useRTCStore = create<RTCState>(set => ({
setHidRpcDisabled: (disabled: boolean) => set({ hidRpcDisabled: disabled }),
rpcHidProtocolVersion: null,
setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }),
setRpcHidProtocolVersion: (version: number | null) => set({ rpcHidProtocolVersion: version }),
rpcHidChannel: null,
setRpcHidChannel: (channel: RTCDataChannel) => set({ rpcHidChannel: channel }),
rpcHidUnreliableChannel: null,
setRpcHidUnreliableChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableChannel: channel }),
rpcHidUnreliableNonOrderedChannel: null,
setRpcHidUnreliableNonOrderedChannel: (channel: RTCDataChannel) => set({ rpcHidUnreliableNonOrderedChannel: channel }),
transceiver: null,
setTransceiver: (transceiver: RTCRtpTransceiver) => set({ transceiver }),

160
ui/src/hooks/useHidRpc.ts
View File

@@ -9,6 +9,7 @@ import {
KeyboardMacroStep,
KeyboardMacroReportMessage,
KeyboardReportMessage,
KeypressKeepAliveMessage,
KeypressReportMessage,
MouseReportMessage,
PointerReportMessage,
@@ -16,42 +17,97 @@ import {
unmarshalHidRpcMessage,
} from "./hidRpc";
const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage();
interface sendMessageParams {
ignoreHandshakeState?: boolean;
useUnreliableChannel?: boolean;
requireOrdered?: boolean;
}
export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, hidRpcDisabled } = useRTCStore();
const {
rpcHidChannel,
rpcHidUnreliableChannel,
rpcHidUnreliableNonOrderedChannel,
setRpcHidProtocolVersion,
rpcHidProtocolVersion, hidRpcDisabled,
} = useRTCStore();
const rpcHidReady = useMemo(() => {
if (hidRpcDisabled) return false;
return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null;
}, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]);
const rpcHidUnreliableReady = useMemo(() => {
return (
rpcHidUnreliableChannel?.readyState === "open" && rpcHidProtocolVersion !== null
);
}, [rpcHidProtocolVersion, rpcHidUnreliableChannel?.readyState]);
const rpcHidUnreliableNonOrderedReady = useMemo(() => {
return (
rpcHidUnreliableNonOrderedChannel?.readyState === "open" &&
rpcHidProtocolVersion !== null
);
}, [rpcHidProtocolVersion, rpcHidUnreliableNonOrderedChannel?.readyState]);
const rpcHidStatus = useMemo(() => {
if (hidRpcDisabled) return "disabled";
if (!rpcHidChannel) return "N/A";
if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState;
if (!rpcHidProtocolVersion) return "handshaking";
return `ready (v${rpcHidProtocolVersion})`;
}, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]);
return `ready (v${rpcHidProtocolVersion}${rpcHidUnreliableReady ? "+u" : ""})`;
}, [rpcHidChannel, rpcHidProtocolVersion, rpcHidUnreliableReady, hidRpcDisabled]);
const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => {
if (hidRpcDisabled) return;
const sendMessage = useCallback(
(
message: RpcMessage,
{
ignoreHandshakeState,
useUnreliableChannel,
requireOrdered = true,
}: sendMessageParams = {},
) => {
if (hidRpcDisabled) return;
if (rpcHidChannel?.readyState !== "open") return;
if (!rpcHidReady && !ignoreHandshakeState) return;
if (!rpcHidReady && !ignoreHandshakeState) return;
let data: Uint8Array | undefined;
try {
data = message.marshal();
} catch (e) {
console.error("Failed to send HID RPC message", e);
}
if (!data) return;
let data: Uint8Array | undefined;
try {
data = message.marshal();
} catch (e) {
console.error("Failed to send HID RPC message", e);
}
if (!data) return;
rpcHidChannel?.send(data as unknown as ArrayBuffer);
}, [rpcHidChannel, rpcHidReady, hidRpcDisabled]);
if (useUnreliableChannel) {
if (requireOrdered && rpcHidUnreliableReady) {
rpcHidUnreliableChannel?.send(data as unknown as ArrayBuffer);
} else if (!requireOrdered && rpcHidUnreliableNonOrderedReady) {
rpcHidUnreliableNonOrderedChannel?.send(data as unknown as ArrayBuffer);
}
return;
}
rpcHidChannel?.send(data as unknown as ArrayBuffer);
},
[
rpcHidChannel,
rpcHidUnreliableChannel,
hidRpcDisabled, rpcHidUnreliableNonOrderedChannel,
rpcHidReady,
rpcHidUnreliableReady,
rpcHidUnreliableNonOrderedReady,
],
);
const reportKeyboardEvent = useCallback(
(keys: number[], modifier: number) => {
sendMessage(new KeyboardReportMessage(keys, modifier));
}, [sendMessage],
},
[sendMessage],
);
const reportKeypressEvent = useCallback(
@@ -63,7 +119,9 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
const reportAbsMouseEvent = useCallback(
(x: number, y: number, buttons: number) => {
sendMessage(new PointerReportMessage(x, y, buttons));
sendMessage(new PointerReportMessage(x, y, buttons), {
useUnreliableChannel: true,
});
},
[sendMessage],
);
@@ -89,32 +147,39 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
[sendMessage],
);
const reportKeypressKeepAlive = useCallback(() => {
sendMessage(KEEPALIVE_MESSAGE);
}, [sendMessage]);
const sendHandshake = useCallback(() => {
if (hidRpcDisabled) return;
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
sendMessage(new HandshakeMessage(HID_RPC_VERSION), true);
sendMessage(new HandshakeMessage(HID_RPC_VERSION), { ignoreHandshakeState: true });
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage, hidRpcDisabled]);
const handleHandshake = useCallback((message: HandshakeMessage) => {
if (hidRpcDisabled) return;
const handleHandshake = useCallback(
(message: HandshakeMessage) => {
if (hidRpcDisabled) return;
if (!message.version) {
console.error("Received handshake message without version", message);
return;
}
console.error("Received handshake message without version", message);
return;
}
if (message.version > HID_RPC_VERSION) {
// we assume that the UI is always using the latest version of the HID RPC protocol
// so we can't support this
// TODO: use capabilities to determine rather than version number
console.error("Server is using a newer HID RPC version than the client", message);
return;
}
if (message.version > HID_RPC_VERSION) {
// we assume that the UI is always using the latest version of the HID RPC protocol
// so we can't support this
// TODO: use capabilities to determine rather than version number
console.error("Server is using a newer HID RPC version than the client", message);
return;
}
setRpcHidProtocolVersion(message.version);
}, [setRpcHidProtocolVersion, hidRpcDisabled]);
setRpcHidProtocolVersion(message.version);
},
[setRpcHidProtocolVersion, hidRpcDisabled],
);
useEffect(() => {
if (!rpcHidChannel) return;
@@ -148,21 +213,33 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
onHidRpcMessage?.(message);
};
const openHandler = () => {
console.info("HID RPC channel opened");
sendHandshake();
};
const closeHandler = () => {
console.info("HID RPC channel closed");
setRpcHidProtocolVersion(null);
};
rpcHidChannel.addEventListener("message", messageHandler);
rpcHidChannel.addEventListener("close", closeHandler);
rpcHidChannel.addEventListener("open", openHandler);
return () => {
rpcHidChannel.removeEventListener("message", messageHandler);
rpcHidChannel.removeEventListener("close", closeHandler);
rpcHidChannel.removeEventListener("open", openHandler);
};
},
[
rpcHidChannel,
onHidRpcMessage,
setRpcHidProtocolVersion,
sendHandshake,
handleHandshake,
}, [
rpcHidChannel,
onHidRpcMessage,
setRpcHidProtocolVersion,
sendHandshake,
handleHandshake,
hidRpcDisabled,
],
);
]);
return {
reportKeyboardEvent,
@@ -171,6 +248,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
reportRelMouseEvent,
reportKeyboardMacroEvent,
cancelOngoingKeyboardMacro,
reportKeypressKeepAlive,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,

296
ui/src/hooks/useKeyboard.ts
View File

@@ -1,5 +1,11 @@
import { useCallback, useRef } from "react";
import {
KeyboardLedStateMessage,
KeyboardMacroStateMessage,
KeyboardMacroStep,
KeysDownStateMessage,
} from "@/hooks/hidRpc";
import {
hidErrorRollOver,
hidKeyBufferSize,
@@ -7,14 +13,8 @@ import {
useHidStore,
useRTCStore,
} from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import {
KeyboardLedStateMessage,
KeyboardMacroStateMessage,
KeyboardMacroStep,
KeysDownStateMessage,
} from "@/hooks/hidRpc";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
const MACRO_RESET_KEYBOARD_STATE = {
@@ -44,6 +44,9 @@ export default function useKeyboard() {
abortController.current = ac;
}, []);
// Keepalive timer management
const keepAliveTimerRef = useRef<number | null>(null);
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
// being tracked on the browser/client-side. When adding the keyPressReport API to the
// device-side code, we have to still support the situation where the browser/client-side code
@@ -51,8 +54,7 @@ export default function useKeyboard() {
// support the keyPressReport API. In that case, we need to handle the key presses locally
// and send the full state to the device, so it can behave like a real USB HID keyboard.
// This flag indicates whether the keyPressReport API is available on the device which is
// dynamically set when the device responds to the first key press event or reports its
// keysDownState when queried since the keyPressReport was introduced together with the
// dynamically set when the device responds to the first key press event or reports its // keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
// HidRPC is a binary format for exchanging keyboard and mouse events
@@ -61,6 +63,7 @@ export default function useKeyboard() {
reportKeypressEvent: sendKeypressEventHidRpc,
reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc,
reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc,
rpcHidReady,
} = useHidRpc(message => {
switch (message.constructor) {
@@ -92,6 +95,7 @@ export default function useKeyboard() {
},
[send, setKeysDownState],
);
const sendKeystrokeLegacy = useCallback(async (keys: number[], modifier: number, ac?: AbortController) => {
return await new Promise<void>((resolve, reject) => {
const abortListener = () => {
@@ -111,10 +115,29 @@ export default function useKeyboard() {
});
}, [send]);
const KEEPALIVE_INTERVAL = 50;
const cancelKeepAlive = useCallback(() => {
if (keepAliveTimerRef.current) {
clearInterval(keepAliveTimerRef.current);
keepAliveTimerRef.current = null;
}
}, []);
const scheduleKeepAlive = useCallback(() => {
// Clears existing keepalive timer
cancelKeepAlive();
keepAliveTimerRef.current = setInterval(() => {
sendKeypressKeepAliveHidRpc();
}, KEEPALIVE_INTERVAL);
}, [cancelKeepAlive, sendKeypressKeepAliveHidRpc]);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
// This is useful for macros and when the browser loses focus to ensure that the keyboard state
// is clean.
// This is useful for macros, in case of client-side rollover, and when the browser loses focus
const resetKeyboardState = useCallback(async () => {
// Cancel keepalive since we're resetting the keyboard state
cancelKeepAlive();
// Reset the keys buffer to zeros and the modifier state to zero
const { keys, modifier } = MACRO_RESET_KEYBOARD_STATE;
if (rpcHidReady) {
@@ -123,7 +146,136 @@ export default function useKeyboard() {
// Older backends don't support the hidRpc API, so we send the full reset state
handleLegacyKeyboardReport(keys, modifier);
}
}, [rpcHidReady, sendKeyboardEventHidRpc, handleLegacyKeyboardReport]);
}, [rpcHidReady, sendKeyboardEventHidRpc, handleLegacyKeyboardReport, cancelKeepAlive]);
// handleKeyPress is used to handle a key press or release event.
// This function handle both key press and key release events.
// It checks if the keyPressReport API is available and sends the key press event.
// If the keyPressReport API is not available, it simulates the device-side key
// handling for legacy devices and updates the keysDownState accordingly.
// It then sends the full keyboard state to the device.
const sendKeypress = useCallback(
(key: number, press: boolean) => {
cancelKeepAlive();
sendKeypressEventHidRpc(key, press);
if (press) {
scheduleKeepAlive();
}
},
[sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive],
);
const handleKeyPress = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
if (rpcHidReady) {
// if the keyPress api is available, we can just send the key press event
// sendKeypressEvent is used to send a single key press/release event to the device.
// It sends the key and whether it is pressed or released.
// Older device version doesn't support this API, so we will switch to local key handling
// In that case we will switch to local key handling and update the keysDownState
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
sendKeypress(key, press);
} else {
// Older backends don't support the hidRpc API, so we need:
// 1. Calculate the state
// 2. Send the newly calculated state to the device
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
keysDownState,
key,
press,
);
handleLegacyKeyboardReport(downState.keys, downState.modifier);
// if we just sent ErrorRollOver, reset to empty state
if (downState.keys[0] === hidErrorRollOver) {
resetKeyboardState();
}
}
},
[
rpcDataChannel?.readyState,
rpcHidReady,
keysDownState,
handleLegacyKeyboardReport,
resetKeyboardState,
sendKeypress,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
function simulateDeviceSideKeyHandlingForLegacyDevices(
state: KeysDownState,
key: number,
press: boolean,
): KeysDownState {
// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
// for handling key presses and releases. It ensures that the USB gadget
// behaves similarly to a real USB HID keyboard. This logic is paralleled
// in the device-side code in hid_keyboard.go so make sure to keep them in sync.
let modifiers = state.modifier;
const keys = state.keys;
const modifierMask = hidKeyToModifierMask[key] || 0;
if (modifierMask !== 0) {
// If the key is a modifier key, we update the keyboardModifier state
// by setting or clearing the corresponding bit in the modifier byte.
// This allows us to track the state of dynamic modifier keys like
// Shift, Control, Alt, and Super.
if (press) {
modifiers |= modifierMask;
} else {
modifiers &= ~modifierMask;
}
} else {
// handle other keys that are not modifier keys by placing or removing them
// from the key buffer since the buffer tracks currently pressed keys
let overrun = true;
for (let i = 0; i < hidKeyBufferSize; i++) {
// If we find the key in the buffer the buffer, we either remove it (if press is false)
// or do nothing (if down is true) because the buffer tracks currently pressed keys
// and if we find a zero byte, we can place the key there (if press is true)
if (keys[i] === key || keys[i] === 0) {
if (press) {
keys[i] = key; // overwrites the zero byte or the same key if already pressed
} else {
// we are releasing the key, remove it from the buffer
if (keys[i] !== 0) {
keys.splice(i, 1);
keys.push(0); // add a zero at the end
}
}
overrun = false; // We found a slot for the key
break;
}
}
// If we reach here it means we didn't find an empty slot or the key in the buffer
if (overrun) {
if (press) {
console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`);
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
keys.length = hidKeyBufferSize;
keys.fill(hidErrorRollOver);
} else {
// If we are releasing a key, and we didn't find it in a slot, who cares?
console.debug(`key ${key} not found in buffer, nothing to release`);
}
}
}
return { modifier: modifiers, keys };
}
// Cleanup function to cancel keepalive timer
const cleanup = useCallback(() => {
cancelKeepAlive();
}, [cancelKeepAlive]);
// executeMacro is used to execute a macro consisting of multiple steps.
@@ -132,13 +284,17 @@ export default function useKeyboard() {
// After the delay, the keys and modifiers are released and the next step is executed.
// If a step has no keys or modifiers, it is treated as a delay-only step.
// A small pause is added between steps to ensure that the device can process the events.
const executeMacroRemote = useCallback(async (steps: MacroSteps) => {
const executeMacroRemote = useCallback(async (
steps: MacroSteps,
) => {
const macro: KeyboardMacroStep[] = [];
for (const [_, step] of steps.entries()) {
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
const modifierMask: number = (step.modifiers || [])
.map(mod => modifiers[mod])
.reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay
@@ -217,117 +373,5 @@ export default function useKeyboard() {
cancelOngoingKeyboardMacroHidRpc();
}, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc, abortController]);
// handleKeyPress is used to handle a key press or release event.
// This function handle both key press and key release events.
// It checks if the keyPressReport API is available and sends the key press event.
// If the keyPressReport API is not available, it simulates the device-side key
// handling for legacy devices and updates the keysDownState accordingly.
// It then sends the full keyboard state to the device.
const handleKeyPress = useCallback(
async (key: number, press: boolean) => {
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
if ((key || 0) === 0) return; // ignore zero key presses (they are bad mappings)
if (rpcHidReady) {
// if the keyPress api is available, we can just send the key press event
// sendKeypressEvent is used to send a single key press/release event to the device.
// It sends the key and whether it is pressed or released.
// Older device version doesn't support this API, so we will switch to local key handling
// In that case we will switch to local key handling and update the keysDownState
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
sendKeypressEventHidRpc(key, press);
} else {
// Older backends don't support the hidRpc API, so we need:
// 1. Calculate the state
// 2. Send the newly calculated state to the device
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
keysDownState,
key,
press,
);
handleLegacyKeyboardReport(downState.keys, downState.modifier);
// if we just sent ErrorRollOver, reset to empty state
if (downState.keys[0] === hidErrorRollOver) {
resetKeyboardState();
}
}
},
[
rpcDataChannel?.readyState,
rpcHidReady,
sendKeypressEventHidRpc,
keysDownState,
handleLegacyKeyboardReport,
resetKeyboardState,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
function simulateDeviceSideKeyHandlingForLegacyDevices(
state: KeysDownState,
key: number,
press: boolean,
): KeysDownState {
// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
// for handling key presses and releases. It ensures that the USB gadget
// behaves similarly to a real USB HID keyboard. This logic is paralleled
// in the device-side code in hid_keyboard.go so make sure to keep them in sync.
let modifiers = state.modifier;
const keys = state.keys;
const modifierMask = hidKeyToModifierMask[key] || 0;
if (modifierMask !== 0) {
// If the key is a modifier key, we update the keyboardModifier state
// by setting or clearing the corresponding bit in the modifier byte.
// This allows us to track the state of dynamic modifier keys like
// Shift, Control, Alt, and Super.
if (press) {
modifiers |= modifierMask;
} else {
modifiers &= ~modifierMask;
}
} else {
// handle other keys that are not modifier keys by placing or removing them
// from the key buffer since the buffer tracks currently pressed keys
let overrun = true;
for (let i = 0; i < hidKeyBufferSize; i++) {
// If we find the key in the buffer the buffer, we either remove it (if press is false)
// or do nothing (if down is true) because the buffer tracks currently pressed keys
// and if we find a zero byte, we can place the key there (if press is true)
if (keys[i] === key || keys[i] === 0) {
if (press) {
keys[i] = key; // overwrites the zero byte or the same key if already pressed
} else {
// we are releasing the key, remove it from the buffer
if (keys[i] !== 0) {
keys.splice(i, 1);
keys.push(0); // add a zero at the end
}
}
overrun = false; // We found a slot for the key
break;
}
}
// If we reach here it means we didn't find an empty slot or the key in the buffer
if (overrun) {
if (press) {
console.warn(
`keyboard buffer overflow current keys ${keys}, key: ${key} not added`,
);
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
keys.length = hidKeyBufferSize;
keys.fill(hidErrorRollOver);
} else {
// If we are releasing a key, and we didn't find it in a slot, who cares?
console.debug(`key ${key} not found in buffer, nothing to release`);
}
}
}
return { modifier: modifiers, keys };
}
return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
return { handleKeyPress, resetKeyboardState, executeMacro, cleanup, cancelExecuteMacro };
}

22
ui/src/routes/devices.$id.tsx
View File

@@ -136,6 +136,8 @@ export default function KvmIdRoute() {
rpcDataChannel,
setTransceiver,
setRpcHidChannel,
setRpcHidUnreliableNonOrderedChannel,
setRpcHidUnreliableChannel,
} = useRTCStore();
const location = useLocation();
@@ -488,6 +490,24 @@ export default function KvmIdRoute() {
setRpcHidChannel(rpcHidChannel);
};
const rpcHidUnreliableChannel = pc.createDataChannel("hidrpc-unreliable-ordered", {
ordered: true,
maxRetransmits: 0,
});
rpcHidUnreliableChannel.binaryType = "arraybuffer";
rpcHidUnreliableChannel.onopen = () => {
setRpcHidUnreliableChannel(rpcHidUnreliableChannel);
};
const rpcHidUnreliableNonOrderedChannel = pc.createDataChannel("hidrpc-unreliable-nonordered", {
ordered: false,
maxRetransmits: 0,
});
rpcHidUnreliableNonOrderedChannel.binaryType = "arraybuffer";
rpcHidUnreliableNonOrderedChannel.onopen = () => {
setRpcHidUnreliableNonOrderedChannel(rpcHidUnreliableNonOrderedChannel);
};
setPeerConnection(pc);
}, [
cleanupAndStopReconnecting,
@@ -499,6 +519,8 @@ export default function KvmIdRoute() {
setPeerConnectionState,
setRpcDataChannel,
setRpcHidChannel,
setRpcHidUnreliableNonOrderedChannel,
setRpcHidUnreliableChannel,
setTransceiver,
]);