clone/kvm
1
0
Fork 0
mirror of https://github.com/luckfox-eng29/kvm.git synced 2026-01-18 03:28:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: send all paste keystrokes to backend (#789)

Browse Source 
* feat: send all paste keystrokes to backend

* feat: cancel paste mode

* wip: send macro using hidRPC channel

* add delay

* feat: allow paste progress to be cancelled

* allow user to override delay

* chore: clear keysDownState

* fix: use currentSession.reportHidRPCKeyboardMacroState

* fix: jsonrpc.go:1142:21: Error return value is not checked (errcheck)

* fix: performance issue of Uint8Array concat

* chore: hide delay option when debugMode isn't enabled

* feat: use clientSide macro if backend doesn't support macros

* fix: update keysDownState handling

* minor issues

* refactor

* fix: send duplicated keyDownState

* chore: add max length for paste text

---------

Co-authored-by: Adam Shiervani <adam.shiervani@gmail.com>
This commit is contained in:
Aveline
2025-09-18 13:00:57 +02:00
committed by GitHub
parent 25b102ac34
commit 72e3013337
14 changed files with 705 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
ui/src/components/InfoBar.tsx
View File

@@ -27,6 +27,7 @@ export default function InfoBar() {
const { rpcDataChannel } = useRTCStore();
const { debugMode, mouseMode, showPressedKeys } = useSettingsStore();
const { isPasteInProgress } = useHidStore();
useEffect(() => {
if (!rpcDataChannel) return;
@@ -108,7 +109,12 @@ export default function InfoBar() {
<span className="text-xs">{rpcHidStatus}</span>
</div>
)}
{isPasteInProgress && (
<div className="flex w-[156px] items-center gap-x-1">
<span className="text-xs font-semibold">Paste Mode:</span>
<span className="text-xs">Enabled</span>
</div>
)}
{showPressedKeys && (
<div className="flex items-center gap-x-1">
<span className="text-xs font-semibold">Keys:</span>

141
ui/src/components/popovers/PasteModal.tsx
View File

@@ -1,42 +1,46 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { LuCornerDownLeft } from "react-icons/lu";
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
import { useClose } from "@headlessui/react";
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { LuCornerDownLeft } from "react-icons/lu";
import { Button } from "@components/Button";
import { GridCard } from "@components/Card";
import { TextAreaWithLabel } from "@components/TextArea";
import { SettingsPageHeader } from "@components/SettingsPageheader";
import { cx } from "@/cva.config";
import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores";
import { keys, modifiers } from "@/keyboardMappings";
import { KeyStroke } from "@/keyboardLayouts";
import useKeyboard, { type MacroStep } from "@/hooks/useKeyboard";
import useKeyboardLayout from "@/hooks/useKeyboardLayout";
import notifications from "@/notifications";
import { Button } from "@components/Button";
import { GridCard } from "@components/Card";
import { InputFieldWithLabel } from "@components/InputField";
import { SettingsPageHeader } from "@components/SettingsPageheader";
import { TextAreaWithLabel } from "@components/TextArea";
const hidKeyboardPayload = (modifier: number, keys: number[]) => {
return { modifier, keys };
};
const modifierCode = (shift?: boolean, altRight?: boolean) => {
return (shift ? modifiers.ShiftLeft : 0)
| (altRight ? modifiers.AltRight : 0)
}
const noModifier = 0
// uint32 max value / 4
const pasteMaxLength = 1073741824;
export default function PasteModal() {
const TextAreaRef = useRef<HTMLTextAreaElement>(null);
const { setPasteModeEnabled } = useHidStore();
const { isPasteInProgress } = useHidStore();
const { setDisableVideoFocusTrap } = useUiStore();
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { executeMacro, cancelExecuteMacro } = useKeyboard();
const [invalidChars, setInvalidChars] = useState<string[]>([]);
const [delayValue, setDelayValue] = useState(100);
const delay = useMemo(() => {
if (delayValue < 50 || delayValue > 65534) {
return 100;
}
return delayValue;
}, [delayValue]);
const close = useClose();
const debugMode = useSettingsStore(state => state.debugMode);
const delayClassName = useMemo(() => debugMode ? "" : "hidden", [debugMode]);
const { setKeyboardLayout } = useSettingsStore();
const { selectedKeyboard } = useKeyboardLayout();
const { selectedKeyboard } = useKeyboardLayout();
useEffect(() => {
send("getKeyboardLayout", {}, (resp: JsonRpcResponse) => {
@@ -46,21 +50,19 @@ export default function PasteModal() {
}, [send, setKeyboardLayout]);
const onCancelPasteMode = useCallback(() => {
setPasteModeEnabled(false);
cancelExecuteMacro();
setDisableVideoFocusTrap(false);
setInvalidChars([]);
}, [setDisableVideoFocusTrap, setPasteModeEnabled]);
}, [setDisableVideoFocusTrap, cancelExecuteMacro]);
const onConfirmPaste = useCallback(async () => {
setPasteModeEnabled(false);
setDisableVideoFocusTrap(false);
if (rpcDataChannel?.readyState !== "open" || !TextAreaRef.current) return;
if (!selectedKeyboard) return;
if (!TextAreaRef.current || !selectedKeyboard) return;
const text = TextAreaRef.current.value;
try {
const macroSteps: MacroStep[] = [];
for (const char of text) {
const keyprops = selectedKeyboard.chars[char];
if (!keyprops) continue;
@@ -70,39 +72,41 @@ export default function PasteModal() {
// if this is an accented character, we need to send that accent FIRST
if (accentKey) {
await sendKeystroke({modifier: modifierCode(accentKey.shift, accentKey.altRight), keys: [ keys[accentKey.key] ] })
const accentModifiers: string[] = [];
if (accentKey.shift) accentModifiers.push("ShiftLeft");
if (accentKey.altRight) accentModifiers.push("AltRight");
macroSteps.push({
keys: [String(accentKey.key)],
modifiers: accentModifiers.length > 0 ? accentModifiers : null,
delay,
});
}
// now send the actual key
await sendKeystroke({ modifier: modifierCode(shift, altRight), keys: [ keys[key] ]});
const modifiers: string[] = [];
if (shift) modifiers.push("ShiftLeft");
if (altRight) modifiers.push("AltRight");
macroSteps.push({
keys: [String(key)],
modifiers: modifiers.length > 0 ? modifiers : null,
delay
});
// if what was requested was a dead key, we need to send an unmodified space to emit
// just the accent character
if (deadKey) {
await sendKeystroke({ modifier: noModifier, keys: [ keys["Space"] ] });
}
if (deadKey) macroSteps.push({ keys: ["Space"], modifiers: null, delay });
}
// now send a message with no keys down to "release" the keys
await sendKeystroke({ modifier: 0, keys: [] });
if (macroSteps.length > 0) {
await executeMacro(macroSteps);
}
} catch (error) {
console.error("Failed to paste text:", error);
notifications.error("Failed to paste text");
}
async function sendKeystroke(stroke: KeyStroke) {
await new Promise<void>((resolve, reject) => {
send(
"keyboardReport",
hidKeyboardPayload(stroke.modifier, stroke.keys),
params => {
if ("error" in params) return reject(params.error);
resolve();
}
);
});
}
}, [selectedKeyboard, rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteModeEnabled]);
}, [selectedKeyboard, executeMacro, delay]);
useEffect(() => {
if (TextAreaRef.current) {
@@ -122,19 +126,25 @@ export default function PasteModal() {
/>
<div
className="animate-fadeIn opacity-0 space-y-2"
className="animate-fadeIn space-y-2 opacity-0"
style={{
animationDuration: "0.7s",
animationDelay: "0.1s",
}}
>
<div>
<div className="w-full" onKeyUp={e => e.stopPropagation()} onKeyDown={e => e.stopPropagation()}>
<div
className="w-full"
onKeyUp={e => e.stopPropagation()}
onKeyDown={e => e.stopPropagation()} onKeyDownCapture={e => e.stopPropagation()}
onKeyUpCapture={e => e.stopPropagation()}
>
<TextAreaWithLabel
ref={TextAreaRef}
label="Paste from host"
rows={4}
onKeyUp={e => e.stopPropagation()}
maxLength={pasteMaxLength}
onKeyDown={e => {
e.stopPropagation();
if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
@@ -171,9 +181,31 @@ export default function PasteModal() {
)}
</div>
</div>
<div className={cx("text-xs text-slate-600 dark:text-slate-400", delayClassName)}>
<InputFieldWithLabel
type="number"
label="Delay between keys"
placeholder="Delay between keys"
min={50}
max={65534}
value={delayValue}
onChange={e => {
setDelayValue(parseInt(e.target.value, 10));
}}
/>
{delayValue < 50 || delayValue > 65534 && (
<div className="mt-2 flex items-center gap-x-2">
<ExclamationCircleIcon className="h-4 w-4 text-red-500 dark:text-red-400" />
<span className="text-xs text-red-500 dark:text-red-400">
Delay must be between 50 and 65534
</span>
</div>
)}
</div>
<div className="space-y-4">
<p className="text-xs text-slate-600 dark:text-slate-400">
Sending text using keyboard layout: {selectedKeyboard.isoCode}-{selectedKeyboard.name}
Sending text using keyboard layout: {selectedKeyboard.isoCode}-
{selectedKeyboard.name}
</p>
</div>
</div>
@@ -181,7 +213,7 @@ export default function PasteModal() {
</div>
</div>
<div
className="flex animate-fadeIn opacity-0 items-center justify-end gap-x-2"
className="flex animate-fadeIn items-center justify-end gap-x-2 opacity-0"
style={{
animationDuration: "0.7s",
animationDelay: "0.2s",
@@ -200,6 +232,7 @@ export default function PasteModal() {
size="SM"
theme="primary"
text="Confirm Paste"
disabled={isPasteInProgress}
onClick={onConfirmPaste}
LeadingIcon={LuCornerDownLeft}
/>

140
ui/src/hooks/hidRpc.ts
View File

@@ -1,4 +1,4 @@
import { KeyboardLedState, KeysDownState } from "./stores";
import { hidKeyBufferSize, KeyboardLedState, KeysDownState } from "./stores";
export const HID_RPC_MESSAGE_TYPES = {
Handshake: 0x01,
@@ -7,8 +7,11 @@ export const HID_RPC_MESSAGE_TYPES = {
WheelReport: 0x04,
KeypressReport: 0x05,
MouseReport: 0x06,
KeyboardMacroReport: 0x07,
CancelKeyboardMacroReport: 0x08,
KeyboardLedState: 0x32,
KeysDownState: 0x33,
KeyboardMacroState: 0x34,
}
export type HidRpcMessageType = typeof HID_RPC_MESSAGE_TYPES[keyof typeof HID_RPC_MESSAGE_TYPES];
@@ -28,7 +31,31 @@ const fromInt32toUint8 = (n: number) => {
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
(n >> 0) & 0xFF,
n & 0xFF,
]);
};
const fromUint16toUint8 = (n: number) => {
if (n > 65535 || n < 0) {
throw new Error(`Number ${n} is not within the uint16 range`);
}
return new Uint8Array([
(n >> 8) & 0xFF,
n & 0xFF,
]);
};
const fromUint32toUint8 = (n: number) => {
if (n > 4294967295 || n < 0) {
throw new Error(`Number ${n} is not within the uint32 range`);
}
return new Uint8Array([
(n >> 24) & 0xFF,
(n >> 16) & 0xFF,
(n >> 8) & 0xFF,
n & 0xFF,
]);
};
@@ -37,7 +64,7 @@ const fromInt8ToUint8 = (n: number) => {
throw new Error(`Number ${n} is not within the int8 range`);
}
return (n >> 0) & 0xFF;
return n & 0xFF;
};
const keyboardLedStateMasks = {
@@ -186,6 +213,99 @@ export class KeyboardReportMessage extends RpcMessage {
}
}
export interface KeyboardMacroStep extends KeysDownState {
delay: number;
}
export class KeyboardMacroReportMessage extends RpcMessage {
isPaste: boolean;
stepCount: number;
steps: KeyboardMacroStep[];
KEYS_LENGTH = hidKeyBufferSize;
constructor(isPaste: boolean, stepCount: number, steps: KeyboardMacroStep[]) {
super(HID_RPC_MESSAGE_TYPES.KeyboardMacroReport);
this.isPaste = isPaste;
this.stepCount = stepCount;
this.steps = steps;
}
marshal(): Uint8Array {
// validate if length is correct
if (this.stepCount !== this.steps.length) {
throw new Error(`Length ${this.stepCount} is not equal to the number of steps ${this.steps.length}`);
}
const data = new Uint8Array(this.stepCount * 9 + 6);
data.set(new Uint8Array([
this.messageType,
this.isPaste ? 1 : 0,
...fromUint32toUint8(this.stepCount),
]), 0);
for (let i = 0; i < this.stepCount; i++) {
const step = this.steps[i];
if (!withinUint8Range(step.modifier)) {
throw new Error(`Modifier ${step.modifier} is not within the uint8 range`);
}
// Ensure the keys are within the KEYS_LENGTH range
const keys = step.keys;
if (keys.length > this.KEYS_LENGTH) {
throw new Error(`Keys ${keys} is not within the hidKeyBufferSize range`);
} else if (keys.length < this.KEYS_LENGTH) {
keys.push(...Array(this.KEYS_LENGTH - keys.length).fill(0));
}
for (const key of keys) {
if (!withinUint8Range(key)) {
throw new Error(`Key ${key} is not within the uint8 range`);
}
}
const macroBinary = new Uint8Array([
step.modifier,
...keys,
...fromUint16toUint8(step.delay),
]);
const offset = 6 + i * 9;
data.set(macroBinary, offset);
}
return data;
}
}
export class KeyboardMacroStateMessage extends RpcMessage {
state: boolean;
isPaste: boolean;
constructor(state: boolean, isPaste: boolean) {
super(HID_RPC_MESSAGE_TYPES.KeyboardMacroState);
this.state = state;
this.isPaste = isPaste;
}
marshal(): Uint8Array {
return new Uint8Array([
this.messageType,
this.state ? 1 : 0,
this.isPaste ? 1 : 0,
]);
}
public static unmarshal(data: Uint8Array): KeyboardMacroStateMessage | undefined {
if (data.length < 1) {
throw new Error(`Invalid keyboard macro state report message length: ${data.length}`);
}
return new KeyboardMacroStateMessage(data[0] === 1, data[1] === 1);
}
}
export class KeyboardLedStateMessage extends RpcMessage {
keyboardLedState: KeyboardLedState;
@@ -256,6 +376,17 @@ export class PointerReportMessage extends RpcMessage {
}
}
export class CancelKeyboardMacroReportMessage extends RpcMessage {
constructor() {
super(HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport);
}
marshal(): Uint8Array {
return new Uint8Array([this.messageType]);
}
}
export class MouseReportMessage extends RpcMessage {
dx: number;
dy: number;
@@ -284,6 +415,9 @@ export const messageRegistry = {
[HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
[HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardMacroReport]: KeyboardMacroReportMessage,
[HID_RPC_MESSAGE_TYPES.CancelKeyboardMacroReport]: CancelKeyboardMacroReportMessage,
[HID_RPC_MESSAGE_TYPES.KeyboardMacroState]: KeyboardMacroStateMessage,
}
export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {

12
ui/src/hooks/stores.ts
View File

@@ -105,6 +105,9 @@ export interface RTCState {
setRpcDataChannel: (channel: RTCDataChannel) => void;
rpcDataChannel: RTCDataChannel | null;
hidRpcDisabled: boolean;
setHidRpcDisabled: (disabled: boolean) => void;
rpcHidProtocolVersion: number | null;
setRpcHidProtocolVersion: (version: number) => void;
@@ -157,6 +160,9 @@ export const useRTCStore = create<RTCState>(set => ({
rpcDataChannel: null,
setRpcDataChannel: (channel: RTCDataChannel) => set({ rpcDataChannel: channel }),
hidRpcDisabled: false,
setHidRpcDisabled: (disabled: boolean) => set({ hidRpcDisabled: disabled }),
rpcHidProtocolVersion: null,
setRpcHidProtocolVersion: (version: number) => set({ rpcHidProtocolVersion: version }),
@@ -464,7 +470,7 @@ export interface HidState {
isVirtualKeyboardEnabled: boolean;
setVirtualKeyboardEnabled: (enabled: boolean) => void;
isPasteModeEnabled: boolean;
isPasteInProgress: boolean;
setPasteModeEnabled: (enabled: boolean) => void;
usbState: USBStates;
@@ -481,8 +487,8 @@ export const useHidStore = create<HidState>(set => ({
isVirtualKeyboardEnabled: false,
setVirtualKeyboardEnabled: (enabled: boolean): void => set({ isVirtualKeyboardEnabled: enabled }),
isPasteModeEnabled: false,
setPasteModeEnabled: (enabled: boolean): void => set({ isPasteModeEnabled: enabled }),
isPasteInProgress: false,
setPasteModeEnabled: (enabled: boolean): void => set({ isPasteInProgress: enabled }),
// Add these new properties for USB state
usbState: "not attached",

40
ui/src/hooks/useHidRpc.ts
View File

@@ -3,8 +3,11 @@ import { useCallback, useEffect, useMemo } from "react";
import { useRTCStore } from "@/hooks/stores";
import {
CancelKeyboardMacroReportMessage,
HID_RPC_VERSION,
HandshakeMessage,
KeyboardMacroStep,
KeyboardMacroReportMessage,
KeyboardReportMessage,
KeypressReportMessage,
MouseReportMessage,
@@ -14,19 +17,23 @@ import {
} from "./hidRpc";
export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore();
const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion, hidRpcDisabled } = useRTCStore();
const rpcHidReady = useMemo(() => {
if (hidRpcDisabled) return false;
return rpcHidChannel?.readyState === "open" && rpcHidProtocolVersion !== null;
}, [rpcHidChannel, rpcHidProtocolVersion]);
}, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]);
const rpcHidStatus = useMemo(() => {
if (hidRpcDisabled) return "disabled";
if (!rpcHidChannel) return "N/A";
if (rpcHidChannel.readyState !== "open") return rpcHidChannel.readyState;
if (!rpcHidProtocolVersion) return "handshaking";
return `ready (v${rpcHidProtocolVersion})`;
}, [rpcHidChannel, rpcHidProtocolVersion]);
}, [rpcHidChannel, rpcHidProtocolVersion, hidRpcDisabled]);
const sendMessage = useCallback((message: RpcMessage, ignoreHandshakeState = false) => {
if (hidRpcDisabled) return;
if (rpcHidChannel?.readyState !== "open") return;
if (!rpcHidReady && !ignoreHandshakeState) return;
@@ -39,7 +46,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
if (!data) return;
rpcHidChannel?.send(data as unknown as ArrayBuffer);
}, [rpcHidChannel, rpcHidReady]);
}, [rpcHidChannel, rpcHidReady, hidRpcDisabled]);
const reportKeyboardEvent = useCallback(
(keys: number[], modifier: number) => {
@@ -68,14 +75,31 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
[sendMessage],
);
const reportKeyboardMacroEvent = useCallback(
(steps: KeyboardMacroStep[]) => {
sendMessage(new KeyboardMacroReportMessage(false, steps.length, steps));
},
[sendMessage],
);
const cancelOngoingKeyboardMacro = useCallback(
() => {
sendMessage(new CancelKeyboardMacroReportMessage());
},
[sendMessage],
);
const sendHandshake = useCallback(() => {
if (hidRpcDisabled) return;
if (rpcHidProtocolVersion) return;
if (!rpcHidChannel) return;
sendMessage(new HandshakeMessage(HID_RPC_VERSION), true);
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage]);
}, [rpcHidChannel, rpcHidProtocolVersion, sendMessage, hidRpcDisabled]);
const handleHandshake = useCallback((message: HandshakeMessage) => {
if (hidRpcDisabled) return;
if (!message.version) {
console.error("Received handshake message without version", message);
return;
@@ -90,10 +114,11 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
}
setRpcHidProtocolVersion(message.version);
}, [setRpcHidProtocolVersion]);
}, [setRpcHidProtocolVersion, hidRpcDisabled]);
useEffect(() => {
if (!rpcHidChannel) return;
if (hidRpcDisabled) return;
// send handshake message
sendHandshake();
@@ -135,6 +160,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
setRpcHidProtocolVersion,
sendHandshake,
handleHandshake,
hidRpcDisabled,
],
);
@@ -143,6 +169,8 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
reportKeypressEvent,
reportAbsMouseEvent,
reportRelMouseEvent,
reportKeyboardMacroEvent,
cancelOngoingKeyboardMacro,
rpcHidProtocolVersion,
rpcHidReady,
rpcHidStatus,

255
ui/src/hooks/useKeyboard.ts
View File

@@ -1,15 +1,48 @@
import { useCallback } from "react";
import { useCallback, useRef } from "react";
import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
import {
hidErrorRollOver,
hidKeyBufferSize,
KeysDownState,
useHidStore,
useRTCStore,
} from "@/hooks/stores";
import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
import { useHidRpc } from "@/hooks/useHidRpc";
import { KeyboardLedStateMessage, KeysDownStateMessage } from "@/hooks/hidRpc";
import {
KeyboardLedStateMessage,
KeyboardMacroStateMessage,
KeyboardMacroStep,
KeysDownStateMessage,
} from "@/hooks/hidRpc";
import { hidKeyToModifierMask, keys, modifiers } from "@/keyboardMappings";
const MACRO_RESET_KEYBOARD_STATE = {
keys: new Array(hidKeyBufferSize).fill(0),
modifier: 0,
delay: 0,
};
export interface MacroStep {
keys: string[] | null;
modifiers: string[] | null;
delay: number;
}
export type MacroSteps = MacroStep[];
const sleep = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));
export default function useKeyboard() {
const { send } = useJsonRpc();
const { rpcDataChannel } = useRTCStore();
const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
const { keysDownState, setKeysDownState, setKeyboardLedState, setPasteModeEnabled } =
useHidStore();
const abortController = useRef<AbortController | null>(null);
const setAbortController = useCallback((ac: AbortController | null) => {
abortController.current = ac;
}, []);
// INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
// being tracked on the browser/client-side. When adding the keyPressReport API to the
@@ -17,17 +50,19 @@ export default function useKeyboard() {
// is running on the cloud against a device that has not been updated yet and thus does not
// support the keyPressReport API. In that case, we need to handle the key presses locally
// and send the full state to the device, so it can behave like a real USB HID keyboard.
// This flag indicates whether the keyPressReport API is available on the device which is
// This flag indicates whether the keyPressReport API is available on the device which is
// dynamically set when the device responds to the first key press event or reports its
// keysDownState when queried since the keyPressReport was introduced together with the
// keysDownState when queried since the keyPressReport was introduced together with the
// getKeysDownState API.
// HidRPC is a binary format for exchanging keyboard and mouse events
const {
reportKeyboardEvent: sendKeyboardEventHidRpc,
reportKeypressEvent: sendKeypressEventHidRpc,
reportKeyboardMacroEvent: sendKeyboardMacroEventHidRpc,
cancelOngoingKeyboardMacro: cancelOngoingKeyboardMacroHidRpc,
rpcHidReady,
} = useHidRpc((message) => {
} = useHidRpc(message => {
switch (message.constructor) {
case KeysDownStateMessage:
setKeysDownState((message as KeysDownStateMessage).keysDownState);
@@ -35,52 +70,61 @@ export default function useKeyboard() {
case KeyboardLedStateMessage:
setKeyboardLedState((message as KeyboardLedStateMessage).keyboardLedState);
break;
case KeyboardMacroStateMessage:
if (!(message as KeyboardMacroStateMessage).isPaste) break;
setPasteModeEnabled((message as KeyboardMacroStateMessage).state);
break;
default:
break;
}
});
// sendKeyboardEvent is used to send the full keyboard state to the device for macro handling
// and resetting keyboard state. It sends the keys currently pressed and the modifier state.
// The device will respond with the keysDownState if it supports the keyPressReport API
// or just accept the state if it does not support (returning no result)
const sendKeyboardEvent = useCallback(
async (state: KeysDownState) => {
if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
console.debug(`Send keyboardReport keys: ${state.keys}, modifier: ${state.modifier}`);
if (rpcHidReady) {
console.debug("Sending keyboard report via HidRPC");
sendKeyboardEventHidRpc(state.keys, state.modifier);
return;
}
send("keyboardReport", { keys: state.keys, modifier: state.modifier }, (resp: JsonRpcResponse) => {
const handleLegacyKeyboardReport = useCallback(
async (keys: number[], modifier: number) => {
send("keyboardReport", { keys, modifier }, (resp: JsonRpcResponse) => {
if ("error" in resp) {
console.error(`Failed to send keyboard report ${state}`, resp.error);
console.error(`Failed to send keyboard report ${keys} ${modifier}`, resp.error);
}
// On older backends, we need to set the keysDownState manually since without the hidRpc API, the state doesn't trickle down from the backend
setKeysDownState({ modifier, keys });
});
},
[
rpcDataChannel?.readyState,
rpcHidReady,
send,
sendKeyboardEventHidRpc,
],
[send, setKeysDownState],
);
const sendKeystrokeLegacy = useCallback(async (keys: number[], modifier: number, ac?: AbortController) => {
return await new Promise<void>((resolve, reject) => {
const abortListener = () => {
reject(new Error("Keyboard report aborted"));
};
ac?.signal?.addEventListener("abort", abortListener);
send(
"keyboardReport",
{ keys, modifier },
params => {
if ("error" in params) return reject(params.error);
resolve();
},
);
});
}, [send]);
// resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
// This is useful for macros and when the browser loses focus to ensure that the keyboard state
// is clean.
const resetKeyboardState = useCallback(
async () => {
// Reset the keys buffer to zeros and the modifier state to zero
keysDownState.keys.length = hidKeyBufferSize;
keysDownState.keys.fill(0);
keysDownState.modifier = 0;
sendKeyboardEvent(keysDownState);
}, [keysDownState, sendKeyboardEvent]);
const resetKeyboardState = useCallback(async () => {
// Reset the keys buffer to zeros and the modifier state to zero
const { keys, modifier } = MACRO_RESET_KEYBOARD_STATE;
if (rpcHidReady) {
sendKeyboardEventHidRpc(keys, modifier);
} else {
// Older backends don't support the hidRpc API, so we send the full reset state
handleLegacyKeyboardReport(keys, modifier);
}
}, [rpcHidReady, sendKeyboardEventHidRpc, handleLegacyKeyboardReport]);
// executeMacro is used to execute a macro consisting of multiple steps.
// Each step can have multiple keys, multiple modifiers and a delay.
@@ -88,28 +132,90 @@ export default function useKeyboard() {
// After the delay, the keys and modifiers are released and the next step is executed.
// If a step has no keys or modifiers, it is treated as a delay-only step.
// A small pause is added between steps to ensure that the device can process the events.
const executeMacro = async (steps: { keys: string[] | null; modifiers: string[] | null; delay: number }[]) => {
for (const [index, step] of steps.entries()) {
const executeMacroRemote = useCallback(async (steps: MacroSteps) => {
const macro: KeyboardMacroStep[] = [];
for (const [_, step] of steps.entries()) {
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
const modifierMask: number = (step.modifiers || []).map(mod => modifiers[mod]).reduce((acc, val) => acc + val, 0);
const modifierMask: number = (step.modifiers || [])
.map(mod => modifiers[mod])
.reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay
if (keyValues.length > 0 || modifierMask > 0) {
sendKeyboardEvent({ keys: keyValues, modifier: modifierMask });
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
resetKeyboardState();
} else {
// This is a delay-only step, just wait for the delay amount
await new Promise(resolve => setTimeout(resolve, step.delay || 50));
}
// Add a small pause between steps if not the last step
if (index < steps.length - 1) {
await new Promise(resolve => setTimeout(resolve, 10));
macro.push({ keys: keyValues, modifier: modifierMask, delay: 20 });
macro.push({ ...MACRO_RESET_KEYBOARD_STATE, delay: step.delay || 100 });
}
}
};
sendKeyboardMacroEventHidRpc(macro);
}, [sendKeyboardMacroEventHidRpc]);
const executeMacroClientSide = useCallback(async (steps: MacroSteps) => {
const promises: (() => Promise<void>)[] = [];
const ac = new AbortController();
setAbortController(ac);
for (const [_, step] of steps.entries()) {
const keyValues = (step.keys || []).map(key => keys[key]).filter(Boolean);
const modifierMask: number = (step.modifiers || [])
.map(mod => modifiers[mod])
.reduce((acc, val) => acc + val, 0);
// If the step has keys and/or modifiers, press them and hold for the delay
if (keyValues.length > 0 || modifierMask > 0) {
promises.push(() => sendKeystrokeLegacy(keyValues, modifierMask, ac));
promises.push(() => resetKeyboardState());
promises.push(() => sleep(step.delay || 100));
}
}
const runAll = async () => {
for (const promise of promises) {
// Check if we've been aborted before executing each promise
if (ac.signal.aborted) {
throw new Error("Macro execution aborted");
}
await promise();
}
}
return await new Promise<void>((resolve, reject) => {
// Set up abort listener
const abortListener = () => {
reject(new Error("Macro execution aborted"));
};
ac.signal.addEventListener("abort", abortListener);
runAll()
.then(() => {
ac.signal.removeEventListener("abort", abortListener);
resolve();
})
.catch((error) => {
ac.signal.removeEventListener("abort", abortListener);
reject(error);
});
});
}, [sendKeystrokeLegacy, resetKeyboardState, setAbortController]);
const executeMacro = useCallback(async (steps: MacroSteps) => {
if (rpcHidReady) {
return executeMacroRemote(steps);
}
return executeMacroClientSide(steps);
}, [rpcHidReady, executeMacroRemote, executeMacroClientSide]);
const cancelExecuteMacro = useCallback(async () => {
if (abortController.current) {
abortController.current.abort();
}
if (!rpcHidReady) return;
// older versions don't support this API,
// and all paste actions are pure-frontend,
// we don't need to cancel it actually
cancelOngoingKeyboardMacroHidRpc();
}, [rpcHidReady, cancelOngoingKeyboardMacroHidRpc, abortController]);
// handleKeyPress is used to handle a key press or release event.
// This function handle both key press and key release events.
@@ -131,9 +237,16 @@ export default function useKeyboard() {
// in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
sendKeypressEventHidRpc(key, press);
} else {
// if the keyPress api is not available, we need to handle the key locally
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
sendKeyboardEvent(downState); // then we send the full state
// Older backends don't support the hidRpc API, so we need:
// 1. Calculate the state
// 2. Send the newly calculated state to the device
const downState = simulateDeviceSideKeyHandlingForLegacyDevices(
keysDownState,
key,
press,
);
handleLegacyKeyboardReport(downState.keys, downState.modifier);
// if we just sent ErrorRollOver, reset to empty state
if (downState.keys[0] === hidErrorRollOver) {
@@ -142,17 +255,21 @@ export default function useKeyboard() {
}
},
[
rpcHidReady,
keysDownState,
resetKeyboardState,
rpcDataChannel?.readyState,
sendKeyboardEvent,
rpcHidReady,
sendKeypressEventHidRpc,
keysDownState,
handleLegacyKeyboardReport,
resetKeyboardState,
],
);
// IMPORTANT: See the keyPressReportApiAvailable comment above for the reason this exists
function simulateDeviceSideKeyHandlingForLegacyDevices(state: KeysDownState, key: number, press: boolean): KeysDownState {
function simulateDeviceSideKeyHandlingForLegacyDevices(
state: KeysDownState,
key: number,
press: boolean,
): KeysDownState {
// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
// for handling key presses and releases. It ensures that the USB gadget
// behaves similarly to a real USB HID keyboard. This logic is paralleled
@@ -164,7 +281,7 @@ export default function useKeyboard() {
if (modifierMask !== 0) {
// If the key is a modifier key, we update the keyboardModifier state
// by setting or clearing the corresponding bit in the modifier byte.
// This allows us to track the state of dynamic modifier keys like
// This allows us to track the state of dynamic modifier keys like
// Shift, Control, Alt, and Super.
if (press) {
modifiers |= modifierMask;
@@ -181,7 +298,7 @@ export default function useKeyboard() {
// and if we find a zero byte, we can place the key there (if press is true)
if (keys[i] === key || keys[i] === 0) {
if (press) {
keys[i] = key // overwrites the zero byte or the same key if already pressed
keys[i] = key; // overwrites the zero byte or the same key if already pressed
} else {
// we are releasing the key, remove it from the buffer
if (keys[i] !== 0) {
@@ -197,18 +314,20 @@ export default function useKeyboard() {
// If we reach here it means we didn't find an empty slot or the key in the buffer
if (overrun) {
if (press) {
console.warn(`keyboard buffer overflow current keys ${keys}, key: ${key} not added`);
console.warn(
`keyboard buffer overflow current keys ${keys}, key: ${key} not added`,
);
// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
keys.length = hidKeyBufferSize;
keys.fill(hidErrorRollOver);
} else {
// If we are releasing a key, and we didn't find it in a slot, who cares?
console.debug(`key ${key} not found in buffer, nothing to release`)
console.debug(`key ${key} not found in buffer, nothing to release`);
}
}
}
return { modifier: modifiers, keys };
}
return { handleKeyPress, resetKeyboardState, executeMacro };
return { handleKeyPress, resetKeyboardState, executeMacro, cancelExecuteMacro };
}

4
ui/src/routes/devices.$id.tsx
View File

@@ -583,6 +583,7 @@ export default function KvmIdRoute() {
keyboardLedState, setKeyboardLedState,
keysDownState, setKeysDownState, setUsbState,
} = useHidStore();
const setHidRpcDisabled = useRTCStore(state => state.setHidRpcDisabled);
const [hasUpdated, setHasUpdated] = useState(false);
const { navigateTo } = useDeviceUiNavigation();
@@ -695,6 +696,7 @@ export default function KvmIdRoute() {
if (resp.error.code === -32601) {
// if we don't support key down state, we know key press is also not available
console.warn("Failed to get key down state, switching to old-school", resp.error);
setHidRpcDisabled(true);
} else {
console.error("Failed to get key down state", resp.error);
}
@@ -705,7 +707,7 @@ export default function KvmIdRoute() {
}
setNeedKeyDownState(false);
});
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState]);
}, [keysDownState, needKeyDownState, rpcDataChannel?.readyState, send, setKeysDownState, setHidRpcDisabled]);
// When the update is successful, we need to refresh the client javascript and show a success modal
useEffect(() => {