feat(keyboard): update keyboard layouts and key display mappings for multiple languages

Signed-off-by: luckfox-eng29 <eng29@luckfox.com>

internal/hidrpc/hidrpc.go

@@ -3,6 +3,7 @@ package hidrpc
 import "fmt"
 
 type Handler interface {
+	HandleHandshake(version byte) error
 	HandleKeyboardReport(modifier byte, keys []byte) error
 	HandleKeypressReport(key byte, press bool) error
 	HandleKeypressKeepAlive() error
@@ -25,6 +26,11 @@ func (s *Server) HandleMessage(data []byte) error {
 	}
 
 	switch msg.Type {
+	case MessageTypeHandshake:
+		if len(msg.Data) < 1 {
+			return fmt.Errorf("invalid handshake length: %d", len(msg.Data))
+		}
+		return s.handler.HandleHandshake(msg.Data[0])
 	case MessageTypeKeyboardReport:
 		if len(msg.Data) < 7 {
 			return fmt.Errorf("invalid keyboard report length: %d", len(msg.Data))

internal/hidrpc/message.go

@@ -30,6 +30,10 @@ func MarshalKeyboardReport(modifier byte, keys []byte) []byte {
 	return data
 }
 
+func MarshalHandshake(version byte) []byte {
+	return []byte{MessageTypeHandshake, version}
+}
+
 func MarshalKeypressReport(key byte, press bool) []byte {
 	data := make([]byte, 3)
 	data[0] = MessageTypeKeypressReport

internal/usbgadget/hid_keyboard.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"reflect"
 	"strings"
 	"time"
@@ -211,15 +212,16 @@ func (u *UsbGadget) OpenKeyboardHidFile() error {
 }
 
 func (u *UsbGadget) keyboardWriteHidFile(data []byte) error {
-	if err := u.openKeyboardHidFile(); err != nil {
+	var parts []string
-		return err
+	for _, b := range data {
+		parts = append(parts, fmt.Sprintf("\\x%02x", b))
 	}
+	hexString := strings.Join(parts, "")
 
-	_, err := u.keyboardHidFile.Write(data)
+	cmd := exec.Command("sh", "-c", fmt.Sprintf("echo -n -e '%s' > /dev/hidg0", hexString))
+	err := cmd.Run()
 	if err != nil {
 		u.logWithSupression("keyboardWriteHidFile", 100, u.log, err, "failed to write to hidg0")
-		u.keyboardHidFile.Close()
-		u.keyboardHidFile = nil
 		return err
 	}
 	u.resetLogSuppressionCounter("keyboardWriteHidFile")

ui/src/keyboardLayouts/cs_CZ.ts

@@ -99,11 +99,11 @@ export const chars = {
   "Ẇ": { key: "KeyW", shift: true, accentKey: keyOverdot },
   X: { key: "KeyX", shift: true },
   "Ẋ": { key: "KeyX", shift: true, accentKey: keyOverdot },
-  Y: { key: "KeyY", shift: true },
+  Y: { key: "KeyZ", shift: true },
-  "Ý": { key: "KeyY", shift: true, accentKey: keyAcute },
+  "Ý": { key: "KeyZ", shift: true, accentKey: keyAcute },
-  "Ẏ": { key: "KeyY", shift: true, accentKey: keyOverdot },
+  "Ẏ": { key: "KeyZ", shift: true, accentKey: keyOverdot },
-  Z: { key: "KeyZ", shift: true },
+  Z: { key: "KeyY", shift: true },
-  "Ż": { key: "KeyZ", shift: true, accentKey: keyOverdot },
+  "Ż": { key: "KeyY", shift: true, accentKey: keyOverdot },
   a: { key: "KeyA" },
   "ä": { key: "KeyA", accentKey: keyTrema },
   "â": { key: "KeyA", accentKey: keyHat },
@@ -191,10 +191,10 @@ export const chars = {
   x: { key: "KeyX" },
   "#": { key: "KeyX", altRight: true },
   "ẋ": { key: "KeyX", accentKey: keyOverdot },
-  y: { key: "KeyY" },
+  y: { key: "KeyZ" },
-  "ẏ": { key: "KeyY", accentKey: keyOverdot },
+  "ẏ": { key: "KeyZ", accentKey: keyOverdot },
-  z: { key: "KeyZ" },
+  z: { key: "KeyY" },
-  "ż": { key: "KeyZ", accentKey: keyOverdot },
+  "ż": { key: "KeyY", accentKey: keyOverdot },
   ";": { key: "Backquote" },
   "°": { key: "Backquote", shift: true, deadKey: true }, 
   "+": { key: "Digit1" },
@@ -245,11 +245,19 @@ export const chars = {
   Tab: { key: "Tab" },
 } as Record<string, KeyCombo>;
 
+const cs_CZ_keyDisplayMap = {
+  ...keyDisplayMap,
+  KeyY: "z",
+  KeyZ: "y",
+  "(KeyY)": "Z",
+  "(KeyZ)": "Y",
+} as Record<string, string>;
+
 export const cs_CZ: KeyboardLayout = {
   isoCode,
   name,
   chars,
-  keyDisplayMap,
+  keyDisplayMap: cs_CZ_keyDisplayMap,
   modifierDisplayMap,
   virtualKeyboard,
 };

ui/src/keyboardLayouts/de_CH.ts

@@ -1,5 +1,6 @@
 import { KeyboardLayout, KeyCombo } from "../keyboardLayouts"
 import { modifierDisplayMap, keyDisplayMap, virtualKeyboard } from "./en_US"
+export { keyDisplayMap } from "./en_US";
 
 const name = "Schwiizerdütsch";
 const isoCode = "de-CH";
@@ -166,11 +167,19 @@ export const chars = {
   Tab: { key: "Tab" },
 } as Record<string, KeyCombo>;
 
+export const de_CH_keyDisplayMap = {
+  ...keyDisplayMap,
+  KeyY: "z",
+  KeyZ: "y",
+  "(KeyY)": "Z",
+  "(KeyZ)": "Y",
+} as Record<string, string>;
+
 export const de_CH: KeyboardLayout = {
   isoCode,
   name,
   chars,
-  keyDisplayMap,
+  keyDisplayMap: de_CH_keyDisplayMap,
   modifierDisplayMap,
   virtualKeyboard,
 };

ui/src/keyboardLayouts/de_DE.ts

@@ -153,11 +153,19 @@ export const chars = {
   Tab: { key: "Tab" },
 } as Record<string, KeyCombo>;
 
+const de_DE_keyDisplayMap = {
+  ...keyDisplayMap,
+  KeyY: "z",
+  KeyZ: "y",
+  "(KeyY)": "Z",
+  "(KeyZ)": "Y",
+} as Record<string, string>;
+
 export const de_DE: KeyboardLayout = {
   isoCode,
   name,
   chars,
-  keyDisplayMap,
+  keyDisplayMap: de_DE_keyDisplayMap,
   modifierDisplayMap,
   virtualKeyboard,
 };

ui/src/keyboardLayouts/fr_BE.ts

@@ -58,10 +58,10 @@ export const chars = {
   "Ù": { key: "KeyU", shift: true, accentKey: keyGrave },
   "Ũ": { key: "KeyU", shift: true, accentKey: keyTilde },
   V: { key: "KeyV", shift: true },
-  W: { key: "KeyW", shift: true },
+  W: { key: "KeyZ", shift: true },
   X: { key: "KeyX", shift: true },
-  Y: { key: "KeyZ", shift: true },
+  Y: { key: "KeyY", shift: true },
-  Z: { key: "KeyY", shift: true },
+  Z: { key: "KeyW", shift: true },
   a: { key: "KeyQ" },
   "ä": { key: "KeyQ", accentKey: keyTrema },
   "â": { key: "KeyQ", accentKey: keyHat },
@@ -106,10 +106,10 @@ export const chars = {
   "ú": { key: "KeyU", accentKey: keyAcute },
   "ũ": { key: "KeyU", accentKey: keyTilde },
   v: { key: "KeyV" },
-  w: { key: "KeyW" },
+  w: { key: "KeyZ" },
   x: { key: "KeyX" },
-  y: { key: "KeyZ" },
+  y: { key: "KeyY" },
-  z: { key: "KeyY" },
+  z: { key: "KeyW" },
   "²": { key: "Backquote" },
   "³": { key: "Backquote", shift: true },
   "&": { key: "Digit1" },
@@ -168,11 +168,27 @@ export const chars = {
   Tab: { key: "Tab" },
 } as Record<string, KeyCombo>;
 
+const fr_BE_keyDisplayMap = {
+  ...keyDisplayMap,
+  KeyA: "q",
+  KeyQ: "a",
+  KeyW: "z",
+  KeyZ: "w",
+  Semicolon: "m",
+  KeyM: ",",
+  "(KeyA)": "Q",
+  "(KeyQ)": "A",
+  "(KeyW)": "Z",
+  "(KeyZ)": "W",
+  "(Semicolon)": "M",
+  "(KeyM)": "?",
+} as Record<string, string>;
+
 export const fr_BE: KeyboardLayout = {
   isoCode,
   name,
   chars,
-  keyDisplayMap,
+  keyDisplayMap: fr_BE_keyDisplayMap,
   modifierDisplayMap,
   virtualKeyboard,
 };

ui/src/keyboardLayouts/fr_CH.ts

@@ -1,6 +1,6 @@
 import { KeyboardLayout, KeyCombo } from "../keyboardLayouts"
-import { chars as chars_de_CH } from "./de_CH"
+import { chars as chars_de_CH, de_CH_keyDisplayMap } from "./de_CH"
-import { modifierDisplayMap, keyDisplayMap, virtualKeyboard } from "./en_US"
+import { modifierDisplayMap, virtualKeyboard } from "./en_US"
 
 const name = "Français de Suisse";
 const isoCode = "fr-CH";
@@ -15,11 +15,21 @@ export const chars = {
   "ä": { key: "Quote", shift: true },
 } as Record<string, KeyCombo>;
 
+const fr_CH_keyDisplayMap = {
+  ...de_CH_keyDisplayMap,
+  BracketLeft: "è",
+  "(BracketLeft)": "ü",
+  Semicolon: "é",
+  "(Semicolon)": "ö",
+  Quote: "à",
+  "(Quote)": "ä",
+} as Record<string, string>;
+
 export const fr_CH: KeyboardLayout = {
   isoCode,
   name,
   chars,
-  keyDisplayMap,
+  keyDisplayMap: fr_CH_keyDisplayMap,
   modifierDisplayMap,
   virtualKeyboard,
 };

ui/src/keyboardLayouts/fr_FR.ts

@@ -140,11 +140,27 @@ export const chars = {
   Tab: { key: "Tab" },
 } as Record<string, KeyCombo>;
 
+const fr_FR_keyDisplayMap = {
+  ...keyDisplayMap,
+  KeyA: "q",
+  KeyQ: "a",
+  KeyW: "z",
+  KeyZ: "w",
+  Semicolon: "m",
+  KeyM: ",",
+  "(KeyA)": "Q",
+  "(KeyQ)": "A",
+  "(KeyW)": "Z",
+  "(KeyZ)": "W",
+  "(Semicolon)": "M",
+  "(KeyM)": "?",
+} as Record<string, string>;
+
 export const fr_FR: KeyboardLayout = {
   isoCode,
   name,
   chars,
-  keyDisplayMap,
+  keyDisplayMap: fr_FR_keyDisplayMap,
   modifierDisplayMap,
   virtualKeyboard,
 };

ui/src/keyboardLayouts/nb_NO.ts

@@ -60,8 +60,8 @@ export const chars = {
   V: { key: "KeyV", shift: true },
   W: { key: "KeyW", shift: true },
   X: { key: "KeyX", shift: true },
-  Y: { key: "KeyZ", shift: true },
+  Y: { key: "KeyY", shift: true },
-  Z: { key: "KeyY", shift: true },
+  Z: { key: "KeyZ", shift: true },
   a: { key: "KeyA" },
   "ä": { key: "KeyA", accentKey: keyTrema },
   "á": { key: "KeyA", accentKey: keyAcute },
@@ -112,8 +112,8 @@ export const chars = {
   v: { key: "KeyV" },
   w: { key: "KeyW" },
   x: { key: "KeyX" },
-  y: { key: "KeyZ" },
+  y: { key: "KeyY" },
-  z: { key: "KeyY" },
+  z: { key: "KeyZ" },
   "|": { key: "Backquote" }, 
   "§": { key: "Backquote", shift: true },
   1: { key: "Digit1" },

ui/src/keyboardLayouts/sl_SI.ts

@@ -146,12 +146,19 @@ export const chars = {
   Delete: { key: "Delete" },
 } as Record<string, KeyCombo>;
 
+const sl_SI_keyDisplayMap = {
+  ...en_US.keyDisplayMap,
+  KeyY: "z",
+  KeyZ: "y",
+  "(KeyY)": "Z",
+  "(KeyZ)": "Y",
+} as Record<string, string>;
+
 export const sl_SI: KeyboardLayout = {
   isoCode: isoCode,
   name: name,
   chars: chars,
-  // TODO need to localize these maps and layouts
+  keyDisplayMap: sl_SI_keyDisplayMap,
-  keyDisplayMap: en_US.keyDisplayMap,
   modifierDisplayMap: en_US.modifierDisplayMap,
   virtualKeyboard: en_US.virtualKeyboard,
 };

ui/src/layout/components_bottom/keyboard/KeyboardPanel.tsx

@@ -6,7 +6,7 @@ import { CloseOutlined } from '@ant-design/icons';
 import { useReactAt } from "i18n-auto-extractor/react";
 
 import ScrollThrottlingSelect, { Option } from "@components/ScrollThrottlingSelect";
-import { layouts } from "@/keyboardLayouts";
+import { layouts, keyboards } from "@/keyboardLayouts";
 import { KeyboardLedSync, useSettingsStore } from "@/hooks/stores";
 import { useJsonRpc } from "@/hooks/useJsonRpc";
 import notifications from "@/notifications";
@@ -24,11 +24,21 @@ const KeyboardPanel: React.FC = () => {
 
   const [layoutOptions, setLayoutOptions] = useState<Option[]>();
   const [maxShowCount, setMaxShowCount] = useState(3);
+
+  const layoutAbbrevMap = useMemo(() => {
+    const map: Record<string, string> = {};
+    keyboards.forEach(kb => {
+      const oldCode = kb.isoCode.replace("-", "_");
+      map[oldCode] = oldCode;
+    });
+    return map;
+  }, []);
+
   useEffect(() => {
     const curLayoutOptions = (() => {
       const options = Object.entries(layouts).map(([code, language]) => ({
         value: code,
-        label: language,
+        label: `${language} (${layoutAbbrevMap[code] || code})`,
       }));
 
       const currentLayout = keyboardLayout ?? "";
@@ -47,7 +57,7 @@ const KeyboardPanel: React.FC = () => {
       return options;
     })();
     setLayoutOptions(curLayoutOptions);
-  }, [layouts, keyboardLayout]);
+  }, [layouts, keyboardLayout, layoutAbbrevMap]);
 
   const safeKeyboardLayout = useMemo(() => {
     if (keyboardLayout && keyboardLayout.length > 0)

ui/src/layout/core/bar_bottom/BottomBarMobile.tsx

@@ -1,4 +1,4 @@
-import React, { useCallback, useEffect, useState } from "react";
+import React, { useCallback, useEffect, useMemo, useState } from "react";
 import { Button as AntdButton, Typography } from "antd";
 import { useReactAt } from "i18n-auto-extractor/react";
 import KeyboardSVG from "@assets/second/keyboard.svg?react";
@@ -25,6 +25,7 @@ import {
   useVpnStore,
 } from "@/hooks/stores";
 import { useJsonRpc } from "@/hooks/useJsonRpc";
+import { keyboards } from "@/keyboardLayouts";
 import {
   button_primary_color,
   dark_bd_style,
@@ -47,7 +48,13 @@ const views = [
 export default function BottomBarMobile() {
   const { $at } = useReactAt();
   const keyboardLedState = useHidStore(state => state.keyboardLedState);
+  const keyboardLayout = useSettingsStore(state => state.keyboardLayout);
   const { isDark } = useTheme();
+
+  const layoutAbbrev = useMemo(() => {
+    if (!keyboardLayout) return "en_US";
+    return keyboardLayout;
+  }, [keyboardLayout]);
   const setDisableFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap);
   const toggleSidebarView = useUiStore(state => state.toggleSidebarView);
   const forceHttp = useSettingsStore(state => state.forceHttp);
@@ -144,6 +151,7 @@ export default function BottomBarMobile() {
           <LedStatusButton ledState={keyboardLedState?.num_lock} text={$at("Num")} />
           <LedStatusButton ledState={keyboardLedState?.caps_lock} text={$at("Caps")} />
           <LedStatusButton ledState={keyboardLedState?.scroll_lock} text={$at("Scroll")} />
+          <span className="pl-2 text-xs opacity-70">{layoutAbbrev}</span>
         </div>
         <div className="w-[20%] flex flex-row flex-wrap items-center justify-end">
           <AntdButton

ui/src/layout/core/bar_bottom/BottomBarPC.tsx

@@ -27,6 +27,7 @@ import {
   useVpnStore,
 } from "@/hooks/stores";
 import { keys, modifiers } from "@/keyboardMappings";
+import { keyboards } from "@/keyboardLayouts";
 import BottomPopoverButton from "@components/PopoverButton";
 import MousePanel from "@components/MousePanel";
 import KeyboardPanel from "@/layout/components_bottom/keyboard/KeyboardPanel";
@@ -57,8 +58,14 @@ export default function BottomBarPC() {
 
 
   const keyboardLedState = useHidStore(state => state.keyboardLedState);
+  const keyboardLayout = useSettingsStore(state => state.keyboardLayout);
   const isTurnServerInUse = useRTCStore(state => state.isTurnServerInUse);
 
+  const layoutAbbrev = useMemo(() => {
+    if (!keyboardLayout) return "en_US";
+    return keyboardLayout;
+  }, [keyboardLayout]);
+
   const [hostname, setHostname] = useState("");
   const [send] = useJsonRpc();
   const peerConnection = useRTCStore(state => state.peerConnection);
@@ -164,6 +171,7 @@ export default function BottomBarPC() {
                   ledState={keyboardLedState?.scroll_lock}
                   text={$at("Scroll")}
                 />
+                <span className="pl-1 text-xs opacity-70" style={{ fontSize: 12 }}>{layoutAbbrev}</span>
               </div>
             }
             align="left"

ui/src/layout/core/desktop/hooks/useKeyboardEvents.ts

@@ -3,6 +3,7 @@ import { useCallback } from "react";
 import  useKeyboard  from "@/hooks/useKeyboard";
 import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores";
 import { keys, modifiers } from "@/keyboardMappings";
+import { keyboards } from "@/keyboardLayouts";
 import { eventMatchesShortcut } from "@/utils/shortcuts";
 
 export const useKeyboardEvents = (
@@ -18,6 +19,28 @@ export const useKeyboardEvents = (
   const pasteShortcutEnabled = useSettingsStore(state => state.pasteShortcutEnabled);
   const pasteShortcut = useSettingsStore(state => state.pasteShortcut);
   const isOcrMode = useUiStore(state => state.isOcrMode);
+  const keyboardLayout = useSettingsStore(state => state.keyboardLayout);
+
+  const remapCode = useCallback((code: string, key: string): string => {
+    const modifierCodes = ["ControlLeft", "ControlRight", "ShiftLeft", "ShiftRight", "AltLeft", "AltRight", "MetaLeft", "MetaRight", "CapsLock", "Tab", "Enter", "Backspace", "Delete", "Insert", "Home", "End", "PageUp", "PageDown", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Escape", "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12", "PrintScreen", "ScrollLock", "Pause", "ContextMenu", "Menu"];
+    if (modifierCodes.includes(code)) return code;
+    if (code.startsWith("Digit") || code.startsWith("Numpad")) return code;
+    if (code.startsWith("Key") && code.length === 4) {
+      const letter = code.charAt(3);
+      if (letter >= "A" && letter <= "Z") {
+        const isoCode = (keyboardLayout || "en-US").replace("_", "-");
+        const layout = keyboards.find(k => k.isoCode === isoCode);
+        if (layout && layout.chars) {
+          const charLower = key.toLowerCase();
+          const charEntry = layout.chars[charLower] || layout.chars[key];
+          if (charEntry && charEntry.key && typeof charEntry.key === "string" && charEntry.key !== code) {
+            return charEntry.key;
+          }
+        }
+      }
+    }
+    return code;
+  }, [keyboardLayout]);
 
   const handleModifierKeys = useCallback((e: KeyboardEvent, activeModifiers: number[]) => {
     const { shiftKey, ctrlKey, altKey, metaKey } = e;
@@ -59,6 +82,8 @@ export const useKeyboardEvents = (
       code = "IntlBackslash";
     }
 
+    code = remapCode(code, key);
+
     const newKeys = [...prev.activeKeys, keys[code]].filter(Boolean);
     const newModifiers = handleModifierKeys(e, [...prev.activeModifiers, modifiers[code]]);
 
@@ -77,13 +102,15 @@ export const useKeyboardEvents = (
     
     // Still update the full state for legacy compatibility and UI display
     sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
-  }, [handleModifierKeys, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, pasteShortcutEnabled, pasteShortcut, pasteCaptureRef, isReinitializingGadget, isOcrMode]);
+  }, [handleModifierKeys, remapCode, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, pasteShortcutEnabled, pasteShortcut, pasteCaptureRef, isReinitializingGadget, isOcrMode]);
 
   const keyUpHandler = useCallback((e: KeyboardEvent) => {
     if (isOcrMode) return;
     if (isReinitializingGadget) return;
     e.preventDefault();
     const prev = useHidStore.getState();
+    const key = e.key;
+    let code = remapCode(e.code, key);
 
     if (!isKeyboardLedManagedByHost) {
       setIsNumLockActive(e.getModifierState("NumLock"));
@@ -91,21 +118,21 @@ export const useKeyboardEvents = (
       setIsScrollLockActive(e.getModifierState("ScrollLock"));
     }
 
-    const newKeys = prev.activeKeys.filter(k => k !== keys[e.code]).filter(Boolean);
+    const newKeys = prev.activeKeys.filter(k => k !== keys[code]).filter(Boolean);
     const newModifiers = handleModifierKeys(
       e,
-      prev.activeModifiers.filter(k => k !== modifiers[e.code]),
+      prev.activeModifiers.filter(k => k !== modifiers[code]),
     );
 
     // Send per-key release event
-    const hidKey = keys[e.code];
+    const hidKey = keys[code];
     if (hidKey !== undefined) {
       sendKeypress(hidKey, false);
     }
     
     // Still update the full state for legacy compatibility and UI display
     sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
-  }, [handleModifierKeys, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, isOcrMode]);
+  }, [handleModifierKeys, remapCode, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, isOcrMode, isReinitializingGadget]);
 
   const setupKeyboardEvents = useCallback(() => {
     const abortController = new AbortController();

usb.go

@@ -157,6 +157,14 @@ type hidRpcHandler struct {
 	session *Session
 }
 
+func (h *hidRpcHandler) HandleHandshake(version byte) error {
+	if h.session.HidChannel != nil {
+		handshakeData := hidrpc.MarshalHandshake(version)
+		return h.session.HidChannel.Send(handshakeData)
+	}
+	return nil
+}
+
 func (h *hidRpcHandler) HandleKeyboardReport(modifier byte, keys []byte) error {
 	return rpcKeyboardReport(modifier, keys)
 }