feat: release keyPress automatically (#796)

* feat: release keyPress automatically * send keepalive when pressing the key * remove logging * clean up logging * chore: use unreliable channel to send keepalive events * chore: use ordered unreliable channel for pointer events * chore: adjust auto release key interval * chore: update logging for kbdAutoReleaseLock * chore: update comment for KEEPALIVE_INTERVAL * fix: should cancelAutorelease when pressed is true * fix: handshake won't happen if webrtc reconnects * chore: add trace log for writeWithTimeout * chore: add timeout for KeypressReport * chore: use the proper key to send release command * refactor: simplify HID RPC keyboard input handling and improve key state management - Updated `handleHidRPCKeyboardInput` to return errors directly instead of keys down state. - Refactored `rpcKeyboardReport` and `rpcKeypressReport` to return errors instead of states. - Introduced a queue for managing key down state updates in the `Session` struct to prevent input handling stalls. - Adjusted the `UpdateKeysDown` method to handle state changes more efficiently. - Removed unnecessary logging and commented-out code for clarity. * refactor: enhance keyboard auto-release functionality and key state management * fix: correct Windows default auto-repeat delay comment from 1ms to 1s * refactor: send keypress as early as possible * refactor: replace console.warn with console.info for HID RPC channel events * refactor: remove unused NewKeypressKeepAliveMessage function from HID RPC * fix: handle error in key release process and log warnings * fix: log warning on keypress report failure * fix: update auto-release keyboard interval to 225 * refactor: enhance keep-alive handling and jitter compensation in HID RPC - Implemented staleness guard to ignore outdated keep-alive packets. - Added jitter compensation logic to adjust timer extensions based on packet arrival times. - Introduced new methods for managing keep-alive state and reset functionality in the Session struct. - Updated auto-release delay mechanism to use dynamic durations based on keep-alive timing. - Adjusted keep-alive interval in the UI to improve responsiveness. * gofmt * clean up code * chore: use dynamic duration for scheduleAutoRelease * Use harcoded timer reset value for now * fix: prevent nil pointer dereference when stopping timers in Close method * refactor: remove nil check for kbdAutoReleaseTimers in DelayAutoReleaseWithDuration * refactor: optimize dependencies in useHidRpc hooks * refactor: streamline keep-alive timer management in useKeyboard hook * refactor: clarify comments in useKeyboard hook for resetKeyboardState function * refactor: reduce keysDownStateQueueSize * refactor: close and reset keysDownStateQueue in newSession function * chore: resolve conflicts * resolve conflicts --------- Co-authored-by: Adam Shiervani <adam.shiervani@gmail.com>
2026-01-18 03:28:19 +01:00 · 2025-09-18 13:35:47 +02:00
parent 72e3013337
commit afb146d78c
18 changed files with 767 additions and 298 deletions
--- a/internal/hidrpc/hidrpc.go
+++ b/internal/hidrpc/hidrpc.go
@@ -15,6 +15,7 @@ const (
 	TypePointerReport             MessageType = 0x03
 	TypeWheelReport               MessageType = 0x04
 	TypeKeypressReport            MessageType = 0x05
+	TypeKeypressKeepAliveReport   MessageType = 0x09
 	TypeMouseReport               MessageType = 0x06
 	TypeKeyboardMacroReport       MessageType = 0x07
 	TypeCancelKeyboardMacroReport MessageType = 0x08
--- a/internal/hidrpc/message.go
+++ b/internal/hidrpc/message.go
@@ -44,6 +44,8 @@ func (m *Message) String() string {
 			return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
 		}
 		return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
+	case TypeKeypressKeepAliveReport:
+		return "KeypressKeepAliveReport"
 	case TypeKeyboardMacroReport:
 		if len(m.d) < 5 {
 			return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d)
--- a/internal/usbgadget/hid_keyboard.go
+++ b/internal/usbgadget/hid_keyboard.go
@@ -5,7 +5,11 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
 	"time"
+
+	"github.com/rs/xid"
+	"github.com/rs/zerolog"
 )

 var keyboardConfig = gadgetConfigItem{
@@ -145,32 +149,95 @@ func (u *UsbGadget) GetKeysDownState() KeysDownState {
 	return u.keysDownState
 }

-func (u *UsbGadget) updateKeyDownState(state KeysDownState) {
-	u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("acquiring keyboardStateLock for updateKeyDownState")
+func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) {
+	u.onKeysDownChange = &f
+}

-	// this is intentional to unlock keyboard state lock before onKeysDownChange callback
-	{
-		u.keyboardStateLock.Lock()
-		defer u.keyboardStateLock.Unlock()
+func (u *UsbGadget) SetOnKeepAliveReset(f func()) {
+	u.onKeepAliveReset = &f
+}

-		if u.keysDownState.Modifier == state.Modifier &&
-			bytes.Equal(u.keysDownState.Keys, state.Keys) {
-			return // No change in key down state
-		}
+// DefaultAutoReleaseDuration is the default duration for auto-release of a key.
+const DefaultAutoReleaseDuration = 100 * time.Millisecond

-		u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated")
-		u.keysDownState = state
+func (u *UsbGadget) scheduleAutoRelease(key byte) {
+	u.kbdAutoReleaseLock.Lock()
+	defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease scheduled")
+
+	if u.kbdAutoReleaseTimers[key] != nil {
+		u.kbdAutoReleaseTimers[key].Stop()
 	}

-	if u.onKeysDownChange != nil {
-		u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange")
-		(*u.onKeysDownChange)(state)
-		u.log.Trace().Interface("state", state).Msg("onKeysDownChange called")
+	// TODO: make this configurable
+	// We currently hardcode the duration to 100ms
+	// However, it should be the same as the duration of the keep-alive reset called baseExtension.
+	u.kbdAutoReleaseTimers[key] = time.AfterFunc(100*time.Millisecond, func() {
+		u.performAutoRelease(key)
+	})
+}
+
+func (u *UsbGadget) cancelAutoRelease(key byte) {
+	u.kbdAutoReleaseLock.Lock()
+	defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease cancelled")
+
+	if timer := u.kbdAutoReleaseTimers[key]; timer != nil {
+		timer.Stop()
+		u.kbdAutoReleaseTimers[key] = nil
+		delete(u.kbdAutoReleaseTimers, key)
+
+		// Reset keep-alive timing when key is released
+		if u.onKeepAliveReset != nil {
+			(*u.onKeepAliveReset)()
+		}
 	}
 }

-func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) {
-	u.onKeysDownChange = &f
+func (u *UsbGadget) DelayAutoReleaseWithDuration(resetDuration time.Duration) {
+	u.kbdAutoReleaseLock.Lock()
+	defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease delayed")
+
+	u.log.Debug().Dur("reset_duration", resetDuration).Msg("delaying auto-release with dynamic duration")
+
+	for _, timer := range u.kbdAutoReleaseTimers {
+		if timer != nil {
+			timer.Reset(resetDuration)
+		}
+	}
+}
+
+func (u *UsbGadget) performAutoRelease(key byte) {
+	u.kbdAutoReleaseLock.Lock()
+
+	if u.kbdAutoReleaseTimers[key] == nil {
+		u.log.Warn().Uint8("key", key).Msg("autoRelease timer not found")
+		u.kbdAutoReleaseLock.Unlock()
+		return
+	}
+
+	u.kbdAutoReleaseTimers[key].Stop()
+	u.kbdAutoReleaseTimers[key] = nil
+	delete(u.kbdAutoReleaseTimers, key)
+	u.kbdAutoReleaseLock.Unlock()
+
+	// Skip if already released
+	state := u.GetKeysDownState()
+	alreadyReleased := true
+
+	for i := range state.Keys {
+		if state.Keys[i] == key {
+			alreadyReleased = false
+			break
+		}
+	}
+
+	if alreadyReleased {
+		return
+	}
+
+	_, err := u.keypressReport(key, false)
+	if err != nil {
+		u.log.Warn().Uint8("key", key).Msg("failed to release key")
+	}
 }

 func (u *UsbGadget) listenKeyboardEvents() {
@@ -242,7 +309,11 @@ func (u *UsbGadget) OpenKeyboardHidFile() error {
 	return u.openKeyboardHidFile()
 }

+var keyboardWriteHidFileLock sync.Mutex
+
 func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error {
+	keyboardWriteHidFileLock.Lock()
+	defer keyboardWriteHidFileLock.Unlock()
 	if err := u.openKeyboardHidFile(); err != nil {
 		return err
 	}
@@ -266,17 +337,29 @@ func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) KeysDownState {
 		}
 	}

-	downState := KeysDownState{
+	state := KeysDownState{
 		Modifier: modifier,
 		Keys:     []byte(keys[:]),
 	}
-	u.updateKeyDownState(downState)
-	return downState
+
+	u.keyboardStateLock.Lock()
+
+	if u.keysDownState.Modifier == state.Modifier &&
+		bytes.Equal(u.keysDownState.Keys, state.Keys) {
+		u.keyboardStateLock.Unlock()
+		return state // No change in key down state
+	}
+
+	u.keysDownState = state
+	u.keyboardStateLock.Unlock()
+
+	if u.onKeysDownChange != nil {
+		(*u.onKeysDownChange)(state) // this enques to the outgoing hidrpc queue via usb.go → currentSession.enqueueKeysDownState(...)
+	}
+	return state
 }

-func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, error) {
-	u.keyboardLock.Lock()
-	defer u.keyboardLock.Unlock()
+func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) error {
 	defer u.resetUserInputTime()

 	if len(keys) > hidKeyBufferSize {
@@ -291,7 +374,8 @@ func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, e
 		u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keyboard report to hidg0")
 	}

-	return u.UpdateKeysDown(modifier, keys), err
+	u.UpdateKeysDown(modifier, keys)
+	return err
 }

 const (
@@ -331,17 +415,23 @@ var KeyCodeToMaskMap = map[byte]byte{
 	RightSuper:   ModifierMaskRightSuper,
 }

-func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) {
-	u.keyboardLock.Lock()
-	defer u.keyboardLock.Unlock()
+func (u *UsbGadget) keypressReport(key byte, press bool) (KeysDownState, error) {
 	defer u.resetUserInputTime()

+	l := u.log.With().Uint8("key", key).Bool("press", press).Logger()
+	if l.GetLevel() <= zerolog.DebugLevel {
+		requestID := xid.New()
+		l = l.With().Str("requestID", requestID.String()).Logger()
+	}
+
 	// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
 	// for handling key presses and releases. It ensures that the USB gadget
 	// behaves similarly to a real USB HID keyboard. This logic is paralleled
 	// in the client/browser-side code in useKeyboard.ts so make sure to keep
 	// them in sync.
-	var state = u.keysDownState
+	var state = u.GetKeysDownState()
+	l.Trace().Interface("state", state).Msg("got keys down state")
+
 	modifier := state.Modifier
 	keys := append([]byte(nil), state.Keys...)

@@ -381,22 +471,36 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error)
 		// If we reach here it means we didn't find an empty slot or the key in the buffer
 		if overrun {
 			if press {
-				u.log.Error().Uint8("key", key).Msg("keyboard buffer overflow, key not added")
+				l.Error().Msg("keyboard buffer overflow, key not added")
 				// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
 				for i := range keys {
 					keys[i] = hidErrorRollOver
 				}
 			} else {
 				// If we are releasing a key, and we didn't find it in a slot, who cares?
-				u.log.Warn().Uint8("key", key).Msg("key not found in buffer, nothing to release")
+				l.Warn().Msg("key not found in buffer, nothing to release")
 			}
 		}
 	}

 	err := u.keyboardWriteHidFile(modifier, keys)
-	if err != nil {
-		u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0")
-	}
-
 	return u.UpdateKeysDown(modifier, keys), err
 }
+
+func (u *UsbGadget) KeypressReport(key byte, press bool) error {
+	state, err := u.keypressReport(key, press)
+	if err != nil {
+		u.log.Warn().Uint8("key", key).Bool("press", press).Msg("failed to report key")
+	}
+	isRolledOver := state.Keys[0] == hidErrorRollOver
+
+	if isRolledOver {
+		u.cancelAutoRelease(key)
+	} else if press {
+		u.scheduleAutoRelease(key)
+	} else {
+		u.cancelAutoRelease(key)
+	}
+
+	return err
+}
--- a/internal/usbgadget/usbgadget.go
+++ b/internal/usbgadget/usbgadget.go
@@ -68,6 +68,9 @@ type UsbGadget struct {
 	keyboardState byte          // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana)
 	keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys)

+	kbdAutoReleaseLock   sync.Mutex
+	kbdAutoReleaseTimers map[byte]*time.Timer
+
 	keyboardStateLock   sync.Mutex
 	keyboardStateCtx    context.Context
 	keyboardStateCancel context.CancelFunc
@@ -85,6 +88,7 @@ type UsbGadget struct {

 	onKeyboardStateChange *func(state KeyboardState)
 	onKeysDownChange      *func(state KeysDownState)
+	onKeepAliveReset      *func()

 	log *zerolog.Logger

@@ -118,23 +122,24 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev
 	keyboardCtx, keyboardCancel := context.WithCancel(context.Background())

 	g := &UsbGadget{
-		name:                name,
-		kvmGadgetPath:       path.Join(gadgetPath, name),
-		configC1Path:        path.Join(gadgetPath, name, "configs/c.1"),
-		configMap:           configMap,
-		customConfig:        *config,
-		configLock:          sync.Mutex{},
-		keyboardLock:        sync.Mutex{},
-		absMouseLock:        sync.Mutex{},
-		relMouseLock:        sync.Mutex{},
-		txLock:              sync.Mutex{},
-		keyboardStateCtx:    keyboardCtx,
-		keyboardStateCancel: keyboardCancel,
-		keyboardState:       0,
-		keysDownState:       KeysDownState{Modifier: 0, Keys: []byte{0, 0, 0, 0, 0, 0}}, // must be initialized to hidKeyBufferSize (6) zero bytes
-		enabledDevices:      *enabledDevices,
-		lastUserInput:       time.Now(),
-		log:                 logger,
+		name:                 name,
+		kvmGadgetPath:        path.Join(gadgetPath, name),
+		configC1Path:         path.Join(gadgetPath, name, "configs/c.1"),
+		configMap:            configMap,
+		customConfig:         *config,
+		configLock:           sync.Mutex{},
+		keyboardLock:         sync.Mutex{},
+		absMouseLock:         sync.Mutex{},
+		relMouseLock:         sync.Mutex{},
+		txLock:               sync.Mutex{},
+		keyboardStateCtx:     keyboardCtx,
+		keyboardStateCancel:  keyboardCancel,
+		keyboardState:        0,
+		keysDownState:        KeysDownState{Modifier: 0, Keys: []byte{0, 0, 0, 0, 0, 0}}, // must be initialized to hidKeyBufferSize (6) zero bytes
+		kbdAutoReleaseTimers: make(map[byte]*time.Timer),
+		enabledDevices:       *enabledDevices,
+		lastUserInput:        time.Now(),
+		log:                  logger,

 		strictMode: config.strictMode,

@@ -149,3 +154,37 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev

 	return g
 }
+
+// Close cleans up resources used by the USB gadget
+func (u *UsbGadget) Close() error {
+	// Cancel keyboard state context
+	if u.keyboardStateCancel != nil {
+		u.keyboardStateCancel()
+	}
+
+	// Stop auto-release timer
+	u.kbdAutoReleaseLock.Lock()
+	for _, timer := range u.kbdAutoReleaseTimers {
+		if timer != nil {
+			timer.Stop()
+		}
+	}
+	u.kbdAutoReleaseTimers = make(map[byte]*time.Timer)
+	u.kbdAutoReleaseLock.Unlock()
+
+	// Close HID files
+	if u.keyboardHidFile != nil {
+		u.keyboardHidFile.Close()
+		u.keyboardHidFile = nil
+	}
+	if u.absMouseHidFile != nil {
+		u.absMouseHidFile.Close()
+		u.absMouseHidFile = nil
+	}
+	if u.relMouseHidFile != nil {
+		u.relMouseHidFile.Close()
+		u.relMouseHidFile = nil
+	}
+
+	return nil
+}
--- a/internal/usbgadget/utils.go
+++ b/internal/usbgadget/utils.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"time"

 	"github.com/rs/zerolog"
@@ -120,6 +121,12 @@ func (u *UsbGadget) writeWithTimeout(file *os.File, data []byte) (n int, err err
 		return
 	}

+	u.log.Trace().
+		Str("file", file.Name()).
+		Bytes("data", data).
+		Err(err).
+		Msg("write failed")
+
 	if errors.Is(err, os.ErrDeadlineExceeded) {
 		u.logWithSuppression(
 			fmt.Sprintf("writeWithTimeout_%s", file.Name()),
@@ -164,3 +171,8 @@ func (u *UsbGadget) resetLogSuppressionCounter(counterName string) {
 		u.logSuppressionCounter[counterName] = 0
 	}
 }
+
+func unlockWithLog(lock *sync.Mutex, logger *zerolog.Logger, msg string, args ...any) {
+	logger.Trace().Msgf(msg, args...)
+	lock.Unlock()
+}