feat: release keyPress automatically (#796)

* feat: release keyPress automatically * send keepalive when pressing the key * remove logging * clean up logging * chore: use unreliable channel to send keepalive events * chore: use ordered unreliable channel for pointer events * chore: adjust auto release key interval * chore: update logging for kbdAutoReleaseLock * chore: update comment for KEEPALIVE_INTERVAL * fix: should cancelAutorelease when pressed is true * fix: handshake won't happen if webrtc reconnects * chore: add trace log for writeWithTimeout * chore: add timeout for KeypressReport * chore: use the proper key to send release command * refactor: simplify HID RPC keyboard input handling and improve key state management - Updated `handleHidRPCKeyboardInput` to return errors directly instead of keys down state. - Refactored `rpcKeyboardReport` and `rpcKeypressReport` to return errors instead of states. - Introduced a queue for managing key down state updates in the `Session` struct to prevent input handling stalls. - Adjusted the `UpdateKeysDown` method to handle state changes more efficiently. - Removed unnecessary logging and commented-out code for clarity. * refactor: enhance keyboard auto-release functionality and key state management * fix: correct Windows default auto-repeat delay comment from 1ms to 1s * refactor: send keypress as early as possible * refactor: replace console.warn with console.info for HID RPC channel events * refactor: remove unused NewKeypressKeepAliveMessage function from HID RPC * fix: handle error in key release process and log warnings * fix: log warning on keypress report failure * fix: update auto-release keyboard interval to 225 * refactor: enhance keep-alive handling and jitter compensation in HID RPC - Implemented staleness guard to ignore outdated keep-alive packets. - Added jitter compensation logic to adjust timer extensions based on packet arrival times. - Introduced new methods for managing keep-alive state and reset functionality in the Session struct. - Updated auto-release delay mechanism to use dynamic durations based on keep-alive timing. - Adjusted keep-alive interval in the UI to improve responsiveness. * gofmt * clean up code * chore: use dynamic duration for scheduleAutoRelease * Use harcoded timer reset value for now * fix: prevent nil pointer dereference when stopping timers in Close method * refactor: remove nil check for kbdAutoReleaseTimers in DelayAutoReleaseWithDuration * refactor: optimize dependencies in useHidRpc hooks * refactor: streamline keep-alive timer management in useKeyboard hook * refactor: clarify comments in useKeyboard hook for resetKeyboardState function * refactor: reduce keysDownStateQueueSize * refactor: close and reset keysDownStateQueue in newSession function * chore: resolve conflicts * resolve conflicts --------- Co-authored-by: Adam Shiervani <adam.shiervani@gmail.com>
2026-01-20 10:14:18 +01:00 · 2025-09-18 13:35:47 +02:00
parent 72e3013337
commit afb146d78c
18 changed files with 767 additions and 298 deletions
--- a/internal/usbgadget/hid_keyboard.go
+++ b/internal/usbgadget/hid_keyboard.go
@@ -5,7 +5,11 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"sync"
 	"time"
+
+	"github.com/rs/xid"
+	"github.com/rs/zerolog"
 )

 var keyboardConfig = gadgetConfigItem{
@@ -145,32 +149,95 @@ func (u *UsbGadget) GetKeysDownState() KeysDownState {
 	return u.keysDownState
 }

-func (u *UsbGadget) updateKeyDownState(state KeysDownState) {
-	u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("acquiring keyboardStateLock for updateKeyDownState")
+func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) {
+	u.onKeysDownChange = &f
+}

-	// this is intentional to unlock keyboard state lock before onKeysDownChange callback
-	{
-		u.keyboardStateLock.Lock()
-		defer u.keyboardStateLock.Unlock()
+func (u *UsbGadget) SetOnKeepAliveReset(f func()) {
+	u.onKeepAliveReset = &f
+}

-		if u.keysDownState.Modifier == state.Modifier &&
-			bytes.Equal(u.keysDownState.Keys, state.Keys) {
-			return // No change in key down state
-		}
+// DefaultAutoReleaseDuration is the default duration for auto-release of a key.
+const DefaultAutoReleaseDuration = 100 * time.Millisecond

-		u.log.Trace().Interface("old", u.keysDownState).Interface("new", state).Msg("keysDownState updated")
-		u.keysDownState = state
+func (u *UsbGadget) scheduleAutoRelease(key byte) {
+	u.kbdAutoReleaseLock.Lock()
+	defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease scheduled")
+
+	if u.kbdAutoReleaseTimers[key] != nil {
+		u.kbdAutoReleaseTimers[key].Stop()
 	}

-	if u.onKeysDownChange != nil {
-		u.log.Trace().Interface("state", state).Msg("calling onKeysDownChange")
-		(*u.onKeysDownChange)(state)
-		u.log.Trace().Interface("state", state).Msg("onKeysDownChange called")
+	// TODO: make this configurable
+	// We currently hardcode the duration to 100ms
+	// However, it should be the same as the duration of the keep-alive reset called baseExtension.
+	u.kbdAutoReleaseTimers[key] = time.AfterFunc(100*time.Millisecond, func() {
+		u.performAutoRelease(key)
+	})
+}
+
+func (u *UsbGadget) cancelAutoRelease(key byte) {
+	u.kbdAutoReleaseLock.Lock()
+	defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease cancelled")
+
+	if timer := u.kbdAutoReleaseTimers[key]; timer != nil {
+		timer.Stop()
+		u.kbdAutoReleaseTimers[key] = nil
+		delete(u.kbdAutoReleaseTimers, key)
+
+		// Reset keep-alive timing when key is released
+		if u.onKeepAliveReset != nil {
+			(*u.onKeepAliveReset)()
+		}
 	}
 }

-func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) {
-	u.onKeysDownChange = &f
+func (u *UsbGadget) DelayAutoReleaseWithDuration(resetDuration time.Duration) {
+	u.kbdAutoReleaseLock.Lock()
+	defer unlockWithLog(&u.kbdAutoReleaseLock, u.log, "autoRelease delayed")
+
+	u.log.Debug().Dur("reset_duration", resetDuration).Msg("delaying auto-release with dynamic duration")
+
+	for _, timer := range u.kbdAutoReleaseTimers {
+		if timer != nil {
+			timer.Reset(resetDuration)
+		}
+	}
+}
+
+func (u *UsbGadget) performAutoRelease(key byte) {
+	u.kbdAutoReleaseLock.Lock()
+
+	if u.kbdAutoReleaseTimers[key] == nil {
+		u.log.Warn().Uint8("key", key).Msg("autoRelease timer not found")
+		u.kbdAutoReleaseLock.Unlock()
+		return
+	}
+
+	u.kbdAutoReleaseTimers[key].Stop()
+	u.kbdAutoReleaseTimers[key] = nil
+	delete(u.kbdAutoReleaseTimers, key)
+	u.kbdAutoReleaseLock.Unlock()
+
+	// Skip if already released
+	state := u.GetKeysDownState()
+	alreadyReleased := true
+
+	for i := range state.Keys {
+		if state.Keys[i] == key {
+			alreadyReleased = false
+			break
+		}
+	}
+
+	if alreadyReleased {
+		return
+	}
+
+	_, err := u.keypressReport(key, false)
+	if err != nil {
+		u.log.Warn().Uint8("key", key).Msg("failed to release key")
+	}
 }

 func (u *UsbGadget) listenKeyboardEvents() {
@@ -242,7 +309,11 @@ func (u *UsbGadget) OpenKeyboardHidFile() error {
 	return u.openKeyboardHidFile()
 }

+var keyboardWriteHidFileLock sync.Mutex
+
 func (u *UsbGadget) keyboardWriteHidFile(modifier byte, keys []byte) error {
+	keyboardWriteHidFileLock.Lock()
+	defer keyboardWriteHidFileLock.Unlock()
 	if err := u.openKeyboardHidFile(); err != nil {
 		return err
 	}
@@ -266,17 +337,29 @@ func (u *UsbGadget) UpdateKeysDown(modifier byte, keys []byte) KeysDownState {
 		}
 	}

-	downState := KeysDownState{
+	state := KeysDownState{
 		Modifier: modifier,
 		Keys:     []byte(keys[:]),
 	}
-	u.updateKeyDownState(downState)
-	return downState
+
+	u.keyboardStateLock.Lock()
+
+	if u.keysDownState.Modifier == state.Modifier &&
+		bytes.Equal(u.keysDownState.Keys, state.Keys) {
+		u.keyboardStateLock.Unlock()
+		return state // No change in key down state
+	}
+
+	u.keysDownState = state
+	u.keyboardStateLock.Unlock()
+
+	if u.onKeysDownChange != nil {
+		(*u.onKeysDownChange)(state) // this enques to the outgoing hidrpc queue via usb.go → currentSession.enqueueKeysDownState(...)
+	}
+	return state
 }

-func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, error) {
-	u.keyboardLock.Lock()
-	defer u.keyboardLock.Unlock()
+func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) error {
 	defer u.resetUserInputTime()

 	if len(keys) > hidKeyBufferSize {
@@ -291,7 +374,8 @@ func (u *UsbGadget) KeyboardReport(modifier byte, keys []byte) (KeysDownState, e
 		u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keyboard report to hidg0")
 	}

-	return u.UpdateKeysDown(modifier, keys), err
+	u.UpdateKeysDown(modifier, keys)
+	return err
 }

 const (
@@ -331,17 +415,23 @@ var KeyCodeToMaskMap = map[byte]byte{
 	RightSuper:   ModifierMaskRightSuper,
 }

-func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) {
-	u.keyboardLock.Lock()
-	defer u.keyboardLock.Unlock()
+func (u *UsbGadget) keypressReport(key byte, press bool) (KeysDownState, error) {
 	defer u.resetUserInputTime()

+	l := u.log.With().Uint8("key", key).Bool("press", press).Logger()
+	if l.GetLevel() <= zerolog.DebugLevel {
+		requestID := xid.New()
+		l = l.With().Str("requestID", requestID.String()).Logger()
+	}
+
 	// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
 	// for handling key presses and releases. It ensures that the USB gadget
 	// behaves similarly to a real USB HID keyboard. This logic is paralleled
 	// in the client/browser-side code in useKeyboard.ts so make sure to keep
 	// them in sync.
-	var state = u.keysDownState
+	var state = u.GetKeysDownState()
+	l.Trace().Interface("state", state).Msg("got keys down state")
+
 	modifier := state.Modifier
 	keys := append([]byte(nil), state.Keys...)

@@ -381,22 +471,36 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error)
 		// If we reach here it means we didn't find an empty slot or the key in the buffer
 		if overrun {
 			if press {
-				u.log.Error().Uint8("key", key).Msg("keyboard buffer overflow, key not added")
+				l.Error().Msg("keyboard buffer overflow, key not added")
 				// Fill all key slots with ErrorRollOver (0x01) to indicate overflow
 				for i := range keys {
 					keys[i] = hidErrorRollOver
 				}
 			} else {
 				// If we are releasing a key, and we didn't find it in a slot, who cares?
-				u.log.Warn().Uint8("key", key).Msg("key not found in buffer, nothing to release")
+				l.Warn().Msg("key not found in buffer, nothing to release")
 			}
 		}
 	}

 	err := u.keyboardWriteHidFile(modifier, keys)
-	if err != nil {
-		u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0")
-	}
-
 	return u.UpdateKeysDown(modifier, keys), err
 }
+
+func (u *UsbGadget) KeypressReport(key byte, press bool) error {
+	state, err := u.keypressReport(key, press)
+	if err != nil {
+		u.log.Warn().Uint8("key", key).Bool("press", press).Msg("failed to report key")
+	}
+	isRolledOver := state.Keys[0] == hidErrorRollOver
+
+	if isRolledOver {
+		u.cancelAutoRelease(key)
+	} else if press {
+		u.scheduleAutoRelease(key)
+	} else {
+		u.cancelAutoRelease(key)
+	}
+
+	return err
+}