mirror of
https://github.com/luckfox-eng29/kvm.git
synced 2026-05-26 08:05:08 +02:00
feat(ui): add OCR detect overlay and capture flow
Signed-off-by: luckfox-eng29 <eng29@luckfox.com>
This commit is contained in:
luckfox-eng29
175
ui/package-lock.json
generated
175
ui/package-lock.json
generated
@@ -43,6 +43,7 @@
|
||||
"recharts": "^2.15.3",
|
||||
"styled-components": "^6.1.19",
|
||||
"tailwind-merge": "^3.3.0",
|
||||
"tesseract.js": "^7.0.0",
|
||||
"usehooks-ts": "^3.1.1",
|
||||
"validator": "^13.15.0",
|
||||
"w-touch": "^2.0.0",
|
||||
@@ -2821,6 +2822,66 @@
|
||||
"node": ">=14.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": {
|
||||
"version": "1.4.3",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/wasi-threads": "1.0.2",
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": {
|
||||
"version": "1.4.3",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": {
|
||||
"version": "1.0.2",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "0.2.9",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"@emnapi/core": "^1.4.0",
|
||||
"@emnapi/runtime": "^1.4.0",
|
||||
"@tybys/wasm-util": "^0.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": {
|
||||
"version": "0.9.0",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"tslib": "^2.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": {
|
||||
"version": "2.8.0",
|
||||
"dev": true,
|
||||
"inBundle": true,
|
||||
"license": "0BSD",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/@tailwindcss/oxide-win32-arm64-msvc": {
|
||||
"version": "4.1.7",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.7.tgz",
|
||||
@@ -3847,6 +3908,12 @@
|
||||
"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/bmp-js": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz",
|
||||
"integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/brace-expansion": {
|
||||
"version": "1.1.12",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
|
||||
@@ -5777,6 +5844,12 @@
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/idb-keyval": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz",
|
||||
"integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/ignore": {
|
||||
"version": "5.3.2",
|
||||
"resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz",
|
||||
@@ -6199,6 +6272,12 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/is-url": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz",
|
||||
"integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/is-weakmap": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz",
|
||||
@@ -6963,6 +7042,26 @@
|
||||
"tslib": "^2.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.19",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
|
||||
@@ -7112,6 +7211,15 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/opencollective-postinstall": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz",
|
||||
"integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"opencollective-postinstall": "index.js"
|
||||
}
|
||||
},
|
||||
"node_modules/optionator": {
|
||||
"version": "0.9.4",
|
||||
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
|
||||
@@ -8430,6 +8538,12 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/regenerator-runtime": {
|
||||
"version": "0.13.11",
|
||||
"resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz",
|
||||
"integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/regexp.prototype.flags": {
|
||||
"version": "1.5.4",
|
||||
"resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",
|
||||
@@ -9203,6 +9317,30 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/tesseract.js": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz",
|
||||
"integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"bmp-js": "^0.1.0",
|
||||
"idb-keyval": "^6.2.0",
|
||||
"is-url": "^1.2.4",
|
||||
"node-fetch": "^2.6.9",
|
||||
"opencollective-postinstall": "^2.0.3",
|
||||
"regenerator-runtime": "^0.13.3",
|
||||
"tesseract.js-core": "^7.0.0",
|
||||
"wasm-feature-detect": "^1.8.0",
|
||||
"zlibjs": "^0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/tesseract.js-core": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz",
|
||||
"integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/text-encoding-utf-8": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmmirror.com/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz",
|
||||
@@ -9284,6 +9422,12 @@
|
||||
"integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/ts-api-utils": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz",
|
||||
@@ -9739,12 +9883,34 @@
|
||||
"integrity": "sha512-PYnngF+KHzZRBFI6qsm5PHwVNO5Lx3dYDrvNv//Ei9fvnYlaOWr9sKAriD/crlshWee9ZPsyvDMA4UnoR1LUHw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/wasm-feature-detect": {
|
||||
"version": "1.8.0",
|
||||
"resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz",
|
||||
"integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/whatwg-fetch": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmmirror.com/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz",
|
||||
"integrity": "sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
@@ -9890,6 +10056,15 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/zlibjs": {
|
||||
"version": "0.3.1",
|
||||
"resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz",
|
||||
"integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/zustand": {
|
||||
"version": "4.5.7",
|
||||
"resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz",
|
||||
|
||||
@@ -54,6 +54,7 @@
|
||||
"recharts": "^2.15.3",
|
||||
"styled-components": "^6.1.19",
|
||||
"tailwind-merge": "^3.3.0",
|
||||
"tesseract.js": "^7.0.0",
|
||||
"usehooks-ts": "^3.1.1",
|
||||
"validator": "^13.15.0",
|
||||
"w-touch": "^2.0.0",
|
||||
|
||||
@@ -18,6 +18,7 @@ interface ConfirmDialogProps {
|
||||
onClose: () => void;
|
||||
title: string;
|
||||
description: React.ReactNode;
|
||||
children?: React.ReactNode;
|
||||
variant?: Variant;
|
||||
confirmText?: string;
|
||||
cancelText?: string | null;
|
||||
@@ -65,6 +66,7 @@ export function ConfirmDialog({
|
||||
onClose,
|
||||
title,
|
||||
description,
|
||||
children,
|
||||
variant = "info",
|
||||
confirmText = "Confirm",
|
||||
cancelText = "Cancel",
|
||||
@@ -107,9 +109,10 @@ export function ConfirmDialog({
|
||||
{description}
|
||||
</div>
|
||||
</div>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-5 sm:mt-6 sm:grid sm:grid-flow-row-dense sm:grid-cols-2 sm:gap-3">
|
||||
<div className={cx("mt-5 sm:mt-6 sm:grid sm:grid-flow-row-dense sm:grid-cols-2 sm:gap-3", isConfirming && "pointer-events-none")}>
|
||||
<Button
|
||||
size="LG"
|
||||
theme={buttonTheme}
|
||||
@@ -157,10 +160,11 @@ export function ConfirmDialog({
|
||||
<div className="mt-2 text-sm leading-snug text-slate-600 dark:text-[#ffffff]">
|
||||
{description}
|
||||
</div>
|
||||
{children}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex justify-end gap-x-2">
|
||||
<div className={cx("flex justify-end gap-x-2", isConfirming && "pointer-events-none")}>
|
||||
{cancelText && (
|
||||
<Button size="SM" theme="light" text={cancelText} onClick={onClose} />
|
||||
)}
|
||||
@@ -177,4 +181,4 @@ export function ConfirmDialog({
|
||||
</div>
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
458
ui/src/components/OcrOverlay.tsx
Normal file
458
ui/src/components/OcrOverlay.tsx
Normal file
@@ -0,0 +1,458 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { useReactAt } from "i18n-auto-extractor/react";
|
||||
import { motion } from "framer-motion";
|
||||
|
||||
import { useSettingsStore, useUiStore, useVideoStore } from "@/hooks/stores";
|
||||
import Card from "@components/Card";
|
||||
import { ConfirmDialog } from "@components/ConfirmDialog";
|
||||
import TextArea from "@components/TextArea";
|
||||
import notifications from "@/notifications";
|
||||
import { eventMatchesShortcut } from "@/utils/shortcuts";
|
||||
|
||||
interface Rect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
type OcrStatus = "idle" | "selecting" | "processing" | "result";
|
||||
|
||||
type TesseractWorker = {
|
||||
recognize: (image: HTMLCanvasElement, options?: unknown, output?: unknown) => Promise<{
|
||||
data: {
|
||||
text?: string;
|
||||
blocks?: Array<{
|
||||
paragraphs: Array<{
|
||||
lines: Array<{
|
||||
text: string;
|
||||
bbox: { x0: number };
|
||||
words: Array<{ text: string; bbox: { x0: number; x1: number } }>;
|
||||
}>;
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
}>;
|
||||
terminate: () => Promise<unknown>;
|
||||
};
|
||||
|
||||
async function loadTesseract() {
|
||||
const { createWorker } = await import("tesseract.js");
|
||||
return createWorker;
|
||||
}
|
||||
|
||||
let workerPromise: Promise<TesseractWorker> | null = null;
|
||||
let cleanupTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
async function initWorker() {
|
||||
const createWorker = await loadTesseract();
|
||||
return createWorker("eng", 1) as Promise<TesseractWorker>;
|
||||
}
|
||||
|
||||
async function terminateWorker() {
|
||||
if (!workerPromise) return;
|
||||
try {
|
||||
const worker = await workerPromise;
|
||||
await worker.terminate();
|
||||
} catch {
|
||||
// Ignore termination errors.
|
||||
}
|
||||
workerPromise = null;
|
||||
}
|
||||
|
||||
function getWorker() {
|
||||
if (cleanupTimer) {
|
||||
clearTimeout(cleanupTimer);
|
||||
cleanupTimer = null;
|
||||
}
|
||||
|
||||
if (!workerPromise) {
|
||||
workerPromise = initWorker().catch(err => {
|
||||
workerPromise = null;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-terminate worker after inactivity to reduce memory footprint.
|
||||
cleanupTimer = setTimeout(() => {
|
||||
void terminateWorker();
|
||||
cleanupTimer = null;
|
||||
}, 60_000);
|
||||
|
||||
return workerPromise;
|
||||
}
|
||||
|
||||
async function performOcr(canvas: HTMLCanvasElement): Promise<string> {
|
||||
const worker = await getWorker();
|
||||
const { data } = await worker.recognize(canvas, {}, { text: true, blocks: true });
|
||||
const lines = data.blocks?.flatMap(b => b.paragraphs.flatMap(p => p.lines)) ?? [];
|
||||
|
||||
if (lines.length === 0) return (data.text || "").trim();
|
||||
|
||||
// Estimate character width from OCR words so left indentation is preserved.
|
||||
let totalCharWidth = 0;
|
||||
let samples = 0;
|
||||
for (const line of lines) {
|
||||
for (const word of line.words) {
|
||||
const len = word.text.trim().length;
|
||||
if (len > 0) {
|
||||
totalCharWidth += (word.bbox.x1 - word.bbox.x0) / len;
|
||||
samples++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (samples === 0) return (data.text || "").trim();
|
||||
|
||||
const charWidth = totalCharWidth / samples;
|
||||
const minX = Math.min(...lines.map(l => l.bbox.x0));
|
||||
|
||||
return lines
|
||||
.map(line => {
|
||||
const indent = Math.round((line.bbox.x0 - minX) / charWidth);
|
||||
return " ".repeat(indent) + line.text.trim();
|
||||
})
|
||||
.join("\n")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function captureRegion(videoEl: HTMLVideoElement, rect: Rect): HTMLCanvasElement {
|
||||
const canvas = document.createElement("canvas");
|
||||
canvas.width = rect.width;
|
||||
canvas.height = rect.height;
|
||||
const ctx = canvas.getContext("2d");
|
||||
if (!ctx) throw new Error("Failed to acquire 2D canvas context");
|
||||
ctx.drawImage(videoEl, rect.x, rect.y, rect.width, rect.height, 0, 0, rect.width, rect.height);
|
||||
return canvas;
|
||||
}
|
||||
|
||||
function getVideoDisplayRect(videoElement: HTMLVideoElement, videoWidth: number, videoHeight: number) {
|
||||
const videoRect = videoElement.getBoundingClientRect();
|
||||
const elementAspectRatio = videoRect.width / videoRect.height;
|
||||
const streamAspectRatio = videoWidth / videoHeight;
|
||||
|
||||
let effectiveWidth = videoRect.width;
|
||||
let effectiveHeight = videoRect.height;
|
||||
let offsetX = 0;
|
||||
let offsetY = 0;
|
||||
|
||||
if (elementAspectRatio > streamAspectRatio) {
|
||||
effectiveWidth = videoRect.height * streamAspectRatio;
|
||||
offsetX = (videoRect.width - effectiveWidth) / 2;
|
||||
} else if (elementAspectRatio < streamAspectRatio) {
|
||||
effectiveHeight = videoRect.width / streamAspectRatio;
|
||||
offsetY = (videoRect.height - effectiveHeight) / 2;
|
||||
}
|
||||
|
||||
return { videoRect, effectiveWidth, effectiveHeight, offsetX, offsetY };
|
||||
}
|
||||
|
||||
interface OcrOverlayProps {
|
||||
videoRef: React.RefObject<HTMLVideoElement>;
|
||||
containerRef: React.RefObject<HTMLDivElement>;
|
||||
}
|
||||
|
||||
export default function OcrOverlay({ videoRef, containerRef }: OcrOverlayProps) {
|
||||
const { width: videoWidth, height: videoHeight } = useVideoStore();
|
||||
const isOcrMode = useUiStore(state => state.isOcrMode);
|
||||
const setOcrMode = useUiStore(state => state.setOcrMode);
|
||||
const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap);
|
||||
const ocrShortcutEnabled = useSettingsStore(state => state.ocrShortcutEnabled);
|
||||
const ocrShortcut = useSettingsStore(state => state.ocrShortcut);
|
||||
const { $at } = useReactAt();
|
||||
|
||||
const mountedRef = useRef(true);
|
||||
const resultRef = useRef<HTMLTextAreaElement>(null);
|
||||
const [status, setStatus] = useState<OcrStatus>("idle");
|
||||
const [selectionStart, setSelectionStart] = useState<{ x: number; y: number } | null>(null);
|
||||
const [selectionRect, setSelectionRect] = useState<Rect | null>(null);
|
||||
const [ocrResult, setOcrResult] = useState("");
|
||||
const [isClosing, setIsClosing] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
mountedRef.current = false;
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if (!ocrShortcutEnabled) return;
|
||||
if (!eventMatchesShortcut(e, ocrShortcut)) return;
|
||||
const activeElement = document.activeElement as HTMLElement | null;
|
||||
const isEditable =
|
||||
!!activeElement
|
||||
&& (activeElement.tagName === "INPUT"
|
||||
|| activeElement.tagName === "TEXTAREA"
|
||||
|| activeElement.isContentEditable);
|
||||
if (isEditable) return;
|
||||
if (videoWidth === 0 || videoHeight === 0) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setOcrMode(!isOcrMode);
|
||||
};
|
||||
document.addEventListener("keydown", handleKeyDown, { capture: true });
|
||||
return () => document.removeEventListener("keydown", handleKeyDown, { capture: true });
|
||||
}, [isOcrMode, ocrShortcut, ocrShortcutEnabled, setOcrMode, videoWidth, videoHeight]);
|
||||
|
||||
const closeOverlay = useCallback(() => {
|
||||
if (status === "processing" || status === "result") {
|
||||
setIsClosing(true);
|
||||
setSelectionRect(null);
|
||||
setSelectionStart(null);
|
||||
setTimeout(() => setOcrMode(false), 200);
|
||||
return;
|
||||
}
|
||||
setOcrMode(false);
|
||||
}, [setOcrMode, status]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOcrMode) {
|
||||
setStatus("idle");
|
||||
setSelectionRect(null);
|
||||
setSelectionStart(null);
|
||||
setOcrResult("");
|
||||
setIsClosing(false);
|
||||
}
|
||||
}, [isOcrMode]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOcrMode) return;
|
||||
if (status === "processing" || status === "result") {
|
||||
setDisableVideoFocusTrap(true);
|
||||
return () => setDisableVideoFocusTrap(false);
|
||||
}
|
||||
}, [isOcrMode, setDisableVideoFocusTrap, status]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOcrMode) return;
|
||||
if (status === "processing" || status === "result") return;
|
||||
const handleKeyDown = (e: KeyboardEvent) => {
|
||||
if (e.key === "Escape") {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setOcrMode(false);
|
||||
}
|
||||
};
|
||||
document.addEventListener("keydown", handleKeyDown, { capture: true });
|
||||
return () => document.removeEventListener("keydown", handleKeyDown, { capture: true });
|
||||
}, [isOcrMode, setOcrMode, status]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isOcrMode) return;
|
||||
if (status !== "result") return;
|
||||
const handleCopy = () => {
|
||||
notifications.success($at("Copied"), { duration: 4000 });
|
||||
closeOverlay();
|
||||
};
|
||||
document.addEventListener("copy", handleCopy);
|
||||
return () => document.removeEventListener("copy", handleCopy);
|
||||
}, [closeOverlay, isOcrMode, status, $at]);
|
||||
|
||||
useEffect(() => {
|
||||
if (status === "result" && resultRef.current) {
|
||||
resultRef.current.focus();
|
||||
resultRef.current.select();
|
||||
}
|
||||
}, [status]);
|
||||
|
||||
const toVideoCoords = useCallback((clientX: number, clientY: number) => {
|
||||
const videoElement = videoRef.current;
|
||||
if (!videoElement) return { x: 0, y: 0 };
|
||||
const { videoRect, effectiveWidth, effectiveHeight, offsetX, offsetY } = getVideoDisplayRect(
|
||||
videoElement,
|
||||
videoWidth,
|
||||
videoHeight,
|
||||
);
|
||||
const relX = clientX - videoRect.left - offsetX;
|
||||
const relY = clientY - videoRect.top - offsetY;
|
||||
const scaleX = videoWidth / effectiveWidth;
|
||||
const scaleY = videoHeight / effectiveHeight;
|
||||
return {
|
||||
x: Math.max(0, Math.min(videoWidth, Math.round(relX * scaleX))),
|
||||
y: Math.max(0, Math.min(videoHeight, Math.round(relY * scaleY))),
|
||||
};
|
||||
}, [videoHeight, videoRef, videoWidth]);
|
||||
|
||||
const finishSelection = useCallback(async () => {
|
||||
const videoElement = videoRef.current;
|
||||
if (status !== "selecting") return;
|
||||
if (!videoElement || !selectionRect) {
|
||||
setStatus("idle");
|
||||
setSelectionStart(null);
|
||||
setSelectionRect(null);
|
||||
return;
|
||||
}
|
||||
if (selectionRect.width < 10 || selectionRect.height < 10) {
|
||||
setStatus("idle");
|
||||
setSelectionStart(null);
|
||||
setSelectionRect(null);
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus("processing");
|
||||
try {
|
||||
const canvas = captureRegion(videoElement, selectionRect);
|
||||
const text = await performOcr(canvas);
|
||||
canvas.width = 0;
|
||||
canvas.height = 0;
|
||||
|
||||
if (!mountedRef.current) return;
|
||||
if (text) {
|
||||
setOcrResult(text);
|
||||
setStatus("result");
|
||||
} else {
|
||||
notifications.error($at("No text detected"));
|
||||
closeOverlay();
|
||||
}
|
||||
} catch (error) {
|
||||
if (!mountedRef.current) return;
|
||||
console.error("OCR failed:", error);
|
||||
notifications.error($at("OCR failed"));
|
||||
closeOverlay();
|
||||
}
|
||||
}, [closeOverlay, selectionRect, status, videoRef, $at]);
|
||||
|
||||
const onPointerDown = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
|
||||
if (status === "processing" || status === "result") return;
|
||||
if (!videoRef.current || videoWidth === 0 || videoHeight === 0) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
e.currentTarget.setPointerCapture(e.pointerId);
|
||||
const coords = toVideoCoords(e.clientX, e.clientY);
|
||||
setSelectionStart(coords);
|
||||
setSelectionRect(null);
|
||||
setStatus("selecting");
|
||||
}, [status, toVideoCoords, videoHeight, videoRef, videoWidth]);
|
||||
|
||||
const onPointerMove = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
|
||||
if (status !== "selecting" || !selectionStart) return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
const coords = toVideoCoords(e.clientX, e.clientY);
|
||||
const x = Math.min(selectionStart.x, coords.x);
|
||||
const y = Math.min(selectionStart.y, coords.y);
|
||||
const width = Math.abs(coords.x - selectionStart.x);
|
||||
const height = Math.abs(coords.y - selectionStart.y);
|
||||
setSelectionRect({ x, y, width, height });
|
||||
}, [selectionStart, status, toVideoCoords]);
|
||||
|
||||
const onPointerUp = useCallback((e: React.PointerEvent<HTMLDivElement>) => {
|
||||
if (status !== "selecting") return;
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
e.currentTarget.releasePointerCapture(e.pointerId);
|
||||
void finishSelection();
|
||||
}, [finishSelection, status]);
|
||||
|
||||
const selectionStyle = useMemo(() => {
|
||||
const videoElement = videoRef.current;
|
||||
const containerElement = containerRef.current;
|
||||
if (!selectionRect || !videoElement || !containerElement || videoWidth === 0 || videoHeight === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const { videoRect, effectiveWidth, effectiveHeight, offsetX, offsetY } = getVideoDisplayRect(
|
||||
videoElement,
|
||||
videoWidth,
|
||||
videoHeight,
|
||||
);
|
||||
const containerRect = containerElement.getBoundingClientRect();
|
||||
const baseX = videoRect.left - containerRect.left + offsetX;
|
||||
const baseY = videoRect.top - containerRect.top + offsetY;
|
||||
|
||||
return {
|
||||
left: `${baseX + (selectionRect.x / videoWidth) * effectiveWidth}px`,
|
||||
top: `${baseY + (selectionRect.y / videoHeight) * effectiveHeight}px`,
|
||||
width: `${(selectionRect.width / videoWidth) * effectiveWidth}px`,
|
||||
height: `${(selectionRect.height / videoHeight) * effectiveHeight}px`,
|
||||
};
|
||||
}, [containerRef, selectionRect, videoHeight, videoRef, videoWidth]);
|
||||
|
||||
if (!isOcrMode) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<motion.div
|
||||
initial={{ opacity: 0 }}
|
||||
animate={{ opacity: 1 }}
|
||||
exit={{ opacity: 0 }}
|
||||
transition={{ duration: 0.15 }}
|
||||
className="absolute inset-0 z-20 touch-none"
|
||||
style={{ cursor: status === "result" ? "default" : "crosshair" }}
|
||||
onPointerDown={onPointerDown}
|
||||
onPointerMove={onPointerMove}
|
||||
onPointerUp={onPointerUp}
|
||||
>
|
||||
<div className="fixed inset-0 bg-black/20" />
|
||||
|
||||
{status === "idle" && (
|
||||
<div className="pointer-events-none absolute inset-x-0 top-4 flex justify-center">
|
||||
<div className="rounded-md bg-black/70 px-3 py-1.5 text-xs font-medium text-white">
|
||||
{$at("Drag to select text area")}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{selectionRect && selectionStyle && status !== "result" && (
|
||||
<div
|
||||
className="absolute border-2 border-dashed border-blue-400 bg-blue-400/10"
|
||||
style={selectionStyle}
|
||||
>
|
||||
{selectionRect.width >= 10 && selectionRect.height >= 10 && (
|
||||
<Card className="absolute right-0 -bottom-6 w-auto px-1.5 py-0.5 text-[10px] font-medium tabular-nums dark:text-white">
|
||||
{selectionRect.width} × {selectionRect.height}
|
||||
</Card>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</motion.div>
|
||||
|
||||
<ConfirmDialog
|
||||
open={(status === "processing" || status === "result") && !isClosing}
|
||||
onClose={closeOverlay}
|
||||
title={status === "result" ? $at("Copy text") : $at("Recognizing text...")}
|
||||
description={status === "result" ? $at("Review the recognized text before copying.") : $at("Please wait while OCR is running.")}
|
||||
confirmText={$at("Copy text")}
|
||||
onConfirm={() => {
|
||||
if (status !== "result") return;
|
||||
if (navigator.clipboard?.writeText && window.isSecureContext) {
|
||||
navigator.clipboard.writeText(ocrResult).then(() => {
|
||||
notifications.success($at("Copied"), { duration: 4000 });
|
||||
closeOverlay();
|
||||
}).catch(() => {
|
||||
if (!resultRef.current) return;
|
||||
resultRef.current.focus();
|
||||
resultRef.current.select();
|
||||
document.execCommand("copy");
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!resultRef.current) return;
|
||||
resultRef.current.focus();
|
||||
resultRef.current.select();
|
||||
document.execCommand("copy");
|
||||
}}
|
||||
isConfirming={status === "processing"}
|
||||
>
|
||||
{status === "processing" ? (
|
||||
<div className="mt-2 space-y-2">
|
||||
<div className="h-4 w-full animate-pulse rounded bg-slate-200 dark:bg-slate-700" />
|
||||
<div className="h-4 w-3/4 animate-pulse rounded bg-slate-200 dark:bg-slate-700" />
|
||||
<div className="h-4 w-5/6 animate-pulse rounded bg-slate-200 dark:bg-slate-700" />
|
||||
</div>
|
||||
) : (
|
||||
<div className="mt-2">
|
||||
<TextArea
|
||||
ref={resultRef}
|
||||
value={ocrResult}
|
||||
readOnly
|
||||
rows={Math.min(10, ocrResult.split("\n").length + 1)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</ConfirmDialog>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -64,6 +64,8 @@ interface UIState {
|
||||
|
||||
disableVideoFocusTrap: boolean;
|
||||
setDisableVideoFocusTrap: (enabled: boolean) => void;
|
||||
isOcrMode: boolean;
|
||||
setOcrMode: (enabled: boolean) => void;
|
||||
|
||||
isWakeOnLanModalVisible: boolean;
|
||||
setWakeOnLanModalVisibility: (enabled: boolean) => void;
|
||||
@@ -91,6 +93,8 @@ export const useUiStore = create<UIState>(set => ({
|
||||
|
||||
disableVideoFocusTrap: false,
|
||||
setDisableVideoFocusTrap: enabled => set({ disableVideoFocusTrap: enabled }),
|
||||
isOcrMode: false,
|
||||
setOcrMode: enabled => set({ isOcrMode: enabled }),
|
||||
isAnimationComplete: false,
|
||||
setIsAnimationComplete: enabled => set({ isAnimationComplete: enabled }),
|
||||
|
||||
@@ -392,6 +396,14 @@ interface SettingsState {
|
||||
|
||||
overrideCtrlV: boolean;
|
||||
setOverrideCtrlV: (enabled: boolean) => void;
|
||||
pasteShortcutEnabled: boolean;
|
||||
setPasteShortcutEnabled: (enabled: boolean) => void;
|
||||
pasteShortcut: string;
|
||||
setPasteShortcut: (shortcut: string) => void;
|
||||
ocrShortcutEnabled: boolean;
|
||||
setOcrShortcutEnabled: (enabled: boolean) => void;
|
||||
ocrShortcut: string;
|
||||
setOcrShortcut: (shortcut: string) => void;
|
||||
|
||||
// Video enhancement settings
|
||||
videoSaturation: number;
|
||||
@@ -461,6 +473,14 @@ export const useSettingsStore = create(
|
||||
|
||||
overrideCtrlV: false,
|
||||
setOverrideCtrlV: enabled => set({ overrideCtrlV: enabled }),
|
||||
pasteShortcutEnabled: true,
|
||||
setPasteShortcutEnabled: enabled => set({ pasteShortcutEnabled: enabled }),
|
||||
pasteShortcut: "Ctrl+V",
|
||||
setPasteShortcut: shortcut => set({ pasteShortcut: shortcut }),
|
||||
ocrShortcutEnabled: true,
|
||||
setOcrShortcutEnabled: enabled => set({ ocrShortcutEnabled: enabled }),
|
||||
ocrShortcut: "Ctrl+C",
|
||||
setOcrShortcut: shortcut => set({ ocrShortcut: shortcut }),
|
||||
|
||||
// Video enhancement settings with default values (1.0 = normal)
|
||||
videoSaturation: 1.0,
|
||||
|
||||
@@ -1,16 +1,18 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { ExclamationCircleIcon } from "@heroicons/react/16/solid";
|
||||
import { useClose } from "@headlessui/react";
|
||||
import { Checkbox, Button } from "antd";
|
||||
import { Checkbox, Button, Input } from "antd";
|
||||
import { useReactAt } from "i18n-auto-extractor/react";
|
||||
import { isMobile } from "react-device-detect";
|
||||
|
||||
import { TextAreaWithLabel } from "@components/TextArea";
|
||||
import { SettingsItem } from "@components/Settings/SettingsView";
|
||||
import { useJsonRpc } from "@/hooks/useJsonRpc";
|
||||
import { useHidStore, useRTCStore, useUiStore, useSettingsStore } from "@/hooks/stores";
|
||||
import { useHidStore, useRTCStore, useUiStore, useSettingsStore, useVideoStore } from "@/hooks/stores";
|
||||
import { keys, modifiers } from "@/keyboardMappings";
|
||||
import { layouts, chars } from "@/keyboardLayouts";
|
||||
import notifications from "@/notifications";
|
||||
import { eventMatchesShortcut, shortcutFromKeyboardEvent } from "@/utils/shortcuts";
|
||||
|
||||
const hidKeyboardPayload = (keys: number[], modifier: number) => {
|
||||
return { keys, modifier };
|
||||
@@ -28,15 +30,24 @@ export default function Clipboard() {
|
||||
const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap);
|
||||
const setSidebarView = useUiStore(state => state.setSidebarView);
|
||||
const toggleTopBarView = useUiStore(state => state.toggleTopBarView);
|
||||
const isOcrMode = useUiStore(state => state.isOcrMode);
|
||||
const setOcrMode = useUiStore(state => state.setOcrMode);
|
||||
const isReinitializingGadget = useHidStore(state => state.isReinitializingGadget);
|
||||
const videoWidth = useVideoStore(state => state.width);
|
||||
const videoHeight = useVideoStore(state => state.height);
|
||||
const [send] = useJsonRpc();
|
||||
const rpcDataChannel = useRTCStore(state => state.rpcDataChannel);
|
||||
|
||||
const [invalidChars, setInvalidChars] = useState<string[]>([]);
|
||||
const close = useClose();
|
||||
const overrideCtrlV = useSettingsStore(state => state.overrideCtrlV);
|
||||
const setOverrideCtrlV = useSettingsStore(state => state.setOverrideCtrlV);
|
||||
const [pasteBuffer, setPasteBuffer] = useState<string>("");
|
||||
const pasteShortcutEnabled = useSettingsStore(state => state.pasteShortcutEnabled);
|
||||
const setPasteShortcutEnabled = useSettingsStore(state => state.setPasteShortcutEnabled);
|
||||
const pasteShortcut = useSettingsStore(state => state.pasteShortcut);
|
||||
const setPasteShortcut = useSettingsStore(state => state.setPasteShortcut);
|
||||
const ocrShortcutEnabled = useSettingsStore(state => state.ocrShortcutEnabled);
|
||||
const setOcrShortcutEnabled = useSettingsStore(state => state.setOcrShortcutEnabled);
|
||||
const ocrShortcut = useSettingsStore(state => state.ocrShortcut);
|
||||
const setOcrShortcut = useSettingsStore(state => state.setOcrShortcut);
|
||||
const [readyToRender, setReadyToRender] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -127,7 +138,6 @@ export default function Clipboard() {
|
||||
}, [rpcDataChannel?.readyState, send, setDisableVideoFocusTrap, setPasteMode, safeKeyboardLayout]);
|
||||
|
||||
const handleTextSend = useCallback(async (text: string) => {
|
||||
setPasteBuffer(text);
|
||||
const segInvalid = [
|
||||
...new Set(
|
||||
// @ts-expect-error TS doesn't recognize Intl.Segmenter in some environments
|
||||
@@ -192,12 +202,35 @@ export default function Clipboard() {
|
||||
}, [handleTextSend]);
|
||||
|
||||
useEffect(() => {
|
||||
// When overrideCtrlV is true, we want to focus the container div to capture paste events
|
||||
// When it is false, we want to focus the textarea if it exists
|
||||
if (!overrideCtrlV && TextAreaRef.current) {
|
||||
if (readyToRender && TextAreaRef.current) {
|
||||
TextAreaRef.current.focus();
|
||||
}
|
||||
}, [readyToRender, overrideCtrlV]);
|
||||
}, [readyToRender]);
|
||||
|
||||
const handleShortcutInput = useCallback(
|
||||
(setter: (shortcut: string) => void) => (e: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
const shortcut = shortcutFromKeyboardEvent(e.nativeEvent);
|
||||
if (!shortcut) return;
|
||||
setter(shortcut);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const handleOpenOcr = useCallback(() => {
|
||||
if (videoWidth === 0 || videoHeight === 0) {
|
||||
notifications.error($at("No video signal"));
|
||||
return;
|
||||
}
|
||||
setOcrMode(!isOcrMode);
|
||||
close();
|
||||
if (isMobile) {
|
||||
toggleTopBarView("ClipboardMobile");
|
||||
} else {
|
||||
setSidebarView(null);
|
||||
}
|
||||
}, [videoWidth, videoHeight, $at, setOcrMode, isOcrMode, close, toggleTopBarView, setSidebarView]);
|
||||
|
||||
return (
|
||||
<div className="space-y-4 py-3" >
|
||||
@@ -205,32 +238,42 @@ export default function Clipboard() {
|
||||
<div className="h-full space-y-4">
|
||||
<div className="space-y-4">
|
||||
|
||||
<div className="flex items-center">
|
||||
<Checkbox
|
||||
checked={overrideCtrlV}
|
||||
onChange={e => setOverrideCtrlV(e.target.checked)}
|
||||
>
|
||||
{$at("Use Ctrl+V to paste clipboard to remote")}
|
||||
</Checkbox>
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_140px] items-center gap-2 sm:grid-cols-[minmax(0,1fr)_180px]">
|
||||
<Checkbox
|
||||
className="min-w-0"
|
||||
checked={pasteShortcutEnabled}
|
||||
onChange={e => setPasteShortcutEnabled(e.target.checked)}
|
||||
>
|
||||
<span className="whitespace-normal break-words">
|
||||
{$at("Enable paste shortcut")}
|
||||
</span>
|
||||
</Checkbox>
|
||||
<Input
|
||||
size="small"
|
||||
value={pasteShortcut}
|
||||
onKeyDown={handleShortcutInput(setPasteShortcut)}
|
||||
onChange={() => void 0}
|
||||
className="w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="w-full px-1 outline-none"
|
||||
tabIndex={overrideCtrlV ? 0 : -1}
|
||||
tabIndex={pasteShortcutEnabled ? 0 : -1}
|
||||
ref={(el) => {
|
||||
if (el && overrideCtrlV && readyToRender) {
|
||||
if (el && pasteShortcutEnabled && readyToRender) {
|
||||
el.focus();
|
||||
}
|
||||
}}
|
||||
onKeyUp={e => e.stopPropagation()}
|
||||
onKeyDown={e => {
|
||||
e.stopPropagation();
|
||||
if (overrideCtrlV && (e.key.toLowerCase() === "v" || e.code === "KeyV") && (e.metaKey || e.ctrlKey)) {
|
||||
if (pasteShortcutEnabled && eventMatchesShortcut(e.nativeEvent, pasteShortcut)) {
|
||||
e.preventDefault();
|
||||
readClipboardToBufferAndSend();
|
||||
}
|
||||
}}
|
||||
onPaste={e => {
|
||||
if (overrideCtrlV) {
|
||||
if (pasteShortcutEnabled) {
|
||||
e.preventDefault();
|
||||
const txt = e.clipboardData?.getData("text") || "";
|
||||
if (txt) {
|
||||
@@ -240,7 +283,7 @@ export default function Clipboard() {
|
||||
}
|
||||
}
|
||||
}}>
|
||||
{!overrideCtrlV && readyToRender && <TextAreaWithLabel
|
||||
{readyToRender && <TextAreaWithLabel
|
||||
ref={TextAreaRef}
|
||||
label={$at("Copy text from your client to the remote host")}
|
||||
rows={4}
|
||||
@@ -295,32 +338,50 @@ export default function Clipboard() {
|
||||
|
||||
</div>
|
||||
<div
|
||||
className="flex animate-fadeIn opacity-0 items-center justify-start gap-x-2"
|
||||
className="flex animate-fadeIn opacity-0 flex-col gap-y-2"
|
||||
style={{
|
||||
animationDuration: "0.7s",
|
||||
animationDelay: "0.2s",
|
||||
}}
|
||||
>
|
||||
|
||||
<Button
|
||||
|
||||
type="primary"
|
||||
className={isMobile ? "w-[49%]" : ""}
|
||||
className="w-full"
|
||||
onClick={onConfirmPaste}
|
||||
>
|
||||
{$at("Confirm paste")}</Button>
|
||||
<Button
|
||||
className={isMobile ? "w-[49%]" : ""}
|
||||
onClick={() => {
|
||||
onCancelPasteMode();
|
||||
close();
|
||||
if(isMobile){
|
||||
toggleTopBarView("ClipboardMobile");
|
||||
}else{
|
||||
setSidebarView(null)
|
||||
}
|
||||
}}
|
||||
>{$at("Cancel")}</Button>
|
||||
|
||||
<div className="grid grid-cols-[minmax(0,1fr)_140px] items-center gap-2 sm:grid-cols-[minmax(0,1fr)_180px]">
|
||||
<Checkbox
|
||||
className="min-w-0"
|
||||
checked={ocrShortcutEnabled}
|
||||
onChange={e => setOcrShortcutEnabled(e.target.checked)}
|
||||
>
|
||||
<span className="whitespace-normal break-words">
|
||||
{$at("Enable OCR shortcut")}
|
||||
</span>
|
||||
</Checkbox>
|
||||
<Input
|
||||
size="small"
|
||||
value={ocrShortcut}
|
||||
onKeyDown={handleShortcutInput(setOcrShortcut)}
|
||||
onChange={() => void 0}
|
||||
className="w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<SettingsItem
|
||||
title={$at("OCR")}
|
||||
description={$at("Open OCR selection mode on the video area")}
|
||||
>
|
||||
<Button
|
||||
type="primary"
|
||||
className={`${isMobile ? "w-full" : ""}`}
|
||||
onClick={handleOpenOcr}
|
||||
>
|
||||
{$at("Open OCR")}
|
||||
</Button>
|
||||
</SettingsItem>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -24,7 +24,7 @@ import KeyboardPanel from "@/layout/components_bottom/keyboard/KeyboardPanel";
|
||||
import Clipboard from "@/layout/components_side/Clipboard/Clipboard";
|
||||
import SettingsModal from "@/layout/components_setting";
|
||||
import { MacroMoreList } from "@/layout/components_side/Macros/MacroTopBar";
|
||||
import { useMacrosSideTitleState , useHidStore, useMouseStore, useSettingsStore } from "@/hooks/stores";
|
||||
import { useMacrosSideTitleState , useHidStore, useMouseStore, useSettingsStore, useUiStore } from "@/hooks/stores";
|
||||
import MobileTerminal from "@/layout/components_bottom/terminal/index.mobile";
|
||||
import { dark_bg_desktop, dark_bg_style_fun } from "@/layout/theme_color";
|
||||
import PowerControl from "@/layout/components_side/Power";
|
||||
@@ -39,6 +39,7 @@ import { usePasteHandler } from "@/layout/core/desktop/hooks/usePasteHandler";
|
||||
import UsbEpModeSelect from "@/layout/components_bottom/usbepmode/UsbEpModeSelect";
|
||||
import VirtualMediaSource from "@/layout/components_side/VirtualMediaSource";
|
||||
import { useJsonRpc } from "@/hooks/useJsonRpc";
|
||||
import OcrOverlay from "@components/OcrOverlay";
|
||||
|
||||
export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number }) {
|
||||
const { $at } = useReactAt();
|
||||
@@ -49,6 +50,7 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
|
||||
const zoomContainerRef = useRef<HTMLDivElement>(null);
|
||||
const pasteCaptureRef = useRef<HTMLTextAreaElement>(null);
|
||||
const isReinitializingGadget = useHidStore(state => state.isReinitializingGadget);
|
||||
const isOcrMode = useUiStore(state => state.isOcrMode);
|
||||
const macrosSideTitle = useMacrosSideTitleState(state => state.sideTitle);
|
||||
|
||||
const videoEffects = useVideoEffects();
|
||||
@@ -267,6 +269,7 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
|
||||
`h-full w-full ${dark_bg_style_fun(isDark)} object-contain transition-all duration-1000`,
|
||||
{
|
||||
"cursor-none": videoEffects.settings.isCursorHidden,
|
||||
"pointer-events-none": isOcrMode,
|
||||
"opacity-0": overlays.shouldHideVideo,
|
||||
"opacity-60!": overlays.showPointerLockBar,
|
||||
"animate-slideUpFade dark:border-slate-300/20":
|
||||
@@ -274,6 +277,10 @@ export default function MobileDesktop({ isFullscreen }: { isFullscreen?: number
|
||||
},
|
||||
)}
|
||||
/>
|
||||
<OcrOverlay
|
||||
videoRef={videoElm as React.RefObject<HTMLVideoElement>}
|
||||
containerRef={zoomContainerRef as React.RefObject<HTMLDivElement>}
|
||||
/>
|
||||
|
||||
{(videoStream.peerConnectionState === "connected" || forceHttp) && (
|
||||
<div
|
||||
|
||||
@@ -25,6 +25,7 @@ import { MacroMoreList } from "@/layout/components_side/Macros/MacroTopBar";
|
||||
import { useUiStore, useHidStore, useSettingsStore } from "@/hooks/stores";
|
||||
import { useTouchZoom } from "@/layout/core/desktop/hooks/useTouchZoom";
|
||||
import { usePasteHandler } from "@/layout/core/desktop/hooks/usePasteHandler";
|
||||
import OcrOverlay from "@components/OcrOverlay";
|
||||
|
||||
export default function PCDesktop({ isFullscreen }: { isFullscreen?: number }) {
|
||||
const videoElm = useRef<HTMLVideoElement>(null);
|
||||
@@ -38,6 +39,7 @@ export default function PCDesktop({ isFullscreen }: { isFullscreen?: number }) {
|
||||
const setTerminalType = useUiStore(state => state.setTerminalType);
|
||||
const terminalType = useUiStore(state => state.terminalType);
|
||||
const setVirtualKeyboardEnabled = useHidStore(state => state.setVirtualKeyboardEnabled);
|
||||
const isOcrMode = useUiStore(state => state.isOcrMode);
|
||||
|
||||
const forceHttp = useSettingsStore(state => state.forceHttp);
|
||||
|
||||
@@ -116,6 +118,7 @@ export default function PCDesktop({ isFullscreen }: { isFullscreen?: number }) {
|
||||
`max-h-full min-h-[384px] max-w-full min-w-[512px] object-contain transition-all duration-1000`,
|
||||
{
|
||||
"cursor-none": videoEffects.settings.isCursorHidden,
|
||||
"pointer-events-none": isOcrMode,
|
||||
"opacity-0": overlays.shouldHideVideo,
|
||||
"opacity-60!": overlays.showPointerLockBar,
|
||||
"animate-slideUpFade shadow-xs ":
|
||||
@@ -123,6 +126,10 @@ export default function PCDesktop({ isFullscreen }: { isFullscreen?: number }) {
|
||||
},
|
||||
)}
|
||||
/>
|
||||
<OcrOverlay
|
||||
videoRef={videoElm as React.RefObject<HTMLVideoElement>}
|
||||
containerRef={zoomContainerRef as React.RefObject<HTMLDivElement>}
|
||||
/>
|
||||
|
||||
{(videoStream.peerConnectionState === "connected" || forceHttp) && (
|
||||
<div
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { useCallback } from "react";
|
||||
|
||||
import useKeyboard from "@/hooks/useKeyboard";
|
||||
import { useHidStore, useSettingsStore } from "@/hooks/stores";
|
||||
import { useHidStore, useSettingsStore, useUiStore } from "@/hooks/stores";
|
||||
import { keys, modifiers } from "@/keyboardMappings";
|
||||
import { eventMatchesShortcut } from "@/utils/shortcuts";
|
||||
|
||||
export const useKeyboardEvents = (
|
||||
pasteCaptureRef?: React.RefObject<HTMLTextAreaElement>,
|
||||
@@ -14,7 +15,9 @@ export const useKeyboardEvents = (
|
||||
const keyboardLedStateSyncAvailable = useHidStore(state => state.keyboardLedStateSyncAvailable);
|
||||
const keyboardLedSync = useSettingsStore(state => state.keyboardLedSync);
|
||||
const isKeyboardLedManagedByHost = keyboardLedSync !== "browser" && keyboardLedStateSyncAvailable;
|
||||
const overrideCtrlV = useSettingsStore(state => state.overrideCtrlV);
|
||||
const pasteShortcutEnabled = useSettingsStore(state => state.pasteShortcutEnabled);
|
||||
const pasteShortcut = useSettingsStore(state => state.pasteShortcut);
|
||||
const isOcrMode = useUiStore(state => state.isOcrMode);
|
||||
|
||||
const handleModifierKeys = useCallback((e: KeyboardEvent, activeModifiers: number[]) => {
|
||||
const { shiftKey, ctrlKey, altKey, metaKey } = e;
|
||||
@@ -28,15 +31,15 @@ export const useKeyboardEvents = (
|
||||
}, []);
|
||||
|
||||
const keyDownHandler = useCallback(async (e: KeyboardEvent) => {
|
||||
if (overrideCtrlV && (e.code === "KeyV" || e.key.toLowerCase() === "v") && (e.ctrlKey || e.metaKey)) {
|
||||
console.log("Override Ctrl V");
|
||||
if (isReinitializingGadget) return;
|
||||
if (pasteCaptureRef && pasteCaptureRef.current) {
|
||||
pasteCaptureRef.current.value = "";
|
||||
pasteCaptureRef.current.focus();
|
||||
}
|
||||
return;
|
||||
if (isOcrMode) return;
|
||||
if (pasteShortcutEnabled && eventMatchesShortcut(e, pasteShortcut)) {
|
||||
if (isReinitializingGadget) return;
|
||||
if (pasteCaptureRef && pasteCaptureRef.current) {
|
||||
pasteCaptureRef.current.value = "";
|
||||
pasteCaptureRef.current.focus();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (isReinitializingGadget) return;
|
||||
|
||||
e.preventDefault();
|
||||
@@ -74,9 +77,10 @@ export const useKeyboardEvents = (
|
||||
|
||||
// Still update the full state for legacy compatibility and UI display
|
||||
sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
|
||||
}, [handleModifierKeys, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, overrideCtrlV, pasteCaptureRef, isReinitializingGadget]);
|
||||
}, [handleModifierKeys, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, pasteShortcutEnabled, pasteShortcut, pasteCaptureRef, isReinitializingGadget, isOcrMode]);
|
||||
|
||||
const keyUpHandler = useCallback((e: KeyboardEvent) => {
|
||||
if (isOcrMode) return;
|
||||
if (isReinitializingGadget) return;
|
||||
e.preventDefault();
|
||||
const prev = useHidStore.getState();
|
||||
@@ -101,7 +105,7 @@ export const useKeyboardEvents = (
|
||||
|
||||
// Still update the full state for legacy compatibility and UI display
|
||||
sendKeyboardEvent([...new Set(newKeys)], [...new Set(newModifiers)]);
|
||||
}, [handleModifierKeys, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive]);
|
||||
}, [handleModifierKeys, sendKeyboardEvent, sendKeypress, isKeyboardLedManagedByHost, setIsNumLockActive, setIsCapsLockActive, setIsScrollLockActive, isOcrMode]);
|
||||
|
||||
const setupKeyboardEvents = useCallback(() => {
|
||||
const abortController = new AbortController();
|
||||
@@ -118,4 +122,4 @@ export const useKeyboardEvents = (
|
||||
return {
|
||||
setupKeyboardEvents,
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@@ -5,10 +5,12 @@ import { useSettingsStore, useHidStore, useUiStore } from "@/hooks/stores";
|
||||
import { keys, modifiers } from "@/keyboardMappings";
|
||||
import { chars } from "@/keyboardLayouts";
|
||||
import notifications from "@/notifications";
|
||||
import { eventMatchesShortcut } from "@/utils/shortcuts";
|
||||
|
||||
export const usePasteHandler = (pasteCaptureRef?: React.RefObject<HTMLTextAreaElement>) => {
|
||||
const [send] = useJsonRpc();
|
||||
const overrideCtrlV = useSettingsStore(state => state.overrideCtrlV);
|
||||
const pasteShortcutEnabled = useSettingsStore(state => state.pasteShortcutEnabled);
|
||||
const pasteShortcut = useSettingsStore(state => state.pasteShortcut);
|
||||
const keyboardLayout = useSettingsStore(state => state.keyboardLayout);
|
||||
const setKeyboardLayout = useSettingsStore(state => state.setKeyboardLayout);
|
||||
const debugMode = useSettingsStore(state => state.debugMode);
|
||||
@@ -140,12 +142,11 @@ export const usePasteHandler = (pasteCaptureRef?: React.RefObject<HTMLTextAreaEl
|
||||
}, [log, send, setKeyboardLayout]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!overrideCtrlV) return;
|
||||
if (!pasteShortcutEnabled) return;
|
||||
|
||||
const onKeyDownCapture = (e: KeyboardEvent) => {
|
||||
if (!overrideCtrlV) return;
|
||||
if (!(e.ctrlKey || e.metaKey)) return;
|
||||
if (e.code !== "KeyV" && e.key.toLowerCase() !== "v") return;
|
||||
if (!pasteShortcutEnabled) return;
|
||||
if (!eventMatchesShortcut(e, pasteShortcut)) return;
|
||||
if (isReinitializingGadget) return;
|
||||
|
||||
const activeElement = document.activeElement as HTMLElement | null;
|
||||
@@ -196,20 +197,20 @@ export const usePasteHandler = (pasteCaptureRef?: React.RefObject<HTMLTextAreaEl
|
||||
return () => {
|
||||
document.removeEventListener("keydown", onKeyDownCapture, { capture: true });
|
||||
};
|
||||
}, [ensureFocusTrapPaused, isReinitializingGadget, log, overrideCtrlV, pasteCaptureRef, safeKeyboardLayout, sendTextToRemote]);
|
||||
}, [ensureFocusTrapPaused, isReinitializingGadget, log, pasteShortcutEnabled, pasteShortcut, pasteCaptureRef, safeKeyboardLayout, sendTextToRemote]);
|
||||
|
||||
const handleGlobalPaste = useCallback(async (e: React.ClipboardEvent<HTMLTextAreaElement> | ClipboardEvent) => {
|
||||
if (!overrideCtrlV) return;
|
||||
if (!pasteShortcutEnabled) return;
|
||||
e.preventDefault();
|
||||
|
||||
const clipboardData = (e as React.ClipboardEvent).clipboardData || (e as ClipboardEvent).clipboardData;
|
||||
const txt = clipboardData?.getData("text") || "";
|
||||
|
||||
await sendTextToRemote(txt);
|
||||
}, [log, overrideCtrlV, safeKeyboardLayout, sendTextToRemote]);
|
||||
}, [log, pasteShortcutEnabled, safeKeyboardLayout, sendTextToRemote]);
|
||||
|
||||
return {
|
||||
handleGlobalPaste,
|
||||
overrideCtrlV,
|
||||
pasteShortcutEnabled,
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user