From fda0138dd1f3c2708fcf4bddb4ba91de72454d43 Mon Sep 17 00:00:00 2001 From: luckfox-eng29 Date: Thu, 30 Apr 2026 09:09:57 +0800 Subject: [PATCH] feat(ui): add OCR detect overlay and capture flow Signed-off-by: luckfox-eng29 --- ui/package-lock.json | 175 +++++++ ui/package.json | 1 + ui/src/components/ConfirmDialog.tsx | 10 +- ui/src/components/OcrOverlay.tsx | 458 ++++++++++++++++++ ui/src/hooks/stores.ts | 20 + .../components_side/Clipboard/Clipboard.tsx | 137 ++++-- ui/src/layout/core/desktop/DesktopMobile.tsx | 9 +- ui/src/layout/core/desktop/DesktopPC.tsx | 7 + .../core/desktop/hooks/useKeyboardEvents.ts | 30 +- .../core/desktop/hooks/usePasteHandler.ts | 19 +- 10 files changed, 802 insertions(+), 64 deletions(-) create mode 100644 ui/src/components/OcrOverlay.tsx diff --git a/ui/package-lock.json b/ui/package-lock.json index 91c94d7..c5a75b1 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -43,6 +43,7 @@ "recharts": "^2.15.3", "styled-components": "^6.1.19", "tailwind-merge": "^3.3.0", + "tesseract.js": "^7.0.0", "usehooks-ts": "^3.1.1", "validator": "^13.15.0", "w-touch": "^2.0.0", @@ -2821,6 +2822,66 @@ "node": ">=14.0.0" } }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.4.3", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.0.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.4.3", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.0.2", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.9", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.0", + "@emnapi/runtime": "^1.4.0", + "@tybys/wasm-util": "^0.9.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.9.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.0", + "dev": true, + "inBundle": true, + "license": "0BSD", + "optional": true + }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.7.tgz", @@ -3847,6 +3908,12 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "license": "MIT" }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -5777,6 +5844,12 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/idb-keyval": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz", + "integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==", + "license": "Apache-2.0" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -6199,6 +6272,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -6963,6 +7042,26 @@ "tslib": "^2.0.3" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-releases": { "version": "2.0.19", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", @@ -7112,6 +7211,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -8430,6 +8538,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", @@ -9203,6 +9317,30 @@ "node": ">=18" } }, + "node_modules/tesseract.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz", + "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^7.0.0", + "wasm-feature-detect": "^1.8.0", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz", + "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==", + "license": "Apache-2.0" + }, "node_modules/text-encoding-utf-8": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/text-encoding-utf-8/-/text-encoding-utf-8-1.0.2.tgz", @@ -9284,6 +9422,12 @@ "integrity": "sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==", "license": "MIT" }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", @@ -9739,12 +9883,34 @@ "integrity": "sha512-PYnngF+KHzZRBFI6qsm5PHwVNO5Lx3dYDrvNv//Ei9fvnYlaOWr9sKAriD/crlshWee9ZPsyvDMA4UnoR1LUHw==", "license": "MIT" }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, "node_modules/whatwg-fetch": { "version": "2.0.4", "resolved": "https://registry.npmmirror.com/whatwg-fetch/-/whatwg-fetch-2.0.4.tgz", "integrity": "sha512-dcQ1GWpOD/eEQ97k66aiEVpNnapVj90/+R+SXTPYGHpYBBypfKJEQjLrvMZ7YXbKm21gXd4NcuxUTjiv1YtLng==", "license": "MIT" }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -9890,6 +10056,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/zustand": { "version": "4.5.7", "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", diff --git a/ui/package.json b/ui/package.json index 67569c0..b3c0cc6 100644 --- a/ui/package.json +++ b/ui/package.json @@ -54,6 +54,7 @@ "recharts": "^2.15.3", "styled-components": "^6.1.19", "tailwind-merge": "^3.3.0", + "tesseract.js": "^7.0.0", "usehooks-ts": "^3.1.1", "validator": "^13.15.0", "w-touch": "^2.0.0", diff --git a/ui/src/components/ConfirmDialog.tsx b/ui/src/components/ConfirmDialog.tsx index 66dbb6d..2b64aa7 100644 --- a/ui/src/components/ConfirmDialog.tsx +++ b/ui/src/components/ConfirmDialog.tsx @@ -18,6 +18,7 @@ interface ConfirmDialogProps { onClose: () => void; title: string; description: React.ReactNode; + children?: React.ReactNode; variant?: Variant; confirmText?: string; cancelText?: string | null; @@ -65,6 +66,7 @@ export function ConfirmDialog({ onClose, title, description, + children, variant = "info", confirmText = "Confirm", cancelText = "Cancel", @@ -107,9 +109,10 @@ export function ConfirmDialog({ {description} + {children} -
+
+ {children}
-
+
{cancelText && (
); -} \ No newline at end of file +} diff --git a/ui/src/components/OcrOverlay.tsx b/ui/src/components/OcrOverlay.tsx new file mode 100644 index 0000000..3b162ca --- /dev/null +++ b/ui/src/components/OcrOverlay.tsx @@ -0,0 +1,458 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useReactAt } from "i18n-auto-extractor/react"; +import { motion } from "framer-motion"; + +import { useSettingsStore, useUiStore, useVideoStore } from "@/hooks/stores"; +import Card from "@components/Card"; +import { ConfirmDialog } from "@components/ConfirmDialog"; +import TextArea from "@components/TextArea"; +import notifications from "@/notifications"; +import { eventMatchesShortcut } from "@/utils/shortcuts"; + +interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +type OcrStatus = "idle" | "selecting" | "processing" | "result"; + +type TesseractWorker = { + recognize: (image: HTMLCanvasElement, options?: unknown, output?: unknown) => Promise<{ + data: { + text?: string; + blocks?: Array<{ + paragraphs: Array<{ + lines: Array<{ + text: string; + bbox: { x0: number }; + words: Array<{ text: string; bbox: { x0: number; x1: number } }>; + }>; + }>; + }>; + }; + }>; + terminate: () => Promise; +}; + +async function loadTesseract() { + const { createWorker } = await import("tesseract.js"); + return createWorker; +} + +let workerPromise: Promise | null = null; +let cleanupTimer: ReturnType | null = null; + +async function initWorker() { + const createWorker = await loadTesseract(); + return createWorker("eng", 1) as Promise; +} + +async function terminateWorker() { + if (!workerPromise) return; + try { + const worker = await workerPromise; + await worker.terminate(); + } catch { + // Ignore termination errors. + } + workerPromise = null; +} + +function getWorker() { + if (cleanupTimer) { + clearTimeout(cleanupTimer); + cleanupTimer = null; + } + + if (!workerPromise) { + workerPromise = initWorker().catch(err => { + workerPromise = null; + throw err; + }); + } + + // Auto-terminate worker after inactivity to reduce memory footprint. + cleanupTimer = setTimeout(() => { + void terminateWorker(); + cleanupTimer = null; + }, 60_000); + + return workerPromise; +} + +async function performOcr(canvas: HTMLCanvasElement): Promise { + const worker = await getWorker(); + const { data } = await worker.recognize(canvas, {}, { text: true, blocks: true }); + const lines = data.blocks?.flatMap(b => b.paragraphs.flatMap(p => p.lines)) ?? []; + + if (lines.length === 0) return (data.text || "").trim(); + + // Estimate character width from OCR words so left indentation is preserved. + let totalCharWidth = 0; + let samples = 0; + for (const line of lines) { + for (const word of line.words) { + const len = word.text.trim().length; + if (len > 0) { + totalCharWidth += (word.bbox.x1 - word.bbox.x0) / len; + samples++; + } + } + } + + if (samples === 0) return (data.text || "").trim(); + + const charWidth = totalCharWidth / samples; + const minX = Math.min(...lines.map(l => l.bbox.x0)); + + return lines + .map(line => { + const indent = Math.round((line.bbox.x0 - minX) / charWidth); + return " ".repeat(indent) + line.text.trim(); + }) + .join("\n") + .trim(); +} + +function captureRegion(videoEl: HTMLVideoElement, rect: Rect): HTMLCanvasElement { + const canvas = document.createElement("canvas"); + canvas.width = rect.width; + canvas.height = rect.height; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("Failed to acquire 2D canvas context"); + ctx.drawImage(videoEl, rect.x, rect.y, rect.width, rect.height, 0, 0, rect.width, rect.height); + return canvas; +} + +function getVideoDisplayRect(videoElement: HTMLVideoElement, videoWidth: number, videoHeight: number) { + const videoRect = videoElement.getBoundingClientRect(); + const elementAspectRatio = videoRect.width / videoRect.height; + const streamAspectRatio = videoWidth / videoHeight; + + let effectiveWidth = videoRect.width; + let effectiveHeight = videoRect.height; + let offsetX = 0; + let offsetY = 0; + + if (elementAspectRatio > streamAspectRatio) { + effectiveWidth = videoRect.height * streamAspectRatio; + offsetX = (videoRect.width - effectiveWidth) / 2; + } else if (elementAspectRatio < streamAspectRatio) { + effectiveHeight = videoRect.width / streamAspectRatio; + offsetY = (videoRect.height - effectiveHeight) / 2; + } + + return { videoRect, effectiveWidth, effectiveHeight, offsetX, offsetY }; +} + +interface OcrOverlayProps { + videoRef: React.RefObject; + containerRef: React.RefObject; +} + +export default function OcrOverlay({ videoRef, containerRef }: OcrOverlayProps) { + const { width: videoWidth, height: videoHeight } = useVideoStore(); + const isOcrMode = useUiStore(state => state.isOcrMode); + const setOcrMode = useUiStore(state => state.setOcrMode); + const setDisableVideoFocusTrap = useUiStore(state => state.setDisableVideoFocusTrap); + const ocrShortcutEnabled = useSettingsStore(state => state.ocrShortcutEnabled); + const ocrShortcut = useSettingsStore(state => state.ocrShortcut); + const { $at } = useReactAt(); + + const mountedRef = useRef(true); + const resultRef = useRef(null); + const [status, setStatus] = useState("idle"); + const [selectionStart, setSelectionStart] = useState<{ x: number; y: number } | null>(null); + const [selectionRect, setSelectionRect] = useState(null); + const [ocrResult, setOcrResult] = useState(""); + const [isClosing, setIsClosing] = useState(false); + + useEffect(() => { + return () => { + mountedRef.current = false; + }; + }, []); + + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + if (!ocrShortcutEnabled) return; + if (!eventMatchesShortcut(e, ocrShortcut)) return; + const activeElement = document.activeElement as HTMLElement | null; + const isEditable = + !!activeElement + && (activeElement.tagName === "INPUT" + || activeElement.tagName === "TEXTAREA" + || activeElement.isContentEditable); + if (isEditable) return; + if (videoWidth === 0 || videoHeight === 0) return; + e.preventDefault(); + e.stopPropagation(); + setOcrMode(!isOcrMode); + }; + document.addEventListener("keydown", handleKeyDown, { capture: true }); + return () => document.removeEventListener("keydown", handleKeyDown, { capture: true }); + }, [isOcrMode, ocrShortcut, ocrShortcutEnabled, setOcrMode, videoWidth, videoHeight]); + + const closeOverlay = useCallback(() => { + if (status === "processing" || status === "result") { + setIsClosing(true); + setSelectionRect(null); + setSelectionStart(null); + setTimeout(() => setOcrMode(false), 200); + return; + } + setOcrMode(false); + }, [setOcrMode, status]); + + useEffect(() => { + if (!isOcrMode) { + setStatus("idle"); + setSelectionRect(null); + setSelectionStart(null); + setOcrResult(""); + setIsClosing(false); + } + }, [isOcrMode]); + + useEffect(() => { + if (!isOcrMode) return; + if (status === "processing" || status === "result") { + setDisableVideoFocusTrap(true); + return () => setDisableVideoFocusTrap(false); + } + }, [isOcrMode, setDisableVideoFocusTrap, status]); + + useEffect(() => { + if (!isOcrMode) return; + if (status === "processing" || status === "result") return; + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") { + e.preventDefault(); + e.stopPropagation(); + setOcrMode(false); + } + }; + document.addEventListener("keydown", handleKeyDown, { capture: true }); + return () => document.removeEventListener("keydown", handleKeyDown, { capture: true }); + }, [isOcrMode, setOcrMode, status]); + + useEffect(() => { + if (!isOcrMode) return; + if (status !== "result") return; + const handleCopy = () => { + notifications.success($at("Copied"), { duration: 4000 }); + closeOverlay(); + }; + document.addEventListener("copy", handleCopy); + return () => document.removeEventListener("copy", handleCopy); + }, [closeOverlay, isOcrMode, status, $at]); + + useEffect(() => { + if (status === "result" && resultRef.current) { + resultRef.current.focus(); + resultRef.current.select(); + } + }, [status]); + + const toVideoCoords = useCallback((clientX: number, clientY: number) => { + const videoElement = videoRef.current; + if (!videoElement) return { x: 0, y: 0 }; + const { videoRect, effectiveWidth, effectiveHeight, offsetX, offsetY } = getVideoDisplayRect( + videoElement, + videoWidth, + videoHeight, + ); + const relX = clientX - videoRect.left - offsetX; + const relY = clientY - videoRect.top - offsetY; + const scaleX = videoWidth / effectiveWidth; + const scaleY = videoHeight / effectiveHeight; + return { + x: Math.max(0, Math.min(videoWidth, Math.round(relX * scaleX))), + y: Math.max(0, Math.min(videoHeight, Math.round(relY * scaleY))), + }; + }, [videoHeight, videoRef, videoWidth]); + + const finishSelection = useCallback(async () => { + const videoElement = videoRef.current; + if (status !== "selecting") return; + if (!videoElement || !selectionRect) { + setStatus("idle"); + setSelectionStart(null); + setSelectionRect(null); + return; + } + if (selectionRect.width < 10 || selectionRect.height < 10) { + setStatus("idle"); + setSelectionStart(null); + setSelectionRect(null); + return; + } + + setStatus("processing"); + try { + const canvas = captureRegion(videoElement, selectionRect); + const text = await performOcr(canvas); + canvas.width = 0; + canvas.height = 0; + + if (!mountedRef.current) return; + if (text) { + setOcrResult(text); + setStatus("result"); + } else { + notifications.error($at("No text detected")); + closeOverlay(); + } + } catch (error) { + if (!mountedRef.current) return; + console.error("OCR failed:", error); + notifications.error($at("OCR failed")); + closeOverlay(); + } + }, [closeOverlay, selectionRect, status, videoRef, $at]); + + const onPointerDown = useCallback((e: React.PointerEvent) => { + if (status === "processing" || status === "result") return; + if (!videoRef.current || videoWidth === 0 || videoHeight === 0) return; + e.preventDefault(); + e.stopPropagation(); + e.currentTarget.setPointerCapture(e.pointerId); + const coords = toVideoCoords(e.clientX, e.clientY); + setSelectionStart(coords); + setSelectionRect(null); + setStatus("selecting"); + }, [status, toVideoCoords, videoHeight, videoRef, videoWidth]); + + const onPointerMove = useCallback((e: React.PointerEvent) => { + if (status !== "selecting" || !selectionStart) return; + e.preventDefault(); + e.stopPropagation(); + const coords = toVideoCoords(e.clientX, e.clientY); + const x = Math.min(selectionStart.x, coords.x); + const y = Math.min(selectionStart.y, coords.y); + const width = Math.abs(coords.x - selectionStart.x); + const height = Math.abs(coords.y - selectionStart.y); + setSelectionRect({ x, y, width, height }); + }, [selectionStart, status, toVideoCoords]); + + const onPointerUp = useCallback((e: React.PointerEvent) => { + if (status !== "selecting") return; + e.preventDefault(); + e.stopPropagation(); + e.currentTarget.releasePointerCapture(e.pointerId); + void finishSelection(); + }, [finishSelection, status]); + + const selectionStyle = useMemo(() => { + const videoElement = videoRef.current; + const containerElement = containerRef.current; + if (!selectionRect || !videoElement || !containerElement || videoWidth === 0 || videoHeight === 0) { + return undefined; + } + + const { videoRect, effectiveWidth, effectiveHeight, offsetX, offsetY } = getVideoDisplayRect( + videoElement, + videoWidth, + videoHeight, + ); + const containerRect = containerElement.getBoundingClientRect(); + const baseX = videoRect.left - containerRect.left + offsetX; + const baseY = videoRect.top - containerRect.top + offsetY; + + return { + left: `${baseX + (selectionRect.x / videoWidth) * effectiveWidth}px`, + top: `${baseY + (selectionRect.y / videoHeight) * effectiveHeight}px`, + width: `${(selectionRect.width / videoWidth) * effectiveWidth}px`, + height: `${(selectionRect.height / videoHeight) * effectiveHeight}px`, + }; + }, [containerRef, selectionRect, videoHeight, videoRef, videoWidth]); + + if (!isOcrMode) return null; + + return ( + <> + +
+ + {status === "idle" && ( +
+
+ {$at("Drag to select text area")} +
+
+ )} + + {selectionRect && selectionStyle && status !== "result" && ( +
+ {selectionRect.width >= 10 && selectionRect.height >= 10 && ( + + {selectionRect.width} × {selectionRect.height} + + )} +
+ )} + + + { + if (status !== "result") return; + if (navigator.clipboard?.writeText && window.isSecureContext) { + navigator.clipboard.writeText(ocrResult).then(() => { + notifications.success($at("Copied"), { duration: 4000 }); + closeOverlay(); + }).catch(() => { + if (!resultRef.current) return; + resultRef.current.focus(); + resultRef.current.select(); + document.execCommand("copy"); + }); + return; + } + + if (!resultRef.current) return; + resultRef.current.focus(); + resultRef.current.select(); + document.execCommand("copy"); + }} + isConfirming={status === "processing"} + > + {status === "processing" ? ( +
+
+
+
+
+ ) : ( +
+