diff --git a/package-lock.json b/package-lock.json index 928a6e1..8de9406 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lms", - "version": "1.2.16", + "version": "1.2.17", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lms", - "version": "1.2.16", + "version": "1.2.17", "dependencies": { "@electron-toolkit/preload": "^3.0.1", "@electron-toolkit/utils": "^4.0.0", diff --git a/package.json b/package.json index 9076918..92c01ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lms", - "version": "1.2.16", + "version": "1.2.17", "description": "Autonomy Studio LMS Electron app", "main": "./out/main/index.js", "author": { diff --git a/scripts/test-agent-request-payload.cjs b/scripts/test-agent-request-payload.cjs index 879f253..326e7ee 100644 --- a/scripts/test-agent-request-payload.cjs +++ b/scripts/test-agent-request-payload.cjs @@ -11,7 +11,7 @@ vm.runInNewContext(ts.transpileModule(fs.readFileSync(require.resolve('../src/re const source = ts.createSourceFile('useAiAgent.ts', fs.readFileSync(require.resolve('../src/renderer/src/features/workspace/useAiAgent.ts'), 'utf8'), ts.ScriptTarget.Latest, true) let payloadExpression function visit(node) { - if (ts.isCallExpression(node) && node.expression.getText(source) === 'JSON.stringify' && node.arguments[0]?.getText(source).includes('options.attachments')) { + if (ts.isCallExpression(node) && node.expression.getText(source) === 'JSON.stringify' && node.arguments[0]?.getText(source).includes('imageTokens')) { payloadExpression = node.getText(source) } ts.forEachChild(node, visit) @@ -19,14 +19,22 @@ function visit(node) { visit(source) assert.ok(payloadExpression, 'Agent request JSON serialization must exist') -function serialize(attachments, finalize = false, request = '설정 위치 알려줘') { +function serialize(attachments, finalize = false, request = '설정 위치 알려줘', imageTokens = []) { return JSON.parse(vm.runInNewContext(payloadExpression, { courseId: 'course-1', weekNumber: 3, request, options: { attachments }, finalize, + textAttachments: attachments ?? [], imageTokens, agentPlanRequestText: contextApi.agentPlanRequestText, context: { modules: [{ name: 'Control', files: [] }], runtimeLogs: [] } })) } +test('sends image reference tokens without image bytes or public URLs on every context round', () => { + const body = serialize(undefined, true, '이미지 분석', ['signed-image-token']) + assert.deepEqual(body.imageTokens, ['signed-image-token']) + assert.equal(Object.hasOwn(body, 'attachments'), false) + assert.equal(JSON.stringify(body).includes('base64'), false) +}) + for (const attachments of [undefined, []]) { test(`omits attachments when ${attachments ? 'empty' : 'undefined'}`, () => { const body = serialize(attachments) diff --git a/scripts/test-agent-response-reveal.cjs b/scripts/test-agent-response-reveal.cjs index 1002220..1ba1892 100644 --- a/scripts/test-agent-response-reveal.cjs +++ b/scripts/test-agent-response-reveal.cjs @@ -72,7 +72,7 @@ test('long answers are grouped into at most twenty steps and finish within five test('only the newly appended paragraph gets the fade class', () => { const { MarkdownMessage } = load('../src/renderer/src/components/ChatWidgets.tsx', { - '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {} + '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {}, './ChatImageAttachments': {} }) const document = createAgentRevealDocument('첫 번째 문단입니다.\n\n두 번째 문단입니다.\n\n세 번째 문단입니다.') const frame = agentRevealFrame(document, 0, AGENT_REVEAL_INTERVAL) @@ -159,7 +159,7 @@ for (const dark of [false, true]) { test(`shared answer can expand and collapse while preserving full source text (${dark ? 'dark' : 'light'})`, () => { let expanded const { ExpandableMarkdownMessage, MarkdownMessage } = load('../src/renderer/src/components/ChatWidgets.tsx', { - '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {}, + '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {}, './ChatImageAttachments': {}, react: { ...React, useEffect() {}, useState(initial) { if (expanded === undefined) expanded = initial return [expanded, update => { expanded = typeof update === 'function' ? update(expanded) : update }] @@ -183,7 +183,7 @@ for (const dark of [false, true]) { test('short answers do not show an unnecessary collapse button', () => { const { ExpandableMarkdownMessage } = load('../src/renderer/src/components/ChatWidgets.tsx', { - '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {}, + '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {}, './ChatImageAttachments': {}, react: { ...React, useEffect() {}, useState: initial => [initial, () => {}] } }) const tree = ExpandableMarkdownMessage({ content: '짧은 답변입니다.', dark: true, defaultCollapsed: true }) diff --git a/scripts/test-ai-response-presentation.cjs b/scripts/test-ai-response-presentation.cjs index 3ac0d63..328af9b 100644 --- a/scripts/test-ai-response-presentation.cjs +++ b/scripts/test-ai-response-presentation.cjs @@ -16,7 +16,7 @@ function compile(source, overrides = {}, globals = {}) { } const widgets = compile(fs.readFileSync(require.resolve('../src/renderer/src/components/ChatWidgets.tsx'), 'utf8'), { - '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {} + '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': {}, './ChatImageAttachments': {} }) const panelSource = ts.createSourceFile('WorkspaceQaPanel.tsx', fs.readFileSync(require.resolve('../src/renderer/src/features/workspace/WorkspaceQaPanel.tsx'), 'utf8'), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX) const declarations = panelSource.statements.filter(node => ts.isFunctionDeclaration(node) && ['AiMessages', 'AgentMessages', 'TeacherMessages'].includes(node.name?.text)) diff --git a/scripts/test-chat-attachments.cjs b/scripts/test-chat-attachments.cjs index a480577..0993f95 100644 --- a/scripts/test-chat-attachments.cjs +++ b/scripts/test-chat-attachments.cjs @@ -16,8 +16,8 @@ test('reads source/config/log files and preserves Korean UTF-8 text', async () = assert.equal(files[2].name, 'runtime.log') }) test('rejects unsupported image/PDF files', async () => { - await assert.rejects(readChatAttachments([file('photo.png', 'image')], []), /이미지와 PDF/) - await assert.rejects(readChatAttachments([file('guide.pdf', 'pdf')], []), /이미지와 PDF/) + await assert.rejects(readChatAttachments([file('photo.png', 'image')], []), /현재 모델/) + await assert.rejects(readChatAttachments([file('guide.pdf', 'pdf')], []), /PDF/) }) test('rejects more than five attachments including existing ones', async () => { await assert.rejects(readChatAttachments([file('a.txt', 'a')], Array(5).fill({ name: 'b.txt', content: '' })), /최대 5개/) diff --git a/scripts/test-chat-clear-confirmation.cjs b/scripts/test-chat-clear-confirmation.cjs index 58a738b..5440aa7 100644 --- a/scripts/test-chat-clear-confirmation.cjs +++ b/scripts/test-chat-clear-confirmation.cjs @@ -38,6 +38,7 @@ function setup(overrides = {}, browser = {}) { '../../components/ProgressiveChatResponse': { ProgressiveChatResponse: () => null }, '../../components/ChatWidgets': { QaInputBox }, '../../components/ChatTextContextMenu': { ChatTextContextMenu }, + '../../components/ChatImageAttachments': {}, 'react-dom': { createPortal: child => child }, react: { ...React, diff --git a/scripts/test-chat-image-attachments.cjs b/scripts/test-chat-image-attachments.cjs new file mode 100644 index 0000000..354ce42 --- /dev/null +++ b/scripts/test-chat-image-attachments.cjs @@ -0,0 +1,48 @@ +const { test } = require('node:test') +const assert = require('node:assert/strict') +const fs = require('node:fs') +const vm = require('node:vm') +const ts = require('typescript') + +function setup() { + const api = {} + const revoked = [] + let count = 0 + vm.runInNewContext(ts.transpileModule(fs.readFileSync(require.resolve('../src/renderer/src/components/chatAttachments.ts'), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } + }).outputText, { + exports: api, TextDecoder, + createImageBitmap: async file => { + if (file.invalid) throw Error('invalid') + return { width: file.width ?? 10, height: file.height ?? 10, close() {} } + }, + URL: { createObjectURL: () => `blob:fixture-${++count}`, revokeObjectURL: url => revoked.push(url) } + }) + return { read: (files, existing = []) => api.readChatAttachments(files, existing, { imagesEnabled: true }), revoked } +} +const image = (extra = {}) => ({ name: 'screen.png', size: 100, ...extra }) + +test('image and text attachments coexist without counting image bytes as text context', async () => { + const h = setup() + const files = await h.read([image({ size: 5 * 1024 * 1024 }), { name: 'config.yaml', size: 10, arrayBuffer: async () => new TextEncoder().encode('robot: mini').buffer }]) + assert.equal(files[0].kind, 'image') + assert.equal(files[0].previewUrl, 'blob:fixture-1') + assert.equal(files[1].content, 'robot: mini') +}) + +test('bounds image counts, byte sizes, decoded pixels and rejects malformed images', async () => { + const h = setup() + await assert.rejects(h.read([image({ size: 5 * 1024 * 1024 + 1 })]), /5MB/) + await assert.rejects(h.read([image({ width: 4001, height: 4001 })]), /1600만/) + await assert.rejects(h.read([image({ invalid: true })]), /읽지 못/) + await assert.rejects(h.read(Array.from({ length: 4 }, () => image())), /3장/) + assert.equal(h.revoked.length, 3) +}) + +test('a mixed batch failure revokes only newly created image previews', async () => { + const h = setup() + const existing = await h.read([image()]) + await assert.rejects(h.read([image(), { name: 'file.pdf', size: 20 }], existing), /PDF/) + assert.deepEqual(h.revoked, ['blob:fixture-2']) + assert.equal(existing[0].previewUrl, 'blob:fixture-1') +}) diff --git a/scripts/test-chat-input-autosize.cjs b/scripts/test-chat-input-autosize.cjs index 29d0a55..3554451 100644 --- a/scripts/test-chat-input-autosize.cjs +++ b/scripts/test-chat-input-autosize.cjs @@ -35,6 +35,7 @@ function harness() { if (name === '../i18n') return { t: key => key, useLocaleRender() {} } if (name === './chatAttachments') return {} if (name === './ChatTextContextMenu') return { ChatTextContextMenu: () => null } + if (name === './ChatImageAttachments') return { ChatImageAttachments: () => null } if (name === 'react') return { ...React, useState(initial) { diff --git a/scripts/test-chat-text-menu.cjs b/scripts/test-chat-text-menu.cjs index 815c8bc..ac157b4 100644 --- a/scripts/test-chat-text-menu.cjs +++ b/scripts/test-chat-text-menu.cjs @@ -125,7 +125,7 @@ for (const selected of [false, true]) { let prevented = false let stopped = false const { QaInputBox } = compile('../src/renderer/src/components/ChatWidgets.tsx', { - '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': { ChatTextContextMenu: () => null }, + '../i18n': { t: key => key, useLocaleRender() {} }, './chatAttachments': {}, './ChatTextContextMenu': { ChatTextContextMenu: () => null }, './ChatImageAttachments': { ChatImageAttachments: () => null }, react: { ...React, useState: initial => [initial, value => { if (value?.kind === 'input') options = value }], useEffect() {}, useLayoutEffect() {}, useRef: () => ({ current: null }) } }, { window: { chatTextMenu: { show: async input => { options = input; return null } } } }) const tree = QaInputBox.type({ dark: true, busy: false, disabled: false, onStop() {}, onSubmit() {} }) diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 6c52553..c995da0 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -627,6 +627,7 @@ function LocalPcWorkspaceMockup({ const [selectedModule, setSelectedModule] = useState(null) const { enabled: agentEnabled, + imagesEnabled: agentImagesEnabled, statusLoading: agentStatusLoading, pending: agentPending, activity: agentActivity, @@ -4712,6 +4713,7 @@ function LocalPcWorkspaceMockup({ visibleTeacherMessages={visibleTeacherMessages} agentMessages={agentMessages} agentEnabled={agentEnabled} + agentImagesEnabled={Boolean(agentImagesEnabled)} agentStatusLoading={agentStatusLoading} agentPending={agentPending} agentActivity={agentActivity} diff --git a/src/renderer/src/components/ChatImageAttachments.tsx b/src/renderer/src/components/ChatImageAttachments.tsx new file mode 100644 index 0000000..c62b595 --- /dev/null +++ b/src/renderer/src/components/ChatImageAttachments.tsx @@ -0,0 +1,26 @@ +import { useEffect, useRef, useState } from 'react' +import { createPortal } from 'react-dom' +import { X } from 'lucide-react' +import { t, useLocaleRender } from '../i18n' + +export function ChatImageAttachments({ images }: { images: Array<{ name: string; url: string }> }): React.JSX.Element | null { + useLocaleRender() + const [preview, setPreview] = useState<{ name: string; url: string } | null>(null) + const dialogRef = useRef(null) + useEffect(() => { + if (preview) dialogRef.current?.showModal() + else dialogRef.current?.close() + }, [preview]) + if (!images.length) return null + return <> +
+ {images.map((image, index) => )} +
+ {preview && createPortal( setPreview(null)} onClick={event => { if (event.target === event.currentTarget) setPreview(null) }} className="fixed inset-0 m-auto max-h-[90vh] max-w-[94vw] overflow-auto rounded-lg border border-white/15 bg-[#202020] p-3 text-white shadow-xl backdrop:bg-black/75"> +
{preview.name}
+ {preview.name} +
, document.body)} + +} diff --git a/src/renderer/src/components/ChatWidgets.tsx b/src/renderer/src/components/ChatWidgets.tsx index 3d163ac..6eb5dc5 100644 --- a/src/renderer/src/components/ChatWidgets.tsx +++ b/src/renderer/src/components/ChatWidgets.tsx @@ -2,7 +2,8 @@ import { t, useLocaleRender } from '../i18n' import { ChevronDown, Paperclip, Send, Square, Trash2, X } from 'lucide-react' import type React from 'react' import { memo, useEffect, useLayoutEffect, useRef, useState } from 'react' -import { CHAT_ATTACHMENT_ACCEPT, readChatAttachments, type ChatAttachment } from './chatAttachments' +import { CHAT_ATTACHMENT_ACCEPT, CHAT_IMAGE_ACCEPT, readChatAttachments, type ChatAttachment } from './chatAttachments' +import { ChatImageAttachments } from './ChatImageAttachments' import ReactMarkdown from 'react-markdown' import remarkGfm from 'remark-gfm' import { ChatTextContextMenu, type ChatTextContextMenuState } from './ChatTextContextMenu' @@ -287,6 +288,7 @@ export const QaInputBox = memo(function QaInputBox({ disabled, accent = 'blue', attachmentsEnabled = false, + imagesEnabled = false, onClearConversation, clearDisabled = false, onStop, @@ -297,6 +299,7 @@ export const QaInputBox = memo(function QaInputBox({ disabled: boolean accent?: 'blue' | 'cyan' attachmentsEnabled?: boolean + imagesEnabled?: boolean onClearConversation?: () => void clearDisabled?: boolean onStop: () => void @@ -310,7 +313,28 @@ export const QaInputBox = memo(function QaInputBox({ const fileInputRef = useRef(null) const textareaRef = useRef(null) const attachmentReadRef = useRef(0) - const canSubmit = busy || (Boolean(draft.trim()) && !disabled && !readingAttachments) + const attachmentUrlsRef = useRef([]) + const canSubmit = busy || ((Boolean(draft.trim()) || attachments.some(file => file.kind === 'image')) && !disabled && !readingAttachments) + + useEffect(() => { + const urls = attachments.flatMap(file => file.kind === 'image' ? [file.previewUrl] : []) + attachmentUrlsRef.current.filter(url => !urls.includes(url)).forEach(url => URL.revokeObjectURL(url)) + attachmentUrlsRef.current = urls + }, [attachments]) + useEffect(() => () => attachmentUrlsRef.current.forEach(url => URL.revokeObjectURL(url)), []) + + const attachFiles = (files: File[]): void => { + if (!attachmentsEnabled || disabled || busy || readingAttachments || !files.length) return + const readId = ++attachmentReadRef.current + setReadingAttachments(true) + setAttachmentError(null) + void readChatAttachments(files, attachments, { imagesEnabled }).then(files => { + if (attachmentReadRef.current === readId) setAttachments(files) + else files.forEach(file => { if (file.kind === 'image' && !attachments.includes(file)) URL.revokeObjectURL(file.previewUrl) }) + }).catch(error => { + if (attachmentReadRef.current === readId) setAttachmentError(error instanceof Error ? error.message : '파일을 읽지 못했습니다.') + }).finally(() => { if (attachmentReadRef.current === readId) setReadingAttachments(false) }) + } useLayoutEffect(() => { if (textareaRef.current) resizeQaInput(textareaRef.current) @@ -331,16 +355,18 @@ export const QaInputBox = memo(function QaInputBox({ useEffect(() => { attachmentReadRef.current += 1 + setReadingAttachments(false) if (!attachmentsEnabled) { setAttachments([]) setAttachmentError(null) setReadingAttachments(false) } + else if (!imagesEnabled) setAttachments(current => current.filter(file => file.kind !== 'image')) return () => { attachmentReadRef.current += 1 } - }, [attachmentsEnabled]) + }, [attachmentsEnabled, imagesEnabled]) const submit = (): void => { - const content = draft.trim() + const content = draft.trim() || (attachments.some(file => file.kind === 'image') ? t('첨부 이미지를 분석해주세요.') : '') if (busy) { onStop() return @@ -353,7 +379,7 @@ export const QaInputBox = memo(function QaInputBox({ } return ( -
+
{ if (attachmentsEnabled && event.dataTransfer.types.includes('Files')) event.preventDefault() }} onDrop={event => { if (!attachmentsEnabled) return; event.preventDefault(); attachFiles(Array.from(event.dataTransfer.files)) }}>