lms-fe/scripts/test-agent-progress.cjs

const { test } = require('node:test')
const assert = require('node:assert/strict')
const fs = require('node:fs')
const vm = require('node:vm')
const ts = require('typescript')

function compile(source, globals = {}) {
  const exports = {}
  vm.runInNewContext(ts.transpileModule(source, {
    compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 }
  }).outputText, { exports, DOMException, Error, ...globals })
  return exports
}
const evidenceApi = compile(fs.readFileSync(require.resolve('../src/renderer/src/features/workspace/agentEvidence.ts'), 'utf8'))
const context = compile(fs.readFileSync(require.resolve('../src/renderer/src/features/workspace/agentContext.ts'), 'utf8'), {
  require: name => { assert.equal(name, './agentEvidence'); return evidenceApi }
})
const source = ts.createSourceFile('useAiAgent.ts', fs.readFileSync(require.resolve('../src/renderer/src/features/workspace/useAiAgent.ts'), 'utf8'), ts.ScriptTarget.Latest, true)
const hook = source.statements.find(node => ts.isFunctionDeclaration(node) && node.name?.text === 'useAiAgent')
const functions = hook.body.statements.filter(node => ts.isVariableStatement(node) && ['requestPlan', 'stop'].includes(node.declarationList.declarations[0].name.getText(source)))
const plain = value => JSON.parse(JSON.stringify(value))
const response = (toolRequests = []) => ({ provider: 'QWEN', model: 'fixture', plan: { summary: '분석 완료', explanation: '최종 설명', requestedModules: [], requestedLogs: [], changes: [], commands: [], createModule: null, actionPlan: [], toolRequests } })
const deferred = () => { let resolve; let reject; const promise = new Promise((yes, no) => { resolve = yes; reject = no }); return { promise, resolve, reject } }
const tick = () => new Promise(resolve => setImmediate(resolve))
const result = (extra = {}) => ({ id: 'tool', name: 'files.read', status: 'ok', target: 'fixture', observedAt: new Date().toISOString(), data: '{"exists":true,"complete":true,"content":"PRIVATE_SOURCE"}', truncated: false, ...extra })

function harness(requestApi) {
  const session = {}
  let now = 1000
  class Clock extends Date { static now() { return now } }
  const state = { pending: false, activity: '', steps: [], messages: [], saved: [], notices: 0, requestStartedAt: null }
  const messageStateRef = { current: { session } }
  const selectedModel = { provider: 'QWEN', model: 'fixture', imagesEnabled: true }
  const api = compile(functions.map(node => `export ${node.getText(source)}`).join('\n'), {
    ...context, session, messageStateRef, AbortController, FormData,
    Date: Clock,
    courseId: 'course', weekNumber: 3, pending: false, historyLoading: false, statusLoading: false,
    selectedModel,
    status: { enabled: true, selectionSupported: true }, abortControllerRef: { current: null },
    setPending: value => { state.pending = value },
    setRequestStartedAt: value => { state.requestStartedAt = value },
    setActivity: value => { state.activity = value },
    setProgressSteps: value => { state.steps = typeof value === 'function' ? value(state.steps) : value },
    updateMessages: update => { state.messages = update(state.messages); state.saved.push(plain(state.messages)) },
    onResponseRef: { current: () => { state.notices++ } }, requestApi
  })
  return { ...api, state, selectedModel, messageStateRef, advance: ms => { now += ms } }
}

test('shows and persists the question, pending state and timer before a slow module snapshot finishes', async () => {
  const snapshot = deferred(), completion = deferred()
  const bodies = []
  let loads = 0
  const h = harness((_, request) => { bodies.push(JSON.parse(request.body)); return completion.promise })
  const running = h.requestPlan('원격 차량 속도 제한 위치', [], [], {
    environment: 'remote', workspaceTarget: 'ssh:100.64.0.10', executeTool: async () => result(),
    loadInitialContext: signal => { loads++; assert.equal(signal.aborted, false); return snapshot.promise }
  })
  assert.equal(h.state.messages[0].content, '원격 차량 속도 제한 위치')
  assert.equal(h.state.saved[0][0].role, 'user')
  assert.equal(h.state.pending, true)
  assert.equal(h.state.requestStartedAt, 1000)
  assert.equal(h.state.activity, '모듈 상태 조회 중')
  assert.equal(bodies.length, 0)
  await h.requestPlan('중복 전송', [], [], { loadInitialContext: () => { loads++; return snapshot.promise } })
  assert.equal(loads, 1)
  assert.equal(h.state.messages.length, 1)
  h.advance(5000)
  h.selectedModel.provider = 'OPENAI'
  snapshot.resolve({ modules: [{ name: 'Control', state: 'running', parameters: ['sim_mode:=false'], files: [] }], runtimeLogs: [] })
  await tick()
  assert.equal(bodies.length, 1)
  assert.equal(bodies[0].provider, 'QWEN')
  assert.equal(bodies[0].modules[0].state, 'running')
  assert.deepEqual(bodies[0].modules[0].parameters, ['sim_mode:=false'])
  h.advance(2000)
  completion.resolve(response())
  await running
  assert.equal(h.state.messages[1].durationMs, 7000)
  assert.equal(h.state.pending, false)
})

for (const reject of [false, true]) test(`cancels during initial module lookup and ignores its late ${reject ? 'failure' : 'result'} without affecting a new request`, async () => {
  const old = deferred(), next = deferred()
  let calls = 0, signal
  const h = harness(async () => { calls++; return response() })
  const first = h.requestPlan('첫 질문', [], [], { loadInitialContext: current => { signal = current; return old.promise } })
  h.stop()
  assert.equal(signal.aborted, true)
  assert.equal(h.state.pending, false)
  const second = h.requestPlan('다음 질문', [], [], { loadInitialContext: () => next.promise })
  if (reject) old.reject(Error('늦게 도착한 SSH 오류'))
  else old.resolve({ modules: [], runtimeLogs: [] })
  await first
  assert.equal(calls, 0)
  assert.equal(h.state.pending, true)
  assert.equal(h.state.activity, '모듈 상태 조회 중')
  assert.equal(h.state.messages.length, 2)
  assert.equal(h.state.notices, 0)
  next.resolve({ modules: [], runtimeLogs: [] })
  await second
  assert.equal(calls, 1)
  assert.equal(h.state.messages.length, 3)
})

test('initial snapshot errors preserve the question and show a timed error without calling the model', async () => {
  const snapshot = deferred()
  let calls = 0
  const h = harness(async () => { calls++; return response() })
  const running = h.requestPlan('질문', [], [], { loadInitialContext: () => snapshot.promise })
  h.advance(3000)
  snapshot.reject(Error('원격 모듈 상태 조회 실패'))
  await running
  assert.equal(calls, 0)
  assert.equal(h.state.messages[0].content, '질문')
  assert.equal(h.state.messages[1].content, '원격 모듈 상태 조회 실패')
  assert.equal(h.state.messages[1].durationMs, 3000)
  assert.equal(h.state.pending, false)
  assert.equal(h.state.requestStartedAt, null)
  assert.equal(h.state.activity, '')
})

test('scope changes during initial snapshot loading suppress later API calls and errors', async () => {
  const snapshot = deferred()
  let calls = 0
  const h = harness(async () => { calls++; return response() })
  const running = h.requestPlan('이전 강좌 질문', [], [], { loadInitialContext: () => snapshot.promise })
  h.messageStateRef.current.session = {}
  snapshot.resolve({ modules: [], runtimeLogs: [] })
  await running
  assert.equal(calls, 0)
  assert.equal(h.state.messages.length, 1)
  assert.equal(h.state.notices, 0)
})

test('publishes completed stages immediately without persisting progress or adding model calls', async () => {
  const first = deferred(), tool = deferred(), final = deferred()
  let calls = 0
  const h = harness(() => (++calls === 1 ? first.promise : final.promise))
  const running = h.requestPlan('카메라 FPS 확인', [{ name: 'Camera', files: [] }], [], { executeTool: () => tool.promise })
  assert.equal(h.state.pending, true)
  assert.equal(h.state.requestStartedAt, 1000)
  assert.equal(h.state.steps.length, 0)
  first.resolve(response([{ id: 'read', name: 'files.read', moduleName: 'Camera', args: { path: 'config/camera.yaml' } }]))
  await tick()
  assert.deepEqual(plain(h.state.steps).map(step => step.kind), ['analysis'])
  assert.match(h.state.activity, /files.read 조회 중/)
  tool.resolve(result())
  await tick()
  assert.deepEqual(plain(h.state.steps).map(step => step.kind), ['analysis', 'tool'])
  assert.equal(h.state.steps[1].path, 'config/camera.yaml')
  assert.equal(JSON.stringify(h.state.steps).includes('PRIVATE_SOURCE'), false)
  assert.equal(h.state.messages.length, 1)
  assert.equal(h.state.saved.flat().some(message => message.progressSteps), false)
  assert.equal(calls, 2)
  h.advance(20_900)
  final.resolve(response())
  await running
  assert.equal(h.state.steps.length, 0)
  assert.equal(h.state.pending, false)
  assert.equal(h.state.activity, '')
  assert.equal(h.state.requestStartedAt, null)
  assert.equal(h.state.messages.length, 2)
  assert.equal(h.state.messages[1].plan.summary, '분석 완료')
  assert.equal(h.state.messages[1].durationMs, 20_900)
  assert.equal(h.state.saved.at(-1)[1].durationMs, 20_900)
  assert.equal(h.state.notices, 1)
})

test('image uploads and every lookup round retain the original provider without uploading the image twice', async () => {
  const bodies = []
  let uploads = 0
  const h = harness(async (path, request) => {
    if (path.endsWith('/attachments/images')) {
      uploads++
      assert.equal(request.body.get('provider'), 'QWEN')
      assert.equal(request.body.get('courseId'), 'course')
      assert.equal(request.body.get('week'), '3')
      assert.equal(request.body.get('file').name, 'screen.png')
      return { token: 'image-reference', name: 'screen.png', url: 'https://images.test/screen.png' }
    }
    bodies.push(JSON.parse(request.body))
    h.selectedModel.provider = 'OPENAI'
    return bodies.length === 1
      ? { ...response(), plan: { ...response().plan, requestedModules: ['Control'] } }
      : response()
  })
  await h.requestPlan('이미지 확인', [{ name: 'Control', files: [] }], [], {
    attachments: [{ kind: 'image', name: 'screen.png', file: new Blob(['fixture'], { type: 'image/png' }) }],
    loadRequestedContext: async () => ({ modules: [{ name: 'Control', files: [{ path: 'config/control.yaml', content: 'speed: 0.3' }] }], runtimeLogs: [] })
  })
  assert.equal(uploads, 1)
  assert.equal(bodies.length, 2)
  assert.ok(bodies.every(body => body.provider === 'QWEN'))
  assert.ok(bodies.every(body => body.imageTokens[0] === 'image-reference'))
  assert.equal(h.state.messages.at(-1).model, 'fixture')
})

test('remote target and observed source ranges reach each provider request including finalization', async () => {
  const bodies = []
  const tool = { id: 'read', name: 'files.read', moduleName: 'Control', args: { path: 'config/car.yaml', startLine: 20, lineCount: 1 } }
  const h = harness(async (_, request) => {
    bodies.push(JSON.parse(request.body))
    return bodies.length < 3 ? response([tool]) : response()
  })
  await h.requestPlan('속도 제한 코드 위치', [{ name: 'Control', state: 'running', parameters: ['sim_mode:=false'], files: [] }], [], {
    environment: 'remote', workspaceTarget: 'ssh:100.64.0.10',
    executeTool: async input => result({ id: input.id, moduleName: 'Control', target: 'ssh:100.64.0.10', truncated: true,
      data: JSON.stringify({ path: 'config/car.yaml', exists: true, hash: 'a'.repeat(64), complete: false, content: 'max_speed: 0.4', startLine: 20, endLine: 20, contentTruncated: false, totalLines: 100 }) })
  })
  assert.equal(bodies.length, 3)
  for (const body of bodies) {
    assert.equal(body.environment, 'remote')
    assert.equal(body.workspaceTarget, 'ssh:100.64.0.10')
    assert.deepEqual(body.modules[0].parameters, ['sim_mode:=false'])
  }
  assert.equal(JSON.parse(bodies[1].toolResults[0].data).content, 'max_speed: 0.4')
  assert.equal(JSON.parse(bodies[2].toolResults[0].data).endLine, 20)
  assert.match(bodies[2].request, /자료 조회는 종료되었습니다/)
})

test('errors replace temporary progress with the existing error reply', async () => {
  let calls = 0
  const h = harness(async () => {
    if (++calls === 1) return response([{ id: 'read', name: 'files.read', moduleName: 'Camera', args: { path: 'config.yaml' } }])
    throw Error('서버 오류')
  })
  await h.requestPlan('확인', [{ name: 'Camera', files: [] }], [], { executeTool: async () => result({ status: 'error' }) })
  assert.equal(h.state.messages.at(-1).content, '서버 오류')
  assert.equal(h.state.steps.length, 0)
  assert.equal(h.state.pending, false)
  assert.equal(h.state.requestStartedAt, null)
})

test('stopping clears progress and a late response cannot overwrite a newer request', async () => {
  const old = deferred(), next = deferred()
  let calls = 0
  const h = harness(() => (++calls === 1 ? old.promise : next.promise))
  const first = h.requestPlan('첫 질문', [], [])
  h.stop()
  assert.equal(h.state.requestStartedAt, null)
  assert.equal(h.state.steps.length, 0)
  assert.equal(h.state.activity, '')
  const second = h.requestPlan('다음 질문', [], [])
  old.resolve(response())
  await first
  assert.equal(h.state.pending, true)
  assert.equal(h.state.messages.filter(message => message.role === 'assistant').length, 0)
  next.resolve(response())
  await second
  assert.equal(h.state.pending, false)
  assert.equal(h.state.steps.length, 0)
  assert.equal(h.state.messages.filter(message => message.role === 'assistant').length, 1)
})

test('scope changes suppress late progress callbacks', async () => {
  const tool = deferred()
  const h = harness(async () => response([{ id: 'read', name: 'files.read', moduleName: 'Camera', args: { path: 'config.yaml' } }]))
  const running = h.requestPlan('확인', [], [], { executeTool: () => tool.promise })
  await tick()
  h.messageStateRef.current.session = {}
  const previous = h.state.steps.length
  h.stop()
  tool.resolve(result())
  await running
  assert.equal(previous, 1)
  assert.equal(h.state.steps.length, 0)
  assert.equal(h.state.notices, 0)
})

test('source context stages report missing and limited modules and empty logs honestly', async () => {
  const steps = []
  let calls = 0
  await context.resolveAgentContext({
    signal: new AbortController().signal,
    initialContext: { modules: [{ name: 'Missing', files: [] }, { name: 'Limited', files: [] }], runtimeLogs: [] },
    onStep: step => steps.push(step),
    requestPlan: async () => ++calls === 1 ? { ...response(), plan: { ...response().plan, requestedModules: ['Missing', 'Limited'], requestedLogs: ['Missing'] } } : response(),
    loadRequestedContext: async () => ({ modules: [{ name: 'Missing', files: [], fileContext: { status: 'uninstalled' } }, { name: 'Limited', files: [{ path: 'x', content: 'PRIVATE_SOURCE' }], fileContext: { status: 'limited' } }], runtimeLogs: [] })
  })
  assert.deepEqual(plain(steps).map(step => [step.kind, step.status]), [['analysis', 'ok'], ['files', 'missing'], ['files', 'partial'], ['logs', 'empty'], ['analysis', 'ok']])
  assert.equal(JSON.stringify(steps).includes('PRIVATE_SOURCE'), false)
})

test('tool stages distinguish missing files, partial reads and counts without exposing raw output', async () => {
  const steps = []
  const tools = [
    { id: 'missing', name: 'files.read', moduleName: 'Camera', args: { path: 'missing.yaml' } },
    { id: 'partial', name: 'files.read', moduleName: 'Camera', args: { path: 'large.py' } },
    { id: 'list', name: 'files.list', moduleName: 'Camera' },
    { id: 'search', name: 'files.search', moduleName: 'Camera', args: { query: 'fps' } }
  ]
  let calls = 0
  await context.resolveAgentContext({
    signal: new AbortController().signal, initialContext: { modules: [], runtimeLogs: [] }, onStep: step => steps.push(step),
    requestPlan: async () => response(++calls === 1 ? tools : []),
    executeTool: async tool => result({ data: JSON.stringify(tool.args?.path === 'missing.yaml' ? { exists: false } : tool.args?.path === 'large.py' ? { complete: false } : tool.name === 'files.list' ? { paths: ['a', 'b'] } : { matches: [] }) })
  })
  assert.deepEqual(plain(steps.filter(step => step.kind === 'tool')).map(step => [step.status, step.count]), [['missing', undefined], ['partial', undefined], ['ok', 2], ['ok', 0]])
})