fix: handle A-Copilot agent responses and image filename encoding

This commit is contained in:
2026-10-19 14:47:21 +09:00
parent 8c051c4bf4
commit 6c70c53b33
9 changed files with 197 additions and 14 deletions
+32 -1
View File
@@ -16,10 +16,41 @@ function setup() {
const credentials = new AiCredentialService(new ConfigService({ AI_CREDENTIAL_ENCRYPTION_KEY: 'fixture-only-encryption-key-123456789' })); const credentials = new AiCredentialService(new ConfigService({ AI_CREDENTIAL_ENCRYPTION_KEY: 'fixture-only-encryption-key-123456789' }));
const service = new AgentImagesService(storage as unknown as R2PublicFileService, credentials); const service = new AgentImagesService(storage as unknown as R2PublicFileService, credentials);
const scope = { studentId: 'student', courseId: 'course', week: 3 }; const scope = { studentId: 'student', courseId: 'course', week: 3 };
return { objects, storage, service, scope }; return { objects, storage, service, scope, credentials };
} }
describe('AgentImagesService', () => { describe('AgentImagesService', () => {
it('repairs multipart UTF-8 filenames without changing correctly decoded names', async () => {
const h = setup();
const buffer = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'white' } }).png().toBuffer();
const koreanName = '스크린샷 2026-10-19 14-30-07.png';
for (const [originalname, expected] of [
[Buffer.from(koreanName, 'utf8').toString('latin1'), koreanName],
[koreanName, koreanName],
['screen.png', 'screen.png'],
['caf\u00e9.png', 'caf\u00e9.png'],
['\uD83D\uDCF7.png', '\uD83D\uDCF7.png'],
['../folder\\screen\u0000.png', '.._folder_screen_.png'],
]) {
const reference = await h.service.upload(h.scope, { originalname, buffer });
expect(reference.name).toBe(expected);
expect(h.service.references([reference], h.scope)[0].name).toBe(expected);
}
});
it('restores legacy filename metadata without changing the token or stored image', async () => {
const h = setup();
const buffer = await sharp({ create: { width: 2, height: 2, channels: 3, background: 'white' } }).png().toBuffer();
const name = '스크린샷 2026-10-19 14-30-07.png';
const reference = await h.service.upload(h.scope, { originalname: name, buffer });
const claim = JSON.parse(h.credentials.decrypt(reference.token));
claim.name = Buffer.from(name, 'utf8').toString('latin1');
const token = h.credentials.encrypt(JSON.stringify(claim));
const restored = h.service.references([{ token }], h.scope);
expect(restored).toEqual([{ ...reference, name, token }]);
expect((await h.service.load(restored))[0].data).toBe(h.objects.get(claim.key)!.toString('base64'));
});
it('reuses the R2 prefix, strips metadata and returns opaque scoped references', async () => { it('reuses the R2 prefix, strips metadata and returns opaque scoped references', async () => {
const h = setup(); const h = setup();
const buffer = await sharp({ create: { width: 24, height: 16, channels: 3, background: '#336699' } }).withMetadata().png().toBuffer(); const buffer = await sharp({ create: { width: 24, height: 16, channels: 3, background: '#336699' } }).withMetadata().png().toBuffer();
+10 -2
View File
@@ -10,6 +10,14 @@ export type AgentImageScope = { studentId: string; courseId: string; week: numbe
export type AgentImageReference = { token: string; name: string; mimeType: 'image/png'; size: number; url: string }; export type AgentImageReference = { token: string; name: string; mimeType: 'image/png'; size: number; url: string };
type ImageClaim = AgentImageScope & { type: 'agent-image'; id: string; name: string; key: string; size: number; sha256: string }; type ImageClaim = AgentImageScope & { type: 'agent-image'; id: string; name: string; key: string; size: number; sha256: string };
function normalizeImageFileName(fileName: string): string {
if (/^[\u0000-\u00ff]*$/.test(fileName)) {
try { fileName = new TextDecoder('utf-8', { fatal: true }).decode(Buffer.from(fileName, 'latin1')); }
catch { /* Preserve genuine Latin-1 names that are not misdecoded UTF-8. */ }
}
return fileName.replace(/[\\/\u0000-\u001f]/g, '_').slice(0, 255) || 'image.png';
}
export function supportsAgentImages(provider: string, model: string): boolean { export function supportsAgentImages(provider: string, model: string): boolean {
return provider === 'QWEN' && ['qwen3.7-plus', 'qwen3.7-flash', 'qwen3.8-max', 'qwen3.8-flash', 'qwen-vl-plus', 'qwen-vl-max'] return provider === 'QWEN' && ['qwen3.7-plus', 'qwen3.7-flash', 'qwen3.8-max', 'qwen3.8-flash', 'qwen-vl-plus', 'qwen-vl-max']
.some(id => model === id || model.startsWith(`${id}-`)); .some(id => model === id || model.startsWith(`${id}-`));
@@ -35,7 +43,7 @@ export class AgentImagesService {
} }
const id = randomUUID(); const id = randomUUID();
const key = this.storage.createLmsAgentImageKey({ courseId: scope.courseId, userId: scope.studentId, imageId: id }); const key = this.storage.createLmsAgentImageKey({ courseId: scope.courseId, userId: scope.studentId, imageId: id });
const name = file.originalname.replace(/[\\/\u0000-\u001f]/g, '_').slice(0, 255) || 'image.png'; const name = normalizeImageFileName(file.originalname);
const claim: ImageClaim = { ...scope, type: 'agent-image', id, name, key, size: body.length, sha256: createHash('sha256').update(body).digest('hex') }; const claim: ImageClaim = { ...scope, type: 'agent-image', id, name, key, size: body.length, sha256: createHash('sha256').update(body).digest('hex') };
const token = this.credentials.encrypt(JSON.stringify(claim)); const token = this.credentials.encrypt(JSON.stringify(claim));
const url = await this.storage.uploadPublicFile({ key, body, contentType: 'image/png' }).catch(() => { throw new BadGatewayException('첨부 이미지 업로드에 실패했습니다. 잠시 후 다시 전송해주세요.'); }); const url = await this.storage.uploadPublicFile({ key, body, contentType: 'image/png' }).catch(() => { throw new BadGatewayException('첨부 이미지 업로드에 실패했습니다. 잠시 후 다시 전송해주세요.'); });
@@ -61,7 +69,7 @@ export class AgentImagesService {
seen.add(claim.id); seen.add(claim.id);
const url = this.storage.publicUrl(claim.key); const url = this.storage.publicUrl(claim.key);
if (!url) throw new BadRequestException('첨부 이미지 주소를 찾을 수 없습니다.'); if (!url) throw new BadRequestException('첨부 이미지 주소를 찾을 수 없습니다.');
return { token: item.token, name: claim.name, mimeType: 'image/png', size: claim.size, url }; return { token: item.token, name: normalizeImageFileName(claim.name), mimeType: 'image/png', size: claim.size, url };
}); });
} }
+6
View File
@@ -38,6 +38,12 @@ describe('AgentService', () => {
const restored = service.toSessionResponse({ id: 'session', messages: [{ ...message, clientMessageId: message.id, summary: null, explanation: null, actions }] }); const restored = service.toSessionResponse({ id: 'session', messages: [{ ...message, clientMessageId: message.id, summary: null, explanation: null, actions }] });
expect(restored.messages[0].imageAttachments).toEqual([reference]); expect(restored.messages[0].imageAttachments).toEqual([reference]);
expect(JSON.stringify(actions)).not.toContain('base64'); expect(JSON.stringify(actions)).not.toContain('base64');
const legacy = { ...reference, name: Buffer.from('스크린샷.png', 'utf8').toString('latin1') };
const normalized = { ...reference, name: '스크린샷.png' };
service.imagesService.references.mockReturnValue([normalized]);
const history = service.toSessionResponse({ id: 'session', messages: [{ ...message, clientMessageId: message.id, summary: null, explanation: null, actions: [{ type: 'IMAGE_ATTACHMENT', payload: legacy }] }] });
expect(service.imagesService.references).toHaveBeenLastCalledWith([legacy], undefined);
expect(history.messages[0].imageAttachments).toEqual([normalized]);
}); });
const parsePlan = (content: Record<string, unknown>) => { const parsePlan = (content: Record<string, unknown>) => {
+2 -1
View File
@@ -702,13 +702,14 @@ export class AgentService {
); );
const hasPlan = const hasPlan =
message.summary !== null && message.explanation !== null; message.summary !== null && message.explanation !== null;
const imageAttachments = this.imageReferences(message.actions.filter(action => action.type === 'IMAGE_ATTACHMENT').map(action => action.payload));
return { return {
id: message.clientMessageId, id: message.clientMessageId,
role: message.role, role: message.role,
content: message.content, content: message.content,
provider: message.provider ?? undefined, provider: message.provider ?? undefined,
model: message.model ?? undefined, model: message.model ?? undefined,
...(message.actions.some(action => action.type === 'IMAGE_ATTACHMENT') ? { imageAttachments: message.actions.filter(action => action.type === 'IMAGE_ATTACHMENT').map(action => action.payload) } : {}), ...(imageAttachments.length ? { imageAttachments } : {}),
usage: usage:
message.usage && typeof message.usage === 'object' message.usage && typeof message.usage === 'object'
? message.usage ? message.usage
+5 -4
View File
@@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config';
import type { AiCompletionResult, AiProvider } from './ai.types'; import type { AiCompletionResult, AiProvider } from './ai.types';
import { getCopilotConfiguration, requestFutureDriveChatCompletion } from './copilot-backend'; import { getCopilotConfiguration, requestFutureDriveChatCompletion } from './copilot-backend';
import { normalizeCopilotPlanResponse, prepareCopilotMessages } from './copilot-agent-format';
export type CompletionInput = { export type CompletionInput = {
provider: AiProvider; provider: AiProvider;
@@ -37,10 +38,10 @@ export class AiProviderService {
const configuration = getCopilotConfiguration(this.configService); const configuration = getCopilotConfiguration(this.configService);
if (!configuration.configured) throw new BadGatewayException('A-Copilot 서버 연결이 설정되어 있지 않습니다.'); if (!configuration.configured) throw new BadGatewayException('A-Copilot 서버 연결이 설정되어 있지 않습니다.');
if (configuration.backend === 'futuredrive') { if (configuration.backend === 'futuredrive') {
return requestFutureDriveChatCompletion(this.configService, [ const response = await requestFutureDriveChatCompletion(this.configService, prepareCopilotMessages(
{ role: 'user', content: input.system }, input.system, input.messages.map(({ role, content }) => ({ role, content })), input.json === true,
...input.messages.map(({ role, content }) => ({ role, content })), ));
]); return input.json ? { ...response, content: normalizeCopilotPlanResponse(response.content) } : response;
} }
return this.openAi({ return this.openAi({
...input, ...input,
+50
View File
@@ -0,0 +1,50 @@
import { COPILOT_MESSAGE_CHAR_LIMIT, normalizeCopilotPlanResponse, prepareCopilotMessages } from './copilot-agent-format';
const plan = { summary: '모듈 확인', explanation: '실제 구성표를 확인합니다.', toolRequests: [{ id: 'inspect', name: 'practice.inspect', args: { profile: 'mkmini-2d' } }], changes: [], commands: [], createModule: null };
describe('A-Copilot agent response protocol', () => {
it('keeps the student question last and explicitly delivers agent JSON inside the RAG answer', () => {
const request = 'MK-mini에서 2D SLAM 실행 모듈 알려줘';
const messages = prepareCopilotMessages('에이전트 지침', [
{ role: 'user', content: JSON.stringify({ request, workspace: { modules: [] } }) },
{ role: 'assistant', content: '잘못된 이전 형식' },
{ role: 'user', content: JSON.stringify({ formatError: 'JSON 오류', instruction: '다시 반환' }) },
], true);
expect(messages[0].content).toContain('에이전트 지침');
expect(messages.at(-1)!.content).toContain(request);
expect(messages.at(-1)!.content).toContain('answer 본문');
expect(messages[3].content).toContain('formatError');
});
it('splits long context and surrogate pairs without dropping or truncating content', () => {
const request = '설정 확인';
const context = JSON.stringify({ request, workspace: { source: 'x'.repeat(100_000) } });
const messages = prepareCopilotMessages('지침', [{ role: 'user', content: context }], true);
expect(messages.every(message => message.content.length <= COPILOT_MESSAGE_CHAR_LIMIT)).toBe(true);
expect(messages.slice(1, -1).map(message => message.content).join('')).toBe(context);
const text = 'x'.repeat(COPILOT_MESSAGE_CHAR_LIMIT - 1) + String.fromCodePoint(0x1f680) + 'end';
const chunks = prepareCopilotMessages('system', [{ role: 'assistant', content: text }], false).slice(1);
expect(chunks.map(message => message.content).join('')).toBe(text);
expect(chunks.every(message => !/[\uD800-\uDBFF]$|^[\uDC00-\uDFFF]/.test(message.content))).toBe(true);
expect(chunks.every(message => message.role === 'assistant')).toBe(true);
});
it.each([
JSON.stringify(plan),
`\`\`\`json\n${JSON.stringify(plan)}\n\`\`\``,
`${JSON.stringify(plan)}\n\`\`\``,
`설명입니다.\n\n\`\`\`json\n${JSON.stringify(plan)}\n\`\`\`\n다음 단계입니다.`,
`설명\n\`\`\`json\r\n${JSON.stringify(plan)}\r\n\`\`\``,
])('extracts a single complete JSON plan from observed RAG output formats', content => {
expect(JSON.parse(normalizeCopilotPlanResponse(content))).toEqual(plan);
});
it('does not fabricate plans from natural-language answers or choose among ambiguous plans', () => {
for (const content of [
'설명만 반환했습니다.',
'설명\n```json\n{invalid}\n```',
'설명\n```json\n{"unrelated":true}\n```',
`설명\n\`\`\`json\n${JSON.stringify(plan)}\n\`\`\`\n다른 제안\n\`\`\`json\n${JSON.stringify({ ...plan, summary: '다른 계획' })}\n\`\`\``,
]) expect(normalizeCopilotPlanResponse(content)).toBe(content);
});
});
+51
View File
@@ -0,0 +1,51 @@
import type { CopilotMessage } from './copilot-backend';
export const COPILOT_MESSAGE_CHAR_LIMIT = 32_000;
export function prepareCopilotMessages(system: string, messages: CopilotMessage[], json: boolean): CopilotMessage[] {
const prepared: CopilotMessage[] = [{ role: 'user', content: system }, ...messages];
if (json) {
let request = messages.findLast(message => message.role === 'user')?.content ?? '';
for (const message of messages) {
if (message.role !== 'user') continue;
try {
const context = JSON.parse(message.content);
if (context && typeof context.request === 'string') { request = context.request; break; }
} catch { /* Plain text is also a valid chat message. */ }
}
prepared[0] = { role: 'user', content: `${system}\n\n긴 문맥 메시지는 API의 메시지별 길이 제한에 맞게 나누어 전달될 수 있습니다. 연속된 같은 역할의 문맥 조각은 순서대로 이어 읽으세요.` };
prepared.push({ role: 'user', content: [
`학생 요청: ${request}`,
'이 API의 일반 응답 answer 본문 안에 위 에이전트 규칙에 맞는 JSON 코드블록 하나를 넣어 답변하세요.',
'에이전트 JSON을 API 최상위 필드로 출력하지 마세요. answer를 비우지 말고 코드블록 밖의 설명은 생략하세요.',
'자료가 부족하면 추측하거나 실행하지 말고 위 스키마의 자료 조회 요청을 반환하세요.',
].join('\n\n') });
}
return prepared.flatMap(message => {
const chunks: CopilotMessage[] = [];
let offset = 0;
do {
let end = Math.min(offset + COPILOT_MESSAGE_CHAR_LIMIT, message.content.length);
const last = message.content.charCodeAt(end - 1);
const next = message.content.charCodeAt(end);
if (last >= 0xd800 && last <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) end--;
chunks.push({ role: message.role, content: message.content.slice(offset, end) });
offset = end;
} while (offset < message.content.length);
return chunks;
});
}
export function normalizeCopilotPlanResponse(content: string): string {
const normalized = content.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
try { JSON.parse(normalized); return normalized; } catch { /* RAG answers can include prose around a JSON code block. */ }
const candidates: string[] = [];
for (const match of content.matchAll(/(?:^|\n)[ \t]*```(?:json)?[ \t]*\r?\n([\s\S]*?)\r?\n[ \t]*```(?=\r?\n|$)/g)) {
try {
const value = JSON.parse(match[1]);
if (value && typeof value === 'object' && !Array.isArray(value)
&& typeof value.summary === 'string' && typeof value.explanation === 'string') candidates.push(JSON.stringify(value));
} catch { /* Non-plan code blocks do not constitute an agent response. */ }
}
return candidates.length === 1 ? candidates[0] : content;
}
+27 -3
View File
@@ -35,9 +35,10 @@ describe('managed A-Copilot backend', () => {
expect(request.body).not.toMatch(/server-only-secret|untrusted/); expect(request.body).not.toMatch(/server-only-secret|untrusted/);
} }
expect(JSON.parse(fetchMock.mock.calls[0][1].body).messages[0].content).toBe('공식 교육 내용'); expect(JSON.parse(fetchMock.mock.calls[0][1].body).messages[0].content).toBe('공식 교육 내용');
expect(JSON.parse(fetchMock.mock.calls[1][1].body).messages).toEqual([ const messages = JSON.parse(fetchMock.mock.calls[1][1].body).messages;
{ role: 'user', content: 'Return an agent JSON plan' }, { role: 'user', content: '워크스페이스 문맥' }, expect(messages[0].content).toContain('Return an agent JSON plan');
]); expect(messages[1]).toEqual({ role: 'user', content: '워크스페이스 문맥' });
expect(messages.at(-1).content).toContain('answer 본문');
}); });
it('follows the AI tab server selection when OpenAI is configured, without accepting course overrides', async () => { it('follows the AI tab server selection when OpenAI is configured, without accepting course overrides', async () => {
@@ -76,6 +77,29 @@ describe('managed A-Copilot backend', () => {
expect(fetchMock).not.toHaveBeenCalled(); expect(fetchMock).not.toHaveBeenCalled();
}); });
it('normalizes a real RAG-style answer containing prose and one agent plan without a repair call', async () => {
const answer = `모듈 구성표를 먼저 조회합니다.\n\n\`\`\`json\n${JSON.stringify(plan)}\n\`\`\``;
const fetchMock = jest.fn().mockResolvedValue({ ok: true, json: async () => ({ answer }) });
global.fetch = fetchMock as typeof fetch;
const provider = new AiProviderService(config(configuration));
const result = await provider.complete({ provider: 'A_COPILOT', model: 'a-copilot', apiKey: '', system: 'JSON plan only', json: true, messages: [{ role: 'user', content: JSON.stringify({ request: '모듈 목록 확인' }) }] });
expect(JSON.parse(result.content)).toEqual(plan);
expect(fetchMock).toHaveBeenCalledTimes(1);
});
it('distinguishes an empty upstream answer and retains its request ID without inventing a response', async () => {
global.fetch = jest.fn().mockResolvedValue({ ok: true, status: 200, json: async () => ({ answer: '', request_id: 'fixture-request-123' }) }) as typeof fetch;
const provider = new AiProviderService(config(configuration));
try {
await provider.complete({ provider: 'A_COPILOT', model: 'a-copilot', apiKey: '', system: 'test', messages: [] });
throw Error('Expected an upstream error');
} catch (error) {
expect((error as { getResponse: () => unknown }).getResponse()).toEqual(expect.objectContaining({
code: 'COPILOT_EMPTY_ANSWER', upstreamRequestId: 'fixture-request-123', message: expect.stringContaining('API 연결은 성공'),
}));
}
});
it('creates and repairs an agent JSON plan using server credentials, without a course credential', async () => { it('creates and repairs an agent JSON plan using server credentials, without a course credential', async () => {
const fetchMock = jest.fn() const fetchMock = jest.fn()
.mockResolvedValueOnce({ ok: true, json: async () => ({ answer: 'invalid plan JSON' }) }) .mockResolvedValueOnce({ ok: true, json: async () => ({ answer: 'invalid plan JSON' }) })
+14 -3
View File
@@ -24,9 +24,20 @@ export async function requestFutureDriveChatCompletion(config: ConfigService, me
headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json', 'ngrok-skip-browser-warning': '1' }, headers: { 'X-API-Key': apiKey, 'Content-Type': 'application/json', 'ngrok-skip-browser-warning': '1' },
body: JSON.stringify({ messages, lang }), body: JSON.stringify({ messages, lang }),
}); });
const data = await response.json().catch(() => ({})) as { answer?: string; error?: { message?: string }; message?: string }; const data = await response.json().catch(() => {
throw new BadGatewayException(`A-Copilot 서버가 JSON 형식이 아닌 응답을 반환했습니다. (HTTP ${response.status})`);
}) as { answer?: string; error?: { message?: string }; message?: string; request_id?: string };
if (!data || typeof data !== 'object' || Array.isArray(data)) throw new BadGatewayException('A-Copilot 서버 응답이 JSON 객체가 아닙니다.');
if (!response.ok) throw new BadGatewayException(data.error?.message ?? data.message ?? `A-Copilot request failed: ${response.status}`); if (!response.ok) throw new BadGatewayException(data.error?.message ?? data.message ?? `A-Copilot request failed: ${response.status}`);
const content = typeof data.answer === 'string' ? data.answer.trim() : ''; if (typeof data.answer !== 'string') throw new BadGatewayException('A-Copilot 서버 응답에 answer 문자열이 없습니다. 서버의 응답 형식을 확인해주세요.');
if (!content) throw new BadGatewayException('A-Copilot 응답이 비어 있습니다.'); const content = data.answer.trim();
if (!content) {
const requestId = typeof data.request_id === 'string' && /^[A-Za-z0-9_-]{1,100}$/.test(data.request_id) ? data.request_id : undefined;
throw new BadGatewayException({
message: 'A-Copilot API 연결은 성공했지만 서버가 빈 answer를 반환했습니다. 공용 AI 서버의 생성 로그를 확인해주세요.',
code: 'COPILOT_EMPTY_ANSWER',
...(requestId ? { upstreamRequestId: requestId } : {}),
});
}
return { content, usage: { promptTokens: undefined, completionTokens: undefined, totalTokens: undefined } }; return { content, usage: { promptTokens: undefined, completionTokens: undefined, totalTokens: undefined } };
} }