From 6007e99985d547b62fbdd8e69b8f2c8d4734dc3b Mon Sep 17 00:00:00 2001 From: chyoo Date: Mon, 2 Nov 2026 17:39:58 +0900 Subject: [PATCH] fix: run Electron artifact builds in the bounded worker [CI SKIP] --- Dockerfile | 2 +- docs/image-build-limits.md | 7 +- scripts/artifact-runner.cjs | 27 ++++++ scripts/artifact.Dockerfile | 14 +++ scripts/build-image.sh | 5 ++ scripts/build-image.test.cjs | 6 ++ src/apps/artifact-build-context.spec.ts | 35 ++++++++ src/apps/artifact-build-context.ts | 52 +++++++++++ src/apps/build-runner.service.ts | 115 ++++-------------------- 9 files changed, 163 insertions(+), 100 deletions(-) create mode 100644 scripts/artifact-runner.cjs create mode 100644 scripts/artifact.Dockerfile create mode 100644 src/apps/artifact-build-context.spec.ts create mode 100644 src/apps/artifact-build-context.ts diff --git a/Dockerfile b/Dockerfile index 760704e..27f804c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN apt-get update \ COPY package*.json ./ RUN npm ci --omit=dev COPY --from=builder /app/dist ./dist -COPY scripts/build-image.sh scripts/buildkitd.toml ./scripts/ +COPY scripts/build-image.sh scripts/buildkitd.toml scripts/artifact.Dockerfile scripts/artifact-runner.cjs ./scripts/ COPY --from=builder /app/prisma ./prisma COPY --from=builder /app/node_modules/.prisma ./node_modules/.prisma COPY --from=builder /app/node_modules/@prisma/launcher-client ./node_modules/@prisma/launcher-client diff --git a/docs/image-build-limits.md b/docs/image-build-limits.md index 6a636cc..cad631b 100644 --- a/docs/image-build-limits.md +++ b/docs/image-build-limits.md @@ -15,6 +15,8 @@ settings or default builder selection are changed. ## Covered source paths - Admin app component image builds (including LMS FE/BE web images). +- Admin Electron/AppImage jobs: dependency installation and packaging run in the + same bounded worker, with secret build configuration and a local artifact exporter. - Launcher backend image builds. - Woodpecker image-build steps for admin FE/BE, auth, platform gateway and manual. @@ -54,7 +56,10 @@ other builders do not automatically use this worker. Those workflows must also be routed explicitly before claiming every image build on the server is limited. Do not let CI cleanup remove the shared builder after each job. -Electron/npm builds outside Docker image builds are separate and unchanged. +Admin Electron jobs have a 30-minute install/build timeout and a 4 GiB Node heap +ceiling inside the aggregate 8 GiB worker limit. Git credentials and the admin +API environment are not sent to the build context. No runtime image setting is +required. The separate launcher-client Electron build path remains unchanged. This does not diagnose or guarantee prevention of power, hardware or kernel failures. Runtime service limits, deployment behavior and database migrations are not modified. diff --git a/scripts/artifact-runner.cjs b/scripts/artifact-runner.cjs new file mode 100644 index 0000000..7d5a598 --- /dev/null +++ b/scripts/artifact-runner.cjs @@ -0,0 +1,27 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const config = JSON.parse(fs.readFileSync('/run/secrets/build-config', 'utf8')); +const env = { + ...process.env, + NODE_ENV: 'development', + ...config.env, + NODE_OPTIONS: '--max-old-space-size=4096', + npm_config_jobs: '2', + MAKEFLAGS: '-j2', + CMAKE_BUILD_PARALLEL_LEVEL: '2', + UV_THREADPOOL_SIZE: '2', + GOMAXPROCS: '2', +}; +for (const [phase, command] of [['install', config.installCommand], ['build', config.buildCommand]]) { + console.log(`[artifact] ${phase} started inside resource-limited worker`); + const result = spawnSync('/bin/sh', ['-c', command], { cwd: '/workspace', env, stdio: 'inherit' }); + if (result.error || result.status !== 0) { + console.error(`[artifact] ${phase} failed: ${result.error?.message || result.signal || result.status}`); + process.exit(result.status || 1); + } +} +const output = fs.realpathSync(path.resolve('/workspace', config.outputDir)); +if (!output.startsWith('/workspace/')) throw new Error('Artifact output escaped the checkout'); +fs.cpSync(output, '/build-output', { recursive: true }); diff --git a/scripts/artifact.Dockerfile b/scripts/artifact.Dockerfile new file mode 100644 index 0000000..22be2a5 --- /dev/null +++ b/scripts/artifact.Dockerfile @@ -0,0 +1,14 @@ +# syntax=docker/dockerfile:1 +FROM node:22-bookworm-slim AS build +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates curl git python3 make g++ \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /workspace +COPY source/ ./ +COPY runner.cjs /build-runner.cjs +ARG BUILD_REQUEST_ID +RUN --mount=type=secret,id=build-config,required=true \ + test -n "$BUILD_REQUEST_ID" && timeout --signal=TERM --kill-after=10s 1800 node /build-runner.cjs + +FROM scratch AS artifact +COPY --from=build /build-output/ / diff --git a/scripts/build-image.sh b/scripts/build-image.sh index cea0fac..53ef115 100644 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -39,4 +39,9 @@ limits=$("$docker_bin" inspect "$container" --format '{{.HostConfig.CpuPeriod}} } echo '[limits] shared image builder: 2 CPUs, 8 GiB RAM, no swap' if [ "${1:-}" = --prepare-only ]; then exit 0; fi +if [ "${1:-}" = --artifact-output ]; then + destination=$2 + shift 2 + exec "$docker_bin" buildx build --builder "$builder" --output "type=local,dest=$destination" --progress plain "$@" +fi exec "$docker_bin" buildx build --builder "$builder" --load --progress plain "$@" diff --git a/scripts/build-image.test.cjs b/scripts/build-image.test.cjs index 27d84b8..a9979f4 100644 --- a/scripts/build-image.test.cjs +++ b/scripts/build-image.test.cjs @@ -62,3 +62,9 @@ test('prepare-only never builds an application', () => { assert.equal(result.status, 0, result.stderr); assert.ok(!result.calls.some((a) => a[0] === 'buildx' && a[1] === 'build')); }); + +test('Electron artifacts use the same bounded worker and local output exporter', () => { + const result = run({}, ['--artifact-output', '/tmp/artifacts', '-f', '/tmp/artifact.Dockerfile', '/tmp/context']); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(result.calls.at(-1), ['buildx', 'build', '--builder', 'studio-limited', '--output', 'type=local,dest=/tmp/artifacts', '--progress', 'plain', '-f', '/tmp/artifact.Dockerfile', '/tmp/context']); +}); diff --git a/src/apps/artifact-build-context.spec.ts b/src/apps/artifact-build-context.spec.ts new file mode 100644 index 0000000..882524f --- /dev/null +++ b/src/apps/artifact-build-context.spec.ts @@ -0,0 +1,35 @@ +import { describe, it, expect } from '@jest/globals'; +import { promises as fs } from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { prepareArtifactBuild, validateArtifactOutput } from './artifact-build-context'; + +describe('bounded artifact build context', () => { + it.each(['../dist', '/tmp/output', '.', 'dist/../etc', 'dist\\etc', ''])('rejects unsafe output: %s', (value) => { + expect(() => validateArtifactOutput(value)).toThrow(); + }); + + it('keeps credentials out of the source and forces a fresh bounded build', async () => { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'artifact-context-test-')); + let root: string | undefined; + try { + for (const dir of ['.git', 'node_modules', 'dist']) { + await fs.mkdir(path.join(source, dir)); + await fs.writeFile(path.join(source, dir, 'private'), 'not-build-input'); + } + await fs.writeFile(path.join(source, 'package.json'), '{"name":"test"}'); + const plan = await prepareArtifactBuild(source, 'dist', 'npm ci', 'npm run build:linux', [{ key: 'BUILD_TOKEN', value: 'private-token' }]); + root = plan.root; + expect(await fs.readdir(path.join(root, 'context/source'))).toEqual(['package.json']); + const config = JSON.parse(await fs.readFile(path.join(root, 'config.json'), 'utf8')); + expect(config.env).toEqual({ BUILD_TOKEN: 'private-token' }); + expect((await fs.stat(path.join(root, 'config.json'))).mode & 0o777).toBe(0o600); + expect(plan.args).toContain('--artifact-output'); + expect(plan.args.join(' ')).not.toContain('private-token'); + expect(plan.args.some((arg) => arg.startsWith('BUILD_REQUEST_ID='))).toBe(true); + } finally { + await fs.rm(source, { recursive: true, force: true }); + if (root) await fs.rm(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/apps/artifact-build-context.ts b/src/apps/artifact-build-context.ts new file mode 100644 index 0000000..6e17bb6 --- /dev/null +++ b/src/apps/artifact-build-context.ts @@ -0,0 +1,52 @@ +import { promises as fs } from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { randomUUID } from 'node:crypto'; + +export function validateArtifactOutput(value: string) { + if (!value || value.startsWith('/') || value.includes('\\') || + value.split('/').some((part) => !part || part === '.' || part === '..')) { + throw new Error('Artifact output must be a relative directory inside the checkout'); + } + return value; +} + +export async function prepareArtifactBuild( + workDir: string, + outputDir: string, + installCommand: string, + buildCommand: string, + variables: { key: string; value: string }[], +) { + validateArtifactOutput(outputDir); + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'studio-artifact-build-')); + try { + const context = path.join(root, 'context'); + await fs.mkdir(context); + await fs.cp(workDir, path.join(context, 'source'), { + recursive: true, + filter: (source) => { + const relative = path.relative(workDir, source); + return !relative.split(path.sep).some((part) => part === '.git' || part === 'node_modules') && + relative !== outputDir && !relative.startsWith(`${outputDir}${path.sep}`); + }, + }); + await fs.copyFile(path.resolve(__dirname, '../../scripts/artifact-runner.cjs'), path.join(context, 'runner.cjs')); + const config = path.join(root, 'config.json'); + const env = Object.fromEntries(variables.filter((v) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(v.key)).map((v) => [v.key, v.value])); + await fs.writeFile(config, JSON.stringify({ outputDir, installCommand, buildCommand, env }), { mode: 0o600 }); + // BuildKit does not include secret contents in cache keys. Never reuse a prior + // install/build result when commands or secret build variables change. + return { + root, + output: path.join(root, 'output'), + args: [path.resolve(__dirname, '../../scripts/build-image.sh'), '--artifact-output', path.join(root, 'output'), + '--secret', `id=build-config,src=${config}`, + '--build-arg', `BUILD_REQUEST_ID=${randomUUID()}`, + '-f', path.resolve(__dirname, '../../scripts/artifact.Dockerfile'), context], + }; + } catch (error) { + await fs.rm(root, { recursive: true, force: true }); + throw error; + } +} diff --git a/src/apps/build-runner.service.ts b/src/apps/build-runner.service.ts index a8507fc..38d2b16 100644 --- a/src/apps/build-runner.service.ts +++ b/src/apps/build-runner.service.ts @@ -17,6 +17,7 @@ import { App, AppComponent, BuildEnvVariable, BuildJob } from '@prisma/client'; import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { PrismaService } from '../prisma/prisma.service'; +import { prepareArtifactBuild, validateArtifactOutput } from './artifact-build-context'; const execFileAsync = promisify(execFile); @@ -375,41 +376,33 @@ export class BuildRunnerService implements OnModuleInit, OnModuleDestroy { 'BUILD_ELECTRON_COMMAND', 'npm run build:linux', ); - const outputDir = this.configService.get( + const outputDir = validateArtifactOutput(this.configService.get( 'BUILD_ARTIFACT_OUTPUT_DIR', 'dist', - ); + )); const artifactDir = this.configService.get( 'BUILD_ARTIFACT_DIR', path.join(os.tmpdir(), 'studio-admin-artifacts'), ); - const buildEnv = resolveBuildEnv(job.component.buildVariables); if (job.component.buildVariables.length > 0) { await append(`[artifact] build env ${job.component.buildVariables.map((variable) => `${variable.key}=${variable.isSecret ? '********' : variable.value}`).join(' ')}`); } - await append(`[artifact] install ${installCommand}`); - await runShellCommand({ - jobId: job.id, - command: installCommand, - cwd: workDir, - env: buildEnv, - append, - onProcess: (child) => this.activeProcesses.set(job.id, child), - onProcessEnd: () => this.activeProcesses.delete(job.id), - }); - - await append(`[artifact] build ${buildCommand}`); - await runShellCommand({ - jobId: job.id, - command: buildCommand, - cwd: workDir, - env: buildEnv, - append, - onProcess: (child) => this.activeProcesses.set(job.id, child), - onProcessEnd: () => this.activeProcesses.delete(job.id), - }); + const context = await prepareArtifactBuild(workDir, outputDir, installCommand, buildCommand, job.component.buildVariables); + try { + if (await this.isCanceled(job.id)) throw new Error('Build cancelled'); + await append('[limits] Electron install/build: shared 2 CPUs, 8 GiB RAM, no swap, 30 minute execution timeout'); + await runCommand({ + jobId: job.id, command: '/bin/sh', args: context.args, cwd: workDir, append, + onProcess: (child) => this.activeProcesses.set(job.id, child), + onProcessEnd: () => this.activeProcesses.delete(job.id), + }); + await fs.rm(path.join(workDir, outputDir), { recursive: true, force: true }); + await fs.cp(context.output, path.join(workDir, outputDir), { recursive: true }); + } finally { + await fs.rm(context.root, { recursive: true, force: true }); + } const artifact = await findNewestArtifact(path.resolve(workDir, outputDir)); if (!artifact) { @@ -712,80 +705,6 @@ async function runCommand({ }); } -async function runShellCommand({ - jobId, - command, - cwd, - env, - append, - onProcess, - onProcessEnd, -}: { - jobId: string; - command: string; - cwd: string; - env?: NodeJS.ProcessEnv; - append: (line: string) => Promise; - onProcess: (child: ChildProcess) => void; - onProcessEnd: () => void; -}) { - await append(`[run] ${command}`); - - await new Promise((resolve, reject) => { - const child = spawn(command, { - cwd, - env: env ?? process.env, - stdio: ['ignore', 'pipe', 'pipe'], - detached: true, - shell: true, - }); - let output = ''; - onProcess(child); - - child.stdout.on('data', (chunk: Buffer) => { - output += chunk.toString(); - }); - - child.stderr.on('data', (chunk: Buffer) => { - output += chunk.toString(); - }); - - child.on('error', (error) => { - onProcessEnd(); - reject(error); - }); - child.on('close', async (code) => { - onProcessEnd(); - try { - for (const line of output.split(/\r?\n/).filter(Boolean)) { - await append(`[out] ${line}`); - } - } catch (error) { - reject(error); - return; - } - - if (code === 0) { - resolve(); - return; - } - - reject(new Error(`${command} for job ${jobId} exited with code ${code}`)); - }); - }); -} - -function resolveBuildEnv(variables: BuildEnvVariable[]): NodeJS.ProcessEnv { - const env: NodeJS.ProcessEnv = { ...process.env }; - - for (const variable of variables) { - if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(variable.key)) continue; - env[variable.key] = variable.value; - } - - return env; -} - async function findNewestArtifact(dir: string) { const candidates = await findArtifactCandidates(dir).catch(() => []); candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);