fix: run Electron artifact builds in the bounded worker [CI SKIP]

This commit is contained in:
2026-11-02 17:39:58 +09:00
parent ca4a819506
commit 6007e99985
9 changed files with 163 additions and 100 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ RUN apt-get update \
COPY package*.json ./ COPY package*.json ./
RUN npm ci --omit=dev RUN npm ci --omit=dev
COPY --from=builder /app/dist ./dist COPY --from=builder /app/dist ./dist
COPY scripts/build-image.sh scripts/buildkitd.toml ./scripts/ COPY scripts/build-image.sh scripts/buildkitd.toml scripts/artifact.Dockerfile scripts/artifact-runner.cjs ./scripts/
COPY --from=builder /app/prisma ./prisma COPY --from=builder /app/prisma ./prisma
COPY --from=builder /app/node_modules/.prisma ./node_modules/.prisma COPY --from=builder /app/node_modules/.prisma ./node_modules/.prisma
COPY --from=builder /app/node_modules/@prisma/launcher-client ./node_modules/@prisma/launcher-client COPY --from=builder /app/node_modules/@prisma/launcher-client ./node_modules/@prisma/launcher-client
+6 -1
View File
@@ -15,6 +15,8 @@ settings or default builder selection are changed.
## Covered source paths ## Covered source paths
- Admin app component image builds (including LMS FE/BE web images). - Admin app component image builds (including LMS FE/BE web images).
- Admin Electron/AppImage jobs: dependency installation and packaging run in the
same bounded worker, with secret build configuration and a local artifact exporter.
- Launcher backend image builds. - Launcher backend image builds.
- Woodpecker image-build steps for admin FE/BE, auth, platform gateway and manual. - Woodpecker image-build steps for admin FE/BE, auth, platform gateway and manual.
@@ -54,7 +56,10 @@ other builders do not automatically use this worker. Those workflows must also
be routed explicitly before claiming every image build on the server is limited. be routed explicitly before claiming every image build on the server is limited.
Do not let CI cleanup remove the shared builder after each job. Do not let CI cleanup remove the shared builder after each job.
Electron/npm builds outside Docker image builds are separate and unchanged. Admin Electron jobs have a 30-minute install/build timeout and a 4 GiB Node heap
ceiling inside the aggregate 8 GiB worker limit. Git credentials and the admin
API environment are not sent to the build context. No runtime image setting is
required. The separate launcher-client Electron build path remains unchanged.
This does not diagnose or guarantee prevention of power, hardware or kernel This does not diagnose or guarantee prevention of power, hardware or kernel
failures. Runtime service limits, deployment behavior and database migrations failures. Runtime service limits, deployment behavior and database migrations
are not modified. are not modified.
+27
View File
@@ -0,0 +1,27 @@
const fs = require('node:fs');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const config = JSON.parse(fs.readFileSync('/run/secrets/build-config', 'utf8'));
const env = {
...process.env,
NODE_ENV: 'development',
...config.env,
NODE_OPTIONS: '--max-old-space-size=4096',
npm_config_jobs: '2',
MAKEFLAGS: '-j2',
CMAKE_BUILD_PARALLEL_LEVEL: '2',
UV_THREADPOOL_SIZE: '2',
GOMAXPROCS: '2',
};
for (const [phase, command] of [['install', config.installCommand], ['build', config.buildCommand]]) {
console.log(`[artifact] ${phase} started inside resource-limited worker`);
const result = spawnSync('/bin/sh', ['-c', command], { cwd: '/workspace', env, stdio: 'inherit' });
if (result.error || result.status !== 0) {
console.error(`[artifact] ${phase} failed: ${result.error?.message || result.signal || result.status}`);
process.exit(result.status || 1);
}
}
const output = fs.realpathSync(path.resolve('/workspace', config.outputDir));
if (!output.startsWith('/workspace/')) throw new Error('Artifact output escaped the checkout');
fs.cpSync(output, '/build-output', { recursive: true });
+14
View File
@@ -0,0 +1,14 @@
# syntax=docker/dockerfile:1
FROM node:22-bookworm-slim AS build
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates curl git python3 make g++ \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /workspace
COPY source/ ./
COPY runner.cjs /build-runner.cjs
ARG BUILD_REQUEST_ID
RUN --mount=type=secret,id=build-config,required=true \
test -n "$BUILD_REQUEST_ID" && timeout --signal=TERM --kill-after=10s 1800 node /build-runner.cjs
FROM scratch AS artifact
COPY --from=build /build-output/ /
+5
View File
@@ -39,4 +39,9 @@ limits=$("$docker_bin" inspect "$container" --format '{{.HostConfig.CpuPeriod}}
} }
echo '[limits] shared image builder: 2 CPUs, 8 GiB RAM, no swap' echo '[limits] shared image builder: 2 CPUs, 8 GiB RAM, no swap'
if [ "${1:-}" = --prepare-only ]; then exit 0; fi if [ "${1:-}" = --prepare-only ]; then exit 0; fi
if [ "${1:-}" = --artifact-output ]; then
destination=$2
shift 2
exec "$docker_bin" buildx build --builder "$builder" --output "type=local,dest=$destination" --progress plain "$@"
fi
exec "$docker_bin" buildx build --builder "$builder" --load --progress plain "$@" exec "$docker_bin" buildx build --builder "$builder" --load --progress plain "$@"
+6
View File
@@ -62,3 +62,9 @@ test('prepare-only never builds an application', () => {
assert.equal(result.status, 0, result.stderr); assert.equal(result.status, 0, result.stderr);
assert.ok(!result.calls.some((a) => a[0] === 'buildx' && a[1] === 'build')); assert.ok(!result.calls.some((a) => a[0] === 'buildx' && a[1] === 'build'));
}); });
test('Electron artifacts use the same bounded worker and local output exporter', () => {
const result = run({}, ['--artifact-output', '/tmp/artifacts', '-f', '/tmp/artifact.Dockerfile', '/tmp/context']);
assert.equal(result.status, 0, result.stderr);
assert.deepEqual(result.calls.at(-1), ['buildx', 'build', '--builder', 'studio-limited', '--output', 'type=local,dest=/tmp/artifacts', '--progress', 'plain', '-f', '/tmp/artifact.Dockerfile', '/tmp/context']);
});
+35
View File
@@ -0,0 +1,35 @@
import { describe, it, expect } from '@jest/globals';
import { promises as fs } from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { prepareArtifactBuild, validateArtifactOutput } from './artifact-build-context';
describe('bounded artifact build context', () => {
it.each(['../dist', '/tmp/output', '.', 'dist/../etc', 'dist\\etc', ''])('rejects unsafe output: %s', (value) => {
expect(() => validateArtifactOutput(value)).toThrow();
});
it('keeps credentials out of the source and forces a fresh bounded build', async () => {
const source = await fs.mkdtemp(path.join(os.tmpdir(), 'artifact-context-test-'));
let root: string | undefined;
try {
for (const dir of ['.git', 'node_modules', 'dist']) {
await fs.mkdir(path.join(source, dir));
await fs.writeFile(path.join(source, dir, 'private'), 'not-build-input');
}
await fs.writeFile(path.join(source, 'package.json'), '{"name":"test"}');
const plan = await prepareArtifactBuild(source, 'dist', 'npm ci', 'npm run build:linux', [{ key: 'BUILD_TOKEN', value: 'private-token' }]);
root = plan.root;
expect(await fs.readdir(path.join(root, 'context/source'))).toEqual(['package.json']);
const config = JSON.parse(await fs.readFile(path.join(root, 'config.json'), 'utf8'));
expect(config.env).toEqual({ BUILD_TOKEN: 'private-token' });
expect((await fs.stat(path.join(root, 'config.json'))).mode & 0o777).toBe(0o600);
expect(plan.args).toContain('--artifact-output');
expect(plan.args.join(' ')).not.toContain('private-token');
expect(plan.args.some((arg) => arg.startsWith('BUILD_REQUEST_ID='))).toBe(true);
} finally {
await fs.rm(source, { recursive: true, force: true });
if (root) await fs.rm(root, { recursive: true, force: true });
}
});
});
+52
View File
@@ -0,0 +1,52 @@
import { promises as fs } from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { randomUUID } from 'node:crypto';
export function validateArtifactOutput(value: string) {
if (!value || value.startsWith('/') || value.includes('\\') ||
value.split('/').some((part) => !part || part === '.' || part === '..')) {
throw new Error('Artifact output must be a relative directory inside the checkout');
}
return value;
}
export async function prepareArtifactBuild(
workDir: string,
outputDir: string,
installCommand: string,
buildCommand: string,
variables: { key: string; value: string }[],
) {
validateArtifactOutput(outputDir);
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'studio-artifact-build-'));
try {
const context = path.join(root, 'context');
await fs.mkdir(context);
await fs.cp(workDir, path.join(context, 'source'), {
recursive: true,
filter: (source) => {
const relative = path.relative(workDir, source);
return !relative.split(path.sep).some((part) => part === '.git' || part === 'node_modules') &&
relative !== outputDir && !relative.startsWith(`${outputDir}${path.sep}`);
},
});
await fs.copyFile(path.resolve(__dirname, '../../scripts/artifact-runner.cjs'), path.join(context, 'runner.cjs'));
const config = path.join(root, 'config.json');
const env = Object.fromEntries(variables.filter((v) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(v.key)).map((v) => [v.key, v.value]));
await fs.writeFile(config, JSON.stringify({ outputDir, installCommand, buildCommand, env }), { mode: 0o600 });
// BuildKit does not include secret contents in cache keys. Never reuse a prior
// install/build result when commands or secret build variables change.
return {
root,
output: path.join(root, 'output'),
args: [path.resolve(__dirname, '../../scripts/build-image.sh'), '--artifact-output', path.join(root, 'output'),
'--secret', `id=build-config,src=${config}`,
'--build-arg', `BUILD_REQUEST_ID=${randomUUID()}`,
'-f', path.resolve(__dirname, '../../scripts/artifact.Dockerfile'), context],
};
} catch (error) {
await fs.rm(root, { recursive: true, force: true });
throw error;
}
}
+17 -98
View File
@@ -17,6 +17,7 @@ import { App, AppComponent, BuildEnvVariable, BuildJob } from '@prisma/client';
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import { prepareArtifactBuild, validateArtifactOutput } from './artifact-build-context';
const execFileAsync = promisify(execFile); const execFileAsync = promisify(execFile);
@@ -375,41 +376,33 @@ export class BuildRunnerService implements OnModuleInit, OnModuleDestroy {
'BUILD_ELECTRON_COMMAND', 'BUILD_ELECTRON_COMMAND',
'npm run build:linux', 'npm run build:linux',
); );
const outputDir = this.configService.get<string>( const outputDir = validateArtifactOutput(this.configService.get<string>(
'BUILD_ARTIFACT_OUTPUT_DIR', 'BUILD_ARTIFACT_OUTPUT_DIR',
'dist', 'dist',
); ));
const artifactDir = this.configService.get<string>( const artifactDir = this.configService.get<string>(
'BUILD_ARTIFACT_DIR', 'BUILD_ARTIFACT_DIR',
path.join(os.tmpdir(), 'studio-admin-artifacts'), path.join(os.tmpdir(), 'studio-admin-artifacts'),
); );
const buildEnv = resolveBuildEnv(job.component.buildVariables);
if (job.component.buildVariables.length > 0) { if (job.component.buildVariables.length > 0) {
await append(`[artifact] build env ${job.component.buildVariables.map((variable) => `${variable.key}=${variable.isSecret ? '********' : variable.value}`).join(' ')}`); await append(`[artifact] build env ${job.component.buildVariables.map((variable) => `${variable.key}=${variable.isSecret ? '********' : variable.value}`).join(' ')}`);
} }
await append(`[artifact] install ${installCommand}`); const context = await prepareArtifactBuild(workDir, outputDir, installCommand, buildCommand, job.component.buildVariables);
await runShellCommand({ try {
jobId: job.id, if (await this.isCanceled(job.id)) throw new Error('Build cancelled');
command: installCommand, await append('[limits] Electron install/build: shared 2 CPUs, 8 GiB RAM, no swap, 30 minute execution timeout');
cwd: workDir, await runCommand({
env: buildEnv, jobId: job.id, command: '/bin/sh', args: context.args, cwd: workDir, append,
append, onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcess: (child) => this.activeProcesses.set(job.id, child), onProcessEnd: () => this.activeProcesses.delete(job.id),
onProcessEnd: () => this.activeProcesses.delete(job.id), });
}); await fs.rm(path.join(workDir, outputDir), { recursive: true, force: true });
await fs.cp(context.output, path.join(workDir, outputDir), { recursive: true });
await append(`[artifact] build ${buildCommand}`); } finally {
await runShellCommand({ await fs.rm(context.root, { recursive: true, force: true });
jobId: job.id, }
command: buildCommand,
cwd: workDir,
env: buildEnv,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
const artifact = await findNewestArtifact(path.resolve(workDir, outputDir)); const artifact = await findNewestArtifact(path.resolve(workDir, outputDir));
if (!artifact) { if (!artifact) {
@@ -712,80 +705,6 @@ async function runCommand({
}); });
} }
async function runShellCommand({
jobId,
command,
cwd,
env,
append,
onProcess,
onProcessEnd,
}: {
jobId: string;
command: string;
cwd: string;
env?: NodeJS.ProcessEnv;
append: (line: string) => Promise<void>;
onProcess: (child: ChildProcess) => void;
onProcessEnd: () => void;
}) {
await append(`[run] ${command}`);
await new Promise<void>((resolve, reject) => {
const child = spawn(command, {
cwd,
env: env ?? process.env,
stdio: ['ignore', 'pipe', 'pipe'],
detached: true,
shell: true,
});
let output = '';
onProcess(child);
child.stdout.on('data', (chunk: Buffer) => {
output += chunk.toString();
});
child.stderr.on('data', (chunk: Buffer) => {
output += chunk.toString();
});
child.on('error', (error) => {
onProcessEnd();
reject(error);
});
child.on('close', async (code) => {
onProcessEnd();
try {
for (const line of output.split(/\r?\n/).filter(Boolean)) {
await append(`[out] ${line}`);
}
} catch (error) {
reject(error);
return;
}
if (code === 0) {
resolve();
return;
}
reject(new Error(`${command} for job ${jobId} exited with code ${code}`));
});
});
}
function resolveBuildEnv(variables: BuildEnvVariable[]): NodeJS.ProcessEnv {
const env: NodeJS.ProcessEnv = { ...process.env };
for (const variable of variables) {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(variable.key)) continue;
env[variable.key] = variable.value;
}
return env;
}
async function findNewestArtifact(dir: string) { async function findNewestArtifact(dir: string) {
const candidates = await findArtifactCandidates(dir).catch(() => []); const candidates = await findArtifactCandidates(dir).catch(() => []);
candidates.sort((a, b) => b.mtimeMs - a.mtimeMs); candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);