fix: run Electron artifact builds in the bounded worker [CI SKIP]
This commit is contained in:
+1
-1
@@ -21,7 +21,7 @@ RUN apt-get update \
|
||||
COPY package*.json ./
|
||||
RUN npm ci --omit=dev
|
||||
COPY --from=builder /app/dist ./dist
|
||||
COPY scripts/build-image.sh scripts/buildkitd.toml ./scripts/
|
||||
COPY scripts/build-image.sh scripts/buildkitd.toml scripts/artifact.Dockerfile scripts/artifact-runner.cjs ./scripts/
|
||||
COPY --from=builder /app/prisma ./prisma
|
||||
COPY --from=builder /app/node_modules/.prisma ./node_modules/.prisma
|
||||
COPY --from=builder /app/node_modules/@prisma/launcher-client ./node_modules/@prisma/launcher-client
|
||||
|
||||
@@ -15,6 +15,8 @@ settings or default builder selection are changed.
|
||||
## Covered source paths
|
||||
|
||||
- Admin app component image builds (including LMS FE/BE web images).
|
||||
- Admin Electron/AppImage jobs: dependency installation and packaging run in the
|
||||
same bounded worker, with secret build configuration and a local artifact exporter.
|
||||
- Launcher backend image builds.
|
||||
- Woodpecker image-build steps for admin FE/BE, auth, platform gateway and manual.
|
||||
|
||||
@@ -54,7 +56,10 @@ other builders do not automatically use this worker. Those workflows must also
|
||||
be routed explicitly before claiming every image build on the server is limited.
|
||||
Do not let CI cleanup remove the shared builder after each job.
|
||||
|
||||
Electron/npm builds outside Docker image builds are separate and unchanged.
|
||||
Admin Electron jobs have a 30-minute install/build timeout and a 4 GiB Node heap
|
||||
ceiling inside the aggregate 8 GiB worker limit. Git credentials and the admin
|
||||
API environment are not sent to the build context. No runtime image setting is
|
||||
required. The separate launcher-client Electron build path remains unchanged.
|
||||
This does not diagnose or guarantee prevention of power, hardware or kernel
|
||||
failures. Runtime service limits, deployment behavior and database migrations
|
||||
are not modified.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
const config = JSON.parse(fs.readFileSync('/run/secrets/build-config', 'utf8'));
|
||||
const env = {
|
||||
...process.env,
|
||||
NODE_ENV: 'development',
|
||||
...config.env,
|
||||
NODE_OPTIONS: '--max-old-space-size=4096',
|
||||
npm_config_jobs: '2',
|
||||
MAKEFLAGS: '-j2',
|
||||
CMAKE_BUILD_PARALLEL_LEVEL: '2',
|
||||
UV_THREADPOOL_SIZE: '2',
|
||||
GOMAXPROCS: '2',
|
||||
};
|
||||
for (const [phase, command] of [['install', config.installCommand], ['build', config.buildCommand]]) {
|
||||
console.log(`[artifact] ${phase} started inside resource-limited worker`);
|
||||
const result = spawnSync('/bin/sh', ['-c', command], { cwd: '/workspace', env, stdio: 'inherit' });
|
||||
if (result.error || result.status !== 0) {
|
||||
console.error(`[artifact] ${phase} failed: ${result.error?.message || result.signal || result.status}`);
|
||||
process.exit(result.status || 1);
|
||||
}
|
||||
}
|
||||
const output = fs.realpathSync(path.resolve('/workspace', config.outputDir));
|
||||
if (!output.startsWith('/workspace/')) throw new Error('Artifact output escaped the checkout');
|
||||
fs.cpSync(output, '/build-output', { recursive: true });
|
||||
@@ -0,0 +1,14 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM node:22-bookworm-slim AS build
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl git python3 make g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /workspace
|
||||
COPY source/ ./
|
||||
COPY runner.cjs /build-runner.cjs
|
||||
ARG BUILD_REQUEST_ID
|
||||
RUN --mount=type=secret,id=build-config,required=true \
|
||||
test -n "$BUILD_REQUEST_ID" && timeout --signal=TERM --kill-after=10s 1800 node /build-runner.cjs
|
||||
|
||||
FROM scratch AS artifact
|
||||
COPY --from=build /build-output/ /
|
||||
@@ -39,4 +39,9 @@ limits=$("$docker_bin" inspect "$container" --format '{{.HostConfig.CpuPeriod}}
|
||||
}
|
||||
echo '[limits] shared image builder: 2 CPUs, 8 GiB RAM, no swap'
|
||||
if [ "${1:-}" = --prepare-only ]; then exit 0; fi
|
||||
if [ "${1:-}" = --artifact-output ]; then
|
||||
destination=$2
|
||||
shift 2
|
||||
exec "$docker_bin" buildx build --builder "$builder" --output "type=local,dest=$destination" --progress plain "$@"
|
||||
fi
|
||||
exec "$docker_bin" buildx build --builder "$builder" --load --progress plain "$@"
|
||||
|
||||
@@ -62,3 +62,9 @@ test('prepare-only never builds an application', () => {
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.ok(!result.calls.some((a) => a[0] === 'buildx' && a[1] === 'build'));
|
||||
});
|
||||
|
||||
test('Electron artifacts use the same bounded worker and local output exporter', () => {
|
||||
const result = run({}, ['--artifact-output', '/tmp/artifacts', '-f', '/tmp/artifact.Dockerfile', '/tmp/context']);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.deepEqual(result.calls.at(-1), ['buildx', 'build', '--builder', 'studio-limited', '--output', 'type=local,dest=/tmp/artifacts', '--progress', 'plain', '-f', '/tmp/artifact.Dockerfile', '/tmp/context']);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, it, expect } from '@jest/globals';
|
||||
import { promises as fs } from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { prepareArtifactBuild, validateArtifactOutput } from './artifact-build-context';
|
||||
|
||||
describe('bounded artifact build context', () => {
|
||||
it.each(['../dist', '/tmp/output', '.', 'dist/../etc', 'dist\\etc', ''])('rejects unsafe output: %s', (value) => {
|
||||
expect(() => validateArtifactOutput(value)).toThrow();
|
||||
});
|
||||
|
||||
it('keeps credentials out of the source and forces a fresh bounded build', async () => {
|
||||
const source = await fs.mkdtemp(path.join(os.tmpdir(), 'artifact-context-test-'));
|
||||
let root: string | undefined;
|
||||
try {
|
||||
for (const dir of ['.git', 'node_modules', 'dist']) {
|
||||
await fs.mkdir(path.join(source, dir));
|
||||
await fs.writeFile(path.join(source, dir, 'private'), 'not-build-input');
|
||||
}
|
||||
await fs.writeFile(path.join(source, 'package.json'), '{"name":"test"}');
|
||||
const plan = await prepareArtifactBuild(source, 'dist', 'npm ci', 'npm run build:linux', [{ key: 'BUILD_TOKEN', value: 'private-token' }]);
|
||||
root = plan.root;
|
||||
expect(await fs.readdir(path.join(root, 'context/source'))).toEqual(['package.json']);
|
||||
const config = JSON.parse(await fs.readFile(path.join(root, 'config.json'), 'utf8'));
|
||||
expect(config.env).toEqual({ BUILD_TOKEN: 'private-token' });
|
||||
expect((await fs.stat(path.join(root, 'config.json'))).mode & 0o777).toBe(0o600);
|
||||
expect(plan.args).toContain('--artifact-output');
|
||||
expect(plan.args.join(' ')).not.toContain('private-token');
|
||||
expect(plan.args.some((arg) => arg.startsWith('BUILD_REQUEST_ID='))).toBe(true);
|
||||
} finally {
|
||||
await fs.rm(source, { recursive: true, force: true });
|
||||
if (root) await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
import { promises as fs } from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
export function validateArtifactOutput(value: string) {
|
||||
if (!value || value.startsWith('/') || value.includes('\\') ||
|
||||
value.split('/').some((part) => !part || part === '.' || part === '..')) {
|
||||
throw new Error('Artifact output must be a relative directory inside the checkout');
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function prepareArtifactBuild(
|
||||
workDir: string,
|
||||
outputDir: string,
|
||||
installCommand: string,
|
||||
buildCommand: string,
|
||||
variables: { key: string; value: string }[],
|
||||
) {
|
||||
validateArtifactOutput(outputDir);
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'studio-artifact-build-'));
|
||||
try {
|
||||
const context = path.join(root, 'context');
|
||||
await fs.mkdir(context);
|
||||
await fs.cp(workDir, path.join(context, 'source'), {
|
||||
recursive: true,
|
||||
filter: (source) => {
|
||||
const relative = path.relative(workDir, source);
|
||||
return !relative.split(path.sep).some((part) => part === '.git' || part === 'node_modules') &&
|
||||
relative !== outputDir && !relative.startsWith(`${outputDir}${path.sep}`);
|
||||
},
|
||||
});
|
||||
await fs.copyFile(path.resolve(__dirname, '../../scripts/artifact-runner.cjs'), path.join(context, 'runner.cjs'));
|
||||
const config = path.join(root, 'config.json');
|
||||
const env = Object.fromEntries(variables.filter((v) => /^[A-Za-z_][A-Za-z0-9_]*$/.test(v.key)).map((v) => [v.key, v.value]));
|
||||
await fs.writeFile(config, JSON.stringify({ outputDir, installCommand, buildCommand, env }), { mode: 0o600 });
|
||||
// BuildKit does not include secret contents in cache keys. Never reuse a prior
|
||||
// install/build result when commands or secret build variables change.
|
||||
return {
|
||||
root,
|
||||
output: path.join(root, 'output'),
|
||||
args: [path.resolve(__dirname, '../../scripts/build-image.sh'), '--artifact-output', path.join(root, 'output'),
|
||||
'--secret', `id=build-config,src=${config}`,
|
||||
'--build-arg', `BUILD_REQUEST_ID=${randomUUID()}`,
|
||||
'-f', path.resolve(__dirname, '../../scripts/artifact.Dockerfile'), context],
|
||||
};
|
||||
} catch (error) {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import { App, AppComponent, BuildEnvVariable, BuildJob } from '@prisma/client';
|
||||
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
import { PrismaService } from '../prisma/prisma.service';
|
||||
import { prepareArtifactBuild, validateArtifactOutput } from './artifact-build-context';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
@@ -375,41 +376,33 @@ export class BuildRunnerService implements OnModuleInit, OnModuleDestroy {
|
||||
'BUILD_ELECTRON_COMMAND',
|
||||
'npm run build:linux',
|
||||
);
|
||||
const outputDir = this.configService.get<string>(
|
||||
const outputDir = validateArtifactOutput(this.configService.get<string>(
|
||||
'BUILD_ARTIFACT_OUTPUT_DIR',
|
||||
'dist',
|
||||
);
|
||||
));
|
||||
const artifactDir = this.configService.get<string>(
|
||||
'BUILD_ARTIFACT_DIR',
|
||||
path.join(os.tmpdir(), 'studio-admin-artifacts'),
|
||||
);
|
||||
const buildEnv = resolveBuildEnv(job.component.buildVariables);
|
||||
|
||||
if (job.component.buildVariables.length > 0) {
|
||||
await append(`[artifact] build env ${job.component.buildVariables.map((variable) => `${variable.key}=${variable.isSecret ? '********' : variable.value}`).join(' ')}`);
|
||||
}
|
||||
|
||||
await append(`[artifact] install ${installCommand}`);
|
||||
await runShellCommand({
|
||||
jobId: job.id,
|
||||
command: installCommand,
|
||||
cwd: workDir,
|
||||
env: buildEnv,
|
||||
append,
|
||||
onProcess: (child) => this.activeProcesses.set(job.id, child),
|
||||
onProcessEnd: () => this.activeProcesses.delete(job.id),
|
||||
});
|
||||
|
||||
await append(`[artifact] build ${buildCommand}`);
|
||||
await runShellCommand({
|
||||
jobId: job.id,
|
||||
command: buildCommand,
|
||||
cwd: workDir,
|
||||
env: buildEnv,
|
||||
append,
|
||||
const context = await prepareArtifactBuild(workDir, outputDir, installCommand, buildCommand, job.component.buildVariables);
|
||||
try {
|
||||
if (await this.isCanceled(job.id)) throw new Error('Build cancelled');
|
||||
await append('[limits] Electron install/build: shared 2 CPUs, 8 GiB RAM, no swap, 30 minute execution timeout');
|
||||
await runCommand({
|
||||
jobId: job.id, command: '/bin/sh', args: context.args, cwd: workDir, append,
|
||||
onProcess: (child) => this.activeProcesses.set(job.id, child),
|
||||
onProcessEnd: () => this.activeProcesses.delete(job.id),
|
||||
});
|
||||
await fs.rm(path.join(workDir, outputDir), { recursive: true, force: true });
|
||||
await fs.cp(context.output, path.join(workDir, outputDir), { recursive: true });
|
||||
} finally {
|
||||
await fs.rm(context.root, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const artifact = await findNewestArtifact(path.resolve(workDir, outputDir));
|
||||
if (!artifact) {
|
||||
@@ -712,80 +705,6 @@ async function runCommand({
|
||||
});
|
||||
}
|
||||
|
||||
async function runShellCommand({
|
||||
jobId,
|
||||
command,
|
||||
cwd,
|
||||
env,
|
||||
append,
|
||||
onProcess,
|
||||
onProcessEnd,
|
||||
}: {
|
||||
jobId: string;
|
||||
command: string;
|
||||
cwd: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
append: (line: string) => Promise<void>;
|
||||
onProcess: (child: ChildProcess) => void;
|
||||
onProcessEnd: () => void;
|
||||
}) {
|
||||
await append(`[run] ${command}`);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const child = spawn(command, {
|
||||
cwd,
|
||||
env: env ?? process.env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: true,
|
||||
shell: true,
|
||||
});
|
||||
let output = '';
|
||||
onProcess(child);
|
||||
|
||||
child.stdout.on('data', (chunk: Buffer) => {
|
||||
output += chunk.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (chunk: Buffer) => {
|
||||
output += chunk.toString();
|
||||
});
|
||||
|
||||
child.on('error', (error) => {
|
||||
onProcessEnd();
|
||||
reject(error);
|
||||
});
|
||||
child.on('close', async (code) => {
|
||||
onProcessEnd();
|
||||
try {
|
||||
for (const line of output.split(/\r?\n/).filter(Boolean)) {
|
||||
await append(`[out] ${line}`);
|
||||
}
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
reject(new Error(`${command} for job ${jobId} exited with code ${code}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function resolveBuildEnv(variables: BuildEnvVariable[]): NodeJS.ProcessEnv {
|
||||
const env: NodeJS.ProcessEnv = { ...process.env };
|
||||
|
||||
for (const variable of variables) {
|
||||
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(variable.key)) continue;
|
||||
env[variable.key] = variable.value;
|
||||
}
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
async function findNewestArtifact(dir: string) {
|
||||
const candidates = await findArtifactCandidates(dir).catch(() => []);
|
||||
candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
||||
|
||||
Reference in New Issue
Block a user