fix: isolate container builds with shared resource limits [CI SKIP]
This commit is contained in:
+1
-1
@@ -13,7 +13,7 @@ steps:
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
commands:
|
||||
- docker build -t localhost:5000/studio-admin-be:0.1.6 .
|
||||
- sh scripts/build-image.sh -t localhost:5000/studio-admin-be:0.1.6 .
|
||||
- docker push localhost:5000/studio-admin-be:0.1.6
|
||||
when:
|
||||
branch: release
|
||||
|
||||
@@ -7,6 +7,7 @@ RUN npm run build
|
||||
|
||||
FROM node:22-bookworm-slim AS runner
|
||||
WORKDIR /app
|
||||
COPY --from=docker:27.5.1-cli /usr/local/libexec/docker/cli-plugins/docker-buildx /usr/local/libexec/docker/cli-plugins/docker-buildx
|
||||
ENV NODE_ENV=production
|
||||
ARG DOCKER_VERSION=27.5.1
|
||||
RUN apt-get update \
|
||||
@@ -20,6 +21,7 @@ RUN apt-get update \
|
||||
COPY package*.json ./
|
||||
RUN npm ci --omit=dev
|
||||
COPY --from=builder /app/dist ./dist
|
||||
COPY scripts/build-image.sh scripts/buildkitd.toml ./scripts/
|
||||
COPY --from=builder /app/prisma ./prisma
|
||||
COPY --from=builder /app/node_modules/.prisma ./node_modules/.prisma
|
||||
COPY --from=builder /app/node_modules/@prisma/launcher-client ./node_modules/@prisma/launcher-client
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
# Container image build limits
|
||||
|
||||
`scripts/build-image.sh` submits image builds to one Docker-container BuildKit
|
||||
worker named `studio-limited0` (container `buildx_buildkit_studio-limited0`).
|
||||
The shared worker has an aggregate 2-CPU quota, 8 GiB memory limit, no swap, and
|
||||
at most two parallel build steps. Concurrent API/CI image builds share these
|
||||
limits instead of each getting a separate allowance.
|
||||
|
||||
The wrapper explicitly selects this worker and verifies its driver, node and
|
||||
Docker resource configuration before building. Missing Buildx or mismatched
|
||||
limits fail the build, with no unrestricted fallback. The result is loaded into
|
||||
Docker so existing push/deployment steps are unchanged. No global Docker daemon
|
||||
settings or default builder selection are changed.
|
||||
|
||||
## Covered source paths
|
||||
|
||||
- Admin app component image builds (including LMS FE/BE web images).
|
||||
- Launcher backend image builds.
|
||||
- Woodpecker image-build steps for admin FE/BE, auth, platform gateway and manual.
|
||||
|
||||
The admin runtime includes Buildx and a copy of the wrapper/config. Other
|
||||
Woodpecker projects mount the host directory below read-only and invoke the same
|
||||
wrapper. Rollout of these source changes is required; preparing the worker alone
|
||||
does not redirect existing callers.
|
||||
|
||||
## Server setup
|
||||
|
||||
Copy `scripts/build-image.sh` and `scripts/buildkitd.toml` to
|
||||
`/home/future2/Studio_v2/build-tools/` before deploying the pipeline changes.
|
||||
Do not remove this directory while those pipelines reference it.
|
||||
|
||||
The old host Buildx 0.10.5 does not support the required CPU driver options.
|
||||
Use the existing Docker 27 CLI image to prepare the worker without replacing the
|
||||
host CLI or restarting services:
|
||||
|
||||
```sh
|
||||
docker run --rm --cpus=0.5 --memory=128m --memory-swap=128m \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
-v /home/future2/Studio_v2/build-tools:/tools:ro \
|
||||
docker:27-cli sh /tools/build-image.sh --prepare-only
|
||||
```
|
||||
|
||||
BuildKit's existing server image is pinned by digest, independently of app
|
||||
versions. Its host network mode supports the existing localhost:5000 registry;
|
||||
this is a trusted build service, not an untrusted-code security sandbox.
|
||||
|
||||
No `BUILD_ARTIFACT_IMAGE` or per-app build image setting is required.
|
||||
|
||||
## Scope and rollout limitations
|
||||
|
||||
This is not a daemon-wide enforcement mechanism: arbitrary `docker build`,
|
||||
unmodified GitHub Actions `setup-buildx-action`/`build-push-action` workflows, and
|
||||
other builders do not automatically use this worker. Those workflows must also
|
||||
be routed explicitly before claiming every image build on the server is limited.
|
||||
Do not let CI cleanup remove the shared builder after each job.
|
||||
|
||||
Electron/npm builds outside Docker image builds are separate and unchanged.
|
||||
This does not diagnose or guarantee prevention of power, hardware or kernel
|
||||
failures. Runtime service limits, deployment behavior and database migrations
|
||||
are not modified.
|
||||
|
||||
Tests: `node --test scripts/build-image.test.cjs` and `npx tsc --noEmit`.
|
||||
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# All clients use the same worker container on the Docker host. Limits are
|
||||
# aggregate, including builds submitted simultaneously by API and CI clients.
|
||||
docker_bin=${DOCKER_BIN:-docker}
|
||||
builder=studio-limited
|
||||
node=${builder}0
|
||||
container=buildx_buildkit_${node}
|
||||
directory=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
image=moby/buildkit@sha256:28a898719c18a33f4e8000685287fa36fd0dd9560c6440227d3a732d79bb41d8
|
||||
|
||||
"$docker_bin" buildx version >/dev/null
|
||||
if ! "$docker_bin" buildx inspect "$builder" >/dev/null 2>&1; then
|
||||
config_flag=--config
|
||||
if "$docker_bin" buildx create --help | grep -q -- --buildkitd-config; then
|
||||
config_flag=--buildkitd-config
|
||||
fi
|
||||
# Another caller may have registered the builder in this client config first.
|
||||
"$docker_bin" buildx create --name "$builder" --node "$node" \
|
||||
--driver docker-container \
|
||||
--driver-opt "image=$image,memory=8g,memory-swap=8g,cpu-period=100000,cpu-quota=200000,network=host,restart-policy=unless-stopped" \
|
||||
"$config_flag" "$directory/buildkitd.toml" >/dev/null \
|
||||
|| "$docker_bin" buildx inspect "$builder" >/dev/null
|
||||
fi
|
||||
|
||||
info=$("$docker_bin" buildx inspect "$builder")
|
||||
driver=$(printf '%s\n' "$info" | awk '$1 == "Driver:" {print $2}')
|
||||
[ "$driver" = docker-container ] || { echo 'Refusing unrestricted build driver' >&2; exit 1; }
|
||||
names=$(printf '%s\n' "$info" | awk '$1 == "Name:" {print $2}')
|
||||
[ "$names" = "$(printf '%s\n%s' "$builder" "$node")" ] || {
|
||||
echo 'Refusing unexpected builder nodes' >&2; exit 1
|
||||
}
|
||||
"$docker_bin" buildx inspect "$builder" --bootstrap >/dev/null
|
||||
limits=$("$docker_bin" inspect "$container" --format '{{.HostConfig.CpuPeriod}} {{.HostConfig.CpuQuota}} {{.HostConfig.Memory}} {{.HostConfig.MemorySwap}}')
|
||||
[ "$limits" = '100000 200000 8589934592 8589934592' ] || {
|
||||
echo "Refusing build: unexpected worker limits: $limits" >&2
|
||||
exit 1
|
||||
}
|
||||
echo '[limits] shared image builder: 2 CPUs, 8 GiB RAM, no swap'
|
||||
if [ "${1:-}" = --prepare-only ]; then exit 0; fi
|
||||
exec "$docker_bin" buildx build --builder "$builder" --load --progress plain "$@"
|
||||
@@ -0,0 +1,64 @@
|
||||
const { test } = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { spawnSync } = require('node:child_process');
|
||||
|
||||
function run(overrides = {}, args = ['-t', 'test:limited', '.']) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'build-limits-test-'));
|
||||
try {
|
||||
const fake = path.join(dir, 'docker');
|
||||
fs.writeFileSync(fake, `#!${process.execPath}
|
||||
const fs = require('node:fs');
|
||||
const a = process.argv.slice(2);
|
||||
fs.appendFileSync(process.env.CALLS, JSON.stringify(a) + '\\n');
|
||||
if (a[0] === 'buildx' && a[1] === 'inspect') {
|
||||
if (process.env.ABSENT && !fs.existsSync(process.env.CREATED)) process.exit(1);
|
||||
console.log('Name: studio-limited\\nDriver: ' + (process.env.DRIVER || 'docker-container') + '\\nNodes:\\nName: ' + (process.env.NODE || 'studio-limited0'));
|
||||
}
|
||||
if (a[0] === 'buildx' && a[1] === 'create') {
|
||||
if (a.includes('--help')) console.log('--buildkitd-config');
|
||||
else fs.writeFileSync(process.env.CREATED, 'yes');
|
||||
}
|
||||
if (a[0] === 'inspect') console.log(process.env.LIMITS || '100000 200000 8589934592 8589934592');
|
||||
`, { mode: 0o755 });
|
||||
const result = spawnSync('/bin/sh', [path.join(__dirname, 'build-image.sh'), ...args], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, DOCKER_BIN: fake, CALLS: path.join(dir, 'calls'), CREATED: path.join(dir, 'created'), ...overrides },
|
||||
});
|
||||
const calls = fs.readFileSync(path.join(dir, 'calls'), 'utf8').trim().split('\n').map(JSON.parse);
|
||||
return { ...result, calls };
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('uses the shared worker and loads the result for existing docker push', () => {
|
||||
const result = run();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.deepEqual(result.calls.at(-1), ['buildx', 'build', '--builder', 'studio-limited', '--load', '--progress', 'plain', '-t', 'test:limited', '.']);
|
||||
});
|
||||
|
||||
test('creates a bounded worker without changing the default builder', () => {
|
||||
const result = run({ ABSENT: '1' });
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const create = result.calls.find((a) => a[1] === 'create' && !a.includes('--help'));
|
||||
const options = create[create.indexOf('--driver-opt') + 1];
|
||||
for (const value of ['memory=8g', 'memory-swap=8g', 'cpu-period=100000', 'cpu-quota=200000']) assert.ok(options.includes(value));
|
||||
assert.ok(!create.includes('--use'));
|
||||
});
|
||||
|
||||
for (const overrides of [{ LIMITS: '0 0 0 0' }, { DRIVER: 'docker' }, { NODE: 'other-node' }]) {
|
||||
test(`refuses an unbounded or unexpected worker: ${JSON.stringify(overrides)}`, () => {
|
||||
const result = run(overrides);
|
||||
assert.notEqual(result.status, 0);
|
||||
assert.ok(!result.calls.some((a) => a[0] === 'buildx' && a[1] === 'build'));
|
||||
});
|
||||
}
|
||||
|
||||
test('prepare-only never builds an application', () => {
|
||||
const result = run({}, ['--prepare-only']);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.ok(!result.calls.some((a) => a[0] === 'buildx' && a[1] === 'build'));
|
||||
});
|
||||
@@ -0,0 +1,5 @@
|
||||
[worker.oci]
|
||||
max-parallelism = 2
|
||||
|
||||
[registry."localhost:5000"]
|
||||
http = true
|
||||
@@ -296,6 +296,9 @@ export class BuildRunnerService implements OnModuleInit, OnModuleDestroy {
|
||||
) {
|
||||
const image = this.resolveImage(job.image!);
|
||||
const builder = this.configService.get<string>('BUILD_IMAGE_BUILDER', 'docker');
|
||||
if (builder !== 'docker') {
|
||||
throw new Error('Image builds require the resource-limited Docker Buildx worker');
|
||||
}
|
||||
const dockerfile = this.configService.get<string>('BUILD_DOCKERFILE', 'Dockerfile');
|
||||
const contextPath = this.configService.get<string>('BUILD_CONTEXT_PATH', '.');
|
||||
const contextDir = path.resolve(workDir, contextPath);
|
||||
@@ -328,38 +331,10 @@ export class BuildRunnerService implements OnModuleInit, OnModuleDestroy {
|
||||
await append(`[image] rewritten to ${image}`);
|
||||
}
|
||||
|
||||
if (builder === 'nerdctl') {
|
||||
const namespace = this.configService.get<string>('BUILD_NERDCTL_NAMESPACE');
|
||||
const prefix = namespace ? ['--namespace', namespace] : [];
|
||||
|
||||
await runCommand({
|
||||
jobId: job.id,
|
||||
command: this.configService.get<string>('NERDCTL_BIN', 'nerdctl'),
|
||||
args: [...prefix, 'build', ...buildArgs, '-t', image, '-f', dockerfile, '.'],
|
||||
cwd: contextDir,
|
||||
append,
|
||||
onProcess: (child) => this.activeProcesses.set(job.id, child),
|
||||
onProcessEnd: () => this.activeProcesses.delete(job.id),
|
||||
});
|
||||
|
||||
if (pushEnabled) {
|
||||
await runCommand({
|
||||
jobId: job.id,
|
||||
command: this.configService.get<string>('NERDCTL_BIN', 'nerdctl'),
|
||||
args: [...prefix, 'push', image],
|
||||
cwd: contextDir,
|
||||
append,
|
||||
onProcess: (child) => this.activeProcesses.set(job.id, child),
|
||||
onProcessEnd: () => this.activeProcesses.delete(job.id),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
await runCommand({
|
||||
jobId: job.id,
|
||||
command: this.configService.get<string>('DOCKER_BIN', 'docker'),
|
||||
args: ['build', ...buildArgs, '-t', image, '-f', dockerfile, '.'],
|
||||
command: '/bin/sh',
|
||||
args: [path.resolve(__dirname, '../../scripts/build-image.sh'), ...buildArgs, '-t', image, '-f', dockerfile, '.'],
|
||||
cwd: contextDir,
|
||||
append,
|
||||
onProcess: (child) => this.activeProcesses.set(job.id, child),
|
||||
|
||||
@@ -4,7 +4,7 @@ import { exec, execFile } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { copyFile, mkdir, readFile, rm, stat } from 'node:fs/promises';
|
||||
import { basename } from 'node:path';
|
||||
import { basename, resolve } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
|
||||
|
||||
@@ -1087,7 +1087,7 @@ export class LauncherService {
|
||||
data: { version: packageVersion, uploadPath: dockerfile, downloadUrl: imageTag },
|
||||
});
|
||||
await runCommand(
|
||||
`docker build -t ${quote(imageTag)} -f ${quote(dockerfile)} .`,
|
||||
`/bin/sh ${quote(resolve(__dirname, '../../scripts/build-image.sh'))} -t ${quote(imageTag)} -f ${quote(dockerfile)} .`,
|
||||
workDir,
|
||||
append,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user