Files
studio-admin-be/src/apps/build-runner.service.ts
T

804 lines
23 KiB
TypeScript

import { spawn, type ChildProcess } from 'node:child_process';
import { createHash } from 'node:crypto';
import { createReadStream } from 'node:fs';
import { promises as fs } from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import {
Injectable,
Logger,
OnModuleDestroy,
OnModuleInit,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { App, AppComponent, BuildJob } from '@prisma/client';
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { PrismaService } from '../prisma/prisma.service';
type BuildJobWithComponent = BuildJob & {
component: AppComponent & {
app: App;
};
};
@Injectable()
export class BuildRunnerService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(BuildRunnerService.name);
private timer?: NodeJS.Timeout;
private running = false;
private giteaUsername?: string;
private r2Client?: S3Client;
private readonly activeProcesses = new Map<string, ChildProcess>();
constructor(
private readonly prisma: PrismaService,
private readonly configService: ConfigService,
) {}
onModuleInit() {
if (this.configService.get<string>('BUILD_RUNNER_ENABLED', 'true') === 'false') {
this.logger.log('Build runner disabled by BUILD_RUNNER_ENABLED=false');
return;
}
const intervalMs = Number(
this.configService.get<string>('BUILD_RUNNER_INTERVAL_MS', '5000'),
);
this.timer = setInterval(() => void this.tick(), intervalMs);
void this.tick();
}
onModuleDestroy() {
if (this.timer) {
clearInterval(this.timer);
}
for (const child of this.activeProcesses.values()) {
terminateProcess(child);
}
}
cancelJob(jobId: string) {
const child = this.activeProcesses.get(jobId);
if (child) {
terminateProcess(child);
}
}
private async tick() {
if (this.running) return;
this.running = true;
try {
const job = await this.claimNextJob();
if (job) {
await this.runJob(job);
}
} catch (error) {
const message = error instanceof Error ? error.message : 'Unknown build runner error';
this.logger.error(message);
} finally {
this.running = false;
}
}
private async claimNextJob() {
const pending = await this.prisma.buildJob.findFirst({
where: { status: 'PENDING' },
orderBy: { createdAt: 'asc' },
include: {
component: {
include: {
app: true,
},
},
},
});
if (!pending) return null;
const claimed = await this.prisma.buildJob.updateMany({
where: { id: pending.id, status: 'PENDING' },
data: {
status: 'RUNNING',
startedAt: new Date(),
},
});
if (claimed.count === 0) return null;
return this.prisma.buildJob.findUnique({
where: { id: pending.id },
include: {
component: {
include: {
app: true,
},
},
},
}) as Promise<BuildJobWithComponent | null>;
}
private async runJob(job: BuildJobWithComponent) {
const lines = splitLogs(job.logs);
const token = this.configService.get<string>('GITEA_ADMIN_TOKEN') || '';
const append = async (line: string) => {
lines.push(redact(line, token));
await this.prisma.buildJob.update({
where: { id: job.id },
data: { logs: truncateLogs(lines.join('\n')) },
});
};
const workRoot = this.configService.get<string>(
'BUILD_WORKDIR',
path.join(os.tmpdir(), 'studio-admin-builds'),
);
const workDir = path.join(workRoot, job.id);
try {
await fs.rm(workDir, { recursive: true, force: true });
await fs.mkdir(workRoot, { recursive: true });
await append(`[running] build runner picked job ${job.id}`);
await this.cloneRepository(job, workDir, append);
await this.executeByComponentType(job, workDir, append);
await append('[done] build job completed');
if (await this.isCanceled(job.id)) {
await append('[cancelled] build job was cancelled');
return;
}
await this.prisma.buildJob.update({
where: { id: job.id },
data: {
status: 'SUCCEEDED',
finishedAt: new Date(),
logs: truncateLogs(lines.join('\n')),
},
});
} catch (error) {
if (await this.isCanceled(job.id)) {
lines.push('[cancelled] build job stopped');
await this.prisma.buildJob.update({
where: { id: job.id },
data: {
status: 'CANCELED',
finishedAt: new Date(),
logs: truncateLogs(lines.join('\n')),
},
});
return;
}
const message = error instanceof Error ? error.message : 'Unknown build failure';
lines.push(`[failed] ${redact(message, token)}`);
await this.prisma.buildJob.update({
where: { id: job.id },
data: {
status: 'FAILED',
finishedAt: new Date(),
logs: truncateLogs(lines.join('\n')),
},
});
} finally {
if (this.configService.get<string>('BUILD_KEEP_WORKDIR', 'false') !== 'true') {
await fs.rm(workDir, { recursive: true, force: true }).catch(() => undefined);
}
}
}
private async isCanceled(jobId: string) {
const job = await this.prisma.buildJob.findUnique({
where: { id: jobId },
select: { status: true },
});
return job?.status === 'CANCELED';
}
private async cloneRepository(
job: BuildJobWithComponent,
workDir: string,
append: (line: string) => Promise<void>,
) {
const safeUrl = this.getHttpCloneUrl(job.component.repository!);
const cloneUrl = await this.getAuthenticatedHttpCloneUrl(
job.component.repository!,
);
await append(`[clone] ${safeUrl}`);
await runCommand({
jobId: job.id,
command: this.configService.get<string>('GIT_BIN', 'git'),
args: ['clone', '--branch', job.branch, cloneUrl, workDir],
safeArgs: ['clone', '--branch', job.branch, safeUrl, workDir],
cwd: process.cwd(),
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
if (job.commitSha) {
await append(`[checkout] ${job.commitSha}`);
await runCommand({
jobId: job.id,
command: this.configService.get<string>('GIT_BIN', 'git'),
args: ['checkout', job.commitSha],
cwd: workDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
}
}
private async executeByComponentType(
job: BuildJobWithComponent,
workDir: string,
append: (line: string) => Promise<void>,
) {
if (job.image) {
await this.buildAndPushImage(job, workDir, append);
return;
}
if (job.component.type === 'MIGRATION') {
await append('[migration] repository checkout verified');
await append('[migration] actual DB migration is handled by deployment step');
return;
}
if (job.component.type === 'ELECTRON') {
await this.buildArtifact(job, workDir, append);
return;
}
await append(`[skip] no build action configured for ${job.component.type}`);
}
private async buildAndPushImage(
job: BuildJobWithComponent,
workDir: string,
append: (line: string) => Promise<void>,
) {
const image = this.resolveImage(job.image!);
const builder = this.configService.get<string>('BUILD_IMAGE_BUILDER', 'docker');
const dockerfile = this.configService.get<string>('BUILD_DOCKERFILE', 'Dockerfile');
const contextPath = this.configService.get<string>('BUILD_CONTEXT_PATH', '.');
const contextDir = path.resolve(workDir, contextPath);
const pushEnabled = this.configService.get<string>('BUILD_PUSH_ENABLED', 'true') !== 'false';
if (image !== job.image) {
await this.prisma.buildJob.update({
where: { id: job.id },
data: { image },
});
await append(`[image] rewritten to ${image}`);
}
if (builder === 'nerdctl') {
const namespace = this.configService.get<string>('BUILD_NERDCTL_NAMESPACE');
const prefix = namespace ? ['--namespace', namespace] : [];
await runCommand({
jobId: job.id,
command: this.configService.get<string>('NERDCTL_BIN', 'nerdctl'),
args: [...prefix, 'build', '-t', image, '-f', dockerfile, '.'],
cwd: contextDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
if (pushEnabled) {
await runCommand({
jobId: job.id,
command: this.configService.get<string>('NERDCTL_BIN', 'nerdctl'),
args: [...prefix, 'push', image],
cwd: contextDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
}
return;
}
await runCommand({
jobId: job.id,
command: this.configService.get<string>('DOCKER_BIN', 'docker'),
args: ['build', '-t', image, '-f', dockerfile, '.'],
cwd: contextDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
if (pushEnabled) {
await runCommand({
jobId: job.id,
command: this.configService.get<string>('DOCKER_BIN', 'docker'),
args: ['push', image],
cwd: contextDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
}
}
private resolveImage(image: string) {
const registry = this.configService.get<string>('BUILD_REGISTRY');
if (!registry) return image;
const imagePath = image.includes('/') ? image.slice(image.indexOf('/') + 1) : image;
return `${registry.replace(/\/$/, '')}/${imagePath}`;
}
private async buildArtifact(
job: BuildJobWithComponent,
workDir: string,
append: (line: string) => Promise<void>,
) {
const installCommand = this.configService.get<string>(
'BUILD_ARTIFACT_INSTALL_COMMAND',
'npm ci',
);
const buildCommand = this.configService.get<string>(
'BUILD_ELECTRON_COMMAND',
'npm run build:linux',
);
const outputDir = this.configService.get<string>(
'BUILD_ARTIFACT_OUTPUT_DIR',
'dist',
);
const artifactDir = this.configService.get<string>(
'BUILD_ARTIFACT_DIR',
path.join(os.tmpdir(), 'studio-admin-artifacts'),
);
await append(`[artifact] install ${installCommand}`);
await runShellCommand({
jobId: job.id,
command: installCommand,
cwd: workDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
await append(`[artifact] build ${buildCommand}`);
await runShellCommand({
jobId: job.id,
command: buildCommand,
cwd: workDir,
append,
onProcess: (child) => this.activeProcesses.set(job.id, child),
onProcessEnd: () => this.activeProcesses.delete(job.id),
});
const artifact = await findNewestArtifact(path.resolve(workDir, outputDir));
if (!artifact) {
throw new Error(`산출물을 찾을 수 없습니다: ${outputDir}`);
}
const artifactVersion = await readPackageVersion(workDir);
await fs.mkdir(artifactDir, { recursive: true });
const destinationName = job.artifact || path.basename(artifact);
const destination = path.join(artifactDir, destinationName);
await fs.copyFile(artifact, destination);
const uploadedArtifact = await this.uploadArtifactIfConfigured(
job,
destination,
destinationName,
artifactVersion,
append,
);
const publicBaseUrl = this.configService.get<string>('BUILD_ARTIFACT_BASE_URL');
const artifactValue = uploadedArtifact
?? (publicBaseUrl
? `${publicBaseUrl.replace(/\/$/, '')}/${encodeURIComponent(destinationName)}`
: destination);
await this.prisma.buildJob.update({
where: { id: job.id },
data: { artifact: artifactValue },
});
job.artifact = artifactValue;
await append(`[artifact] found ${path.relative(workDir, artifact)}`);
await append(`[artifact] copied to ${artifactValue}`);
}
private async uploadArtifactIfConfigured(
job: BuildJobWithComponent,
artifactPath: string,
artifactName: string,
artifactVersion: string | null,
append: (line: string) => Promise<void>,
) {
const bucket = this.configService.get<string>('R2_BUCKET');
const publicBaseUrl = this.configService.get<string>('R2_PUBLIC_BASE_URL');
const accessKeyId = this.configService.get<string>('R2_ACCESS_KEY_ID');
const secretAccessKey = this.configService.get<string>('R2_SECRET_ACCESS_KEY');
if (!bucket || !publicBaseUrl || !accessKeyId || !secretAccessKey) {
return null;
}
const key = this.createArtifactObjectKey(job, artifactName, artifactVersion);
const stat = await fs.stat(artifactPath);
const sha256 = await sha256File(artifactPath);
const client = this.getR2Client(accessKeyId, secretAccessKey);
await append(`[r2] upload ${key}`);
await client.send(new PutObjectCommand({
Bucket: bucket,
Key: key,
Body: createReadStream(artifactPath),
ContentLength: stat.size,
ContentType: contentTypeForFile(artifactName),
ContentDisposition: contentDispositionForFile(artifactName),
Metadata: {
sha256,
app: safeMetadataValue(job.component.app.key),
component: safeMetadataValue(job.component.name),
branch: safeMetadataValue(job.branch),
commit: safeMetadataValue(job.commitSha ?? ''),
version: safeMetadataValue(artifactVersion ?? ''),
},
}));
await client.send(new PutObjectCommand({
Bucket: bucket,
Key: `${key}.sha256`,
Body: `${sha256} ${artifactName}\n`,
ContentType: 'text/plain; charset=utf-8',
}));
const url = `${publicBaseUrl.replace(/\/$/, '')}/${encodeObjectKey(key)}`;
await append(`[r2] uploaded ${url}`);
await append(`[r2] sha256 ${sha256}`);
return url;
}
private getR2Client(accessKeyId: string, secretAccessKey: string) {
if (this.r2Client) return this.r2Client;
const endpoint = this.configService.get<string>('R2_ENDPOINT')
|| `https://${this.configService.get<string>('R2_ACCOUNT_ID')}.r2.cloudflarestorage.com`;
this.r2Client = new S3Client({
region: this.configService.get<string>('R2_REGION', 'auto'),
endpoint,
credentials: {
accessKeyId,
secretAccessKey,
},
forcePathStyle: this.configService.get<string>('R2_FORCE_PATH_STYLE', 'true') !== 'false',
});
return this.r2Client;
}
private createArtifactObjectKey(
job: BuildJobWithComponent,
artifactName: string,
artifactVersion: string | null,
) {
const prefix = this.configService.get<string>('R2_ARTIFACT_PREFIX', 'studio-artifacts');
const versionPath = artifactVersion
? [artifactVersion]
: [job.branch, 'latest'];
return [
prefix,
'apps',
job.component.app.key,
job.component.name,
...versionPath,
artifactName,
].map(sanitizeObjectKeyPart).join('/');
}
private getHttpCloneUrl(repository: string) {
return `${this.configService
.get<string>('GITEA_BASE_URL', 'http://localhost:3000')
.replace(/\/$/, '')}/${repository}.git`;
}
private async getAuthenticatedHttpCloneUrl(repository: string) {
const baseUrl = this.getHttpCloneUrl(repository);
const token = this.configService.get<string>('GITEA_ADMIN_TOKEN');
if (!token || this.configService.get<string>('BUILD_GIT_AUTH', 'token') === 'none') {
return baseUrl;
}
const username =
this.configService.get<string>('BUILD_GIT_USERNAME') ??
(await this.getGiteaUsername(token));
const url = new URL(baseUrl);
url.username = username;
url.password = token;
return url.toString();
}
private async getGiteaUsername(token: string) {
if (this.giteaUsername) return this.giteaUsername;
const response = await fetch(
`${this.configService
.get<string>('GITEA_BASE_URL', 'http://localhost:3000')
.replace(/\/$/, '')}/api/v1/user`,
{
headers: {
Accept: 'application/json',
Authorization: `token ${token}`,
},
},
);
if (!response.ok) {
throw new Error(`Gitea 사용자 확인 실패: ${response.status}`);
}
const data = (await response.json()) as { login?: string };
if (!data.login) {
throw new Error('Gitea 토큰 사용자명을 확인할 수 없습니다.');
}
this.giteaUsername = data.login;
return data.login;
}
}
async function runCommand({
jobId,
command,
args,
safeArgs = args,
cwd,
append,
onProcess,
onProcessEnd,
}: {
jobId: string;
command: string;
args: string[];
safeArgs?: string[];
cwd: string;
append: (line: string) => Promise<void>;
onProcess: (child: ChildProcess) => void;
onProcessEnd: () => void;
}) {
await append(`[run] ${command} ${safeArgs.join(' ')}`);
await new Promise<void>((resolve, reject) => {
const child = spawn(command, args, {
cwd,
env: process.env,
stdio: ['ignore', 'pipe', 'pipe'],
detached: true,
});
let output = '';
onProcess(child);
child.stdout.on('data', (chunk: Buffer) => {
output += chunk.toString();
});
child.stderr.on('data', (chunk: Buffer) => {
output += chunk.toString();
});
child.on('error', (error) => {
onProcessEnd();
reject(error);
});
child.on('close', async (code) => {
onProcessEnd();
try {
for (const line of output.split(/\r?\n/).filter(Boolean)) {
await append(`[out] ${line}`);
}
} catch (error) {
reject(error);
return;
}
if (code === 0) {
resolve();
return;
}
reject(new Error(`${command} for job ${jobId} exited with code ${code}`));
});
});
}
async function runShellCommand({
jobId,
command,
cwd,
append,
onProcess,
onProcessEnd,
}: {
jobId: string;
command: string;
cwd: string;
append: (line: string) => Promise<void>;
onProcess: (child: ChildProcess) => void;
onProcessEnd: () => void;
}) {
await append(`[run] ${command}`);
await new Promise<void>((resolve, reject) => {
const child = spawn(command, {
cwd,
env: process.env,
stdio: ['ignore', 'pipe', 'pipe'],
detached: true,
shell: true,
});
let output = '';
onProcess(child);
child.stdout.on('data', (chunk: Buffer) => {
output += chunk.toString();
});
child.stderr.on('data', (chunk: Buffer) => {
output += chunk.toString();
});
child.on('error', (error) => {
onProcessEnd();
reject(error);
});
child.on('close', async (code) => {
onProcessEnd();
try {
for (const line of output.split(/\r?\n/).filter(Boolean)) {
await append(`[out] ${line}`);
}
} catch (error) {
reject(error);
return;
}
if (code === 0) {
resolve();
return;
}
reject(new Error(`${command} for job ${jobId} exited with code ${code}`));
});
});
}
async function findNewestArtifact(dir: string) {
const candidates = await findArtifactCandidates(dir).catch(() => []);
candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);
return candidates[0]?.path ?? null;
}
async function readPackageVersion(workDir: string) {
try {
const packageJson = await fs.readFile(path.join(workDir, 'package.json'), 'utf8');
const parsed = JSON.parse(packageJson) as { version?: unknown };
return typeof parsed.version === 'string' && parsed.version.trim()
? parsed.version.trim()
: null;
} catch {
return null;
}
}
async function findArtifactCandidates(dir: string): Promise<Array<{ path: string; mtimeMs: number }>> {
const entries = await fs.readdir(dir, { withFileTypes: true });
const results: Array<{ path: string; mtimeMs: number }> = [];
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
results.push(...await findArtifactCandidates(entryPath));
continue;
}
if (!/\.(AppImage|deb|rpm|zip|tar\.gz)$/i.test(entry.name)) continue;
const stat = await fs.stat(entryPath);
results.push({ path: entryPath, mtimeMs: stat.mtimeMs });
}
return results;
}
async function sha256File(filePath: string) {
const hash = createHash('sha256');
await new Promise<void>((resolve, reject) => {
const stream = createReadStream(filePath);
stream.on('data', (chunk) => hash.update(chunk));
stream.on('error', reject);
stream.on('end', resolve);
});
return hash.digest('hex');
}
function contentTypeForFile(fileName: string) {
const lowerName = fileName.toLowerCase();
if (lowerName.endsWith('.tar.gz')) return 'application/gzip';
const extension = path.extname(lowerName);
const types: Record<string, string> = {
'.appimage': 'application/octet-stream',
'.deb': 'application/vnd.debian.binary-package',
'.rpm': 'application/x-rpm',
'.zip': 'application/zip',
'.json': 'application/json',
'.txt': 'text/plain; charset=utf-8',
'.yml': 'application/x-yaml',
'.yaml': 'application/x-yaml',
};
return types[extension] ?? 'application/octet-stream';
}
function sanitizeObjectKeyPart(value: string) {
return value
.trim()
.replace(/^\/+|\/+$/g, '')
.replace(/[^a-zA-Z0-9._@+=-]+/g, '-')
|| 'unknown';
}
function encodeObjectKey(key: string) {
return key.split('/').map(encodeURIComponent).join('/');
}
function contentDispositionForFile(fileName: string) {
const fallbackName = fileName.replace(/[\\\"\r\n]/g, '_');
return `attachment; filename="${fallbackName}"; filename*=UTF-8''${encodeURIComponent(fileName)}`;
}
function safeMetadataValue(value: string) {
return value.replace(/[^\x20-\x7E]/g, '').slice(0, 512);
}
function terminateProcess(child: ChildProcess) {
if (!child.pid || child.killed) return;
try {
process.kill(-child.pid, 'SIGTERM');
setTimeout(() => {
if (!child.killed && child.pid) {
try {
process.kill(-child.pid, 'SIGKILL');
} catch {
// Process may have already exited.
}
}
}, 3000).unref();
} catch {
child.kill('SIGTERM');
}
}
function splitLogs(logs: string | null) {
return logs ? logs.split('\n').filter(Boolean) : [];
}
function truncateLogs(logs: string) {
return logs.length > 200_000 ? logs.slice(-200_000) : logs;
}
function redact(value: string, secret: string) {
if (!secret) return value;
return value.split(secret).join('<redacted>');
}