(fix) module list status bug..
This commit is contained in:
+131
-14
@@ -22,6 +22,7 @@ const terminalStopTimers = new Map<string, NodeJS.Timeout>()
|
||||
const intentionalTerminalStops = new Set<string>()
|
||||
const nav2RuntimeObstacleNames = new Set<string>()
|
||||
const remoteSshQueues = new Map<string, Promise<void>>()
|
||||
const remoteDockerStatsRetryAfter = new Map<string, number>()
|
||||
const remoteGitModuleRoots = new Map<string, string>()
|
||||
const remoteSshControlDirectory = resolve(homedir(), '.autonomy-studio', 'ssh', 'control')
|
||||
const moduleCacheRoot = resolve(homedir(), '.caches')
|
||||
@@ -1377,12 +1378,15 @@ async function collectRemoteAvailableModules(
|
||||
}
|
||||
})
|
||||
.sort((a, b) => a.sortOrder - b.sortOrder)
|
||||
const output = await runSshOutput(targetInput ?? target, buildRemoteModuleProbeScript(target.workspaceRoot, moduleInputs), 30_000)
|
||||
const [output, statsOutput] = await Promise.all([
|
||||
runSshOutput(targetInput ?? target, buildRemoteModuleProbeScript(target.workspaceRoot, moduleInputs), 30_000),
|
||||
readRemoteDockerStatsOutput(target),
|
||||
])
|
||||
const dockerStatuses = new Map<string, { running: boolean; exists: boolean; status: string }>()
|
||||
const dockerStats = new Map<string, { cpu: number; memory: number }>()
|
||||
const installedModules = new Map<string, { path: string; parameters: string[] }>()
|
||||
|
||||
for (const line of output.split('\n')) {
|
||||
for (const line of `${output}\n${statsOutput}`.split('\n')) {
|
||||
if (!line.trim()) continue
|
||||
const [kind, first = '', second = '', third = ''] = line.split('\t')
|
||||
if (kind === 'DOCKER' && first) {
|
||||
@@ -1501,8 +1505,10 @@ function remoteRepositoryUrl(repositoryUrl: string): string {
|
||||
async function runCommandOutput(command: string, args: string[], cwd?: string, env?: NodeJS.ProcessEnv, timeoutMs = 120_000): Promise<string> {
|
||||
return new Promise<string>((resolvePromise, rejectPromise) => {
|
||||
let output = ''
|
||||
let terminated = false
|
||||
const child = spawn(command, args, {
|
||||
cwd,
|
||||
detached: process.platform !== 'win32',
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
env: {
|
||||
...process.env,
|
||||
@@ -1511,7 +1517,18 @@ async function runCommandOutput(command: string, args: string[], cwd?: string, e
|
||||
}
|
||||
})
|
||||
const timeout = setTimeout(() => {
|
||||
if (!child.killed) child.kill('SIGTERM')
|
||||
terminated = true
|
||||
if (process.platform !== 'win32' && child.pid) {
|
||||
const processId = child.pid
|
||||
// SSH/SCP spawn a child process. Kill the whole group so a timed-out
|
||||
// transfer cannot leave an SSH child alive in the background.
|
||||
try { process.kill(-processId, 'SIGTERM') } catch { /* already exited */ }
|
||||
setTimeout(() => {
|
||||
try { process.kill(-processId, 'SIGKILL') } catch { /* already exited */ }
|
||||
}, 1_000).unref()
|
||||
} else if (!child.killed) {
|
||||
child.kill('SIGTERM')
|
||||
}
|
||||
rejectPromise(new Error(`${command} ${args.join(' ')} timed out`))
|
||||
}, timeoutMs)
|
||||
|
||||
@@ -1527,6 +1544,7 @@ async function runCommandOutput(command: string, args: string[], cwd?: string, e
|
||||
})
|
||||
child.on('close', (code) => {
|
||||
clearTimeout(timeout)
|
||||
if (terminated) return
|
||||
if (code === 0) resolvePromise(output)
|
||||
else rejectPromise(new Error(output.trim() || `${command} ${args.join(' ')} failed with exit code ${code ?? 'unknown'}`))
|
||||
})
|
||||
@@ -1890,10 +1908,106 @@ async function runSshOutput(target: RemoteWorkspaceTarget, command: string, time
|
||||
))
|
||||
}
|
||||
|
||||
async function runUnqueuedSshOutput(target: RemoteWorkspaceTarget, command: string, timeoutMs: number): Promise<string> {
|
||||
const normalized = normalizeRemoteTarget(target)
|
||||
return runCommandOutput(
|
||||
'ssh',
|
||||
[
|
||||
'-p',
|
||||
String(normalized.port),
|
||||
'-o',
|
||||
'BatchMode=yes',
|
||||
'-o',
|
||||
'StrictHostKeyChecking=accept-new',
|
||||
'-o',
|
||||
'ConnectTimeout=5',
|
||||
'-o',
|
||||
'PreferredAuthentications=publickey',
|
||||
'-o',
|
||||
'PasswordAuthentication=no',
|
||||
'-o',
|
||||
'KbdInteractiveAuthentication=no',
|
||||
'-o',
|
||||
'ControlMaster=no',
|
||||
'-i',
|
||||
normalized.identityFile,
|
||||
`${normalized.user}@${normalized.host}`,
|
||||
command,
|
||||
],
|
||||
undefined,
|
||||
undefined,
|
||||
timeoutMs,
|
||||
)
|
||||
}
|
||||
|
||||
async function readRemoteDockerStatsOutput(target: RemoteWorkspaceTarget): Promise<string> {
|
||||
const normalized = normalizeRemoteTarget(target)
|
||||
const key = `${normalized.user}@${normalized.host}:${normalized.port}`
|
||||
if ((remoteDockerStatsRetryAfter.get(key) ?? 0) > Date.now()) return ''
|
||||
|
||||
try {
|
||||
const output = await runUnqueuedSshOutput(
|
||||
normalized,
|
||||
'timeout -k 1s 2s docker stats --no-stream --format "STATS\\t{{.Name}}\\t{{.CPUPerc}}\\t{{.MemPerc}}" 2>/dev/null || true',
|
||||
4_000,
|
||||
)
|
||||
remoteDockerStatsRetryAfter.delete(key)
|
||||
return output
|
||||
} catch {
|
||||
// Do not spawn another blocked Docker client on every ten-second module
|
||||
// refresh when the Jetson Docker daemon is under load.
|
||||
remoteDockerStatsRetryAfter.set(key, Date.now() + 60_000)
|
||||
return ''
|
||||
}
|
||||
}
|
||||
|
||||
async function runSsh(target: RemoteWorkspaceTarget, command: string, timeoutMs?: number): Promise<void> {
|
||||
await runSshOutput(target, command, timeoutMs)
|
||||
}
|
||||
|
||||
async function copyRemoteFileToLocal(
|
||||
target: RemoteWorkspaceTarget,
|
||||
remotePath: string,
|
||||
localPath: string,
|
||||
timeoutMs = 900_000,
|
||||
): Promise<void> {
|
||||
const normalized = normalizeRemoteTarget(target)
|
||||
const host = normalized.host.includes(':') ? `[${normalized.host}]` : normalized.host
|
||||
// A large 3D map database can take several minutes over a relayed Tailscale
|
||||
// link. Keep that file transfer outside the serialized command queue so it
|
||||
// cannot prevent module state, stop, or control requests from reaching the
|
||||
// platform in the meantime.
|
||||
await runCommand(
|
||||
'scp',
|
||||
[
|
||||
'-q',
|
||||
'-P',
|
||||
String(normalized.port),
|
||||
'-o',
|
||||
'BatchMode=yes',
|
||||
'-o',
|
||||
'StrictHostKeyChecking=accept-new',
|
||||
'-o',
|
||||
'ConnectTimeout=5',
|
||||
'-o',
|
||||
'PreferredAuthentications=publickey',
|
||||
'-o',
|
||||
'PasswordAuthentication=no',
|
||||
'-o',
|
||||
'KbdInteractiveAuthentication=no',
|
||||
'-o',
|
||||
'ControlMaster=no',
|
||||
'-i',
|
||||
normalized.identityFile,
|
||||
`${normalized.user}@${host}:${remotePath}`,
|
||||
localPath,
|
||||
],
|
||||
undefined,
|
||||
undefined,
|
||||
timeoutMs,
|
||||
)
|
||||
}
|
||||
|
||||
async function readRemoteSavedMapArtifacts(
|
||||
targetInput: RemoteWorkspaceTarget,
|
||||
artifacts: SavedMapArtifactPath[],
|
||||
@@ -1929,25 +2043,31 @@ async function copyRemoteActiveRtabmapDatabaseToLocal(
|
||||
'destination_db.close()',
|
||||
'source_db.close()',
|
||||
].join('; ')
|
||||
const remoteCommand = [
|
||||
const createSnapshotCommand = [
|
||||
'set -e',
|
||||
`SOURCE=${quoteShellSingle(sourcePath)}`,
|
||||
'test -f "$SOURCE" || { echo "3D SLAM DB를 찾을 수 없습니다: $SOURCE" >&2; exit 1; }',
|
||||
'SNAPSHOT=$(mktemp)',
|
||||
'SNAPSHOT=$(mktemp --suffix=.db)',
|
||||
`python3 -c ${quoteShellSingle(snapshotScript)} "$SOURCE" "$SNAPSHOT"`,
|
||||
'base64 -w0 "$SNAPSHOT"',
|
||||
'rm -f "$SNAPSHOT"',
|
||||
'printf "%s" "$SNAPSHOT"',
|
||||
].join('\n')
|
||||
const base64 = (await runSshOutput(target, remoteCommand, 120_000)).trim()
|
||||
if (!base64) throw new Error('3D SLAM DB 스냅샷을 읽지 못했습니다.')
|
||||
|
||||
await syncWorkspaceSavedMapArtifacts(localWorkspaceRoot, [])
|
||||
const root = expandUserPath(localWorkspaceRoot)
|
||||
if (!root) throw new Error('로컬 워크스페이스 경로가 설정되어 있지 않습니다.')
|
||||
const mapsDirectory = resolve(root, 'maps')
|
||||
const destination = resolve(mapsDirectory, destinationFileName)
|
||||
if (!destination.startsWith(`${mapsDirectory}/`)) throw new Error('저장된 3D 맵 경로가 올바르지 않습니다.')
|
||||
await writeFile(destination, Buffer.from(base64, 'base64'))
|
||||
const snapshotPath = (await runSshOutput(target, createSnapshotCommand, 120_000)).trim()
|
||||
if (!snapshotPath.startsWith('/tmp/')) throw new Error('3D SLAM DB 스냅샷을 만들지 못했습니다.')
|
||||
const temporaryDestination = `${destination}.download-${process.pid}-${Date.now()}`
|
||||
|
||||
try {
|
||||
await copyRemoteFileToLocal(target, snapshotPath, temporaryDestination)
|
||||
await rename(temporaryDestination, destination)
|
||||
} finally {
|
||||
await rm(temporaryDestination, { force: true }).catch(() => undefined)
|
||||
await runSsh(target, `rm -f ${quoteShellSingle(snapshotPath)}`, 15_000).catch(() => undefined)
|
||||
}
|
||||
return { databasePath: `/workspace/maps/${destinationFileName}` }
|
||||
}
|
||||
|
||||
@@ -2149,9 +2269,6 @@ function buildRemoteModuleProbeScript(workspaceRoot: string, modules: CourseModu
|
||||
`ROOT=${quoteShellSingle(workspaceRoot)}`,
|
||||
...expandRemotePathVar('ROOT'),
|
||||
'timeout 5s docker ps -a --format "DOCKER\\t{{.Names}}\\t{{.Status}}" 2>/dev/null || true',
|
||||
// docker stats can stall on a remote Jetson. Module state must not wait
|
||||
// behind a long statistics sample.
|
||||
'timeout 2s docker stats --no-stream --format "STATS\\t{{.Name}}\\t{{.CPUPerc}}\\t{{.MemPerc}}" 2>/dev/null || true',
|
||||
...moduleFolders.map((folderName) => {
|
||||
const modulePath = remotePathJoin('$ROOT', folderName)
|
||||
return [
|
||||
|
||||
Reference in New Issue
Block a user