Fix actix worker wedge on session teardown; grant CAP_KILL; bump to 0.3.12
Terminal::stopping did child.kill() (error ignored) and then a blocking child.wait() on the actix worker thread. Since the 0.3.11 privilege drop, sessions run as `demo` and the server had no CAP_KILL, so kill() failed with EPERM and wait() blocked forever. With two workers, exactly every other request to :8082 then timed out — 5 days of readiness/liveness flapping on socktop.io and orphaned restricted-shell/socktop processes piling up in the pods. - Signal the session's whole process group (portable-pty setsid()s the child), not just the shell: a non-interactive bash defers signals while a foreground command runs, so the old SIGHUP/SIGKILL to the shell alone orphaned socktop anyway. - Reap on a dedicated thread: SIGHUP, 2 s grace, SIGKILL, 10 s deadline, then a blocking wait on that thread only. Signal failures are logged. - Drop the pty handles before handing off the child. - kubernetes/03-deployment.yaml: add CAP_KILL (sessions still lose every cap via setpriv). CI never applies the manifest — it was patched live. - tests/reaper_tests.rs pins the non-blocking return, process-group kill, and SIGHUP→SIGKILL escalation. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,144 @@
|
||||
// Copyright (c) 2024 Jason Witty <jasonpwitty+socktop@proton.me>.
|
||||
// All rights reserved.
|
||||
//
|
||||
// Regression tests for session teardown (`webterm::reap_child`).
|
||||
//
|
||||
// Background: `Terminal::stopping` used to call `child.kill()` then a blocking
|
||||
// `child.wait()` on the actix worker thread. When the signal was refused
|
||||
// (sessions run as another uid, server without CAP_KILL) or only reached the
|
||||
// shell (a non-interactive bash defers signals while a foreground command
|
||||
// runs), the child survived and the worker hung forever — every other request
|
||||
// to the server then timed out. These tests pin the fixed behaviour: the whole
|
||||
// process group is signalled, SIGHUP escalates to SIGKILL, and the caller is
|
||||
// never blocked.
|
||||
|
||||
use std::io::Read;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use portable_pty::{native_pty_system, CommandBuilder, PtySize};
|
||||
|
||||
/// True while `pid` exists and is not a zombie (kill(pid, 0) also succeeds on
|
||||
/// zombies, so read /proc directly).
|
||||
fn alive(pid: i32) -> bool {
|
||||
match std::fs::read_to_string(format!("/proc/{pid}/status")) {
|
||||
Ok(s) => !s
|
||||
.lines()
|
||||
.any(|l| l.starts_with("State:") && l.contains("Z (zombie)")),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_gone(pid: i32, timeout: Duration) -> bool {
|
||||
let start = Instant::now();
|
||||
while start.elapsed() < timeout {
|
||||
if !alive(pid) {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
!alive(pid)
|
||||
}
|
||||
|
||||
/// Spawn `script` under `sh -c` in a fresh pty. The script must print the pid
|
||||
/// of its long-running grandchild as its first line; that pid is returned along
|
||||
/// with the pty child handle.
|
||||
fn spawn_in_pty(script: &str) -> (Box<dyn portable_pty::Child + Send>, i32) {
|
||||
let pty = native_pty_system();
|
||||
let pair = pty
|
||||
.openpty(PtySize {
|
||||
rows: 24,
|
||||
cols: 80,
|
||||
pixel_width: 0,
|
||||
pixel_height: 0,
|
||||
})
|
||||
.expect("openpty");
|
||||
|
||||
let mut cmd = CommandBuilder::new("sh");
|
||||
cmd.arg("-c");
|
||||
cmd.arg(script);
|
||||
let child = pair.slave.spawn_command(cmd).expect("spawn");
|
||||
drop(pair.slave);
|
||||
|
||||
let mut reader = pair.master.try_clone_reader().expect("reader");
|
||||
let mut buf = Vec::new();
|
||||
let mut byte = [0u8; 1];
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
while Instant::now() < deadline {
|
||||
match reader.read(&mut byte) {
|
||||
Ok(1) if byte[0] == b'\n' => break,
|
||||
Ok(1) => buf.push(byte[0]),
|
||||
_ => break,
|
||||
}
|
||||
}
|
||||
let grandchild: i32 = String::from_utf8_lossy(&buf)
|
||||
.trim()
|
||||
.parse()
|
||||
.expect("grandchild pid line");
|
||||
assert!(
|
||||
alive(grandchild),
|
||||
"grandchild {grandchild} should be running"
|
||||
);
|
||||
|
||||
// Keep the master alive for the duration of the test (mirrors the
|
||||
// server's reader thread holding a cloned fd), so the kernel does not
|
||||
// hang up the pty for us and the reaper alone has to do the work.
|
||||
std::mem::forget(pair.master);
|
||||
std::mem::forget(reader);
|
||||
|
||||
(child, grandchild)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reap_child_returns_immediately_and_kills_whole_process_group() {
|
||||
// sh waits on a foreground sleep: signalling sh alone would be deferred.
|
||||
let (child, sleeper) = spawn_in_pty("sleep 30 & echo $!; wait");
|
||||
let shell = child.process_id().expect("pid") as i32;
|
||||
|
||||
let t0 = Instant::now();
|
||||
let reaper = webterm::reap_child(child);
|
||||
assert!(
|
||||
t0.elapsed() < Duration::from_millis(500),
|
||||
"reap_child must not block the caller (took {:?})",
|
||||
t0.elapsed()
|
||||
);
|
||||
|
||||
reaper.join().expect("reaper thread");
|
||||
assert!(
|
||||
wait_gone(shell, Duration::from_secs(2)),
|
||||
"shell {shell} survived"
|
||||
);
|
||||
assert!(
|
||||
wait_gone(sleeper, Duration::from_secs(2)),
|
||||
"grandchild {sleeper} was orphaned instead of killed with its group"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reap_child_escalates_to_sigkill_when_sighup_is_ignored() {
|
||||
// `trap '' HUP` makes sh ignore SIGHUP and the ignored disposition is
|
||||
// inherited across exec, so sleep ignores it too — exactly the shape of a
|
||||
// TUI that swallows HUP.
|
||||
let (child, sleeper) = spawn_in_pty("trap '' HUP; sleep 30 & echo $!; wait");
|
||||
let shell = child.process_id().expect("pid") as i32;
|
||||
|
||||
let t0 = Instant::now();
|
||||
webterm::reap_child(child).join().expect("reaper thread");
|
||||
let took = t0.elapsed();
|
||||
|
||||
assert!(
|
||||
took >= Duration::from_secs(2),
|
||||
"should have waited out the SIGHUP grace period (took {took:?})"
|
||||
);
|
||||
assert!(
|
||||
took < Duration::from_secs(8),
|
||||
"SIGKILL escalation should finish well inside the deadline (took {took:?})"
|
||||
);
|
||||
assert!(
|
||||
wait_gone(shell, Duration::from_secs(2)),
|
||||
"shell {shell} survived SIGKILL"
|
||||
);
|
||||
assert!(
|
||||
wait_gone(sleeper, Duration::from_secs(2)),
|
||||
"grandchild {sleeper} survived SIGKILL"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user