Properly terminate remote command even when it ignores SIGHUP

diff --git a/novaboot b/novaboot
index 60da56a46556206619559404777d53b4882541d1..8e991dc742488173348f2edb33d40cbeb2a32e01 100755 (executable)
--- a/novaboot
+++ b/novaboot
@@ -660,6 +660,30 @@ if (exists $variables->{WVDESC}) {
 
 my $exp; # Expect object to communicate with the target over serial line
 
+sub kill_exp_on_signal() {
+    # Sometimes, under unclear circumstances (e.g. when running under
+    # both Jenkins and Robotframework), novaboot does not terminate
+    # console command when killed. The console is then blocked by the
+    # stale process forever. Theoretically, this should not happen,
+    # because when novaboot is killed, console command's controlling
+    # terminal sends SIGHUP to the console command and the command
+    # should terminate. It seems that at least SSH sometimes ignores
+    # HUP and does not terminate. The code below seems to work around
+    # that problem by killing the process immediately with SIGTERM,
+    # which is not ignored.
+
+    sub kill_console { kill TERM => $exp->pid if $exp->pid; die "Terminated by SIG$_[0]"; };
+
+    # For our Jenkins/Robotframework use case, it was sufficient to
+    # handle the TERM signal, but to be on the safe side, we also
+    # catch other signals.
+    $SIG{TERM} = \&kill_console;
+    $SIG{HUP} = \&kill_console;
+    $SIG{INT} = \&kill_console;
+    $SIG{QUIT} = \&kill_console;
+}
+
+
 my ($amt_user, $amt_password, $amt_host, $amt_port);
 
 if (defined $iprelay || defined $iprelay_cmd) {
@@ -682,6 +706,7 @@ if (defined $iprelay || defined $iprelay_cmd) {
        $exp = new Expect;
        $exp->raw_pty(1);
        $exp->spawn($iprelay_cmd);
+       kill_exp_on_signal();
     }
 
     while (1) {
@@ -753,6 +778,7 @@ elsif ($serial) {
 elsif (@remote_cmd) {
     print STDERR "novaboot: Running: ".shell_cmd_string(@remote_cmd)."\n";
     $exp = Expect->spawn(@remote_cmd);
+    kill_exp_on_signal();
 }
 elsif (defined $amt) {
     require LWP::UserAgent;
@@ -1285,6 +1311,10 @@ if ($interaction && defined $exp) {
     }
 }
 
+# When exp-spawned command ignores SIGHUP, Expect waits 5 seconds
+# before killing it. We kill it by SIGTERM immediately.
+kill TERM => $exp->pid if defined $exp && $exp->pid;
+
 ## Kill dhcpc or tftpd
 if (defined $dhcp_tftp || defined $tftp) {
     die("novaboot: This should kill servers on background\n");

diff --git a/tests/novaboot.wv b/tests/novaboot.wv
index 1cfea8ea7108aafb0337405b5376ba4adb524d54..8304d72b7a86afd3c59c53c7e1535a90f8cd030f 100755 (executable)
--- a/tests/novaboot.wv
+++ b/tests/novaboot.wv
@@ -125,4 +125,22 @@ WVPASS test "$(cat t)" = Target3
 WVSTART --iprelay-cmd
 WVPASS novaboot --iprelay-cmd="/bin/echo -e '<iprelayd: connected>\xFF\xFA\x2C\x97\xDF\xFF\xF0 \xFF\xFA\x2C\x97\xFF\xFF\xF0'" --on
 
+WVSTART Killing of --remote-cmd when it ignores SIGHUP
+WVPASS tee script <<'EOF'
+$SIG{HUP}='IGNORE';
+open(FH, ">", "pid") or die;
+print FH "$$";
+close FH;
+print "ready pid=$$\n";
+print "exit\n";
+sleep;
+EOF
+WVPASS novaboot --remote-cmd='perl script' --remote-expect=ready --exiton=exit /dev/null
+WVFAIL test -d /proc/$(cat pid)
+coproc novaboot --remote-cmd='perl script' --remote-expect=ready /dev/null
+WVPASS sed -e '/ready/q0' <&${COPROC[0]}
+WVPASS kill $COPROC_PID
+WVFAIL wait $COPROC_PID # Signal termination is considered a failure
+WVFAIL test -d /proc/$(cat pid)
+
 # Hi-lock: (("^.*\\(?:WVSTART\\).*$" (0 (quote hi-black-b) t)))