Signals & Jobs

Master signals for graceful script termination, cleanup handlers, and job control for managing background processes.

Common Signals

Signal Number Description

SIGHUP

1

Hangup - terminal closed

SIGINT

2

Interrupt - Ctrl+C

SIGQUIT

3

Quit - Ctrl+\

SIGKILL

9

Kill - cannot be caught

SIGTERM

15

Terminate - graceful stop

SIGSTOP

19

Stop - cannot be caught

SIGTSTP

20

Terminal stop - Ctrl+Z

SIGCONT

18

Continue stopped process

SIGUSR1

10

User-defined 1

SIGUSR2

12

User-defined 2

Sending Signals

kill Command

# By PID
kill PID                    # SIGTERM (default)
kill -15 PID                # SIGTERM explicit
kill -9 PID                 # SIGKILL (force)
kill -HUP PID               # SIGHUP (reload config)
kill -USR1 PID              # User signal 1

# Multiple PIDs
kill PID1 PID2 PID3

# Process group
kill -TERM -PGID            # Negative for group

pkill and killall

# By name pattern
pkill -f "python script.py"
pkill -u username           # By user

# Exact name match
killall nginx

# With signal
pkill -HUP nginx
killall -USR1 nginx

Keyboard Shortcuts

Key Signal

Ctrl+C

SIGINT (interrupt)

Ctrl+Z

SIGTSTP (suspend)

Ctrl+\

SIGQUIT (quit with core dump)

Trapping Signals

Basic Trap

#!/bin/bash

# Define cleanup function
cleanup() {
    echo "Cleaning up..."
    rm -f /tmp/myapp.lock
    exit 0
}

# Set trap
trap cleanup SIGINT SIGTERM

# Main script
echo "Running... (Ctrl+C to stop)"
while true; do
    sleep 1
done

Trap on EXIT

#!/bin/bash

# Always runs on exit (any reason)
trap 'rm -f /tmp/myapp.$$' EXIT

# Create temp file
echo "data" > /tmp/myapp.$$

# ... script continues ...
# Temp file cleaned up automatically on exit

Multiple Signals

trap 'echo "Interrupt"; exit 1' SIGINT
trap 'echo "Terminate"; exit 0' SIGTERM
trap 'echo "Hangup - reloading config"; reload_config' SIGHUP

Ignore Signal

trap '' SIGINT              # Ignore Ctrl+C
trap '' SIGHUP              # Ignore hangup

Reset to Default

trap - SIGINT               # Reset SIGINT to default
trap - EXIT                 # Remove EXIT trap

List Traps

trap -p                     # Show all traps
trap -l                     # List all signal names

Job Control

Background Jobs

# Start in background
command &

# Suspend foreground job
# Press Ctrl+Z

# Resume in background
bg

# Resume in foreground
fg

List Jobs

jobs                        # List jobs
jobs -l                     # With PIDs
jobs -p                     # PIDs only

Reference Jobs

fg %1                       # Job 1 to foreground
bg %2                       # Job 2 to background
kill %1                     # Kill job 1
fg %-                       # Previous job
fg %+                       # Current job
fg %?pattern                # Job matching pattern

Wait for Jobs

# Wait for specific PID
wait $pid

# Wait for all background jobs
wait

# Wait with timeout (bash 4.3+)
timeout 10 bash -c 'wait $pid'

Disown

# Run command, then disown
long_command &
disown                      # Remove from job table

# Disown specific job
disown %1

# Start without job control
nohup long_command &        # Immune to hangup

Script Patterns

Graceful Shutdown

#!/bin/bash

RUNNING=true

cleanup() {
    echo "Shutting down gracefully..."
    RUNNING=false
    # Close connections, save state, etc.
    exit 0
}

trap cleanup SIGINT SIGTERM

while $RUNNING; do
    # Main work
    process_item
    sleep 1
done

Lock File with Cleanup

#!/bin/bash

LOCKFILE="/var/run/myapp.lock"

cleanup() {
    rm -f "$LOCKFILE"
    exit
}

trap cleanup EXIT SIGINT SIGTERM

# Acquire lock
if ! mkdir "$LOCKFILE" 2>/dev/null; then
    echo "Already running"
    exit 1
fi

# Main script
# Lock automatically cleaned up on exit

Temp File Cleanup

#!/bin/bash

TMPDIR=$(mktemp -d)
trap 'rm -rf "$TMPDIR"' EXIT

# Use temp files
cd "$TMPDIR"
# ... work ...
# Cleaned up automatically

Parallel with Wait

#!/bin/bash

# Start parallel jobs
for host in server1 server2 server3; do
    ssh "$host" "backup_script" &
done

# Wait for all
wait
echo "All backups complete"

Background with PID Tracking

#!/bin/bash

pids=()

# Start jobs
for i in {1..5}; do
    do_work "$i" &
    pids+=($!)
done

# Wait for all
for pid in "${pids[@]}"; do
    wait "$pid"
    status=$?
    if [ $status -ne 0 ]; then
        echo "Job $pid failed with $status"
    fi
done

Timeout Pattern

#!/bin/bash

# Run with timeout
timeout 30 long_command
status=$?

if [ $status -eq 124 ]; then
    echo "Command timed out"
elif [ $status -ne 0 ]; then
    echo "Command failed"
fi

Watchdog

#!/bin/bash

watchdog() {
    while true; do
        if ! pgrep -f "important_process" > /dev/null; then
            echo "Process died, restarting..."
            important_process &
        fi
        sleep 60
    done
}

trap 'killall important_process; exit' SIGTERM
watchdog

Infrastructure Patterns

Service Script

#!/bin/bash

PID_FILE="/var/run/myservice.pid"

start() {
    if [ -f "$PID_FILE" ]; then
        echo "Already running"
        return 1
    fi
    myservice &
    echo $! > "$PID_FILE"
}

stop() {
    if [ -f "$PID_FILE" ]; then
        kill $(cat "$PID_FILE")
        rm "$PID_FILE"
    fi
}

reload() {
    if [ -f "$PID_FILE" ]; then
        kill -HUP $(cat "$PID_FILE")
    fi
}

case "$1" in
    start) start ;;
    stop) stop ;;
    reload) reload ;;
    restart) stop; start ;;
    *) echo "Usage: $0 {start|stop|reload|restart}" ;;
esac

Parallel SSH

#!/bin/bash

HOSTS="server1 server2 server3"
COMMAND="$1"
pids=()

for host in $HOSTS; do
    echo "Running on $host..."
    ssh "$host" "$COMMAND" &
    pids+=($!)
done

# Wait and check
failed=0
for i in "${!pids[@]}"; do
    wait "${pids[$i]}"
    if [ $? -ne 0 ]; then
        echo "Failed on ${HOSTS[$i]}"
        ((failed++))
    fi
done

exit $failed

Progress Indicator

#!/bin/bash

progress() {
    local pid=$1
    local spin='-\|/'
    local i=0
    while kill -0 $pid 2>/dev/null; do
        i=$(( (i+1) % 4 ))
        printf "\r${spin:$i:1}"
        sleep 0.1
    done
    printf "\r"
}

long_command &
progress $!
wait $!

Safe Background Execution

#!/bin/bash

# Start in background, immune to hangup
nohup ./my_script.sh > /var/log/myscript.log 2>&1 &
echo $! > /var/run/myscript.pid
disown

echo "Started in background, PID: $(cat /var/run/myscript.pid)"

Debugging

Show Signals

# List all signals
kill -l

# Signal number to name
kill -l 15                  # TERM

Trace Signals

# strace shows signals
strace -e signal command

# Monitor with trap
trap 'echo "Got signal"' DEBUG

Quick Reference

# Common signals
SIGINT (2)   - Ctrl+C
SIGTERM (15) - Graceful stop
SIGKILL (9)  - Force kill
SIGHUP (1)   - Reload config
SIGSTOP (19) - Pause
SIGCONT (18) - Resume

# Trap syntax
trap 'commands' SIGNAL      # Set handler
trap '' SIGNAL              # Ignore
trap - SIGNAL               # Reset to default
trap -p                     # List traps

# Job control
command &                   # Background
Ctrl+Z                      # Suspend
bg                         # Resume background
fg                         # Resume foreground
jobs                       # List jobs
wait                       # Wait for jobs

# Killing
kill PID                   # SIGTERM
kill -9 PID                # SIGKILL
pkill -f pattern           # By name
killall name               # By exact name

Key Takeaways

  1. trap cleanup EXIT - Always clean up on exit

  2. trap 'handler' SIGTERM - Handle graceful shutdown

  3. SIGKILL cannot be caught - Use SIGTERM first

  4. & and wait - Parallel execution

  5. nohup and disown - Survive logout

  6. jobs, fg, bg - Interactive job control

  7. Lock files - Prevent multiple instances