Bash Streams & Redirection

# Three standard streams
# stdin  (0) - Input
# stdout (1) - Output
# stderr (2) - Errors

# Basic redirection
command > file                               # stdout to file (overwrite)
command >> file                              # stdout to file (append)
command 2> file                              # stderr to file
command 2>> file                             # stderr append
command < file                               # stdin from file

# Combine streams
command &> file                              # stdout AND stderr to file
command &>> file                             # Both append
command > file 2>&1                          # Same as &> (POSIX portable)
command 2>&1 > file                          # WRONG ORDER: stderr still to terminal!

# Discard output
command > /dev/null                          # Discard stdout
command 2> /dev/null                         # Discard stderr
command &> /dev/null                         # Discard both

# Redirect stderr to stdout for pipeline
command 2>&1 | next_command                  # Both streams in pipeline
command |& next_command                      # Bash shorthand for same

# Here-document: Multi-line input
cat << 'EOF'
This is literal text.
Variables like $HOME are NOT expanded.
Useful for templates.
EOF

cat << EOF
This text DOES expand variables.
Home is: $HOME
User is: $USER
EOF

# Indent-friendly (<<-)
cat <<- EOF
	Tabs at start are stripped.
	Content stays aligned.
	Great for functions.
EOF

# Here-document for SSH commands
ssh vault-01 << 'EOF'
hostname
uptime
df -h /
EOF

# Here-document for config files
cat > /tmp/nginx.conf << 'EOF'
server {
    listen 80;
    server_name example.com;
    root /var/www/html;
}
EOF

# Here-string: Single-line input
grep "pattern" <<< "search in this string"

while read -r word; do
    echo "Word: $word"
done <<< "one two three"

# Variable content to stdin
json='{"key": "value"}'
jq '.key' <<< "$json"

# Command output as input
bc <<< "scale=2; 100/3"

# Here-document in function
create_service_file() {
    local name="$1"
    local exec="$2"
    cat << EOF
[Unit]
Description=$name Service
After=network.target

[Service]
ExecStart=$exec
Restart=always

[Install]
WantedBy=multi-user.target
EOF
}

create_service_file "MyApp" "/usr/bin/myapp" > /tmp/myapp.service

# Open file descriptor for reading (3-9 are free)
exec 3< /etc/passwd
while read -r line <&3; do
    echo "$line"
done
exec 3<&-                                    # Close descriptor

# Open for writing
exec 4> /tmp/output.log
echo "Log entry 1" >&4
echo "Log entry 2" >&4
exec 4>&-                                    # Close

# Open for read/write
exec 5<> /tmp/data.txt
read -r line <&5                             # Read
echo "appended" >&5                          # Write
exec 5>&-

# Duplicate stdin/stdout for restoration
exec 3>&1                                    # Save stdout to fd 3
exec 1> /tmp/output.log                      # Redirect stdout
echo "This goes to file"
exec 1>&3                                    # Restore stdout
exec 3>&-                                    # Close fd 3
echo "This goes to terminal"

# Swap stdout and stderr
command 3>&1 1>&2 2>&3 3>&-

# Log to file while preserving terminal output
exec 3>&1 4>&2                               # Save original stdout/stderr
exec 1> >(tee -a /tmp/script.log) 2>&1       # Redirect with tee
echo "This logs and displays"
exec 1>&3 2>&4                               # Restore
exec 3>&- 4>&-                               # Close

# Multiple log files
exec 3>> /var/log/app/info.log
exec 4>> /var/log/app/error.log
echo "Info message" >&3
echo "Error message" >&4
exec 3>&- 4>&-

# <() creates readable pseudo-file from command
diff <(cat file1) <(cat file2)

# Compare remote files
diff <(ssh vault-01 cat /etc/hosts) <(ssh vault-02 cat /etc/hosts)

# Compare sorted
diff <(sort file1.txt) <(sort file2.txt)

# >() creates writable pseudo-file
command | tee >(gzip > output.gz)

# Fan-out to multiple processors
generate_data | tee >(process_a > a.out) >(process_b > b.out) > combined.out

# Log levels to separate files
./app 2>&1 | tee >(grep ERROR >> errors.log) >(grep WARN >> warnings.log)

# Feed multiple commands from same source
<(cat file.txt) >(grep pattern1) >(grep pattern2)

# Avoid subshell with process substitution
count=0
while read -r line; do
    ((count++))
done < <(cat /etc/passwd)                    # No subshell!
echo "Count: $count"                         # Correct value

# Redirect for entire script
exec > /var/log/myscript.log 2>&1
echo "All output goes to log file"

# Redirect for code block
{
    echo "Line 1"
    echo "Line 2"
    echo "Line 3"
} > output.txt

# Redirect for loop
for host in vault-01 ise-01 bind-01; do
    echo "Checking $host..."
    ping -c1 "$host"
done > hosts_status.log 2>&1

# Conditional redirection
verbose=true
if $verbose; then
    exec 3>&1                                # Normal stdout
else
    exec 3> /dev/null                        # Discard
fi
echo "Maybe visible" >&3

# Redirect stdin for entire script
exec < /etc/passwd
while read -r line; do
    echo "$line"
done

# Read password without echo (terminal control)
read -s -p "Password: " password
echo ""                                      # Newline after hidden input

# Redirect from string
read -r name age <<< "John 25"
echo "Name: $name, Age: $age"

# Append with sudo (can't just sudo >>)
echo "new line" | sudo tee -a /etc/myconfig > /dev/null

# Start coprocess
coproc my_coproc { bc -l; }

# Write to coprocess stdin
echo "scale=4; 22/7" >&"${my_coproc[1]}"

# Read from coprocess stdout
read -r result <&"${my_coproc[0]}"
echo "Result: $result"                       # 3.1428

# Close and wait
exec {my_coproc[1]}>&-
wait "$my_coproc_PID"

# Coprocess with named pipes
coproc MATH { bc -l; }

# Multiple calculations
for expr in "2+2" "10/3" "sqrt(2)"; do
    echo "scale=4; $expr" >&"${MATH[1]}"
    read -r result <&"${MATH[0]}"
    echo "$expr = $result"
done

exec {MATH[1]}>&-

# Interactive session
coproc SSH { ssh -tt vault-01; }
echo "hostname" >&"${SSH[1]}"
read -r hostname <&"${SSH[0]}"
echo "Host is: $hostname"
exec {SSH[1]}>&-

# Session logging with timestamps
exec > >(while read -r line; do echo "$(date '+%Y-%m-%d %H:%M:%S') $line"; done | tee -a /var/log/session.log) 2>&1

# Separate stdout/stderr logs
exec 1> >(tee -a /var/log/app/stdout.log)
exec 2> >(tee -a /var/log/app/stderr.log >&2)

# Configuration from heredoc
cat << 'EOF' | ssh vault-01 "sudo tee /etc/vault.d/config.hcl > /dev/null"
listener "tcp" {
  address     = "0.0.0.0:8200"
  tls_disable = false
}

storage "raft" {
  path = "/opt/vault/data"
}
EOF

# Parallel command output to separate files
for host in vault-01 ise-01 bind-01; do
    ssh "$host" "df -h /" > "/tmp/${host}_disk.txt" 2>&1 &
done
wait

# Feed dACL content to netapi
netapi ise ers dacl create --name "Linux-AD-Auth" --content - << 'EOF'
permit udp any any eq domain
permit tcp any any eq domain
permit udp any host ${AD_IP} eq 88
permit tcp any host ${AD_IP} eq 88
permit udp any host ${AD_IP} eq 123
deny ip any any
EOF

# k8s secrets from heredoc
kubectl create secret generic vault-config --from-file=config.hcl=/dev/stdin << 'EOF'
listener "tcp" {
  address = "0.0.0.0:8200"
}
EOF

# Multi-line git commit message
git commit -m "$(cat << 'EOF'
feat(auth): Add LDAP authentication

- Integrate with Active Directory
- Support nested groups
- Add caching for performance
EOF
)"

# Capture stderr only
errors=$(command 2>&1 >/dev/null)

# Capture stdout and stderr separately
{ stdout=$(command 2>&1 1>&3 3>&-); } 3>&1
stderr=$(command 2>&1 >/dev/null)

# Check if command produced errors
if stderr=$(command 2>&1 >/dev/null) && [[ -z "$stderr" ]]; then
    echo "No errors"
else
    echo "Errors: $stderr"
fi

# Log errors with context
run_with_logging() {
    local cmd="$*"
    local output
    local exit_code

    output=$("$@" 2>&1)
    exit_code=$?

    if (( exit_code != 0 )); then
        echo "[ERROR] Command failed: $cmd" >&2
        echo "[ERROR] Exit code: $exit_code" >&2
        echo "[ERROR] Output: $output" >&2
    fi

    return $exit_code
}

# Conditional error display
command 2>&1 | grep -v "expected warning"

# Prefix stderr lines
command 2> >(sed 's/^/[stderr] /' >&2)

# Timestamped error log
exec 2> >(while read -r line; do
    echo "$(date '+%Y-%m-%d %H:%M:%S') [ERROR] $line"
done >> /var/log/errors.log)

# WRONG: Order matters!
command 2>&1 > file                          # stderr to terminal, stdout to file
command > file 2>&1                          # CORRECT: Both to file

# WRONG: Redirect in wrong place
sudo echo "text" > /etc/protected            # Redirection runs as user, not root!

# CORRECT: Use tee
echo "text" | sudo tee /etc/protected > /dev/null

# WRONG: Lost file descriptor
{
    exec 3> /tmp/out
    echo "test" >&3
}                                            # fd 3 closed at block end!

# CORRECT: Manage scope properly
exec 3> /tmp/out
echo "test" >&3
exec 3>&-

# WRONG: Here-document in subshell
(
    cat << EOF
$var is expanded in subshell
EOF
)                                            # $var might not be what you expect

# WRONG: Expecting read from closed descriptor
exec 3< file.txt
cat <&3                                      # Reads entire file
read -r line <&3                             # Nothing left to read!

# CORRECT: Seek not possible, reopen
exec 3< file.txt
read -r line <&3
exec 3< file.txt                             # Reopen to "reset"

# WRONG: Buffering in pipeline to file
tail -f log | grep pattern > matches.txt    # Buffered, delayed writes!

# CORRECT: Unbuffered
tail -f log | stdbuf -oL grep pattern > matches.txt

# WRONG: Here-doc in function loses stdin
myfunc() {
    cat << EOF
Some content
EOF
    read -r input                            # Can't read! stdin consumed by heredoc
}

# CORRECT: Use fd 0 explicitly if needed
myfunc() {
    cat << EOF
Some content
EOF
    read -r input <&0                        # Still won't work - stdin is heredoc
}

command > /dev/null 2>&1              # Suppress everything
command &> /dev/null                  # Same (bash shorthand)
which git > /dev/null 2>&1 && echo "installed"   # Quiet existence check (zsh-compatible)
cat /dev/null > file.txt              # Truncate a file

dd if=/dev/zero of=bigfile.bin bs=1M count=1024   # Create 1GB file
dd if=/dev/zero of=sparse.img bs=1 count=0 seek=10G  # Sparse 10GB image

cat /dev/urandom | tr -dc 'a-zA-Z0-9' | head -c 16    # Random password (e.g. QDaXzhMCFf6wAmwG)
head -c 16 /dev/urandom | od -An -tx1 | tr -d ' \n'    # Random hex string (no xxd needed)
od -An -tu4 -N4 /dev/urandom | tr -d ' '               # Random integer

# Inside a pipeline, stdin is the pipe — /dev/tty reaches the actual terminal
cat file.txt | while read line; do
    echo "Processing: $line"
    read -p "Continue? " answer < /dev/tty
done

tar czf /dev/stdout files/ | ssh server 'cat > backup.tar.gz'
echo "Error: critical" > /dev/stderr
program --input /dev/stdin < inputfile.txt

command 2>&1 | while IFS= read -r line; do
    echo "$(date '+%Y-%m-%d %H:%M:%S') $line"
done | tee -a /var/log/myapp.log

command 2> >(tee -a errors.log >&2)

cat data.txt |
    tee /tmp/step1.txt |
    grep "pattern" |
    tee /tmp/step2.txt |
    sort |
    tee /tmp/step3.txt |
    uniq -c
# Inspect /tmp/step{1,2,3}.txt to find where data drops

n=0
while read line; do
    ((n++))
    ((n % 1000 == 0)) && echo "Processed $n lines" >&2
done < input.txt > output.txt

#!/usr/bin/env bash
VERBOSE=${VERBOSE:-0}

log() {
    if ((VERBOSE)); then
        echo "[$(date '+%H:%M:%S')] $*" >&2
    fi
}

log "Starting process"
VERBOSE=1 ./myscript.sh     # Enable with env var

exec > >(tee -a output.log | logger -t myscript) 2>&1
echo "This goes to file, syslog, and screen"

#!/usr/bin/env bash
exec 3<&0                           # Save original stdin

read -s -p "Password: " pass < /dev/tty
echo

while read -u 3 line; do            # Read data from saved fd 3
    echo "Processing: $line"
done

exec 3<&-                           # Close fd 3