journalctl & systemd Journal

systemd-journald.service    # Collection daemon
/run/log/journal/           # Volatile storage (RAM)
/var/log/journal/           # Persistent storage (disk)
/etc/systemd/journald.conf  # Configuration

# Default pager (less)
journalctl

# Navigation in less:
#   g     - go to beginning
#   G     - go to end
#   /     - search forward
#   ?     - search backward
#   n     - next match
#   N     - previous match
#   q     - quit

# No pager (direct output)
journalctl --no-pager

# Specific number of lines
journalctl -n 100
journalctl --lines=100

# Oldest first (default)
journalctl

# Newest first (reverse)
journalctl -r
journalctl --reverse

# Follow new entries (like tail -f)
journalctl -f
journalctl --follow

# Follow specific unit
journalctl -u nginx -f

# Follow multiple units
journalctl -u nginx -u php-fpm -f

# Last N time units
journalctl --since "5 minutes ago"
journalctl --since "2 hours ago"
journalctl --since "1 day ago"
journalctl --since "1 week ago"

# Combined relative
journalctl --since "1 hour ago" --until "30 minutes ago"

# Specific date/time
journalctl --since "2024-01-15 10:00:00"
journalctl --since "2024-01-15" --until "2024-01-16"

# Date only (midnight)
journalctl --since "2024-01-15"

# Time shortcuts
journalctl --since today
journalctl --since yesterday
journalctl --since "yesterday" --until "today"

# Current boot
journalctl -b
journalctl -b 0

# Previous boot
journalctl -b -1

# Two boots ago
journalctl -b -2

# List available boots
journalctl --list-boots

# Output:
# -2 abc123... Sat 2024-01-13 09:00:00 UTC—Sat 2024-01-13 18:30:00 UTC
# -1 def456... Sun 2024-01-14 10:00:00 UTC—Sun 2024-01-14 22:00:00 UTC
#  0 ghi789... Mon 2024-01-15 08:00:00 UTC—Mon 2024-01-15 15:30:00 UTC

# Specific boot by ID
journalctl -b abc123

# Combined with time
journalctl -b -1 --since "10:00" --until "12:00"

# Specific service
journalctl -u nginx.service
journalctl -u nginx           # .service is implied

# Multiple units
journalctl -u nginx -u php-fpm -u mysql

# Unit with wildcard
journalctl -u "docker*"
journalctl -u "systemd-*"

# User units
journalctl --user -u myapp.service
journalctl --user-unit myapp.service

# By PID
journalctl _PID=1234

# By executable path
journalctl _EXE=/usr/bin/nginx
journalctl /usr/bin/nginx     # Shorthand

# By command name
journalctl _COMM=sshd

# Current shell's PID
journalctl _PID=$$

# By UID
journalctl _UID=1000

# By username (resolve to UID)
journalctl _UID=$(id -u username)

# By GID
journalctl _GID=1000

# Root user
journalctl _UID=0

# Kernel messages (dmesg equivalent)
journalctl -k
journalctl --dmesg

# Kernel for current boot
journalctl -k -b

# By syslog identifier
journalctl -t sudo
journalctl -t sshd
journalctl --identifier=sudo

# By transport
journalctl _TRANSPORT=kernel      # Kernel messages
journalctl _TRANSPORT=syslog      # Syslog protocol
journalctl _TRANSPORT=journal     # Native journal
journalctl _TRANSPORT=stdout      # Stdout capture
journalctl _TRANSPORT=audit       # Audit messages

# Specific host (for centralized logging)
journalctl _HOSTNAME=webserver01

# Current host
journalctl _HOSTNAME=$(hostname)

# Specific level and above (<=)
journalctl -p err             # err, crit, alert, emerg
journalctl -p 3               # Same as above
journalctl -p warning         # warning and above
journalctl -p crit            # crit, alert, emerg

# Priority range
journalctl -p warning..err    # warning, err only
journalctl -p 4..3            # Same as above

# Emergency only
journalctl -p emerg
journalctl -p 0

# All errors since boot
journalctl -b -p err

# Warnings and errors for specific service
journalctl -u nginx -p warning

# Critical issues in last hour
journalctl --since "1 hour ago" -p crit

# Daily error summary
journalctl --since today -p err --no-pager | wc -l

# Default short format
journalctl -n 1
# Jan 15 10:30:15 myhost sshd[1234]: Accepted publickey for user...

# ISO timestamps
journalctl -o short-iso -n 1
# 2024-01-15T10:30:15+0000 myhost sshd[1234]: Accepted publickey...

# Verbose (all fields)
journalctl -o verbose -n 1
# Mon 2024-01-15 10:30:15.123456 UTC [...]
#     _TRANSPORT=syslog
#     _UID=0
#     _GID=0
#     _PID=1234
#     _COMM=sshd
#     MESSAGE=Accepted publickey...
#     ...

# JSON output
journalctl -o json -n 1
# {"__CURSOR":"...","_PID":"1234","MESSAGE":"Accepted..."}

# Pretty JSON
journalctl -o json-pretty -n 1

# Message only (for processing)
journalctl -o cat -n 1
# Accepted publickey for user from 192.168.1.100 port 54321

# Parse with jq
journalctl -o json -n 10 | jq '.MESSAGE'

# Filter JSON fields
journalctl -o json -n 10 | jq '{time: .__REALTIME_TIMESTAMP, msg: .MESSAGE}'

# Count messages by unit
journalctl -o json --since today | jq -r '._SYSTEMD_UNIT' | sort | uniq -c | sort -rn

# Extract specific field
journalctl -u nginx -o json | jq -r 'select(.PRIORITY == "3") | .MESSAGE'

# List all field names
journalctl --fields

# Show values for specific field
journalctl -F _SYSTEMD_UNIT
journalctl -F _COMM
journalctl -F PRIORITY
journalctl -F _UID

# Match specific field value
journalctl PRIORITY=3                    # Errors
journalctl _SYSTEMD_UNIT=nginx.service
journalctl _COMM=sshd
journalctl SYSLOG_FACILITY=10            # authpriv

# Multiple field matches (AND)
journalctl _SYSTEMD_UNIT=nginx.service PRIORITY=3

# Multiple values same field (OR)
journalctl _SYSTEMD_UNIT=nginx.service + _SYSTEMD_UNIT=php-fpm.service

# Combine with other filters
journalctl _SYSTEMD_UNIT=sshd.service -p warning --since today

# Show cursor for each entry
journalctl --show-cursor -n 5

# Output includes:
# -- cursor: s=abc123...;i=1234;b=xyz789...;m=...;t=...;x=...

# Start after specific cursor
journalctl --after-cursor="s=abc123...;i=1234;..."

# Start at specific cursor (inclusive)
journalctl --cursor="s=abc123...;i=1234;..."

# Save cursor position
journalctl -n 1 --show-cursor -o cat | tail -1 > /var/lib/myapp/cursor

# Read from saved position
CURSOR=$(cat /var/lib/myapp/cursor)
journalctl --after-cursor="$CURSOR" -o json

# View current config
cat /etc/systemd/journald.conf

# Or with drop-ins
systemd-analyze cat-config systemd/journald.conf

[Journal]
# Storage mode: volatile, persistent, auto, none
Storage=persistent

# Compress stored journals
Compress=yes

# Forward to syslog (rsyslog compatibility)
ForwardToSyslog=no

# Forward to kmsg
ForwardToKMsg=no

# Forward to console
ForwardToConsole=no

# Max disk usage (default: 10% of filesystem)
SystemMaxUse=500M

# Max size per journal file
SystemMaxFileSize=50M

# Keep at least this much free
SystemKeepFree=1G

# Max retention time
MaxRetentionSec=1month

# Max file retention time
MaxFileSec=1week

# Rate limiting (per-service)
RateLimitIntervalSec=30s
RateLimitBurst=10000

# Audit forwarding
Audit=yes

# Volatile (lost on reboot)
/run/log/journal/

# Persistent (survives reboot)
/var/log/journal/

# Create persistent directory (enables persistence)
sudo mkdir -p /var/log/journal
sudo systemd-tmpfiles --create --prefix /var/log/journal

# Check current storage mode
journalctl --disk-usage

# Reload configuration
sudo systemctl restart systemd-journald

# Or send SIGUSR2 for rotation
sudo systemctl kill -s SIGUSR2 systemd-journald

# Show journal disk usage
journalctl --disk-usage

# Output:
# Archived and active journals take up 256.0M in the file system.

# Show file details
ls -lh /var/log/journal/*/

# Reduce to specific size
sudo journalctl --vacuum-size=100M

# Remove entries older than duration
sudo journalctl --vacuum-time=2weeks
sudo journalctl --vacuum-time=30d

# Remove old archived files (keep N)
sudo journalctl --vacuum-files=5

# Combine limits
sudo journalctl --vacuum-size=500M --vacuum-time=1month

# Verify after vacuum
journalctl --verify

# Force rotation
sudo journalctl --rotate

# Rotate and vacuum
sudo journalctl --rotate --vacuum-time=1week

# Verify journal files
journalctl --verify

# Output shows any corruption:
# PASS: /var/log/journal/abc123.../system.journal
# PASS: /var/log/journal/abc123.../user-1000.journal

# Set up sealing
sudo journalctl --setup-keys

# Output provides key to store securely:
# Sealing key fingerprint: abc123...
# Store this securely for verification

# Verify sealed journals
journalctl --verify-key=abc123...

# Show catalog entries (explanations)
journalctl -x
journalctl --catalog

# Example with error explanation
journalctl -xe

# Update catalog database
sudo journalctl --update-catalog

# Read journal from different root
journalctl -D /mnt/rescue/var/log/journal

# Merge with local journal
journalctl --merge -D /mnt/other/var/log/journal

# Show logs from specific machine
journalctl -M container1

# List container/VM journals
machinectl list

#!/bin/bash
# Monitor for errors and alert

journalctl -f -p err -o json | while read -r line; do
    message=$(echo "$line" | jq -r '.MESSAGE')
    unit=$(echo "$line" | jq -r '._SYSTEMD_UNIT // "unknown"')

    echo "ERROR in $unit: $message"
    # Add alerting logic here
done

#!/bin/bash
# Generate daily error report

YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
TODAY=$(date +%Y-%m-%d)

echo "=== Error Report for $YESTERDAY ==="

echo -e "\n--- Errors by Service ---"
journalctl --since "$YESTERDAY" --until "$TODAY" -p err -o json \
    | jq -r '._SYSTEMD_UNIT // "_no_unit_"' \
    | sort | uniq -c | sort -rn | head -20

echo -e "\n--- Recent Errors ---"
journalctl --since "$YESTERDAY" --until "$TODAY" -p err -n 50 --no-pager

# Export to binary format (for backup/transfer)
journalctl --since "1 week ago" -o export > journal-export.bin

# Import on another system
journalctl --file=journal-export.bin

# Export to JSON for external analysis
journalctl --since today -o json > /tmp/today.json

# Filter and export
journalctl -u nginx --since today -o json | gzip > nginx-today.json.gz

# Watch for failed SSH logins
journalctl -u sshd -f | grep -i "failed"

# Watch for OOM kills
journalctl -k -f | grep -i "killed process"

# Watch for disk errors
journalctl -k -f | grep -iE "error|fail|i/o"

# 1. Check recent service logs
journalctl -u myservice -n 50 --no-pager

# 2. Check with errors highlighted
journalctl -xe -u myservice

# 3. Check since last restart attempt
journalctl -u myservice --since "5 minutes ago"

# 4. Full verbose output
journalctl -u myservice -o verbose -n 20

# 1. Check previous boot (if current boot fails)
journalctl -b -1

# 2. Kernel messages during boot
journalctl -b -k

# 3. Critical messages during boot
journalctl -b -p crit

# 4. Boot timing
systemd-analyze blame
systemd-analyze critical-chain

# PAM/auth logs
journalctl -t sudo
journalctl -t sshd
journalctl _COMM=login
journalctl SYSLOG_FACILITY=10    # authpriv

# Failed auth in last hour
journalctl --since "1 hour ago" | grep -iE "fail|denied|invalid"

# All kernel storage messages
journalctl -k | grep -iE "sd[a-z]|nvme|scsi|ata|disk"

# Filesystem errors
journalctl -k | grep -iE "ext4|xfs|btrfs|error|warning"

# SMART issues
journalctl | grep -i smart

# NetworkManager logs
journalctl -u NetworkManager

# DHCP issues
journalctl -u NetworkManager | grep -i dhcp

# DNS issues
journalctl -u systemd-resolved

# Firewall logs
journalctl -k | grep -i "firewall\|nft\|iptables"

# OOM killer activity
journalctl -k | grep -i "out of memory\|oom\|killed process"

# Memory pressure
journalctl -k | grep -i "memory"

# Swap activity
journalctl -k | grep -i swap

[Journal]
ForwardToSyslog=yes

# Send arbitrary message to journal
echo "Deployment started" | systemd-cat -t myapp -p info

# Tag custom scripts
./my-script.sh 2>&1 | systemd-cat -t myscript -p warning

# With priority
systemd-cat -t backup -p notice /usr/local/bin/backup.sh

# Traditional logger (goes to journal via syslog)
logger "My message"
logger -t myapp "Application event"
logger -p auth.warning "Security warning"