systemd Deep Dive

Quick Reference

# Service basics
systemctl status service-name
systemctl start|stop|restart service-name
systemctl enable|disable service-name

# View logs
journalctl -u service-name -f
journalctl -u service-name --since "10 min ago"

# Find why it failed
systemctl status service-name
journalctl -u service-name -b --no-pager

# Boot analysis
systemd-analyze blame
systemd-analyze critical-chain

Unit Management

Unit Types

Type Extension Purpose

Service

.service

Daemons, background processes

Socket

.socket

Socket-based activation

Timer

.timer

Scheduled tasks (cron replacement)

Mount

.mount

Filesystem mounts

Target

.target

Grouping units (like runlevels)

Path

.path

Path-based activation

Device

.device

Device-based activation

Slice

.slice

Resource management

Listing Units

# All loaded units
systemctl list-units

# All unit files (loaded or not)
systemctl list-unit-files

# Filter by type
systemctl list-units --type=service
systemctl list-units --type=timer

# Failed units
systemctl --failed

# Units in specific state
systemctl list-units --state=running
systemctl list-units --state=failed

Unit Status

# Detailed status
systemctl status nginx.service

# Check if enabled/active
systemctl is-enabled nginx
systemctl is-active nginx
systemctl is-failed nginx

# Show all properties
systemctl show nginx.service

# Specific property
systemctl show nginx.service -p MainPID

Managing Services

# Start/stop
sudo systemctl start nginx
sudo systemctl stop nginx
sudo systemctl restart nginx
sudo systemctl reload nginx        # Reload config without restart
sudo systemctl reload-or-restart nginx

# Enable/disable at boot
sudo systemctl enable nginx
sudo systemctl disable nginx
sudo systemctl enable --now nginx  # Enable AND start

# Mask (completely prevent starting)
sudo systemctl mask nginx
sudo systemctl unmask nginx

Unit Files

Locations (Priority Order)

Path Purpose Priority

/etc/systemd/system/

Local admin overrides

Highest

/run/systemd/system/

Runtime units

Medium

/usr/lib/systemd/system/

Package-installed units

Lowest

 
# Find unit file location
systemctl show nginx.service -p FragmentPath

# Show unit file contents
systemctl cat nginx.service

Service Unit Structure

# /etc/systemd/system/myapp.service

[Unit]
Description=My Application
Documentation=https://myapp.example.com/docs
After=network.target postgresql.service
Requires=postgresql.service
Wants=redis.service

[Service]
Type=simple
User=myapp
Group=myapp
WorkingDirectory=/opt/myapp
Environment=NODE_ENV=production
EnvironmentFile=/etc/myapp/env
ExecStartPre=/opt/myapp/bin/preflight.sh
ExecStart=/opt/myapp/bin/start.sh
ExecReload=/bin/kill -HUP $MAINPID
ExecStop=/opt/myapp/bin/stop.sh
Restart=on-failure
RestartSec=5
TimeoutStartSec=30
TimeoutStopSec=30

[Install]
WantedBy=multi-user.target

Service Types

Type Behavior Use Case

simple

Process started by ExecStart is main process

Default, most services

forking

Process forks, parent exits

Traditional daemons

oneshot

Process expected to exit

Setup scripts

notify

Sends sd_notify when ready

systemd-aware apps

dbus

Acquires D-Bus name when ready

D-Bus services

idle

Like simple, but waits for jobs

Console output

Restart Options

[Service]
# When to restart
Restart=always          # Always restart
Restart=on-failure      # Only on non-zero exit
Restart=on-abnormal     # Signal, timeout, watchdog
Restart=no              # Never auto-restart

# Restart timing
RestartSec=5            # Wait 5s before restart

# Restart limits
StartLimitIntervalSec=60
StartLimitBurst=5       # Max 5 starts in 60s

Security Hardening

[Service]
# User/Group isolation
User=myapp
Group=myapp
DynamicUser=yes         # Auto-create transient user

# Filesystem restrictions
ProtectSystem=strict    # Read-only except /dev, /proc, /sys
ProtectHome=yes         # No access to /home, /root, /run/user
ReadWritePaths=/var/lib/myapp
PrivateTmp=yes          # Private /tmp

# Network restrictions
PrivateNetwork=yes      # No network (if not needed)
RestrictAddressFamilies=AF_INET AF_INET6

# Capability restrictions
CapabilityBoundingSet=
NoNewPrivileges=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes

# System call filtering
SystemCallFilter=@system-service
SystemCallArchitectures=native

Dependencies

[Unit]
# Ordering (when to start)
After=network.target       # Start after network
Before=httpd.service       # Start before httpd

# Requirements (what must exist)
Requires=postgresql.service  # Hard dependency
Wants=redis.service          # Soft dependency
BindsTo=other.service        # Start/stop together

# Conflicts
Conflicts=other.service      # Cannot run simultaneously

Override Files

# Create override directory
sudo systemctl edit nginx.service

# This creates /etc/systemd/system/nginx.service.d/override.conf
# Only specify what you want to change

# Example: Increase timeout
[Service]
TimeoutStartSec=120

# After editing
sudo systemctl daemon-reload
sudo systemctl restart nginx

# View effective configuration
systemctl cat nginx.service

Timers

Timer vs Cron

Feature systemd Timer cron

Logging

journalctl integration

Separate log files

Dependencies

Full systemd dependency system

None

Missed runs

Persistent=true catches up

Lost

Resource control

cgroups, CPU/memory limits

None

Boot-relative

OnBootSec, OnUnitActiveSec

Not possible

Timer Unit Structure

# /etc/systemd/system/backup.timer
[Unit]
Description=Daily backup timer

[Timer]
OnCalendar=daily
OnCalendar=*-*-* 02:00:00    # Alternative: specific time
Persistent=true               # Run missed jobs after boot
RandomizedDelaySec=1h         # Spread load

[Install]
WantedBy=timers.target
# /etc/systemd/system/backup.service
[Unit]
Description=Daily backup

[Service]
Type=oneshot
ExecStart=/usr/local/bin/backup.sh

Timer Expressions

# OnCalendar examples
OnCalendar=daily              # Every day at 00:00
OnCalendar=weekly             # Every Monday at 00:00
OnCalendar=monthly            # First of month
OnCalendar=*-*-* 02:00:00     # Every day at 02:00
OnCalendar=Mon-Fri 09:00      # Weekdays at 09:00
OnCalendar=*:0/15             # Every 15 minutes
OnCalendar=hourly             # Every hour

# Monotonic timers (relative)
OnBootSec=5min                # 5 min after boot
OnUnitActiveSec=1h            # 1h after last run
OnStartupSec=10min            # 10 min after systemd start

# Test expression
systemd-analyze calendar "Mon-Fri 09:00"

Managing Timers

# List active timers
systemctl list-timers

# Enable timer
sudo systemctl enable --now backup.timer

# View timer status
systemctl status backup.timer

# View last run
journalctl -u backup.service -n 20

journalctl

Basic Usage

# All logs
journalctl

# Follow (like tail -f)
journalctl -f

# Specific unit
journalctl -u nginx.service

# Current boot only
journalctl -b

# Previous boot
journalctl -b -1

# Kernel messages (dmesg replacement)
journalctl -k

Filtering

# By time
journalctl --since "2024-01-15"
journalctl --since "10 min ago"
journalctl --since "09:00" --until "10:00"

# By priority (0=emerg through 7=debug)
journalctl -p err         # err and above
journalctl -p warning     # warning and above

# By field
journalctl _UID=1000
journalctl _PID=1234
journalctl _COMM=sshd

# Multiple units
journalctl -u nginx -u php-fpm

Output Formats

# JSON output
journalctl -o json
journalctl -o json-pretty

# Verbose (all fields)
journalctl -o verbose

# Short (default)
journalctl -o short

# With timestamps
journalctl -o short-iso

Log Management

# Disk usage
journalctl --disk-usage

# Vacuum (clean) logs
sudo journalctl --vacuum-time=7d    # Keep 7 days
sudo journalctl --vacuum-size=1G    # Keep 1GB

# Configure in /etc/systemd/journald.conf
[Journal]
SystemMaxUse=1G
MaxRetentionSec=1month

Boot Analysis

Timing Analysis

# Overall boot time
systemd-analyze

# Blame (slowest units)
systemd-analyze blame

# Critical chain (blocking dependencies)
systemd-analyze critical-chain

# Critical chain for specific unit
systemd-analyze critical-chain nginx.service

# Generate SVG boot chart
systemd-analyze plot > boot.svg

Debugging Boot

# Find what's blocking
systemd-analyze critical-chain multi-user.target

# Verify unit files
systemd-analyze verify /etc/systemd/system/*.service

# Security analysis
systemd-analyze security nginx.service

Targets (Runlevels)

Common Targets

Target SysV Level Description

poweroff.target

0

Halt

rescue.target

1

Single user

multi-user.target

3

Multi-user, no GUI

graphical.target

5

Multi-user with GUI

reboot.target

6

Reboot

Managing Targets

# Current target
systemctl get-default

# Change default
sudo systemctl set-default multi-user.target

# Switch target now
sudo systemctl isolate multi-user.target

# Emergency mode
sudo systemctl isolate rescue.target

Resource Control

cgroups v2

# View cgroup hierarchy
systemd-cgls

# Resource usage
systemd-cgtop

Limiting Resources

[Service]
# CPU
CPUQuota=50%              # Max 50% of one CPU
CPUWeight=100             # Relative weight (default 100)

# Memory
MemoryMax=1G              # Hard limit
MemoryHigh=800M           # Soft limit (throttle)

# IO
IOWeight=100              # Relative weight
IOReadBandwidthMax=/dev/sda 10M

Troubleshooting

Service Won’t Start

# 1. Check status
systemctl status service.service

# 2. Check logs
journalctl -u service.service -b --no-pager

# 3. Check dependencies
systemctl list-dependencies service.service

# 4. Verify unit file
systemd-analyze verify /etc/systemd/system/service.service

# 5. Check if masked
systemctl is-enabled service.service
# If "masked", unmask:
sudo systemctl unmask service.service

Service Keeps Failing

# Check restart limits
systemctl show service.service -p StartLimitBurst -p StartLimitIntervalSec

# Reset failure count
sudo systemctl reset-failed service.service

# Check resource limits
systemctl show service.service -p LimitNOFILE -p LimitNPROC

Boot Issues

# Check failed units
systemctl --failed

# Check boot log
journalctl -b -p err

# Check specific target
systemctl status multi-user.target

Quick Reference

Essential Commands

# Service control
systemctl start|stop|restart|reload|status unit
systemctl enable|disable|mask|unmask unit

# Unit info
systemctl cat unit              # Show unit file
systemctl show unit             # All properties
systemctl list-dependencies unit

# System state
systemctl list-units            # Loaded units
systemctl list-unit-files       # All unit files
systemctl --failed              # Failed units

# Logs
journalctl -u unit -f           # Follow logs
journalctl -b                   # Current boot
journalctl --since "10 min ago"

# Boot
systemd-analyze blame
systemd-analyze critical-chain

# Reload after changes
sudo systemctl daemon-reload

Unit File Locations

# System (package installed)
/usr/lib/systemd/system/

# System (admin created/overrides)
/etc/systemd/system/

# User units
~/.config/systemd/user/

See Also