Boot Issues Troubleshooting

┌─────────────────────────────────────────────────────────────────┐
│  1. FIRMWARE (BIOS/UEFI)                                        │
│     - POST (Power-On Self Test)                                 │
│     - Hardware initialization                                    │
│     - Load bootloader from disk                                 │
└────────────────────────────────┬────────────────────────────────┘
                                 │
┌────────────────────────────────▼────────────────────────────────┐
│  2. BOOTLOADER (GRUB2)                                          │
│     - Display boot menu                                         │
│     - Load kernel and initramfs                                 │
│     - Pass parameters to kernel                                 │
└────────────────────────────────┬────────────────────────────────┘
                                 │
┌────────────────────────────────▼────────────────────────────────┐
│  3. KERNEL + INITRAMFS                                          │
│     - Initialize hardware drivers                               │
│     - Mount root filesystem                                     │
│     - Start init process (PID 1)                                │
└────────────────────────────────┬────────────────────────────────┘
                                 │
┌────────────────────────────────▼────────────────────────────────┐
│  4. SYSTEMD INITIALIZATION                                      │
│     - Reach default target                                      │
│     - Start services in parallel                                │
│     - Present login prompt                                      │
└─────────────────────────────────────────────────────────────────┘

# Check UEFI boot entries
efibootmgr -v

# Add a boot entry
efibootmgr -c -d /dev/sda -p 1 -L "Linux" -l "\EFI\fedora\shimx64.efi"

# Change boot order
efibootmgr -o 0001,0002,0003

# Delete a boot entry
efibootmgr -b 0002 -B

# Check if booted in UEFI mode
[ -d /sys/firmware/efi ] && echo "UEFI" || echo "BIOS"

# List UEFI variables
ls /sys/firmware/efi/efivars/

# Check Secure Boot status
mokutil --sb-state

# Check secure boot status
mokutil --sb-state

# List enrolled keys
mokutil --list-enrolled

# Import MOK (Machine Owner Key)
mokutil --import /path/to/key.der

# Disable secure boot validation (dangerous)
mokutil --disable-validation

# List partitions
grub> ls
grub> ls (hd0,gpt2)/

# Set root and boot
grub> set root=(hd0,gpt2)
grub> linux /boot/vmlinuz-5.15.0 root=/dev/sda2
grub> initrd /boot/initramfs-5.15.0.img
grub> boot

# Find boot partition
grub rescue> ls
grub rescue> ls (hd0,gpt2)/boot/grub

# Load normal module
grub rescue> set prefix=(hd0,gpt2)/boot/grub
grub rescue> insmod normal
grub rescue> normal

# Boot from live USB, then:

# For BIOS systems
mount /dev/sda2 /mnt
grub-install --root-directory=/mnt /dev/sda

# For UEFI systems
mount /dev/sda2 /mnt
mount /dev/sda1 /mnt/boot/efi
for dir in dev proc sys run; do mount --bind /$dir /mnt/$dir; done
chroot /mnt
grub-install --target=x86_64-efi --efi-directory=/boot/efi
grub-mkconfig -o /boot/grub/grub.cfg

# Exit chroot
exit
umount -R /mnt

# RHEL/CentOS/Fedora
grub2-mkconfig -o /boot/grub2/grub.cfg      # BIOS
grub2-mkconfig -o /boot/efi/EFI/fedora/grub.cfg  # UEFI

# Debian/Ubuntu
update-grub
# Or manually:
grub-mkconfig -o /boot/grub/grub.cfg

# Arch Linux
grub-mkconfig -o /boot/grub/grub.cfg

# Add to linux line for debugging:
linux /boot/vmlinuz-... root=/dev/sda2 debug systemd.log_level=debug

# Boot to rescue mode:
linux /boot/vmlinuz-... root=/dev/sda2 systemd.unit=rescue.target

# Boot to emergency mode:
linux /boot/vmlinuz-... root=/dev/sda2 systemd.unit=emergency.target

# Boot with specific init:
linux /boot/vmlinuz-... root=/dev/sda2 init=/bin/bash

# Disable quiet and splash for more output:
# Remove: quiet splash
# Add: debug

# Common settings
GRUB_TIMEOUT=5
GRUB_CMDLINE_LINUX_DEFAULT="quiet"
GRUB_CMDLINE_LINUX=""

# For debugging, change to:
GRUB_TIMEOUT=10
GRUB_CMDLINE_LINUX_DEFAULT=""
GRUB_CMDLINE_LINUX="debug"

# Regenerate after changes
grub-mkconfig -o /boot/grub/grub.cfg

Common panic messages:

"VFS: Unable to mount root fs"
  → Wrong root= parameter or missing driver in initramfs

"Kernel panic - not syncing: No init found"
  → init=/bin/systemd missing or corrupt

"Unable to find root device"
  → Missing disk driver in initramfs

"ACPI: Unable to load the System Description Tables"
  → BIOS/ACPI issues

# Boot with older kernel from GRUB menu

# Rebuild initramfs

# RHEL/Fedora
dracut -f /boot/initramfs-$(uname -r).img $(uname -r)

# Arch Linux
mkinitcpio -p linux

# Debian/Ubuntu
update-initramfs -u -k $(uname -r)
# Or all kernels:
update-initramfs -u -k all

# Check root parameter at boot
# Edit GRUB and verify root= is correct

# Common formats:
root=/dev/sda2
root=UUID=xxxxx-xxxxx-xxxxx
root=LABEL=root
root=/dev/mapper/vg-root

# Find correct UUID
blkid

# If using LVM, ensure lvm module is in initramfs
dracut --add lvm -f

# Check available block devices
ls -la /dev/disk/by-uuid/
ls -la /dev/mapper/

# Check if LVM is activated
lvm vgscan
lvm vgchange -ay

# Check if LUKS needs unlocking
cryptsetup luksOpen /dev/sda3 cryptroot

# Try mounting root manually
mount /dev/mapper/vg-root /sysroot
exit  # Continue boot

# === RHEL/Fedora (dracut) ===
# Regenerate for current kernel
dracut -f

# Regenerate for specific kernel
dracut -f /boot/initramfs-5.15.0.img 5.15.0

# Include specific modules
dracut --add "lvm crypt" -f

# Verbose rebuild
dracut -f -v

# === Arch Linux (mkinitcpio) ===
# Regenerate all presets
mkinitcpio -P

# Specific preset
mkinitcpio -p linux

# === Debian/Ubuntu (initramfs-tools) ===
# Update current kernel
update-initramfs -u

# Update all kernels
update-initramfs -u -k all

# Regenerate
update-initramfs -c -k $(uname -r)

# List failed units
systemctl --failed

# Check specific service
systemctl status <service>
journalctl -u <service>

# Reset failed state
systemctl reset-failed

# Start service manually
systemctl start <service>

# Check what failed
journalctl -xb

# Common causes and fixes:

# 1. Bad /etc/fstab entry
cat /etc/fstab
# Comment out problematic entries with #

# 2. Filesystem corruption
fsck /dev/sda2

# 3. SELinux issues (RHEL/Fedora)
# Boot with: selinux=0 or enforcing=0
# Then relabel:
touch /.autorelabel
reboot

# From rescue mode, remount root read-write
mount -o remount,rw /

# Then make fixes and reboot
reboot

# Check service dependencies
systemctl list-dependencies <service>

# Reverse dependencies (what depends on this)
systemctl list-dependencies --reverse <service>

# Show boot ordering
systemd-analyze critical-chain

# Show what's blocking boot
systemd-analyze verify default.target

# Boot to emergency mode, then:
mount -o remount,rw /

# Edit fstab
vi /etc/fstab

# Comment out problematic entries
# Bad entry:
# /dev/sdb1 /data ext4 defaults 0 2
# Fixed:
# #/dev/sdb1 /data ext4 defaults 0 2

# Test fstab
mount -a

# Reboot
reboot

# Never run fsck on mounted filesystem!

# Boot to emergency mode or live USB

# Check ext4 filesystem
fsck.ext4 -y /dev/sda2

# Check XFS filesystem
xfs_repair /dev/sda2

# Check btrfs filesystem
btrfs check /dev/sda2
# Repair:
btrfs check --repair /dev/sda2

# Force check on next boot
touch /forcefsck
# Or add to kernel command line: fsck.mode=force

# Scan for volume groups
vgscan

# Activate volume groups
vgchange -ay

# Check logical volumes
lvs
lvdisplay

# If LV is inactive
lvchange -ay /dev/vg/lv

# Repair LVM
vgck vg_name
vgreduce --removemissing vg_name

# If encryption not unlocked at boot

# From initramfs shell or live USB:
cryptsetup luksOpen /dev/sda3 cryptroot
# Enter passphrase

# Then mount and chroot
mount /dev/mapper/cryptroot /mnt
# Or for LVM on LUKS:
vgchange -ay
mount /dev/mapper/vg-root /mnt

# Check /etc/crypttab
cat /mnt/etc/crypttab
# Format: <name> <device> <key> <options>
# cryptroot UUID=xxx-xxx none luks

# Check what's waiting
systemctl list-jobs

# Common culprits:
systemctl status NetworkManager-wait-online.service
systemctl status systemd-networkd-wait-online.service

# Disable if not needed
systemctl disable NetworkManager-wait-online.service

# Or reduce timeout
mkdir -p /etc/systemd/system/NetworkManager-wait-online.service.d/
cat > /etc/systemd/system/NetworkManager-wait-online.service.d/timeout.conf << 'EOF'
[Service]
TimeoutStartSec=10
EOF

# Check network status
ip link
ip addr

# Check NetworkManager
nmcli general status
nmcli device status

# Check for errors
journalctl -u NetworkManager
journalctl -u systemd-networkd

# Overall boot time
systemd-analyze

# Service startup times
systemd-analyze blame

# Critical path
systemd-analyze critical-chain

# Plot boot chart
systemd-analyze plot > boot.svg

# Verify unit files
systemd-analyze verify default.target

# Show service startup time
systemd-analyze blame | head -20

# Security analysis
systemd-analyze security
systemd-analyze security sshd.service

# Current boot
journalctl -b

# Previous boot
journalctl -b -1

# List available boots
journalctl --list-boots

# Errors only
journalctl -b -p err

# Kernel messages
journalctl -b -k

# Specific service
journalctl -b -u ssh

# Since boot, follow
journalctl -b -f

# Export for analysis
journalctl -b --no-pager > boot.log

# Show kernel ring buffer
dmesg

# With timestamps
dmesg -T

# Human-readable
dmesg -H

# Errors and warnings
dmesg --level=err,warn

# Follow new messages
dmesg -w

# Clear buffer (requires root)
dmesg -c

# Show driver messages
dmesg | grep -i driver

# Show storage messages
dmesg | grep -E 'sd[a-z]|nvme'

# 1. Boot older kernel from GRUB menu

# 2. If that works, check what changed
rpm -qa --last | head -20    # RHEL/Fedora
apt list --installed 2>/dev/null | head -20  # Debian

# 3. Rollback kernel
dnf history list
dnf history undo <id>

# Or remove new kernel
rpm -e kernel-5.xx.xx

# 4. Rebuild initramfs for working kernel
dracut -f /boot/initramfs-$(uname -r).img $(uname -r)

# 1. At GRUB, edit kernel line
# Add: init=/bin/bash

# 2. Press Ctrl+X to boot

# 3. Remount root read-write
mount -o remount,rw /

# 4. Change password
passwd root

# 5. SELinux relabel (RHEL/Fedora)
touch /.autorelabel

# 6. Reboot
exec /sbin/init
# Or:
reboot -f

# 1. Try recovery mode from GRUB menu

# 2. Edit kernel line, add:
nomodeset

# 3. Or try different video driver:
nouveau.modeset=0    # Disable nouveau
nvidia-drm.modeset=0  # Disable nvidia drm

# 4. Once booted, check GPU driver
lspci -k | grep -A 2 VGA
journalctl -b | grep -i drm
journalctl -b | grep -i nvidia

# 5. Reconfigure X/Wayland
# Remove problematic driver
dnf remove nvidia*
# Or reinstall
dnf reinstall xorg-x11-drv-*

# 1. Wait for timeout or press Ctrl+C

# 2. Check what's hanging
journalctl -b | grep -i timeout

# 3. Common causes:
#    - Network wait (see Network Boot Issues)
#    - Unmountable filesystem (check fstab)
#    - Slow disk (check dmesg for disk errors)

# 4. Reduce timeouts
systemctl edit <service>
# Add:
# [Service]
# TimeoutStartSec=30

# Disable Plymouth for debugging
# Add to kernel line:
plymouth.enable=0

# Or remove splash:
# Remove: splash quiet

# Reinstall Plymouth
dnf reinstall plymouth plymouth-scripts

# Rebuild initramfs
dracut -f

# 1. Boot from live USB

# 2. Identify root partition
lsblk
blkid

# 3. Mount root
mount /dev/sda2 /mnt

# 4. For LVM:
vgchange -ay
mount /dev/mapper/vg-root /mnt

# 5. For LUKS:
cryptsetup luksOpen /dev/sda3 cryptroot
mount /dev/mapper/cryptroot /mnt

# 6. Mount additional partitions
mount /dev/sda1 /mnt/boot
mount /dev/sda1 /mnt/boot/efi  # UEFI

# 7. Bind mount system directories
for dir in dev proc sys run; do mount --bind /$dir /mnt/$dir; done

# 8. Chroot
chroot /mnt

# 9. Now fix issues...

# 10. Exit and unmount
exit
umount -R /mnt

# Simpler chroot for Arch
mount /dev/sda2 /mnt
arch-chroot /mnt

# Fix issues, then exit
exit
umount -R /mnt