Network Troubleshooting

Quick Reference

# Layer 1-2: Physical/Link
ip link show
ethtool eth0

# Layer 3: Network
ip addr show
ip route show
ping -c 3 gateway_ip

# Layer 4: Transport
ss -tlnp
nc -zv host port

# DNS
dig example.com
nslookup example.com
cat /etc/resolv.conf

# Firewall
iptables -L -n
nft list ruleset
firewall-cmd --list-all

Troubleshooting Methodology

OSI Layer Approach

Work through network layers systematically:

Layer Check Tools

1. Physical

Cable, NIC, link status

ip link, ethtool, dmesg

2. Data Link

MAC address, ARP

ip neigh, arp, bridge

3. Network

IP address, routing

ip addr, ip route, ping

4. Transport

Ports, connections

ss, nc, telnet

5-7. Application

Services, DNS

curl, dig, nslookup

Basic Connectivity Check

# 1. Check interface status
ip link show

# 2. Check IP address
ip addr show

# 3. Check default route
ip route show

# 4. Ping gateway
ping -c 3 $(ip route | grep default | awk '{print $3}')

# 5. Ping external (if gateway works)
ping -c 3 8.8.8.8

# 6. Check DNS
dig google.com

Interface Status

# List all interfaces
ip link show

# Check specific interface
ip link show eth0

# States to look for:
# state UP = interface is up
# state DOWN = interface is down
# NO-CARRIER = cable unplugged

# Check with ethtool
ethtool eth0
# Speed: 1000Mb/s
# Duplex: Full
# Link detected: yes

Interface Management

# Bring interface up
ip link set eth0 up

# Bring interface down
ip link set eth0 down

# Check driver and hardware info
ethtool -i eth0

# Show statistics
ip -s link show eth0

# Check for errors
ethtool -S eth0 | grep -i error

Cable and Hardware

# Check dmesg for hardware issues
dmesg | grep -i eth0
dmesg | grep -i "link up\|link down"

# Check for NIC errors
cat /sys/class/net/eth0/statistics/rx_errors
cat /sys/class/net/eth0/statistics/tx_errors

# Test cable (if supported)
ethtool --test eth0

ARP and Neighbor Cache

# View ARP cache
ip neigh show
arp -a

# Flush ARP cache
ip neigh flush all

# Add static ARP entry
ip neigh add 192.168.1.100 lladdr aa:bb:cc:dd:ee:ff dev eth0

# Check for duplicate IPs (arping)
arping -D -I eth0 192.168.1.50

Layer 3: Network

IP Address Configuration

# Show all IP addresses
ip addr show

# Show specific interface
ip addr show eth0

# Check if IP is assigned
ip addr show eth0 | grep "inet "

# Add IP address (temporary)
ip addr add 192.168.1.100/24 dev eth0

# Delete IP address
ip addr del 192.168.1.100/24 dev eth0

# Check DHCP lease
cat /var/lib/dhclient/dhclient.leases
journalctl -u NetworkManager | grep -i dhcp

Routing

# View routing table
ip route show

# Check default gateway
ip route | grep default

# Add route
ip route add 10.0.0.0/8 via 192.168.1.1

# Add default gateway
ip route add default via 192.168.1.1

# Delete route
ip route del 10.0.0.0/8

# Check route to specific host
ip route get 8.8.8.8

Ping Tests

# Basic ping
ping -c 3 192.168.1.1

# Ping with specific interface
ping -I eth0 192.168.1.1

# Ping with specific source IP
ping -I 192.168.1.100 8.8.8.8

# Flood ping (root)
ping -f -c 100 192.168.1.1

# Ping with packet size
ping -s 1472 192.168.1.1    # Test MTU

# Don't fragment
ping -M do -s 1472 192.168.1.1

Traceroute

# Basic traceroute
traceroute 8.8.8.8

# Use ICMP instead of UDP
traceroute -I 8.8.8.8

# Use TCP
traceroute -T -p 443 example.com

# mtr (combined ping + traceroute)
mtr 8.8.8.8
mtr --report -c 10 8.8.8.8

MTU Issues

# Check MTU
ip link show eth0 | grep mtu

# Test MTU with ping
ping -M do -s 1472 192.168.1.1    # 1472 + 28 = 1500
# If fails, reduce size until works

# Set MTU
ip link set eth0 mtu 1400

# Check path MTU
tracepath example.com

Layer 4: Transport

Check Listening Ports

# All listening ports
ss -tlnp

# TCP listening ports
ss -tln

# UDP listening ports
ss -uln

# Include process info
ss -tlnp

# Specific port
ss -tlnp | grep :22
ss -tlnp | grep :80

Check Established Connections

# All connections
ss -tn

# Connections to specific port
ss -tn | grep :443

# Connection states
ss -t state established
ss -t state time-wait
ss -t state close-wait

Test Port Connectivity

# Using nc (netcat)
nc -zv example.com 443
nc -zv -w 5 192.168.1.1 22    # With timeout

# Using telnet
telnet example.com 80

# Using curl
curl -v telnet://example.com:22

# Using bash
echo > /dev/tcp/example.com/443 && echo "Port open"

TCP Connection Issues

# Check for connection timeouts
ss -tn | grep SYN

# Check TCP retransmits
netstat -s | grep -i retransmit

# Check socket buffers
sysctl net.core.rmem_max
sysctl net.core.wmem_max

# Check for TIME_WAIT buildup
ss -tan | grep TIME-WAIT | wc -l

DNS Troubleshooting

Check DNS Configuration

# Current DNS servers
cat /etc/resolv.conf

# systemd-resolved status
resolvectl status

# NetworkManager DNS
nmcli dev show | grep DNS

DNS Resolution Tests

# Using dig
dig example.com
dig example.com +short
dig @8.8.8.8 example.com    # Specific DNS server

# Using nslookup
nslookup example.com
nslookup example.com 8.8.8.8

# Using host
host example.com
host example.com 8.8.8.8

# Reverse lookup
dig -x 8.8.8.8
host 8.8.8.8

DNS Record Types

# A record (IPv4)
dig example.com A

# AAAA record (IPv6)
dig example.com AAAA

# MX records
dig example.com MX

# TXT records
dig example.com TXT

# NS records
dig example.com NS

# SOA record
dig example.com SOA

Common DNS Issues

# DNS server not responding
dig @192.168.1.1 example.com
# Check if DNS port is reachable
nc -zvu 192.168.1.1 53

# Wrong or stale DNS cache
# Flush systemd-resolved cache
resolvectl flush-caches
# Flush nscd cache
nscd -i hosts

# DNS timeout
dig +time=10 +tries=3 example.com

# Check hosts file
cat /etc/hosts
getent hosts example.com

Firewall Issues

Check iptables Rules

# List all rules
iptables -L -n -v

# List NAT rules
iptables -t nat -L -n -v

# List specific chain
iptables -L INPUT -n -v

# Check if traffic is blocked
iptables -L -n -v | grep DROP
iptables -L -n -v | grep REJECT

Check nftables

# List all rules
nft list ruleset

# List specific table
nft list table inet filter

# Check counters
nft list ruleset | grep -A2 "counter"

Check firewalld

# Status
firewall-cmd --state

# List all rules
firewall-cmd --list-all

# List specific zone
firewall-cmd --zone=public --list-all

# Check if port is allowed
firewall-cmd --query-port=80/tcp
firewall-cmd --query-service=http

Temporarily Disable Firewall

# TESTING ONLY - re-enable afterward!

# iptables - flush rules
iptables -F
iptables -P INPUT ACCEPT
iptables -P FORWARD ACCEPT
iptables -P OUTPUT ACCEPT

# nftables
nft flush ruleset

# firewalld
systemctl stop firewalld

# ufw
ufw disable

Network Captures

tcpdump Basic Usage

# Capture on interface
tcpdump -i eth0

# Capture specific host
tcpdump -i eth0 host 192.168.1.100

# Capture specific port
tcpdump -i eth0 port 80
tcpdump -i eth0 port 443 or port 80

# Capture specific protocol
tcpdump -i eth0 icmp
tcpdump -i eth0 tcp
tcpdump -i eth0 udp

# Save to file
tcpdump -i eth0 -w capture.pcap

# Read capture file
tcpdump -r capture.pcap

tcpdump Filters

# Source or destination
tcpdump -i eth0 src host 192.168.1.100
tcpdump -i eth0 dst host 192.168.1.100

# Source or destination port
tcpdump -i eth0 src port 22
tcpdump -i eth0 dst port 443

# Network range
tcpdump -i eth0 net 192.168.1.0/24

# Combination
tcpdump -i eth0 'host 192.168.1.100 and port 443'
tcpdump -i eth0 'tcp and port 80 and host 192.168.1.100'

# Show packet contents
tcpdump -i eth0 -X port 80
tcpdump -i eth0 -A port 80    # ASCII only

Common Issues

No Network After Boot

# Check interface exists
ip link show

# Check interface is up
ip link set eth0 up

# Check for IP address
ip addr show eth0

# Check NetworkManager status
systemctl status NetworkManager

# Try to get DHCP address
dhclient eth0

Can Ping IP but Not Hostname

# DNS issue
# Check DNS servers
cat /etc/resolv.conf

# Test DNS directly
dig @8.8.8.8 example.com

# Try different DNS
echo "nameserver 8.8.8.8" | sudo tee /etc/resolv.conf

# Check /etc/hosts
cat /etc/hosts

Intermittent Connectivity

# Check for packet loss
ping -c 100 gateway_ip | grep loss

# Check for interface flapping
dmesg | grep -i "link"

# Monitor interface
watch -n 1 'ip link show eth0'

# Check for duplex mismatch
ethtool eth0

# Check for errors
ip -s link show eth0

Slow Network

# Test bandwidth
iperf3 -c server_ip

# Check for packet loss
mtr --report server_ip

# Check latency
ping -c 50 server_ip

# Check for TCP issues
ss -ti

# Check MTU
tracepath example.com

Can’t Reach External Network

# Check default gateway
ip route | grep default

# Ping gateway
ping -c 3 $(ip route | grep default | awk '{print $3}')

# Check NAT (if applicable)
iptables -t nat -L -n

# Check IP forwarding
cat /proc/sys/net/ipv4/ip_forward

Service-Specific Issues

SSH Connection Issues

# Verbose SSH connection
ssh -vvv user@host

# Check SSH service
systemctl status sshd

# Check SSH listening
ss -tlnp | grep :22

# Check firewall
iptables -L -n | grep 22

Web Server Issues

# Test HTTP
curl -v http://example.com

# Test HTTPS
curl -v https://example.com

# Check listening
ss -tlnp | grep -E ':80|:443'

# Check service
systemctl status nginx
systemctl status httpd

Database Connection Issues

# Test MySQL/MariaDB
nc -zv db-server 3306
mysql -h db-server -u user -p

# Test PostgreSQL
nc -zv db-server 5432
psql -h db-server -U user -d database

# Check from application perspective
# Check firewall between app and db
# Check SELinux (httpd_can_network_connect_db)

Quick Command Reference

# Interface management
ip link show                              # List interfaces
ip link set eth0 up/down                  # Enable/disable
ethtool eth0                              # Hardware info

# IP and routing
ip addr show                              # Show IPs
ip route show                             # Show routes
ip route get 8.8.8.8                      # Test route

# Connectivity tests
ping -c 3 HOST                            # ICMP test
traceroute HOST                           # Path trace
mtr HOST                                  # Combined tool

# Port testing
ss -tlnp                                  # Listening ports
nc -zv HOST PORT                          # Test port
curl -v URL                               # Test HTTP

# DNS
dig DOMAIN                                # DNS lookup
dig @DNS_SERVER DOMAIN                    # Specific server
cat /etc/resolv.conf                      # DNS config

# Firewall
iptables -L -n                            # List rules
nft list ruleset                          # nftables rules
firewall-cmd --list-all                   # firewalld rules

# Capture
tcpdump -i eth0                           # Capture traffic
tcpdump -i eth0 -w file.pcap              # Save capture