ZFS

Quick Reference

# Pool operations
zpool create tank /dev/sda                    # Single disk
zpool create tank mirror /dev/sda /dev/sdb    # Mirror
zpool create tank raidz /dev/sda /dev/sdb /dev/sdc  # RAIDZ
zpool status
zpool list

# Dataset operations
zfs create tank/data
zfs list
zfs set compression=zstd tank/data
zfs get all tank/data

# Snapshots
zfs snapshot tank/data@backup
zfs list -t snapshot
zfs rollback tank/data@backup

# Send/receive
zfs send tank/data@snap | zfs receive backup/data

Understanding ZFS

Key Concepts

ZFS combines a filesystem and volume manager into a single system:

Pool (zpool) - Storage pool aggregating multiple devices
Dataset - Filesystem or volume within a pool
Snapshot - Read-only point-in-time copy
Clone - Writable copy from a snapshot
vdev - Virtual device (single disk, mirror, raidz)

ZFS vs Traditional Stack

Traditional storage requires separate layers: partitions, RAID, volume manager, filesystem. ZFS integrates all layers, providing checksumming across the entire data path, eliminating silent data corruption.

Installation

Arch Linux

# Install from AUR (DKMS method)
yay -S zfs-dkms

# Load module
sudo modprobe zfs

# Enable services
sudo systemctl enable zfs-import-cache.service
sudo systemctl enable zfs-mount.service
sudo systemctl enable zfs.target

Debian/Ubuntu

# Add contrib repository
sudo apt install software-properties-common
sudo apt-add-repository contrib

# Install ZFS
sudo apt install zfsutils-linux

# Verify
zfs version

RHEL/CentOS

# Add ZFS repository
sudo dnf install https://zfsonlinux.org/epel/zfs-release-2-3.el9.noarch.rpm

# Install DKMS version
sudo dnf install kernel-devel zfs

# Load module
sudo modprobe zfs

Pool Management

Create Pools

# Single disk (no redundancy)
zpool create tank /dev/sda

# Mirror (RAID1 equivalent)
zpool create tank mirror /dev/sda /dev/sdb

# Three-way mirror
zpool create tank mirror /dev/sda /dev/sdb /dev/sdc

# RAIDZ (RAID5 equivalent, single parity)
zpool create tank raidz /dev/sda /dev/sdb /dev/sdc

# RAIDZ2 (RAID6 equivalent, double parity)
zpool create tank raidz2 /dev/sda /dev/sdb /dev/sdc /dev/sdd

# RAIDZ3 (triple parity)
zpool create tank raidz3 /dev/sd{a,b,c,d,e}

# Striped mirrors (RAID10 equivalent)
zpool create tank mirror /dev/sda /dev/sdb mirror /dev/sdc /dev/sdd

# With specific mount point
zpool create -m /data tank /dev/sda

# Without mounting
zpool create -m none tank /dev/sda

# Force creation (overwrite existing)
zpool create -f tank /dev/sda

Pool with Special Devices

# Add SSD cache (L2ARC)
zpool add tank cache /dev/nvme0n1

# Add SSD log (ZIL/SLOG)
zpool add tank log /dev/nvme0n1

# Mirrored log (recommended for production)
zpool add tank log mirror /dev/nvme0n1 /dev/nvme1n1

# Add hot spare
zpool add tank spare /dev/sdf

# Remove cache/log/spare
zpool remove tank /dev/nvme0n1

Pool Status and Information

# List pools
zpool list
zpool list -v         # Verbose with vdevs

# Pool status
zpool status          # All pools
zpool status tank     # Specific pool
zpool status -x       # Only pools with errors

# Pool history
zpool history tank

# I/O statistics
zpool iostat
zpool iostat tank 5   # Every 5 seconds
zpool iostat -v tank  # Per-vdev stats

Pool Maintenance

# Scrub (verify data integrity)
zpool scrub tank
zpool scrub -s tank   # Stop scrub

# Check scrub progress
zpool status tank

# Clear errors
zpool clear tank

# Export pool (safe removal)
zpool export tank

# Import pool
zpool import                  # List available
zpool import tank             # Import by name
zpool import -d /dev/disk/by-id tank  # Specify device path
zpool import -f tank          # Force import

# Upgrade pool version
zpool upgrade tank
zpool upgrade -a      # Upgrade all

Replace and Resilver

# Replace failed disk
zpool replace tank /dev/sda /dev/sdd

# Offline disk
zpool offline tank /dev/sda

# Online disk
zpool online tank /dev/sda

# Attach disk (convert single to mirror)
zpool attach tank /dev/sda /dev/sdb

# Detach disk (convert mirror to single)
zpool detach tank /dev/sdb

# Remove vdev (only works for certain configurations)
zpool remove tank /dev/sda

Dataset Management

Create Datasets

# Create dataset
zfs create tank/data
zfs create tank/data/documents

# Create with properties
zfs create -o compression=zstd -o mountpoint=/data tank/data

# Create volume (zvol) for block device
zfs create -V 50G tank/vm-disk

# Create sparse volume
zfs create -s -V 100G tank/thin-disk

List and Properties

# List all datasets
zfs list

# List with specific properties
zfs list -o name,used,avail,refer,mountpoint

# List snapshots
zfs list -t snapshot

# List volumes
zfs list -t volume

# Show all properties
zfs get all tank/data

# Show specific property
zfs get compression tank/data
zfs get compression,compressratio tank/data

# Show inherited properties
zfs get -s inherited all tank/data

Set Properties

# Enable compression
zfs set compression=zstd tank/data
zfs set compression=lz4 tank/data   # Faster, lower ratio
zfs set compression=gzip-9 tank/data  # Highest ratio

# Quota and reservation
zfs set quota=100G tank/data        # Hard limit
zfs set refquota=100G tank/data     # Limit excluding snapshots
zfs set reservation=50G tank/data   # Guaranteed space

# Mount point
zfs set mountpoint=/data tank/data
zfs set mountpoint=none tank/data   # Don't mount

# Access time
zfs set atime=off tank/data

# Deduplication (high memory usage)
zfs set dedup=on tank/data

# Record size (for specific workloads)
zfs set recordsize=1M tank/media    # Large files
zfs set recordsize=16K tank/db      # Databases

# Sync behavior
zfs set sync=disabled tank/scratch  # Dangerous, fast
zfs set sync=standard tank/data     # Default

Mount Operations

# Mount dataset
zfs mount tank/data

# Mount all datasets
zfs mount -a

# Unmount
zfs unmount tank/data
zfs unmount -f tank/data    # Force

# Show mount points
zfs list -o name,mountpoint,mounted

# Legacy mount (for fstab)
zfs set mountpoint=legacy tank/data
# Then add to /etc/fstab:
# tank/data /data zfs defaults 0 0

Rename and Destroy

# Rename dataset
zfs rename tank/old tank/new

# Rename with mountpoint change
zfs rename -u tank/old tank/new

# Destroy dataset
zfs destroy tank/old

# Destroy recursively
zfs destroy -r tank/old

# Destroy with dependents (clones)
zfs destroy -R tank/old

# Dry run
zfs destroy -nv tank/old

Snapshots

Create Snapshots

# Create snapshot
zfs snapshot tank/data@today
zfs snapshot tank/data@$(date +%Y%m%d-%H%M%S)

# Recursive snapshot
zfs snapshot -r tank/data@backup

# Snapshot with hold (prevent deletion)
zfs snapshot tank/data@important
zfs hold keep tank/data@important

List and Manage Snapshots

# List snapshots
zfs list -t snapshot
zfs list -t snapshot -r tank/data

# Show snapshot space usage
zfs list -t snapshot -o name,used,refer

# Compare snapshots
zfs diff tank/data@snap1 tank/data@snap2

# Show what changed since snapshot
zfs diff tank/data@snap1

# Delete snapshot
zfs destroy tank/data@old

# Delete range of snapshots
zfs destroy tank/data@snap1%snap5

# Release hold
zfs release keep tank/data@important

Rollback

# Rollback to snapshot (destroys changes)
zfs rollback tank/data@snap

# Rollback destroying intermediate snapshots
zfs rollback -r tank/data@snap

# Rollback destroying clones too
zfs rollback -R tank/data@snap

Accessing Snapshots

# Snapshots accessible via .zfs directory
ls /tank/data/.zfs/snapshot/
ls /tank/data/.zfs/snapshot/today/

# Restore single file
cp /tank/data/.zfs/snapshot/today/file.txt /tank/data/

# Show hidden .zfs directory
zfs set snapdir=visible tank/data

Clone from Snapshot

# Create clone (writable copy)
zfs clone tank/data@snap tank/data-clone

# Promote clone (make independent)
zfs promote tank/data-clone

# List clones
zfs list -o name,origin

Send/Receive (Backup)

Local Backup

# Send to file
zfs send tank/data@snap > /backup/data.zfs

# Send to another pool
zfs send tank/data@snap | zfs receive backup/data

# Send with properties
zfs send -p tank/data@snap | zfs receive backup/data

# Send recursive
zfs send -R tank/data@snap | zfs receive -d backup

# Resume interrupted receive
zfs send -t <token> | zfs receive backup/data

Incremental Backup

# Send incremental (changes between snapshots)
zfs send -i tank/data@snap1 tank/data@snap2 | zfs receive backup/data

# Incremental from any snapshot (replication stream)
zfs send -I tank/data@snap1 tank/data@snap2 | zfs receive backup/data

# Find common snapshot for incremental
zfs list -t snapshot -o name -s creation tank/data
zfs list -t snapshot -o name -s creation backup/data

Remote Backup

# Send over SSH
zfs send tank/data@snap | ssh user@remote zfs receive tank/data

# With compression
zfs send tank/data@snap | zstd | ssh user@remote 'zstd -d | zfs receive tank/data'

# With mbuffer for network stability
zfs send tank/data@snap | mbuffer -s 128k -m 1G | \
    ssh user@remote 'mbuffer -s 128k -m 1G | zfs receive tank/data'

# Incremental remote
zfs send -i tank/data@old tank/data@new | \
    ssh user@remote zfs receive -F tank/data

Automated Backup with Sanoid

# Install sanoid
# Arch
yay -S sanoid

# Debian/Ubuntu
sudo apt install sanoid

# Configure /etc/sanoid/sanoid.conf
[tank/data]
    use_template = production
    recursive = yes

[template_production]
    frequently = 0
    hourly = 24
    daily = 30
    monthly = 12
    yearly = 2
    autosnap = yes
    autoprune = yes

# Run sanoid
sanoid --cron

# Replicate with syncoid
syncoid tank/data user@remote:backup/data

Performance Tuning

ARC (Adaptive Replacement Cache)

# View ARC stats
cat /proc/spl/kstat/zfs/arcstats

# Set maximum ARC size (in /etc/modprobe.d/zfs.conf)
options zfs zfs_arc_max=8589934592    # 8GB

# Runtime adjustment
echo 8589934592 > /sys/module/zfs/parameters/zfs_arc_max

# View current ARC size
arc_summary    # Or:
cat /proc/spl/kstat/zfs/arcstats | grep "^size"

L2ARC (SSD Cache)

# Add L2ARC cache
zpool add tank cache /dev/nvme0n1

# View L2ARC stats
cat /proc/spl/kstat/zfs/arcstats | grep l2

# Tune L2ARC write speed
echo 524288000 > /sys/module/zfs/parameters/l2arc_write_max

ZIL/SLOG (Sync Write Acceleration)

# Add SLOG (mirrored for safety)
zpool add tank log mirror /dev/nvme0n1 /dev/nvme1n1

# For non-critical data, disable sync
zfs set sync=disabled tank/scratch

Dataset Tuning

# Large sequential files (media, backups)
zfs set recordsize=1M tank/media

# Databases
zfs set recordsize=16K tank/postgresql
zfs set primarycache=metadata tank/postgresql
zfs set logbias=throughput tank/postgresql

# VMs
zfs set recordsize=64K tank/vms
zfs set primarycache=metadata tank/vms

# High-performance
zfs set compression=lz4 tank/data
zfs set atime=off tank/data
zfs set xattr=sa tank/data

Troubleshooting

Check Pool Health

# Status with errors
zpool status -v tank

# Only show problems
zpool status -x

# Clear transient errors
zpool clear tank

# View error log
dmesg | grep -i zfs

Corrupted Data

# Scrub to detect corruption
zpool scrub tank

# If corruption found in RAIDZ:
# Data will be automatically repaired

# If corruption in non-redundant pool:
# Check for backup/snapshot
zfs list -t snapshot

# Restore from snapshot
zfs rollback tank/data@lastgood

Replace Failed Device

# Identify failed device
zpool status tank

# Replace with new device
zpool replace tank /dev/sda-old /dev/sda-new

# Monitor resilver progress
zpool status tank

# If device disappeared
zpool online tank /dev/sda    # Try bringing back

# Mark as faulted manually
zpool offline tank /dev/sda

Import Problems

# List importable pools
zpool import

# Import by ID (if name conflict)
zpool import -d /dev/disk/by-id 12345678901234567890

# Force import (use with caution)
zpool import -f tank

# Import read-only
zpool import -o readonly=on tank

# Import with missing device
zpool import -F tank

Recovery Options

# Mount read-only for recovery
zpool import -o readonly=on tank
cp -a /tank/data /recovery/

# Rollback to earlier transaction
zpool import -F -T 12345 tank

# Clear unimportable state
zpool import -F -n tank    # Dry run
zpool import -F tank       # Execute

Security

Encryption

# Create encrypted dataset
zfs create -o encryption=aes-256-gcm -o keyformat=passphrase tank/secure

# Create with key file
dd if=/dev/urandom of=/root/zfs.key bs=32 count=1
zfs create -o encryption=aes-256-gcm -o keyformat=raw \
    -o keylocation=file:///root/zfs.key tank/secure

# Load key for encrypted dataset
zfs load-key tank/secure
zfs mount tank/secure

# Unload key (unmount first)
zfs unmount tank/secure
zfs unload-key tank/secure

# Change key
zfs change-key tank/secure

# Key status
zfs get keystatus tank/secure

Permissions

# Allow user to manage datasets
zfs allow user create,destroy,mount,snapshot tank

# Allow user full control of specific dataset
zfs allow user @data tank/userdata

# Remove permissions
zfs unallow user create,destroy tank

# Show permissions
zfs allow tank

Quick Command Reference

# Pool
zpool create tank mirror /dev/sd{a,b}         # Create mirror
zpool status                                   # Pool health
zpool list                                     # Pool usage
zpool scrub tank                              # Verify integrity
zpool destroy tank                            # Remove pool

# Dataset
zfs create tank/data                          # Create dataset
zfs list                                      # List datasets
zfs set compression=zstd tank/data            # Set property
zfs get all tank/data                         # Get properties
zfs destroy tank/data                         # Remove dataset

# Snapshot
zfs snapshot tank/data@name                   # Create snapshot
zfs list -t snapshot                          # List snapshots
zfs rollback tank/data@name                   # Restore snapshot
zfs destroy tank/data@name                    # Remove snapshot

# Send/Receive
zfs send tank/data@snap | zfs receive pool/data      # Full send
zfs send -i @old @new | zfs receive pool/data        # Incremental

ZFS