sed

sed

Attribute Value

Goal

Expert sed for text transformation

Interest Link

Systems Tools > Text Processing

Status

In Progress

Documentation

Codex sed section, domus-linux-ops sed-deep-dive

Skill Areas

Area Description Status

Substitution

s/old/new/g, flags, delimiters

[x] Proficient

Addressing

Line numbers, ranges, patterns

[x] Proficient

Commands

d, p, a, i, c, y

[ ] In Progress

Hold Space

h, H, g, G, x

[ ] Learning

Branching

b, t, labels

[ ] Not Started

Substitution Basics

# Basic substitution
sed 's/old/new/' file                    # First occurrence per line
sed 's/old/new/g' file                   # All occurrences (global)
sed 's/old/new/2' file                   # Second occurrence only
sed 's/old/new/gi' file                  # Global, case insensitive

# In-place editing
sed -i 's/old/new/g' file                # Edit file directly
sed -i.bak 's/old/new/g' file            # With backup (.bak)

# Alternative delimiters (when pattern contains /)
sed 's|/var/log|/opt/log|g' file         # Pipe delimiter
sed 's#/home/user#/home/admin#g' file    # Hash delimiter
sed 's@http://@https://@g' file          # At sign delimiter

# Escape special characters
sed 's/\./,/g' file                      # Replace literal dot
sed 's/\*/x/g' file                      # Replace literal asterisk
sed 's/\[ERROR\]/[WARN]/g' file          # Replace brackets

# Use regex
sed 's/[0-9]\+/NUM/g' file               # Replace numbers
sed 's/vault-[0-9]*/vault-XX/g' file     # Anonymize host numbers
sed 's/[[:space:]]\+/ /g' file           # Collapse whitespace

# Capture groups (backreferences)
sed 's/\(.*\)@\(.*\)/User: \1, Domain: \2/' file    # Split email
sed 's/^\([^:]*\):/[\1]:/' file                     # Bracket first field
sed 's/\([0-9]\{4\}\)-\([0-9]\{2\}\)-\([0-9]\{2\}\)/\3\/\2\/\1/' file  # Date format

# Extended regex (-E or -r)
sed -E 's/([0-9]+)-([0-9]+)/\2-\1/g' file          # Swap numbers
sed -E 's/vault-([0-9]+)/server-\1/g' file         # Rename pattern

# Infrastructure: Update IP in config
sed -i 's/10\.50\.1\.20/10.50.1.21/g' /etc/hosts

# Infrastructure: Update hostname
sed -i 's/vault-01/vault-02/g' config.yaml

Line Addressing

# By line number
sed -n '5p' file                         # Print line 5 only
sed -n '5,10p' file                      # Print lines 5-10
sed '5d' file                            # Delete line 5
sed '5,10d' file                         # Delete lines 5-10

# From line to end
sed -n '5,$p' file                       # From line 5 to end
sed '5,$d' file                          # Delete from line 5 to end

# First/last line
sed -n '1p' file                         # First line
sed -n '$p' file                         # Last line
sed '1d' file                            # Delete first line
sed '$d' file                            # Delete last line

# Every nth line
sed -n '0~2p' file                       # Even lines
sed -n '1~2p' file                       # Odd lines
sed -n '0~5p' file                       # Every 5th line

# By pattern
sed -n '/ERROR/p' file                   # Lines containing ERROR
sed '/ERROR/d' file                      # Delete lines with ERROR
sed -n '/START/,/END/p' file             # Range between patterns

# Pattern + offset
sed -n '/pattern/,+3p' file              # Pattern line + next 3 lines
sed '/pattern/,+3d' file                 # Delete pattern + next 3 lines

# Negation
sed -n '/ERROR/!p' file                  # Lines NOT containing ERROR
sed '1!d' file                           # Delete all except first line

# Combined addressing
sed -n '5,/pattern/p' file               # From line 5 to pattern
sed '/START/,/END/{s/old/new/g}' file    # Substitute only in range

# Infrastructure: Extract config section
sed -n '/^\[database\]/,/^\[/p' config.ini | head -n -1

# Infrastructure: Skip header
sed '1d' data.csv

# Infrastructure: Get specific line for verification
sed -n '73p' /etc/ssh/sshd_config

Infrastructure Patterns

# Update SSH config
sudo sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config
sudo sed -i 's/^#\?PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config

# Verify line before change
sed -n '73p' /etc/ssh/sshd_config
# Change specific line
sudo sed -i '73s/^#GSSAPIAuthentication no/GSSAPIAuthentication yes/' /etc/ssh/sshd_config
# Verify after change
sed -n '73p' /etc/ssh/sshd_config

# Update hosts file
sudo sed -i '/old-host/d' /etc/hosts
echo "10.50.1.60 vault-01.inside.domusdigitalis.dev vault-01" | sudo tee -a /etc/hosts

# Comment/uncomment lines
sed -i 's/^dangerous_option/#&/' config.conf          # Comment out
sed -i 's/^#\(wanted_option\)/\1/' config.conf        # Uncomment

# Update version numbers
sed -i 's/version: [0-9]\+\.[0-9]\+\.[0-9]\+/version: 2.0.0/' Chart.yaml

# Replace in systemd unit file
sudo sed -i 's|ExecStart=.*|ExecStart=/usr/bin/myapp --config /etc/myapp/config.yaml|' /etc/systemd/system/myapp.service
sudo systemctl daemon-reload

# Update DNS resolvers
sudo sed -i 's/^nameserver.*/nameserver 10.50.1.90/' /etc/resolv.conf

# Fix file permissions in sudoers
sudo sed -i 's/ALL$/NOPASSWD: ALL/' /etc/sudoers.d/admin

# Update Kubernetes manifest
sed -i 's/replicas: [0-9]\+/replicas: 3/' deployment.yaml

# Replace in ansible inventory
sed -i "s/ansible_host=.*/ansible_host=$NEW_IP/" inventory.ini

# Update certificate paths
sed -i "s|/etc/ssl/certs/old.crt|/etc/ssl/certs/new.crt|g" /etc/nginx/nginx.conf

# Mass update across files (with find)
find /etc/myapp -name "*.conf" -exec sed -i 's/old_setting/new_setting/g' {} \;

# Update YAML value (simple case)
sed -i 's/\(port:\) [0-9]\+/\1 8443/' config.yaml

# Add entry if not exists
grep -q "new_entry" /etc/config || sed -i '$a\new_entry=value' /etc/config

# Remove duplicate empty lines
sed -i '/^$/N;/^\n$/d' file

# Convert Windows to Unix line endings
sed -i 's/\r$//' file

# Backup before bulk edit
for f in *.conf; do
    cp "$f" "$f.bak"
    sed -i 's/old/new/g' "$f"
done

sed Gotchas

# WRONG: In-place without backup can lose data
sed -i 's/old/new/g' important_file      # No backup!

# CORRECT: Always backup when editing important files
sed -i.bak 's/old/new/g' important_file

# WRONG: Delimiter in pattern without escaping
sed 's/path/to/file/new/path/g' file     # Error!

# CORRECT: Use alternative delimiter
sed 's|path/to/file|new/path|g' file

# WRONG: Forgetting -n with p
sed '/pattern/p' file                    # Prints matching lines TWICE

# CORRECT: Use -n to suppress default output
sed -n '/pattern/p' file

# WRONG: Literal interpretation of special chars
sed 's/.*//g' file                       # Deletes everything!

# CORRECT: Escape or be explicit
sed 's/\.\*//g' file                     # Delete literal .*

# WRONG: BRE vs ERE confusion
sed 's/(foo|bar)/baz/g' file             # Literal parentheses!

# CORRECT: Use -E for extended regex
sed -E 's/(foo|bar)/baz/g' file

# WRONG: Expecting in-place to work on symlinks
sed -i 's/old/new/g' /etc/symlink        # Breaks symlink!

# CORRECT: Follow symlink explicitly or edit target
sed -i 's/old/new/g' "$(readlink -f /etc/symlink)"

# WRONG: Multiline matching by default
sed 's/start.*end/MATCH/' file           # Only works on same line

# CORRECT: Use N to read multiple lines
sed 'N;s/start\n.*end/MATCH/' file

# WRONG: Greedy matching when you don't want it
echo "<a>text</a>" | sed 's/<.*>/TAG/'   # <a>text</a> -> TAG

# CORRECT: Use negated character class
echo "<a>text</a>" | sed 's/<[^>]*>/TAG/g'  # <a> -> TAG, </a> -> TAG

# WRONG: Quotes with shell variables
var="value"
sed 's/$var/replacement/' file           # Literal $var

# CORRECT: Use double quotes for expansion
sed "s/$var/replacement/" file

# WRONG: Special chars in replacement
sed "s/old/path/to/new/" file            # Interpreted as delimiters!

# CORRECT: Escape or use different delimiter
sed "s|old|path/to/new|" file

# WRONG: Pattern contains / (common with vim patterns, URLs)
sed 's/{pattern}/\\{pattern}/g' file     # Error: unknown option to 's'
# The /{pattern} is interpreted as: s/{pattern}  /  \\  {pattern}  /g

# CORRECT: Use | delimiter when pattern contains /
sed 's|{pattern}|\\{pattern}|g' file     # Works!

# REAL-WORLD: Escape AsciiDoc attribute placeholders in docs
# Patterns like /{pattern}, ?{pattern}, `f{char}` contain /
sed -i 's|{char}|\\{char}|g' shortcuts.adoc
sed -i 's|{motion}|\\{motion}|g' shortcuts.adoc
sed -i 's|{pattern}|\\{pattern}|g' shortcuts.adoc

# WRONG: Line numbers from grep
grep -n "pattern" file | sed 's/old/new/'  # Includes line number in output!

# CORRECT: Use grep -o or sed alone
sed -n '/pattern/{s/old/new/p}' file

# macOS vs Linux sed differences
# macOS requires argument to -i
sed -i '' 's/old/new/' file              # macOS
sed -i 's/old/new/' file                 # Linux
# For portable scripts:
sed -i.bak 's/old/new/' file && rm -f file.bak