Essential Arch Tools

# Basic search
grep "pattern" file
grep "error" /var/log/syslog

# Case insensitive
grep -i "error" file

# Recursive directory search
grep -r "pattern" /path/
grep -rn "pattern" /path/     # With line numbers

# Extended regex (ERE)
grep -E "error|warning|critical" file
egrep "error|warning" file    # Equivalent

# Fixed strings (no regex)
grep -F "literal.string" file
fgrep "literal.string" file   # Equivalent

# Invert match (exclude)
grep -v "pattern" file

# Count matches
grep -c "pattern" file

# Show only matching part
grep -o "pattern" file

# Context lines
grep -B 3 "error" file        # 3 lines Before
grep -A 3 "error" file        # 3 lines After
grep -C 3 "error" file        # 3 lines Context (both)

# Multiple patterns
grep -e "pattern1" -e "pattern2" file

# Search multiple files
grep "pattern" file1 file2 file3
grep "pattern" *.log

# List files containing match
grep -l "pattern" *.conf

# List files NOT containing match
grep -L "pattern" *.conf

# Word boundaries
grep -w "error" file          # Matches "error" not "errors"

# Quiet mode (just exit code)
grep -q "pattern" file && echo "found"

# Basic substitution
sed 's/old/new/' file         # First occurrence per line
sed 's/old/new/g' file        # All occurrences (global)

# Edit in place
sed -i 's/old/new/g' file
sed -i.bak 's/old/new/g' file # Backup original

# Delete lines
sed '/pattern/d' file         # Delete matching lines
sed '5d' file                 # Delete line 5
sed '5,10d' file              # Delete lines 5-10
sed '1d' file                 # Delete first line
sed '$d' file                 # Delete last line

# Print specific lines
sed -n '5p' file              # Print line 5
sed -n '5,10p' file           # Print lines 5-10
sed -n '/pattern/p' file      # Print matching lines

# Insert/Append
sed '3i\New line' file        # Insert before line 3
sed '3a\New line' file        # Append after line 3
sed '/pattern/i\Text' file    # Insert before pattern
sed '/pattern/a\Text' file    # Append after pattern

# Replace entire line
sed '/pattern/c\New line' file

# Multiple operations
sed -e 's/foo/bar/g' -e 's/baz/qux/g' file
sed 's/foo/bar/g; s/baz/qux/g' file

# Address ranges
sed '10,20s/old/new/g' file   # Lines 10-20 only
sed '/start/,/end/s/old/new/g' file  # Between patterns

# Regex groups (capture)
sed 's/\(.*\):\(.*\)/\2:\1/' file    # Swap fields
sed -E 's/(.*):(.*):\2:\1/' file     # Extended regex

# Delete blank lines
sed '/^$/d' file

# Remove leading/trailing whitespace
sed 's/^[ \t]*//' file        # Leading
sed 's/[ \t]*$//' file        # Trailing
sed 's/^[ \t]*//;s/[ \t]*$//' file  # Both

# Print between patterns (inclusive)
sed -n '/START/,/END/p' file

# Print specific columns
awk '{print $1}' file         # First column
awk '{print $1, $3}' file     # First and third
awk '{print $NF}' file        # Last column
awk '{print $(NF-1)}' file    # Second to last

# Custom field separator
awk -F: '{print $1}' /etc/passwd
awk -F',' '{print $2}' file.csv
awk -F'\t' '{print $1}' file.tsv

# Pattern matching
awk '/pattern/' file          # Print matching lines
awk '/pattern/ {print $1}' file
awk '!/pattern/' file         # Non-matching lines

# Conditional logic
awk '$3 > 100 {print $1, $3}' file
awk '$1 == "root"' /etc/passwd
awk '$NF ~ /error/' file      # Regex match on field

# Line numbers
awk '{print NR, $0}' file     # Add line numbers
awk 'NR==5' file              # Print line 5
awk 'NR>=5 && NR<=10' file    # Lines 5-10

# Field calculations
awk '{sum += $1} END {print sum}' file
awk '{sum += $1; count++} END {print sum/count}' file  # Average

# Format output
awk '{printf "%-10s %5d\n", $1, $2}' file

# Multiple patterns
awk '/start/,/end/' file      # Range between patterns

# BEGIN/END blocks
awk 'BEGIN {print "Header"} {print} END {print "Footer"}' file
awk 'BEGIN {FS=":"; OFS="\t"} {print $1, $3}' /etc/passwd

# Built-in variables
# NR - Record number (line number)
# NF - Number of fields
# FS - Field separator (input)
# OFS - Output field separator
# RS - Record separator
# ORS - Output record separator
# FILENAME - Current filename

# Extract unique values
awk '!seen[$1]++' file        # Unique first column

# Sum by group
awk '{sum[$1] += $2} END {for (k in sum) print k, sum[k]}' file

# In-place edit (GNU awk)
awk -i inplace '{gsub(/old/,"new")}1' file

# cut - Extract columns
cut -d: -f1 /etc/passwd       # Field 1, delimiter :
cut -d',' -f1,3 file.csv      # Fields 1 and 3
cut -c1-10 file               # Characters 1-10
cut -c5- file                 # Character 5 to end

# sort
sort file                     # Alphabetical
sort -n file                  # Numeric
sort -r file                  # Reverse
sort -k2 file                 # By column 2
sort -k2,2n -k1,1 file        # Numeric col 2, then col 1
sort -t: -k3 -n /etc/passwd   # By UID
sort -u file                  # Unique (sort + uniq)
sort -h file                  # Human-readable (1K, 2M)

# uniq (requires sorted input!)
sort file | uniq              # Remove duplicates
sort file | uniq -c           # Count occurrences
sort file | uniq -d           # Show only duplicates
sort file | uniq -u           # Show only unique

# tr - Translate characters
tr 'a-z' 'A-Z' < file         # Lowercase to uppercase
tr -d '\r' < file             # Remove carriage returns
tr -s ' ' < file              # Squeeze repeated spaces
tr ':' '\t' < file            # Replace : with tab
tr -d '[:digit:]' < file      # Remove all digits
tr -cd '[:alnum:]\n' < file   # Keep only alphanumeric

# Find by name
find /path -name "*.log"
find /path -iname "*.LOG"     # Case insensitive

# Find by type
find /path -type f            # Files
find /path -type d            # Directories
find /path -type l            # Symlinks

# Find by size
find /path -size +100M        # Larger than 100MB
find /path -size -1k          # Smaller than 1KB

# Find by time
find /path -mtime -7          # Modified in last 7 days
find /path -mtime +30         # Modified more than 30 days ago
find /path -mmin -60          # Modified in last 60 minutes
find /path -newer reference   # Newer than reference file

# Find by permission
find /path -perm 644
find /path -perm -u+x         # User executable
find /path -perm /u+x,g+x     # User OR group executable

# Find by owner
find /path -user root
find /path -group wheel

# Execute command on results
find /path -name "*.tmp" -exec rm {} \;
find /path -name "*.log" -exec gzip {} \;
find /path -type f -exec chmod 644 {} +    # More efficient

# Combine with xargs (faster for many files)
find /path -name "*.txt" | xargs grep "pattern"
find /path -name "*.txt" -print0 | xargs -0 grep "pattern"  # Handle spaces

# Multiple conditions
find /path -name "*.log" -size +10M
find /path \( -name "*.txt" -o -name "*.md" \)
find /path -name "*.bak" ! -mtime -30      # NOT modified in 30 days

# Limit depth
find /path -maxdepth 2 -name "*.conf"

# Basic usage
echo "file1 file2 file3" | xargs rm

# With find
find . -name "*.tmp" | xargs rm
find . -name "*.txt" -print0 | xargs -0 cat  # Handle spaces/newlines

# Limit arguments per command
echo {1..100} | xargs -n 10 echo

# Run in parallel
find . -name "*.jpg" | xargs -P 4 -I {} convert {} -resize 50% resized_{}

# Placeholder for argument
ls *.txt | xargs -I {} cp {} backup/{}
find . -name "*.log" | xargs -I {} sh -c 'gzip -c {} > {}.gz'

# Prompt before execution
ls | xargs -p rm

# Maximum command length
xargs --show-limits

wc file                       # Lines, words, bytes
wc -l file                    # Line count
wc -w file                    # Word count
wc -c file                    # Byte count
wc -m file                    # Character count

# Count files
ls -1 | wc -l
find . -type f | wc -l

# Multiple files
wc -l *.txt

# File existence and type
[ -e file ]      # Exists (any type)
[ -f file ]      # Regular file
[ -d dir ]       # Directory
[ -L link ]      # Symbolic link
[ -h link ]      # Symbolic link (same as -L)
[ -p pipe ]      # Named pipe (FIFO)
[ -S socket ]    # Socket
[ -b device ]    # Block device
[ -c device ]    # Character device

# File properties
[ -s file ]      # Size > 0 (not empty)
[ -r file ]      # Readable
[ -w file ]      # Writable
[ -x file ]      # Executable
[ -u file ]      # SUID bit set
[ -g file ]      # SGID bit set
[ -k file ]      # Sticky bit set
[ -O file ]      # Owned by current user
[ -G file ]      # Owned by current group

# File comparisons
[ file1 -nt file2 ]    # file1 Newer Than file2
[ file1 -ot file2 ]    # file1 Older Than file2
[ file1 -ef file2 ]    # Same inode (hard link)

# Examples
[ -f /etc/passwd ] && echo "passwd exists"
[ -d /tmp ] && echo "/tmp is a directory"
[ -x /usr/bin/vim ] && echo "vim is executable"
[ -s /var/log/syslog ] && echo "syslog has content"

# String comparisons
[ -z "$var" ]         # Zero length (empty)
[ -n "$var" ]         # Non-zero length (not empty)
[ "$a" = "$b" ]       # Equal (POSIX)
[ "$a" == "$b" ]      # Equal (bash)
[ "$a" != "$b" ]      # Not equal
[ "$a" \< "$b" ]      # Less than (alphabetically)
[ "$a" \> "$b" ]      # Greater than (alphabetically)

# ALWAYS quote variables to handle spaces/empty
[ -z "$MYVAR" ] && echo "MYVAR is empty or unset"
[ -n "$MYVAR" ] && echo "MYVAR has a value: $MYVAR"

# Check if variable is set (different from empty)
[ -v MYVAR ]          # Variable is set (bash 4.2+)
[ "${MYVAR+x}" ]      # Variable is set (POSIX)

# Examples
[ -z "$1" ] && echo "No argument provided" && exit 1
[ "$USER" = "root" ] && echo "Running as root"
[ "$SHELL" != "/bin/bash" ] && echo "Not using bash"

# Integer comparisons (use these, not < >)
[ "$a" -eq "$b" ]     # Equal
[ "$a" -ne "$b" ]     # Not equal
[ "$a" -lt "$b" ]     # Less than
[ "$a" -le "$b" ]     # Less than or equal
[ "$a" -gt "$b" ]     # Greater than
[ "$a" -ge "$b" ]     # Greater than or equal

# Examples
[ $? -eq 0 ] && echo "Last command succeeded"
[ $# -lt 2 ] && echo "Need at least 2 arguments"
[ $(wc -l < file) -gt 100 ] && echo "File has more than 100 lines"

count=$(ls -1 | wc -l)
[ "$count" -ge 10 ] && echo "10 or more files"

# Inside [ ] (POSIX)
[ expr1 -a expr2 ]    # AND
[ expr1 -o expr2 ]    # OR
[ ! expr ]            # NOT

# Outside [ ] (preferred, short-circuit)
[ expr1 ] && [ expr2 ]    # AND
[ expr1 ] || [ expr2 ]    # OR
! [ expr ]                # NOT

# Examples
[ -f file ] && [ -r file ] && echo "File exists and is readable"
[ -z "$1" ] || [ -z "$2" ] && echo "Missing argument(s)"
[ ! -d /backup ] && mkdir /backup

# Complex conditions
[ -f "$1" ] && [ -r "$1" ] && [ -s "$1" ] && echo "Valid non-empty readable file"

# Pattern matching (glob)
[[ "$file" == *.txt ]]        # Ends with .txt
[[ "$file" == img_* ]]        # Starts with img_
[[ "$file" != *.bak ]]        # Doesn't end with .bak

# Regex matching
[[ "$email" =~ ^[a-z]+@[a-z]+\.[a-z]+$ ]]
[[ "$ip" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]

# Capture groups (BASH_REMATCH)
if [[ "$line" =~ ^([0-9]+):(.*)$ ]]; then
    num="${BASH_REMATCH[1]}"
    rest="${BASH_REMATCH[2]}"
fi

# No word splitting (safer, no quotes needed)
[[ $var == "test" ]]          # Works even if var has spaces

# Logical operators (cleaner syntax)
[[ -f file && -r file ]]      # AND
[[ -z $1 || -z $2 ]]          # OR

# Examples
[[ "$OSTYPE" == linux* ]] && echo "Linux system"
[[ "$HOSTNAME" =~ ^web[0-9]+$ ]] && echo "Web server"
[[ -d "$dir" && -w "$dir" ]] && echo "Writable directory"

# Basic if
if [ -f /etc/passwd ]; then
    echo "passwd exists"
fi

# if-else
if [ -d "$1" ]; then
    echo "$1 is a directory"
else
    echo "$1 is not a directory"
fi

# if-elif-else
if [ "$1" = "start" ]; then
    echo "Starting..."
elif [ "$1" = "stop" ]; then
    echo "Stopping..."
elif [ "$1" = "restart" ]; then
    echo "Restarting..."
else
    echo "Usage: $0 {start|stop|restart}"
    exit 1
fi

# Compact form
if [ -f file ]; then echo "exists"; else echo "missing"; fi

# Nested
if [ -f "$1" ]; then
    if [ -r "$1" ]; then
        cat "$1"
    else
        echo "Cannot read $1"
    fi
else
    echo "$1 not found"
fi

# Basic case
case "$1" in
    start)
        echo "Starting service..."
        ;;
    stop)
        echo "Stopping service..."
        ;;
    restart)
        echo "Restarting service..."
        ;;
    *)
        echo "Usage: $0 {start|stop|restart}"
        exit 1
        ;;
esac

# Multiple patterns
case "$file" in
    *.tar.gz|*.tgz)
        tar -xzf "$file"
        ;;
    *.tar.bz2|*.tbz2)
        tar -xjf "$file"
        ;;
    *.tar.xz|*.txz)
        tar -xJf "$file"
        ;;
    *.zip)
        unzip "$file"
        ;;
    *)
        echo "Unknown archive format"
        ;;
esac

# Character classes
case "$char" in
    [a-z])   echo "lowercase letter" ;;
    [A-Z])   echo "uppercase letter" ;;
    [0-9])   echo "digit" ;;
    *)       echo "other" ;;
esac

# Yes/No prompts
read -p "Continue? [y/N] " answer
case "$answer" in
    [Yy]|[Yy][Ee][Ss])
        echo "Proceeding..."
        ;;
    *)
        echo "Aborted."
        exit 0
        ;;
esac

# for loop - iterate over list
for file in *.txt; do
    echo "Processing $file"
done

# for loop - iterate over arguments
for arg in "$@"; do
    echo "Argument: $arg"
done

# for loop - C-style (bash)
for ((i=1; i<=10; i++)); do
    echo "Count: $i"
done

# for loop - sequence
for i in {1..10}; do
    echo "Number: $i"
done

for i in $(seq 1 10); do    # POSIX compatible
    echo "Number: $i"
done

# while loop
count=0
while [ "$count" -lt 5 ]; do
    echo "Count: $count"
    count=$((count + 1))
done

# while read (process file line by line)
while IFS= read -r line; do
    echo "Line: $line"
done < file.txt

# while read with pipe (subshell warning!)
cat file.txt | while read -r line; do
    echo "$line"
done

# until loop (opposite of while)
until [ -f /tmp/ready ]; do
    echo "Waiting for /tmp/ready..."
    sleep 1
done

# Infinite loop with break
while true; do
    read -p "Enter command (q to quit): " cmd
    [ "$cmd" = "q" ] && break
    echo "You entered: $cmd"
done

# Loop control
for i in {1..10}; do
    [ "$i" -eq 5 ] && continue    # Skip 5
    [ "$i" -eq 8 ] && break       # Stop at 8
    echo "$i"
done

# Check if command exists
command -v git >/dev/null 2>&1 && echo "git installed"
type -P vim >/dev/null 2>&1 || echo "vim not found"
which python3 >/dev/null 2>&1 || { echo "python3 required"; exit 1; }

# Check if running as root
[ "$(id -u)" -eq 0 ] || { echo "Must run as root"; exit 1; }
[ "$EUID" -eq 0 ] || { echo "Must run as root"; exit 1; }

# Check argument count
[ $# -eq 0 ] && { echo "Usage: $0 <file>"; exit 1; }
[ $# -lt 2 ] && { echo "Need 2 arguments"; exit 1; }

# Default values
name="${1:-default}"              # Use $1 or "default"
dir="${DIR:-/tmp}"                # Use $DIR or "/tmp"
config="${CONFIG_FILE:=/etc/app.conf}"  # Set if unset

# File exists or create
[ -f "$config" ] || touch "$config"
[ -d "$logdir" ] || mkdir -p "$logdir"

# Safe file operations
[ -f "$file" ] && rm "$file"
[ -d "$dir" ] && rm -rf "$dir"

# Process all arguments
for file in "$@"; do
    [ -f "$file" ] || { echo "Skip: $file not found"; continue; }
    process "$file"
done

# Check exit status
if command; then
    echo "Success"
else
    echo "Failed with exit code $?"
fi

# Trap errors
trap 'echo "Error on line $LINENO"; exit 1' ERR

# Cleanup on exit
trap 'rm -f /tmp/myapp.$$.*' EXIT

# Lock file (prevent concurrent runs)
lockfile="/var/run/myscript.lock"
[ -f "$lockfile" ] && { echo "Already running"; exit 1; }
trap 'rm -f "$lockfile"' EXIT
touch "$lockfile"

# Read config file
if [ -f /etc/myapp.conf ]; then
    . /etc/myapp.conf    # Source it
fi

# Check network connectivity
ping -c 1 -W 2 8.8.8.8 >/dev/null 2>&1 && echo "Network OK"

# Wait for service
timeout=30
while [ $timeout -gt 0 ]; do
    nc -z localhost 8080 && break
    sleep 1
    timeout=$((timeout - 1))
done
[ $timeout -eq 0 ] && { echo "Timeout"; exit 1; }

# Parallel execution with wait
for host in server1 server2 server3; do
    ssh "$host" 'uptime' &
done
wait    # Wait for all background jobs

# Array iteration (bash)
files=("file1.txt" "file2.txt" "file3.txt")
for f in "${files[@]}"; do
    echo "$f"
done

# Associative array (bash 4+)
declare -A config
config[host]="localhost"
config[port]="8080"
echo "${config[host]}:${config[port]}"

# Standard exit codes
exit 0      # Success
exit 1      # General error
exit 2      # Misuse of shell command
exit 126    # Command cannot execute
exit 127    # Command not found
exit 128+n  # Fatal signal n (128+9=137 for SIGKILL)

# Check last exit code
command
if [ $? -eq 0 ]; then
    echo "Success"
fi

# Exit on first error
set -e

# Exit on undefined variable
set -u

# Exit on pipe failure
set -o pipefail

# Common combination (strict mode)
set -euo pipefail

# Return from function
my_function() {
    [ -f "$1" ] || return 1
    cat "$1"
    return 0
}

# Top 10 largest files
find . -type f -exec ls -s {} + | sort -rn | head -10

# Count unique IP addresses in log
awk '{print $1}' access.log | sort | uniq -c | sort -rn | head -20

# Find all TODO comments
grep -rn "TODO" --include="*.py" . | awk -F: '{print $1}' | sort -u

# Extract and sort error messages
grep -i error /var/log/syslog | awk '{print $5}' | sort | uniq -c | sort -rn

# Sum numbers in a file
awk '{sum+=$1} END {print sum}' numbers.txt

# Replace in multiple files
find . -name "*.conf" -exec sed -i 's/old/new/g' {} \;

# List processes using most memory
ps aux | sort -k4 -rn | head -10

# Disk usage by directory
du -sh */ | sort -rh | head -10

# Extract specific log time range
awk '/2024-01-15 10:00/,/2024-01-15 11:00/' app.log

# Parse CSV and filter
awk -F',' '$3 > 1000 {print $1, $3}' data.csv

# Find duplicate files (by name)
find . -type f -name "*" | xargs -I {} basename {} | sort | uniq -d

# Monitor log in real-time with filtering
tail -f /var/log/syslog | grep --line-buffered "error"