Heredoc Patterns

Overview

Heredocs (here documents) allow embedding multi-line text directly in scripts. Essential for configuration generation, remote commands, and clean multi-line strings.

Basic Syntax

command <<DELIMITER
content line 1
content line 2
DELIMITER

The delimiter can be any word. Common choices: EOF, END, DOC, SQL.

Delimiter Variations

Standard (Variables Expanded)

NAME="World"
cat <<EOF
Hello, $NAME
Current directory: $(pwd)
EOF

Output:

Hello, World
Current directory: /home/user

Quoted Delimiter (Literal Content)

cat <<'EOF'
Variables like $NAME are NOT expanded
Commands like $(pwd) are literal
Backslashes \ stay as-is
EOF

Output:

Variables like $NAME are NOT expanded
Commands like $(pwd) are literal
Backslashes \ stay as-is

Use <<'EOF' (quoted) when content should be literal. Use <<EOF (unquoted) when you want variable expansion.

Strip Leading Tabs (<←)

if true; then
	cat <<-EOF
	This text has leading tabs stripped
	Indentation in scripts looks cleaner
	Only TABS work, not spaces
	EOF
fi

Variable Expansion

With Expansion (default)

HOST="server01"
PORT="8080"

cat <<EOF
server {
    hostname = $HOST
    port = $PORT
    workers = $(($(nproc) * 2))
}
EOF

Partial Expansion

HOST="server01"

cat <<EOF
Known host: $HOST
Literal dollar: \$PATH
Escaped variable: \${UNDEFINED}
EOF

No Expansion (Quoted)

cat <<'EOF'
#!/bin/bash
for file in *.txt; do
    echo "$file"
done
EOF

Writing to Files

Overwrite File

cat > /etc/myapp/config.conf <<EOF
[server]
host = 0.0.0.0
port = 8080
EOF

Append to File

cat >> /etc/myapp/config.conf <<EOF

[logging]
level = info
EOF

With sudo

# This WON'T work (redirection happens before sudo)
# sudo cat > /etc/protected.conf <<EOF
# content
# EOF

# CORRECT: Use tee
sudo tee /etc/protected.conf > /dev/null <<EOF
[settings]
value = secure
EOF

# CORRECT: Use cat with sudo tee
cat <<EOF | sudo tee /etc/protected.conf > /dev/null
[settings]
value = secure
EOF

Git Commit Messages

Multi-line Commit

git commit -m "$(cat <<EOF
feat(auth): Add OAuth2 support

- Implement authorization code flow
- Add token refresh mechanism
- Configure redirect URIs
EOF
)"

With Variables

TICKET="PROJ-123"
git commit -m "$(cat <<EOF
fix($TICKET): Resolve memory leak

- Close database connections properly
- Add connection pooling
- Update timeout settings
EOF
)"

Clean Formatting

git commit -m "$(cat <<'EOF'
docs(api): Update REST API documentation

- Add authentication examples
- Document rate limiting
- Include error responses
EOF
)"

Remote Commands (SSH)

Execute Script on Remote Host

ssh user@server <<'EOF'
cd /var/log
grep "ERROR" *.log | tail -20
systemctl status nginx
EOF

With Variables (local expansion)

LOGDIR="/var/log/myapp"
DATE=$(date +%Y-%m-%d)

ssh user@server <<EOF
find $LOGDIR -name "*.log" -mtime -1
grep "$DATE" $LOGDIR/app.log | tail -50
EOF

With Variables (remote expansion)

ssh user@server <<'EOF'
cd $HOME
echo "Remote home: $HOME"
echo "Remote user: $USER"
df -h $HOME
EOF

Combined Expansion

LOCAL_VAR="from-local"

ssh user@server /bin/bash <<EOF
# This is expanded locally
echo "Local value: $LOCAL_VAR"
# These are expanded remotely (escaped)
echo "Remote home: \$HOME"
echo "Remote user: \$USER"
EOF

Configuration Generation

Nginx Config

DOMAIN="example.com"
PORT="8080"

cat > /etc/nginx/sites-available/$DOMAIN.conf <<EOF
server {
    listen 80;
    server_name $DOMAIN www.$DOMAIN;

    location / {
        proxy_pass http://127.0.0.1:$PORT;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
    }
}
EOF

Note: $DOMAIN and $PORT expand locally, while $host and $remote_addr are escaped to remain as Nginx variables.

Systemd Unit

SERVICE_NAME="myapp"
USER="appuser"
EXEC_PATH="/usr/local/bin/myapp"

cat > /etc/systemd/system/${SERVICE_NAME}.service <<EOF
[Unit]
Description=$SERVICE_NAME Service
After=network.target

[Service]
Type=simple
User=$USER
ExecStart=$EXEC_PATH
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable $SERVICE_NAME

Database Schema

DB_NAME="production"
TABLE_PREFIX="app_"

psql -U admin $DB_NAME <<EOF
CREATE TABLE IF NOT EXISTS ${TABLE_PREFIX}users (
    id SERIAL PRIMARY KEY,
    username VARCHAR(100) NOT NULL,
    email VARCHAR(255) UNIQUE,
    created_at TIMESTAMP DEFAULT NOW()
);

CREATE INDEX idx_${TABLE_PREFIX}users_email
    ON ${TABLE_PREFIX}users(email);
EOF

Herestrings

Single-line variant using <<<:

# Pass string as stdin
grep "pattern" <<< "search in this string"

# Variable as input
DATA="line1
line2
line3"
grep "line2" <<< "$DATA"

# To command
base64 <<< "encode this"

# Read into variable
read VAR <<< "value"

Functions with Heredocs

Return Multi-line String

generate_config() {
    local hostname=$1
    local port=$2

    cat <<EOF
server {
    hostname = $hostname
    port = $port
}
EOF
}

# Usage
CONFIG=$(generate_config "server01" "8080")
echo "$CONFIG" > config.conf

Template Function

render_template() {
    eval "cat <<EOF
$(cat $1)
EOF"
}

# template.txt contains:
# Hello, $NAME
# Today is $(date +%A)

NAME="User"
render_template template.txt

Pipeline Integration

Into Commands

# Email with heredoc
mail -s "Report" admin@example.com <<EOF
Daily report for $(date +%Y-%m-%d)

$(df -h)

$(free -h)
EOF

# SQL query
mysql -u user -p database <<EOF | column -t
SELECT username, email
FROM users
WHERE created_at > DATE_SUB(NOW(), INTERVAL 7 DAY);
EOF

Through Pipes

# Generate and filter
cat <<EOF | grep -v "^#" | grep -v "^$"
# Comment line
server1
server2
# Another comment

server3
EOF

# Generate and sort
cat <<EOF | sort | uniq
charlie
alice
bob
alice
david
EOF

JSON Generation

Simple JSON

NAME="John"
AGE=30

cat <<EOF
{
    "name": "$NAME",
    "age": $AGE,
    "active": true
}
EOF

With jq Processing

cat <<EOF | jq .
{
    "users": [
        {"name": "Alice", "role": "admin"},
        {"name": "Bob", "role": "user"}
    ]
}
EOF

YAML Generation

APP_NAME="myapp"
VERSION="1.0.0"
REPLICAS=3

cat <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $APP_NAME
spec:
  replicas: $REPLICAS
  template:
    spec:
      containers:
      - name: $APP_NAME
        image: registry/$APP_NAME:$VERSION
        ports:
        - containerPort: 8080
EOF

Common Patterns

Conditional Content

DEBUG=${DEBUG:-false}

cat <<EOF
[application]
name = myapp
$(if [ "$DEBUG" = "true" ]; then
    echo "log_level = debug"
    echo "verbose = true"
else
    echo "log_level = info"
fi)
EOF

Loop-Generated Content

SERVERS="server1 server2 server3"

cat <<EOF
[servers]
$(for srv in $SERVERS; do
    echo "$srv.example.com"
done)
EOF

Include External File

cat <<EOF
# Main configuration

# Include external settings
$(cat external_settings.conf)

# End of configuration
EOF

Quick Reference

# Basic heredoc (variables expand)
cat <<EOF
Hello $NAME
EOF

# Literal heredoc (no expansion)
cat <<'EOF'
Literal $NAME
EOF

# Strip leading tabs
cat <<-EOF
	Indented content
EOF

# Write to file
cat > file.txt <<EOF
Content
EOF

# Append to file
cat >> file.txt <<EOF
More content
EOF

# With sudo
sudo tee /etc/file > /dev/null <<EOF
Content
EOF

# Git commit
git commit -m "$(cat <<'EOF'
Multi-line message
EOF
)"

# SSH remote execution
ssh user@host <<'EOF'
remote commands
EOF

# Herestring (single line)
grep pattern <<< "search string"

Troubleshooting

Common Issues

Issue Solution

Issue	Solution
Variables not expanding	Use `<<EOF` instead of `<<'EOF'`
Variables expanding unexpectedly	Use `<<'EOF'` (quoted) for literal content
Whitespace issues	Use `<←` with tabs for indentation
Redirect with sudo fails	Use `sudo tee` instead of `sudo cat >`
Heredoc in function fails	Ensure proper quoting and escaping

Variables not expanding

Use <<EOF instead of <<'EOF'

Variables expanding unexpectedly

Use <<'EOF' (quoted) for literal content

Whitespace issues

Use <← with tabs for indentation

Redirect with sudo fails

Use sudo tee instead of sudo cat >

Heredoc in function fails

Ensure proper quoting and escaping

Debugging

# Test expansion without writing
cat <<EOF
Debug: $VARIABLE
Command: $(echo test)
EOF

# Check what's generated
diff <(cat <<EOF
expected content
EOF
) actual_file.txt

API Testing with curl

REST API Requests

# POST with JSON body
curl -X POST "$API_URL/users" \
    -H "Content-Type: application/json" \
    -H "Authorization: Bearer $TOKEN" \
    -d "$(cat <<EOF
{
    "username": "$USERNAME",
    "email": "$EMAIL",
    "role": "user"
}
EOF
)"

# PUT update
curl -X PUT "$API_URL/users/$USER_ID" \
    -H "Content-Type: application/json" \
    -d "$(cat <<EOF
{
    "status": "active",
    "updated_at": "$(date -Iseconds)"
}
EOF
)"

GraphQL Queries

QUERY=$(cat <<'EOF'
{
    "query": "query GetUser($id: ID!) { user(id: $id) { name email } }",
    "variables": { "id": "123" }
}
EOF
)
curl -X POST "$GRAPHQL_URL" \
    -H "Content-Type: application/json" \
    -d "$QUERY"

Multi-Part Forms

# Generate boundary
BOUNDARY="----FormBoundary$(date +%s)"

curl -X POST "$UPLOAD_URL" \
    -H "Content-Type: multipart/form-data; boundary=$BOUNDARY" \
    --data-binary "$(cat <<EOF
--$BOUNDARY
Content-Disposition: form-data; name="title"

My Upload
--$BOUNDARY
Content-Disposition: form-data; name="file"; filename="data.txt"
Content-Type: text/plain

$(cat data.txt)
--$BOUNDARY--
EOF
)"

Docker and Containers

Dockerfile Generation

APP_NAME="myapp"
APP_PORT="8080"
NODE_VERSION="20"

cat > Dockerfile <<EOF
FROM node:${NODE_VERSION}-alpine

WORKDIR /app

COPY package*.json ./
RUN npm ci --only=production

COPY . .

EXPOSE ${APP_PORT}
ENV NODE_ENV=production
ENV PORT=${APP_PORT}

USER node
CMD ["node", "server.js"]
EOF

Docker Compose

PROJECT_NAME="myproject"
DB_PASSWORD=$(openssl rand -base64 24)

cat > docker-compose.yml <<EOF
version: '3.8'

services:
  app:
    build: .
    ports:
      - "8080:8080"
    environment:
      - DATABASE_URL=postgres://user:${DB_PASSWORD}@db:5432/${PROJECT_NAME}
    depends_on:
      - db

  db:
    image: postgres:15-alpine
    environment:
      POSTGRES_DB: ${PROJECT_NAME}
      POSTGRES_PASSWORD: ${DB_PASSWORD}
    volumes:
      - pgdata:/var/lib/postgresql/data

volumes:
  pgdata:
EOF

Container Exec Commands

# Run script inside container
docker exec -i mycontainer bash <<'EOF'
cd /app
npm run migrate
npm run seed
echo "Database initialized"
EOF

# Database operations
docker exec -i postgres psql -U admin <<EOF
CREATE DATABASE IF NOT EXISTS ${DB_NAME};
GRANT ALL PRIVILEGES ON DATABASE ${DB_NAME} TO ${DB_USER};
EOF

CTF and Security Automation

Encoding/Decoding

# Batch base64 decode
cat <<'EOF' | while read encoded; do
    echo "=== $encoded ==="
    echo "$encoded" | base64 -d 2>/dev/null
    echo
done
Zm xhZ3t0ZXN0fQ==
c2VjcmV0X2tleQ==
cGFzc3dvcmQxMjM=
EOF

# ROT13 decoder
cat <<'EOF' | tr 'A-Za-z' 'N-ZA-Mn-za-m'
Guvf vf n frperg zrffntr
EOF

# Hex to ASCII
cat <<'EOF' | xxd -r -p
666c61677b746573745f666c61677d
EOF

# Multiple encoding attempts
ENCODED="VGVzdCBTdHJpbmc="
cat <<'EOF' | while read method; do
    echo "=== $method ==="
    case $method in
        base64) echo "$ENCODED" | base64 -d 2>/dev/null ;;
        hex) echo "$ENCODED" | xxd -r -p 2>/dev/null ;;
        rot13) echo "$ENCODED" | tr 'A-Za-z' 'N-ZA-Mn-za-m' ;;
    esac
done
base64
hex
rot13
EOF

Password Generation and Testing

# Generate wordlist mutations
BASE_WORD="password"
cat <<'EOF' | while read pattern; do
    eval "echo $pattern"
done > mutations.txt
${BASE_WORD}
${BASE_WORD}123
${BASE_WORD}!
${BASE_WORD}@2026
${BASE_WORD^^}
${BASE_WORD^}
EOF

# Hash cracking prep
TARGET_HASH="5f4dcc3b5aa765d61d8327deb882cf99"
cat <<'EOF' | while read word; do
    HASH=$(echo -n "$word" | md5sum | cut -d' ' -f1)
    if [ "$HASH" = "$TARGET_HASH" ]; then
        echo "FOUND: $word"
    fi
done
password
password123
secret
admin
EOF

Network Recon Scripts

# Port scan wrapper
TARGET="192.168.1.1"
cat <<'EOF' | while read port; do
    timeout 1 bash -c "echo >/dev/tcp/$TARGET/$port" 2>/dev/null && echo "Port $port: OPEN"
done
22
80
443
8080
3306
5432
EOF

# DNS enumeration
DOMAIN="example.com"
cat <<'EOF' | while read sub; do
    host "${sub}.${DOMAIN}" 2>/dev/null | grep -q "has address" && echo "${sub}.${DOMAIN}"
done
www
mail
ftp
admin
api
dev
staging
EOF

Database Operations

MySQL/MariaDB

mysql -u "$DB_USER" -p"$DB_PASS" "$DB_NAME" <<EOF
-- Create tables
CREATE TABLE IF NOT EXISTS users (
    id INT AUTO_INCREMENT PRIMARY KEY,
    username VARCHAR(50) NOT NULL UNIQUE,
    email VARCHAR(100),
    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

-- Insert data
INSERT INTO users (username, email) VALUES
    ('admin', 'admin@example.com'),
    ('user1', 'user1@example.com')
ON DUPLICATE KEY UPDATE email = VALUES(email);

-- Query and export
SELECT * FROM users INTO OUTFILE '/tmp/users.csv'
FIELDS TERMINATED BY ',' ENCLOSED BY '"'
LINES TERMINATED BY '\n';
EOF

PostgreSQL

PGPASSWORD="$DB_PASS" psql -h localhost -U "$DB_USER" "$DB_NAME" <<EOF
-- Create schema
CREATE SCHEMA IF NOT EXISTS app;

-- Create tables
CREATE TABLE IF NOT EXISTS app.events (
    id SERIAL PRIMARY KEY,
    event_type VARCHAR(50),
    payload JSONB,
    created_at TIMESTAMPTZ DEFAULT NOW()
);

-- Insert with JSON
INSERT INTO app.events (event_type, payload) VALUES
    ('login', '{"user": "admin", "ip": "192.168.1.1"}'),
    ('action', '{"type": "update", "target": "profile"}');

-- Query JSON
SELECT * FROM app.events
WHERE payload->>'user' = 'admin';
EOF

SQLite

DB_FILE="app.db"
sqlite3 "$DB_FILE" <<'EOF'
CREATE TABLE IF NOT EXISTS logs (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    timestamp TEXT DEFAULT CURRENT_TIMESTAMP,
    level TEXT,
    message TEXT
);

INSERT INTO logs (level, message) VALUES
    ('INFO', 'Application started'),
    ('WARN', 'High memory usage');

.mode csv
.headers on
.output logs.csv
SELECT * FROM logs;
.output stdout
.quit
EOF

Testing and Automation

Test Data Generation

# Generate test users JSON
cat <<'EOF' | while read i; do
    echo "{\"id\": $i, \"name\": \"user$i\", \"email\": \"user$i@test.com\"}"
done | jq -s '.' > test_users.json
1
2
3
4
5
EOF

# Generate CSV test data
cat > test_data.csv <<EOF
id,name,value,timestamp
$(for i in {1..10}; do
    echo "$i,item$i,$((RANDOM % 100)),$(date -d "-$i days" +%Y-%m-%d)"
done)
EOF

Expect-like Scripts

# Automated SSH setup
cat <<'EOF' | ssh-keygen -t ed25519 -C "automated" -f ~/.ssh/auto_key
y

EOF

# Automated input to interactive command
./interactive_program <<EOF
yes
myusername
mypassword
n
quit
EOF

CI/CD Pipeline Snippets

# Generate GitHub Actions workflow
cat > .github/workflows/ci.yml <<'EOF'
name: CI
on: [push, pull_request]
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: '20'
      - run: npm ci
      - run: npm test
      - run: npm run build
EOF

# Generate GitLab CI
cat > .gitlab-ci.yml <<'EOF'
stages:
  - test
  - build
  - deploy

test:
  stage: test
  script:
    - npm ci
    - npm test

build:
  stage: build
  script:
    - npm run build
  artifacts:
    paths:
      - dist/
EOF

Error Handling Patterns

Validate Before Write

CONFIG_CONTENT=$(cat <<EOF
[server]
host = $HOST
port = $PORT
EOF
)

# Validate variables exist
if [ -z "$HOST" ] || [ -z "$PORT" ]; then
    echo "ERROR: HOST and PORT must be set" >&2
    exit 1
fi

# Write only if valid
echo "$CONFIG_CONTENT" > /etc/app/config.ini

Backup Before Overwrite

TARGET="/etc/nginx/nginx.conf"
BACKUP="${TARGET}.bak.$(date +%Y%m%d%H%M%S)"

# Backup existing
[ -f "$TARGET" ] && cp "$TARGET" "$BACKUP"

# Write new config
cat > "$TARGET" <<EOF
worker_processes auto;
events {
    worker_connections 1024;
}
http {
    include /etc/nginx/conf.d/*.conf;
}
EOF

# Validate and rollback if needed
if ! nginx -t 2>/dev/null; then
    echo "Invalid config, rolling back"
    mv "$BACKUP" "$TARGET"
    exit 1
fi

Atomic Writes

TARGET="/etc/app/config.yml"
TMPFILE=$(mktemp)

# Write to temp file
cat > "$TMPFILE" <<EOF
database:
  host: $DB_HOST
  port: $DB_PORT
EOF

# Atomic move (same filesystem)
mv "$TMPFILE" "$TARGET"

# Or copy if different filesystems
# cp "$TMPFILE" "$TARGET" && rm "$TMPFILE"

Multi-Line Variable Assignment

Simple Assignment

# Using $()
SCRIPT=$(cat <<'EOF'
#!/bin/bash
echo "This is a script"
exit 0
EOF
)

# Using read
read -r -d '' HTML <<'EOF'
<!DOCTYPE html>
<html>
<head><title>Test</title></head>
<body><h1>Hello</h1></body>
</html>
EOF

# Using printf
MESSAGE=$(printf '%s\n' \
    "Line 1" \
    "Line 2" \
    "Line 3"
)

Arrays from Heredoc

# Read lines into array
mapfile -t SERVERS <<'EOF'
server1.example.com
server2.example.com
server3.example.com
EOF

# Process array
for server in "${SERVERS[@]}"; do
    echo "Checking $server..."
    ping -c 1 "$server" &>/dev/null && echo "  UP" || echo "  DOWN"
done