Encoding & Decoding

Encoding and decoding operations for data transformation.

Encoding Fundamentals

# ENCODING != ENCRYPTION
# Encoding: transform format (reversible by anyone)
# Encryption: protect confidentiality (requires key)
# NEVER use encoding for security!

# COMMON ENCODINGS
# Base64:     A-Za-z0-9+/=     (6 bits per char, 33% overhead)
# Base32:     A-Z2-7=          (5 bits per char, 60% overhead)
# Hex:        0-9a-f           (4 bits per char, 100% overhead)
# URL:        %XX              (percent-encoding special chars)

# BASE64 BASICS
echo -n "Hello World" | base64
# Output: SGVsbG8gV29ybGQ=

echo "SGVsbG8gV29ybGQ=" | base64 -d
# Output: Hello World

# CRITICAL: -n prevents newline!
echo "Hello" | base64    # SGVsbG8K (includes newline!)
echo -n "Hello" | base64 # SGVsbG8= (correct)

# BASE64 FILE
base64 file.bin > file.b64
base64 -d file.b64 > file.bin

# BASE64 URL-SAFE (RFC 4648)
# Replace + with -, / with _, remove =
echo -n "data" | base64 | tr '+/' '-_' | tr -d '='
# Python: secrets.token_urlsafe()

# HEX ENCODING
echo -n "Hello" | xxd -p
# Output: 48656c6c6f

echo "48656c6c6f" | xxd -r -p
# Output: Hello

# HEX WITH SPACES
echo -n "Hello" | xxd -p | fold -w2 | paste -sd' '
# Output: 48 65 6c 6c 6f

# ALTERNATIVE: od (octal dump)
echo -n "Hello" | od -A x -t x1z -v
# Shows hex with ASCII

# BASE32 (used in TOTP)
echo -n "Hello" | base32
# Output: JBSWY3DP

echo "JBSWY3DP" | base32 -d
# Output: Hello

# ASCII <-> NUMERIC
printf '%d\n' "'A"        # 65 (char to decimal)
printf '\\x41'            # A (hex to char)
printf '%c' 65            # A (decimal to char)

URL Encoding

# URL ENCODING (percent-encoding)
# Encodes special characters as %XX (hex)

# PYTHON (most reliable)
python3 -c "import urllib.parse; print(urllib.parse.quote('hello world'))"
# Output: hello%20world

python3 -c "import urllib.parse; print(urllib.parse.quote_plus('hello world'))"
# Output: hello+world (form encoding)

# FULL ENCODING (including /)
python3 -c "import urllib.parse; print(urllib.parse.quote('path/to/file', safe=''))"
# Output: path%2Fto%2Ffile

# URL DECODE
python3 -c "import urllib.parse; print(urllib.parse.unquote('hello%20world'))"
# Output: hello world

# BASH-ONLY URL ENCODE (basic)
url_encode() {
    local string="$1"
    local length="${#string}"
    local encoded=""
    for (( i = 0; i < length; i++ )); do
        local c="${string:$i:1}"
        case "$c" in
            [a-zA-Z0-9.~_-]) encoded+="$c" ;;
            ' ') encoded+="%20" ;;
            *) encoded+=$(printf '%%%02X' "'$c") ;;
        esac
    done
    echo "$encoded"
}
url_encode "hello world & more"
# Output: hello%20world%20%26%20more

# CURL URL ENCODING
curl --data-urlencode "param=hello world" https://api.example.com
# Automatically encodes the parameter

# JQ URL ENCODING
echo '"hello world"' | jq -r '@uri'
# Output: hello%20world

# QUERY STRING BUILDING
python3 << 'EOF'
import urllib.parse
params = {
    'name': 'John Doe',
    'query': 'foo&bar=baz',
    'special': 'hello@world.com'
}
print(urllib.parse.urlencode(params))
EOF
# Output: name=John+Doe&query=foo%26bar%3Dbaz&special=hello%40world.com

# INFRASTRUCTURE PATTERNS

# Encode ISE filter parameter
FILTER="name.CONTAINS.server"
ENCODED=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$FILTER'))")
curl -k "https://ise-01/ers/config/endpoint?filter=$ENCODED"

# Encode pfSense API parameter
netapi pfsense dns add -h "host" -d "inside.domusdigitalis.dev" \
    -i "10.50.1.100" --descr "Test host"
# netapi handles encoding internally

JWT (JSON Web Token) Operations

# JWT STRUCTURE
# header.payload.signature (base64url encoded)
# Example: eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.signature

# DECODE JWT (without verification)
jwt_decode() {
    local token="$1"
    local part="$2"  # 1=header, 2=payload

    echo "$token" | cut -d. -f"$part" | \
        tr '_-' '/+' | \
        base64 -d 2>/dev/null | \
        jq '.'
}

TOKEN="eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4ifQ.sig"
jwt_decode "$TOKEN" 1  # Header
jwt_decode "$TOKEN" 2  # Payload

# ONE-LINER DECODE
echo "eyJhbGciOiJIUzI1NiJ9" | tr '_-' '/+' | base64 -d | jq '.'

# PYTHON JWT DECODE (full)
python3 << 'EOF'
import base64
import json

token = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.sig"
parts = token.split('.')

def decode_part(part):
    # Add padding
    padding = 4 - len(part) % 4
    if padding != 4:
        part += '=' * padding
    # URL-safe to standard base64
    part = part.replace('-', '+').replace('_', '/')
    return json.loads(base64.b64decode(part))

print("Header:", decode_part(parts[0]))
print("Payload:", decode_part(parts[1]))
EOF

# CHECK JWT EXPIRY
jwt_check_expiry() {
    local token="$1"
    local payload=$(echo "$token" | cut -d. -f2 | tr '_-' '/+' | base64 -d 2>/dev/null)
    local exp=$(echo "$payload" | jq -r '.exp // empty')

    if [ -z "$exp" ]; then
        echo "No expiry claim"
        return
    fi

    local now=$(date +%s)
    if [ "$exp" -lt "$now" ]; then
        echo "EXPIRED $(( (now - exp) / 60 )) minutes ago"
    else
        echo "Valid for $(( (exp - now) / 60 )) more minutes"
    fi
}

# CREATE JWT (HMAC-SHA256)
create_jwt() {
    local secret="$1"
    local payload="$2"

    local header='{"alg":"HS256","typ":"JWT"}'
    local h=$(echo -n "$header" | base64 | tr '+/' '-_' | tr -d '=')
    local p=$(echo -n "$payload" | base64 | tr '+/' '-_' | tr -d '=')
    local unsigned="$h.$p"
    local sig=$(echo -n "$unsigned" | openssl dgst -sha256 -hmac "$secret" -binary | base64 | tr '+/' '-_' | tr -d '=')

    echo "$unsigned.$sig"
}

# Example: create JWT
PAYLOAD='{"sub":"user123","iat":1234567890,"exp":1234571490}'
create_jwt "my-secret-key" "$PAYLOAD"

# INFRASTRUCTURE PATTERNS

# Decode Wazuh JWT
WAZUH_TOKEN=$(curl -sk -X POST "https://wazuh-api:55000/security/user/authenticate" \
    -d '{"username":"admin","password":"admin"}' | jq -r '.data.token')
jwt_decode "$WAZUH_TOKEN" 2 | jq '.exp | strftime("%Y-%m-%d %H:%M:%S")'

# Decode Keycloak access token
ACCESS_TOKEN=$(curl -s "https://keycloak/realms/master/protocol/openid-connect/token" \
    -d "client_id=admin-cli" -d "username=admin" -d "password=$PASS" \
    -d "grant_type=password" | jq -r '.access_token')
jwt_decode "$ACCESS_TOKEN" 2 | jq '{sub,preferred_username,exp}'

Binary/Hex/Base Conversions

# DECIMAL TO BINARY
echo "obase=2; 255" | bc
# Output: 11111111

# BINARY TO DECIMAL
echo "ibase=2; 11111111" | bc
# Output: 255

# DECIMAL TO HEX
printf '%x\n' 255
# Output: ff

printf '%02x\n' 15
# Output: 0f (zero-padded)

# HEX TO DECIMAL
printf '%d\n' 0xff
# Output: 255

echo $((16#ff))
# Output: 255 (bash arithmetic)

# HEX TO BINARY
echo "ibase=16; obase=2; FF" | bc
# Output: 11111111

# BASE CONVERSIONS WITH PYTHON
python3 -c "print(bin(255))"      # 0b11111111
python3 -c "print(oct(255))"      # 0o377
python3 -c "print(hex(255))"      # 0xff
python3 -c "print(int('ff', 16))" # 255
python3 -c "print(int('11111111', 2))"  # 255

# IP ADDRESS CONVERSIONS
ip_to_int() {
    local ip=$1
    local IFS='.'
    read -r a b c d <<< "$ip"
    echo $(( (a << 24) + (b << 16) + (c << 8) + d ))
}
ip_to_int "10.50.1.100"
# Output: 172818788

int_to_ip() {
    local int=$1
    echo "$(( (int >> 24) & 255 )).$(( (int >> 16) & 255 )).$(( (int >> 8) & 255 )).$(( int & 255 ))"
}
int_to_ip 172818788
# Output: 10.50.1.100

# MAC ADDRESS FORMATS
mac="14:F6:D8:7B:31:80"

# Colon to dash
echo "$mac" | tr ':' '-'
# 14-F6-D8-7B-31-80

# Colon to Cisco format
echo "$mac" | tr -d ':' | sed 's/\(....\)/\1./g' | sed 's/\.$//'
# 14F6.D87B.3180

# To lowercase
echo "$mac" | tr '[:upper:]' '[:lower:]'
# 14:f6:d8:7b:31:80

# BYTE ORDER (Endianness)
# Little-endian: least significant byte first
# Big-endian: most significant byte first

# Show byte order
echo -n "ABCD" | xxd -e -g4
# Shows endian interpretation

# UNICODE CONVERSIONS
# UTF-8 to hex
echo -n "é" | xxd -p
# Output: c3a9 (2 bytes in UTF-8)

# Hex to character
printf '\xc3\xa9'
# Output: é

# Unicode codepoint to character
printf '\u00e9'
# Output: é

python3 -c "print('\u00e9')"
# Output: é

# Character to codepoint
printf '%d\n' "'é"
# Output: 233 (U+00E9)

Certificate Encoding Conversions

# CERTIFICATE ENCODING FORMATS

# PEM (Privacy Enhanced Mail)
# Text format, base64 with headers
# -----BEGIN CERTIFICATE-----
# MIIBk...
# -----END CERTIFICATE-----

# DER (Distinguished Encoding Rules)
# Binary format, no headers

# EXTRACT RAW BASE64 FROM PEM
sed -n '/BEGIN/,/END/p' cert.pem | sed '1d;$d' | tr -d '\n'
# Raw base64 without headers/newlines

# DECODE PEM CONTENT
sed -n '/BEGIN/,/END/p' cert.pem | sed '1d;$d' | base64 -d | xxd | head
# Shows binary ASN.1 structure

# PEM TO SINGLE LINE (for JSON)
awk 'NF {sub(/\r/, ""); printf "%s\\n",$0}' cert.pem
# Escapes newlines for JSON embedding

# SINGLE LINE BACK TO PEM
echo -e "-----BEGIN CERTIFICATE-----\nMIIB...\n-----END CERTIFICATE-----"

# VIEW ASN.1 STRUCTURE
openssl asn1parse -in cert.pem
# Shows parsed ASN.1 structure

openssl asn1parse -in cert.der -inform DER
# For DER format

# EXTRACT PUBLIC KEY
openssl x509 -in cert.pem -noout -pubkey
# Output: PEM-encoded public key

# PUBLIC KEY TO RAW BYTES
openssl x509 -in cert.pem -noout -pubkey | \
    openssl rsa -pubin -outform DER 2>/dev/null | \
    xxd -p

# FINGERPRINT FORMATS
openssl x509 -in cert.pem -noout -fingerprint
# SHA-1 fingerprint (default)

openssl x509 -in cert.pem -noout -fingerprint -sha256
# SHA-256 fingerprint

# Remove colons from fingerprint
openssl x509 -in cert.pem -noout -fingerprint -sha256 | \
    cut -d= -f2 | tr -d ':'

# SERIAL NUMBER FORMATS
openssl x509 -in cert.pem -noout -serial
# Output: serial=01A2B3C4...

# Decimal serial
openssl x509 -in cert.pem -noout -serial | cut -d= -f2 | \
    python3 -c "import sys; print(int(sys.stdin.read().strip(), 16))"

# INFRASTRUCTURE PATTERNS

# Convert Vault PEM to single-line for ISE import
CERT=$(vault read -field=certificate pki_int/cert/ca)
echo "$CERT" | awk 'NF {sub(/\r/, ""); printf "%s\\n",$0}'

# Extract certificate from PKCS#12 and base64 for API
openssl pkcs12 -in cert.p12 -nokeys -clcerts | \
    sed -n '/BEGIN/,/END/p' | \
    base64 -w0

# Prepare CA bundle for k8s secret
cat DOMUS-ISSUING-CA.pem DOMUS-ROOT-CA.pem | base64 -w0
# Use in: kubectl create secret generic ca-bundle --from-literal=ca.crt=...

Data Format Encoding

# JSON ENCODING

# Escape string for JSON
python3 -c 'import json; print(json.dumps("hello\nworld"))'
# Output: "hello\nworld"

# JSON to base64
echo '{"key": "value"}' | base64
# Or compact:
echo '{"key":"value"}' | jq -c '.' | base64

# Base64 in JSON
jq -n --arg data "$(echo 'secret' | base64)" '{"encoded": $data}'
# Output: {"encoded": "c2VjcmV0Cg=="}

# YAML ENCODING
# Multiline string (literal block)
cat << 'EOF'
data: |
  line1
  line2
  line3
EOF

# Multiline string (folded)
cat << 'EOF'
data: >
  this will
  be one line
EOF

# BASE64 IN YAML (k8s secrets)
echo -n "secret" | base64
# c2VjcmV0

cat << EOF
apiVersion: v1
kind: Secret
data:
  password: c2VjcmV0
EOF

# XML ENCODING
# Escape special characters
python3 << 'EOF'
import html
text = '<script>alert("xss")</script>'
print(html.escape(text))
EOF
# Output: &lt;script&gt;alert(&quot;xss&quot;)&lt;/script&gt;

# SHELL ESCAPING
# Escape for shell
printf '%q\n' "hello 'world' \$var"
# Output: hello\ \'world\'\ \$var

# INFRASTRUCTURE PATTERNS

# Encode secret for k8s
kubectl create secret generic my-secret \
    --from-literal=username=admin \
    --from-literal=password="$(gopass show -o v3/domains/d000/k8s/service)" \
    --dry-run=client -o yaml

# Decode k8s secret
kubectl get secret my-secret -o jsonpath='{.data.password}' | base64 -d

# Encode for Ansible vault inline
echo "secret_value" | ansible-vault encrypt_string --stdin-name 'my_secret'

# Encode cloud-init userdata
gzip -c userdata.yaml | base64 -w0
# For cloud providers requiring compressed userdata

# Encode for environment variable (handle special chars)
export COMPLEX_VAR="$(printf '%s' 'value with $pecial chars!' | base64)"
echo "$COMPLEX_VAR" | base64 -d  # Decode when needed

TOTP Secret Encoding

# TOTP USES BASE32 ENCODING
# Alphabet: A-Z2-7 (no 0, 1, 8, 9 to avoid confusion)

# HEX TO BASE32 (common conversion)
HEX_SECRET="48656c6c6f"
echo "$HEX_SECRET" | xxd -r -p | base32
# Output: JBSWY3DP (without padding)

# BASE32 TO HEX
echo "JBSWY3DP" | base32 -d | xxd -p
# Output: 48656c6c6f

# GENERATE TOTP SECRET
# Standard: 20 bytes (160 bits) = 32 base32 chars
openssl rand 20 | base32 | tr -d '='
# Output: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

# WITH PROPER PADDING
openssl rand 20 | base32
# May include = padding

# VERIFY BASE32 FORMAT
validate_base32() {
    local secret="$1"
    # Remove padding, check characters
    local clean=$(echo "$secret" | tr -d '=' | tr '[:lower:]' '[:upper:]')
    if echo "$clean" | grep -q '[^A-Z2-7]'; then
        echo "Invalid base32 characters"
        return 1
    fi
    # Check length (must be multiple of 8 after padding)
    local len=${#clean}
    local padded_len=$(( (len + 7) / 8 * 8 ))
    echo "Valid base32, ${len} chars (${padded_len} with padding)"
}

validate_base32 "JBSWY3DPEHPK3PXP"

# OTPAUTH URI FORMAT
# otpauth://totp/LABEL?secret=BASE32&issuer=ISSUER&algorithm=SHA1&digits=6&period=30

# Generate full URI
ISSUER="MyService"
ACCOUNT="user@example.com"
SECRET=$(openssl rand 20 | base32 | tr -d '=')
LABEL=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$ISSUER:$ACCOUNT'))")

echo "otpauth://totp/${LABEL}?secret=${SECRET}&issuer=${ISSUER}&algorithm=SHA1&digits=6&period=30"

# GENERATE QR CODE
qrencode -o totp.png "otpauth://totp/MyService:user@example.com?secret=${SECRET}&issuer=MyService"

# CONVERT BETWEEN HEX AND OTPAUTH
# Some services give hex, authenticators want base32
hex_to_otpauth() {
    local hex="$1"
    local issuer="$2"
    local account="$3"

    local secret=$(echo "$hex" | xxd -r -p | base32 | tr -d '=')
    echo "otpauth://totp/${issuer}:${account}?secret=${secret}&issuer=${issuer}"
}

hex_to_otpauth "48656c6c6f576f726c64" "Test" "user@test.com"

# INFRASTRUCTURE PATTERNS

# Store TOTP in gopass
gopass edit v3/domains/d000/identity/github-2fa
# Add: totp: JBSWY3DPEHPK3PXP

# Generate code
gopass otp v3/domains/d000/identity/github-2fa

Encoding Gotchas

# WRONG: Thinking base64 is encryption
echo "password" | base64
# Anyone can decode! This is NOT security!

# CORRECT: Use actual encryption
echo "password" | age -r age1...

# WRONG: Forgetting -n with echo
echo "secret" | base64
# Includes newline! c2VjcmV0Cg== instead of c2VjcmV0

# CORRECT: Use -n
echo -n "secret" | base64
# Output: c2VjcmV0

# WRONG: Base64 with special chars in URL
TOKEN="abc+/="
curl "https://api.example.com?token=$TOKEN"
# + and / break URL!

# CORRECT: URL-safe base64
TOKEN=$(echo -n "data" | base64 | tr '+/' '-_' | tr -d '=')

# WRONG: Hex case mismatch
echo "ABC123" | xxd -r -p  # Works
echo "abc123" | xxd -r -p  # Also works
# But comparing them as strings fails!

# CORRECT: Normalize case
echo "ABC123" | tr '[:upper:]' '[:lower:]'

# WRONG: Treating base32 padding as optional
echo "JBSWY3DP" | base32 -d     # OK
echo "JBSWY3D" | base32 -d      # May fail (wrong length)!

# CORRECT: Add padding if needed
add_base32_padding() {
    local s="$1"
    local pad=$(( (8 - ${#s} % 8) % 8 ))
    printf '%s%s' "$s" "$(printf '=%.0s' $(seq 1 $pad))"
}

# WRONG: Double encoding
url_encode(url_encode("hello world"))
# hello%2520world (% got encoded!)

# CORRECT: Encode once at the boundary
url_encode("hello world")
# hello%20world

# WRONG: Mixed encoding in same string
"Hello%20World%E2%80%99s"  # Apostrophe encoded, space plain
# Inconsistent!

# CORRECT: Encode all or none
"Hello%20World%E2%80%99s"  # All encoded
"Hello World's"             # None encoded

# WRONG: JWT without signature verification
payload=$(jwt_decode "$TOKEN" 2)
# Attacker could have modified payload!

# CORRECT: Always verify signature
# Use a library that validates signatures

# WRONG: Assuming UTF-8
echo "café" | xxd -p
# c3a9 - this IS UTF-8, but:
echo "café" | iconv -t ISO-8859-1 | xxd -p
# e9 - different encoding, same visual!

# CORRECT: Explicitly specify encoding
iconv -f UTF-8 -t UTF-8 file.txt > normalized.txt

Quick Reference

# BASE64
echo -n "text" | base64                 # Encode (NO newline!)
echo "dGV4dA==" | base64 -d             # Decode
base64 file.bin > file.b64             # File encode
base64 -d file.b64 > file.bin          # File decode

# HEX
echo -n "text" | xxd -p                 # String to hex
echo "74657874" | xxd -r -p             # Hex to string
xxd file.bin > file.hex                # File to hex
xxd -r file.hex > file.bin             # Hex to file

# URL ENCODING
python3 -c "import urllib.parse; print(urllib.parse.quote('text'))"
python3 -c "import urllib.parse; print(urllib.parse.unquote('text%20here'))"

# BASE32 (TOTP)
echo -n "text" | base32                 # Encode
echo "ORSXG5A=" | base32 -d             # Decode
openssl rand 20 | base32 | tr -d '='    # Generate TOTP secret

# JWT DECODE
echo "$TOKEN" | cut -d. -f2 | tr '_-' '/+' | base64 -d | jq '.'

# URL-SAFE BASE64
echo -n "data" | base64 | tr '+/' '-_' | tr -d '='

# NUMBER CONVERSIONS
printf '%x\n' 255                       # Decimal to hex
printf '%d\n' 0xff                      # Hex to decimal
echo "obase=2; 255" | bc                # Decimal to binary

# K8S SECRET
echo -n "value" | base64                # Encode for secret
kubectl get secret X -o jsonpath='{.data.key}' | base64 -d  # Decode

# CERTIFICATE SINGLE-LINE
awk 'NF {sub(/\r/,""); printf "%s\\n",$0}' cert.pem  # For JSON

See Also