jq JSON Processing

# Pretty print
cat file.json | jq .
echo '{"a":1}' | jq .

# Access object keys
jq '.key' file.json                      # Top-level key
jq '.nested.key' file.json               # Nested key
jq '.a.b.c' file.json                    # Deep nesting

# Handle null/missing keys
jq '.missing' file.json                  # Returns null
jq '.missing // "default"' file.json    # Default value

# Keys with special characters
jq '.["key-with-dash"]' file.json
jq '.["key with space"]' file.json
jq '."key-with-dash"' file.json          # Alternative

# Multiple keys
jq '.key1, .key2' file.json              # Both values (separate outputs)
jq '{a: .key1, b: .key2}' file.json      # Combined into object

# Raw output (no quotes for strings)
jq -r '.name' file.json

# Compact output (no pretty print)
jq -c '.' file.json

# Infrastructure: Get ISE session user
netapi ise mnt sessions --format json | jq -r '.[0].user_name'

# Infrastructure: Get Vault token info
vault token lookup -format=json | jq -r '.data.display_name'

# Access array elements
jq '.[0]' file.json                      # First element
jq '.[-1]' file.json                     # Last element
jq '.[2:5]' file.json                    # Slice (elements 2,3,4)
jq '.[:3]' file.json                     # First 3 elements
jq '.[-3:]' file.json                    # Last 3 elements

# Iterate all elements
jq '.[]' file.json                       # Each element (separate outputs)
jq '.[] | .name' file.json               # Each element's name

# Array length
jq 'length' file.json                    # Array length
jq '.items | length' file.json           # Nested array length

# Array operations
jq 'first' file.json                     # First element (same as .[0])
jq 'last' file.json                      # Last element
jq 'reverse' file.json                   # Reverse array
jq 'unique' file.json                    # Remove duplicates
jq 'sort' file.json                      # Sort (strings/numbers)
jq 'sort_by(.name)' file.json            # Sort by field
jq 'group_by(.type)' file.json           # Group by field

# Add element
jq '. + ["new"]' file.json               # Append
jq '["first"] + .' file.json             # Prepend

# Flatten nested arrays
jq 'flatten' file.json                   # Flatten all levels
jq 'flatten(1)' file.json                # Flatten 1 level

# Infrastructure: Get all ISE session MACs
netapi ise mnt sessions --format json | jq -r '.[].calling_station_id'

# Infrastructure: Get k8s pod names
kubectl get pods -o json | jq -r '.items[].metadata.name'

# Infrastructure: Sort ISE sessions by user
netapi ise mnt sessions --format json | jq 'sort_by(.user_name)'

# Basic filter
jq '.[] | select(.status == "active")' file.json
jq '.[] | select(.value > 10)' file.json
jq '.[] | select(.name != null)' file.json

# String operations
jq '.[] | select(.name | startswith("vault"))' file.json
jq '.[] | select(.name | endswith(".dev"))' file.json
jq '.[] | select(.name | contains("prod"))' file.json
jq '.[] | select(.name | test("^vault-[0-9]+$"))' file.json  # Regex

# Negation
jq '.[] | select(.status != "failed")' file.json
jq '.[] | select(.name | contains("test") | not)' file.json

# Multiple conditions (AND)
jq '.[] | select(.status == "active" and .value > 10)' file.json

# Multiple conditions (OR)
jq '.[] | select(.status == "active" or .status == "pending")' file.json

# Check if field exists
jq '.[] | select(has("email"))' file.json
jq '.[] | select(.email != null)' file.json

# Type checking
jq '.[] | select(.value | type == "number")' file.json
jq '.[] | select(.data | type == "array")' file.json

# In array (membership)
jq '.[] | select(.role | IN("admin", "manager"))' file.json

# Infrastructure: Failed ISE authentications
netapi ise mnt sessions --format json | jq '.[] | select(.authentication_status == "FAILED")'

# Infrastructure: Pods not running
kubectl get pods -o json | jq '.items[] | select(.status.phase != "Running") | .metadata.name'

# Infrastructure: High CPU pods
kubectl top pods -o json | jq '.items[] | select(.containers[0].usage.cpu | rtrimstr("m") | tonumber > 500)'

# Infrastructure: ISE sessions from specific switch
netapi ise mnt sessions --format json | jq '.[] | select(.nas_ip_address == "10.50.1.11")'

# Create new object
jq '{name: .title, id: .uuid}' file.json

# Rename keys
jq '{new_name: .old_name}' file.json

# Preserve original field name (shorthand)
jq '{name, id, status}' file.json        # Same as {name: .name, id: .id, status: .status}

# Add computed fields
jq '{name, full_name: (.first + " " + .last)}' file.json

# Conditional fields
jq '{name, status: (if .active then "up" else "down" end)}' file.json

# Nested object creation
jq '{
  meta: {name: .name, id: .id},
  data: {value: .value, count: .items | length}
}' file.json

# From array to object
jq '[.[] | {key: .name, value: .count}] | from_entries' file.json
jq 'map({(.name): .value}) | add' file.json

# Extract multiple into array of objects
jq '[.[] | {name, status}]' file.json

# Infrastructure: ISE session summary
netapi ise mnt sessions --format json | jq '[.[] | {
  mac: .calling_station_id,
  user: .user_name,
  switch: .nas_ip_address,
  status: .authentication_status
}]'

# Infrastructure: k8s pod summary
kubectl get pods -o json | jq '[.items[] | {
  name: .metadata.name,
  namespace: .metadata.namespace,
  status: .status.phase,
  ip: .status.podIP
}]'

# Infrastructure: Vault secret metadata
vault kv metadata list -format=json secret/ | jq '[.data.keys[] | {path: ., full: ("secret/" + .)}]'

# map - apply to each array element
jq 'map(.name)' file.json                # Array of names
jq 'map(. + 1)' file.json                # Increment each number
jq 'map(select(.active))' file.json      # Filter, keep array structure

# map_values - apply to each object value
jq 'map_values(. * 2)' file.json         # Double each value

# reduce - aggregate values
jq 'reduce .[] as $x (0; . + $x)' file.json              # Sum
jq 'reduce .[] as $x (0; . + $x.value)' file.json        # Sum field
jq 'reduce .[] as $x (""; . + $x.name + ",")' file.json  # Concatenate

# group_by + map for aggregation
jq 'group_by(.type) | map({type: .[0].type, count: length})' file.json

# Pivot table style
jq 'group_by(.category) | map({
  category: .[0].category,
  total: map(.value) | add,
  count: length,
  avg: (map(.value) | add / length)
})' file.json

# min/max
jq 'min' file.json                       # Min of array
jq 'max_by(.value)' file.json            # Element with max value
jq '[.[] | .value] | min' file.json      # Min value from objects

# add - sum/concatenate
jq '[.[] | .count] | add' file.json      # Sum counts
jq '[.[] | .name] | join(", ")' file.json # Join names

# Infrastructure: Total ISE sessions by status
netapi ise mnt sessions --format json | jq 'group_by(.authentication_status) | map({
  status: .[0].authentication_status,
  count: length
})'

# Infrastructure: k8s resource summary by namespace
kubectl get pods -o json | jq '.items | group_by(.metadata.namespace) | map({
  namespace: .[0].metadata.namespace,
  pods: length,
  running: map(select(.status.phase == "Running")) | length
})'

# Concatenation
jq '.first + " " + .last' file.json

# Interpolation
jq '"User: \(.name), ID: \(.id)"' file.json

# Split and join
jq '.name | split("-")' file.json        # "vault-01" -> ["vault", "01"]
jq '.items | join(", ")' file.json       # ["a", "b"] -> "a, b"

# Case conversion
jq '.name | ascii_downcase' file.json
jq '.name | ascii_upcase' file.json

# Trim whitespace
jq '.name | ltrimstr(" ")' file.json     # Left trim
jq '.name | rtrimstr(" ")' file.json     # Right trim
jq '.name | gsub("^\\s+|\\s+$"; "")' file.json  # Both sides

# Substring
jq '.name | .[0:5]' file.json            # First 5 chars
jq '.name | .[-3:]' file.json            # Last 3 chars

# Replace
jq '.name | gsub("-"; "_")' file.json    # Replace all
jq '.name | sub("-"; "_")' file.json     # Replace first

# Regex match
jq '.name | test("vault-[0-9]+")' file.json        # Boolean
jq '.name | match("vault-([0-9]+)").captures[0].string' file.json  # Capture group

# Parse numbers from strings
jq '.cpu | rtrimstr("m") | tonumber' file.json     # "500m" -> 500
jq '.mem | rtrimstr("Mi") | tonumber' file.json    # "256Mi" -> 256

# Format numbers as strings
jq '.value | tostring' file.json
jq '"Value: \(.value)"' file.json

# Infrastructure: Extract hostname from FQDN
netapi ise mnt sessions --format json | jq -r '.[].nas_identifier | split(".")[0]'

# Infrastructure: Normalize MAC addresses
netapi ise mnt sessions --format json | jq -r '.[].calling_station_id | gsub(":"; "-") | ascii_downcase'

# if-then-else
jq 'if .value > 10 then "high" else "low" end' file.json

# Nested conditionals
jq 'if .value > 100 then "critical"
    elif .value > 50 then "warning"
    else "ok" end' file.json

# Conditional in object construction
jq '{
  name,
  level: (if .value > 100 then "high" elif .value > 50 then "medium" else "low" end)
}' file.json

# Empty/null handling
jq '.items // []' file.json              # Default to empty array
jq '.name // "unknown"' file.json        # Default string
jq 'if .name then .name else "N/A" end' file.json

# Alternative operator (or)
jq '.primary // .secondary // "fallback"' file.json

# Error suppression
jq '.items[]? | .name' file.json         # ? suppresses errors

# try-catch
jq 'try .value catch "error"' file.json

# Infrastructure: ISE status indicator
netapi ise mnt sessions --format json | jq -r '.[] |
  if .authentication_status == "AUTHENTICATED" then "✓"
  elif .authentication_status == "FAILED" then "✗"
  else "?" end + " " + .calling_station_id'

# Infrastructure: k8s pod health
kubectl get pods -o json | jq '.items[] | {
  name: .metadata.name,
  healthy: (if .status.phase == "Running" and (.status.containerStatuses // []) | all(.ready) then true else false end)
}'

# Path expressions
jq 'paths' file.json                     # All paths
jq 'path(.a.b)' file.json                # Path to specific key
jq 'getpath(["a", "b"])' file.json       # Get value at path

# Recursive descent
jq '.. | numbers' file.json              # All numbers anywhere
jq '.. | strings | select(test("error"))' file.json  # All error strings

# Walk and transform
jq 'walk(if type == "string" then ascii_downcase else . end)' file.json

# Update in place
jq '.items[0].status = "updated"' file.json
jq '.items[] |= . + {processed: true}' file.json
jq '.count += 1' file.json

# Delete keys
jq 'del(.unwanted)' file.json
jq 'del(.items[].internal)' file.json

# Merge objects
jq '. + {new_field: "value"}' file.json
jq '. * {nested: {deep: "update"}}' file.json  # Deep merge

# Env variables
jq --arg name "$USER" '{user: $name}' file.json
jq --argjson count 5 '{count: $count}' file.json

# From file
jq --slurpfile config config.json '. + $config[0]' file.json

# Multiple inputs
jq -s '.[0] + .[1]' file1.json file2.json  # Merge two files
jq -s 'add' file1.json file2.json          # Combine arrays

# Stream processing (for huge files)
jq -c --stream 'select(.[0][0] == "items")' huge.json

# Infrastructure: Update Vault policy
vault policy read -format=json default | jq '.rules |= . + "\npath \"secret/*\" { capabilities = [\"read\"] }"'

# Infrastructure: Merge k8s configs
jq -s '.[0] * .[1]' base.json overlay.json

# Raw strings (no quotes)
jq -r '.name' file.json

# Compact (single line)
jq -c '.' file.json

# Tab-separated
jq -r '[.name, .value, .status] | @tsv' file.json

# CSV
jq -r '[.name, .value, .status] | @csv' file.json

# With headers
jq -r '["NAME", "VALUE", "STATUS"], (.[] | [.name, .value, .status]) | @csv' file.json

# URI encoding
jq -r '.query | @uri' file.json

# Base64
jq -r '.data | @base64' file.json
jq -r '.encoded | @base64d' file.json    # Decode

# HTML encoding
jq -r '.text | @html' file.json

# JSON text (for nested strings)
jq -r '.config | @json' file.json

# Shell-safe
jq -r '.args | @sh' file.json            # Quote for shell

# Infrastructure: ISE sessions as TSV
netapi ise mnt sessions --format json | jq -r '.[] | [.calling_station_id, .user_name, .authentication_status] | @tsv'

# Infrastructure: Export to CSV for Excel
kubectl get pods -o json | jq -r '
  ["NAMESPACE","NAME","STATUS","IP"],
  (.items[] | [.metadata.namespace, .metadata.name, .status.phase, .status.podIP // "N/A"])
  | @csv' > pods.csv

# ISE: Failed auth analysis
netapi ise mnt sessions --format json | jq '
  [.[] | select(.authentication_status == "FAILED")] |
  group_by(.failure_reason) |
  map({reason: .[0].failure_reason, count: length, users: [.[].user_name] | unique}) |
  sort_by(-.count)'

# ISE: Sessions per switch
netapi ise mnt sessions --format json | jq '
  group_by(.nas_ip_address) |
  map({
    switch: .[0].nas_ip_address,
    total: length,
    authenticated: map(select(.authentication_status == "AUTHENTICATED")) | length,
    failed: map(select(.authentication_status == "FAILED")) | length
  }) |
  sort_by(-.total)'

# ISE: Policy set hit counts
netapi ise openapi GET '/api/v1/policy/network-access/policy-set' | jq '
  .response | map({name, hitCount: .hitCounts.hitCount}) | sort_by(-.hitCount)'

# k8s: Pod resource usage
kubectl top pods -o json | jq '.items | map({
  name: .metadata.name,
  cpu: (.containers[0].usage.cpu | rtrimstr("m") | tonumber),
  memory: (.containers[0].usage.memory | rtrimstr("Mi") | tonumber)
}) | sort_by(-.cpu) | .[0:10]'

# k8s: Deployments not at desired replicas
kubectl get deployments -o json | jq '.items | map(select(.status.replicas != .status.readyReplicas)) | map({
  name: .metadata.name,
  desired: .spec.replicas,
  ready: .status.readyReplicas
})'

# Vault: List all secrets recursively
vault kv list -format=json secret/ | jq -r '.data.keys[]' | while read key; do
  if [[ "$key" == */ ]]; then
    vault kv list -format=json "secret/$key" | jq -r --arg p "$key" '.data.keys[] | "secret/\($p)\(.)"'
  else
    echo "secret/$key"
  fi
done

# Vault: Certificate expiry from PKI
vault list -format=json pki_int/certs | jq -r '.data.keys[]' | head -5 | while read serial; do
  vault read -format=json "pki_int/cert/$serial" | jq -r '{
    serial: .data.serial_number,
    cn: .data.certificate | @base64d | capture("CN=(?<cn>[^,]+)").cn,
    expiry: .data.expiration
  }'
done

# Wazuh: Agent status
netapi wazuh agents | jq '.data.affected_items | map({
  name,
  status,
  ip,
  last_keepalive
}) | sort_by(.status)'

# API response pagination (combine pages)
for page in 1 2 3; do
  curl -s "https://api.example.com/items?page=$page"
done | jq -s '[.[].items[]] | flatten'

# Deep diff between two JSON files
diff <(jq -S '.' file1.json) <(jq -S '.' file2.json)

# Find differences in arrays
jq -s '(.[0] - .[1]) as $removed | (.[1] - .[0]) as $added | {removed: $removed, added: $added}' old.json new.json

# WRONG: Forgetting -r for strings
name=$(echo '{"name":"vault"}' | jq '.name')
echo "$name"                             # "vault" (with quotes!)

# CORRECT: Use -r for raw strings
name=$(echo '{"name":"vault"}' | jq -r '.name')
echo "$name"                             # vault

# WRONG: Not handling null
echo '{"a": null}' | jq '.a.b'          # null, not error
echo '{}' | jq '.a.b'                   # null, not error

# CORRECT: Use // for defaults
echo '{}' | jq '.a.b // "default"'      # "default"

# WRONG: Piping when you mean array
jq '.[] | select(.x > 0)' file.json     # Separate outputs
jq '[.[] | select(.x > 0)]' file.json   # Single array output

# WRONG: Arithmetic on strings
echo '{"cpu": "500m"}' | jq '.cpu + 100' # Error!

# CORRECT: Parse first
echo '{"cpu": "500m"}' | jq '.cpu | rtrimstr("m") | tonumber | . + 100'

# WRONG: Comparing different types
echo '{"val": "10"}' | jq '.val > 5'     # false! String vs number

# CORRECT: Convert types
echo '{"val": "10"}' | jq '.val | tonumber > 5'  # true

# WRONG: Empty input
echo "" | jq '.'                         # parse error

# CORRECT: Handle empty
echo "" | jq -e '.' 2>/dev/null || echo "No JSON"

# WRONG: Multiple JSON objects (not in array)
echo '{"a":1}{"b":2}' | jq '.'           # Only processes first

# CORRECT: Use -s to slurp
echo '{"a":1}{"b":2}' | jq -s '.'        # [{a:1},{b:2}]

# WRONG: Shell variable in single quotes
name="vault"
jq '.[$name]' file.json                  # Error!

# CORRECT: Use --arg
jq --arg n "$name" '.[$n]' file.json

# WRONG: Expecting map to modify in place
jq '.items | map(.x = 1)' file.json      # Only items returned

# CORRECT: Use |=
jq '.items |= map(. + {x: 1})' file.json # Full object returned

# WRONG: Deep merge with +
echo '{"a":{"b":1}}' | jq '. + {"a":{"c":2}}'  # {"a":{"c":2}} - overwrites!

# CORRECT: Use * for deep merge
echo '{"a":{"b":1}}' | jq '. * {"a":{"c":2}}'  # {"a":{"b":1,"c":2}}