BGP Active-Active Dual-Homed Home Enterprise

# Verify port status on each hypervisor
for port in eno{1..8}; do
  state=$(cat /sys/class/net/$port/operstate 2>/dev/null || echo "missing")
  speed=$(cat /sys/class/net/$port/speed 2>/dev/null || echo "?")
  master=$(ip link show $port 2>/dev/null | awk -F'master ' '{print $2}' | awk '{print $1}')
  printf "%-6s  %-6s  %5sMbps  %s\n" "$port" "$state" "$speed" "${master:-[none]}"
done

# Verify after bridge mode: VyOS eth0 should show public IP
show interfaces ethernet eth0
# Address should be a public IPv4, not 192.168.x.x

   [ISP-A modem/ONT]              [T-Mobile 5G gateway]
          |                               |
       [eno7]                          [eno1]
     [br-wan]                       [br-ispb]
          |           \        /          |
          |            \      /           |
     [vyos-01]                       [vyos-02]
     eth0=WAN-A primary              eth0=LAN (SWAPPED - needs fix)
     eth1=LAN                        eth1=WAN-A
          |                               |
     VRF WAN-A (table 100)          VRF WAN-B (table 200)
     wg0 → iFog Fremont             wg0 → iFog Fremont
     wg1 → ROUTE64                  wg1 → ROUTE64
          |                               |
          +-------------+  +-------------+
                        |  |
                  [eth3 HA link]
                  10.50.99.0/30
                  conntrack-sync
                        |
               [eth2 internal trunk]
               VRRP split-master
               dual-stack downstream

# Verify existing bridges on hypervisor
bridge link show

# Create new br-ispb for T-Mobile gateway
sudo nmcli con add type bridge con-name br-ispb ifname br-ispb
sudo nmcli con add type ethernet con-name br-ispb-port ifname eno1 master br-ispb
sudo nmcli con up br-ispb

# Create br-ha (internal only, no physical port)
sudo nmcli con add type bridge con-name br-ha ifname br-ha
sudo nmcli con up br-ha

[VyOS node]
    | WireGuard UDP (encrypted, over home ISP circuit)
[iFog or ROUTE64 Linux PE — datacenter x86]
    | Their upstream transit (Cogent, Telia, HE, etc.)
[Internet DFZ]

Organization name : Domus Digitalis Network
                    (individual registration accepted)
Upstream ASN 1    : AS34927  (iFog BGPTunnel)
Upstream ASN 2    : <ROUTE64 ASN — verify on bgp.tools>
Use description   : Home enterprise network, BGP multihoming,
                    non-commercial research and learning
Country           : United States
                    (RIPE sponsoring is not region-restricted)

ifog-vyos1  : endpoint, peer-pubkey, my-link-addr/127, peer-addr
ifog-vyos2  : endpoint, peer-pubkey, my-link-addr/127, peer-addr
r64-vyos1   : endpoint, peer-pubkey, my-link-addr/127, peer-addr
r64-vyos2   : endpoint, peer-pubkey, my-link-addr/127, peer-addr

# On kvm-02 — verify vyos-02 is BACKUP
ssh vyos-02 "show vrrp" | awk '/State/ {print $3}'
# Expected: BACKUP (all VRIDs)

# Shutdown vyos-02
sudo virsh shutdown vyos-02

# Wait for clean shutdown
watch -n1 'sudo virsh list --all | grep vyos-02'

# Remove old interfaces (wrong order)
sudo virsh detach-interface vyos-02 bridge --mac $(sudo virsh domiflist vyos-02 | awk 'NR==3 {print $5}') --persistent
sudo virsh detach-interface vyos-02 bridge --mac $(sudo virsh domiflist vyos-02 | awk 'NR==3 {print $5}') --persistent

# Attach new interfaces in correct order
sudo virsh attach-interface vyos-02 bridge br-wan --model virtio --persistent
sudo virsh attach-interface vyos-02 bridge br-ispb --model virtio --persistent
sudo virsh attach-interface vyos-02 bridge br-mgmt --model virtio --persistent
sudo virsh attach-interface vyos-02 bridge br-ha --model virtio --persistent

# Verify interface order
sudo virsh domiflist vyos-02
# Expected:
# eth0 → br-wan
# eth1 → br-ispb
# eth2 → br-mgmt
# eth3 → br-ha

# Start vyos-02
sudo virsh start vyos-02

# Verify VyOS sees interfaces
ssh vyos-02 "show interfaces"

# SSH to vyos-02 — reconfigure interface addresses
configure

# eth0 is now WAN (was LAN)
delete interfaces ethernet eth0 address
set interfaces ethernet eth0 description 'WAN-A-ISPA'
set interfaces ethernet eth0 address 'dhcp'

# eth1 is ISP-B (new)
set interfaces ethernet eth1 description 'WAN-B-ISPB'
set interfaces ethernet eth1 address 'dhcp'

# eth2 is LAN (was eth0)
delete interfaces ethernet eth1 address 10.50.1.3/24
set interfaces ethernet eth2 description 'INTERNAL-TRUNK'
set interfaces ethernet eth2 address '10.50.1.3/24'

# eth3 is HA sync (new)
set interfaces ethernet eth3 description 'HA-SYNC'
set interfaces ethernet eth3 address '10.50.99.2/30'

# Update VRRP interface references
set high-availability vrrp group MGMT interface 'eth2'
set high-availability vrrp group MGMT hello-source-address '10.50.1.3'

commit
save

# Verify VRRP is BACKUP
show vrrp
# All groups should show BACKUP

# On vyos-01 — reduce VRRP priority to trigger failover
configure
set high-availability vrrp group MGMT priority 50
commit

# Verify vyos-02 becomes MASTER
ssh vyos-02 "show vrrp"
# Expected: MASTER

# On kvm-01
sudo virsh shutdown vyos-01

# Detach old interfaces
sudo virsh detach-interface vyos-01 bridge --mac $(sudo virsh domiflist vyos-01 | awk 'NR==3 {print $5}') --persistent
sudo virsh detach-interface vyos-01 bridge --mac $(sudo virsh domiflist vyos-01 | awk 'NR==3 {print $5}') --persistent

# Attach new interfaces in correct order
sudo virsh attach-interface vyos-01 bridge br-wan --model virtio --persistent
sudo virsh attach-interface vyos-01 bridge br-ispb --model virtio --persistent
sudo virsh attach-interface vyos-01 bridge br-mgmt --model virtio --persistent
sudo virsh attach-interface vyos-01 bridge br-ha --model virtio --persistent

# Start
sudo virsh start vyos-01

# SSH to vyos-01
configure

# eth2 is now LAN (was eth1)
delete interfaces ethernet eth1 address 10.50.1.2/24
set interfaces ethernet eth2 description 'INTERNAL-TRUNK'
set interfaces ethernet eth2 address '10.50.1.2/24'

# eth1 is ISP-B (new)
set interfaces ethernet eth1 description 'WAN-B-ISPB'
set interfaces ethernet eth1 address 'dhcp'

# eth3 is HA sync (new)
set interfaces ethernet eth3 description 'HA-SYNC'
set interfaces ethernet eth3 address '10.50.99.1/30'

# Update VRRP interface references
set high-availability vrrp group MGMT interface 'eth2'
set high-availability vrrp group MGMT hello-source-address '10.50.1.2'

# Restore priority
set high-availability vrrp group MGMT priority 200

commit
save

# On both nodes
show interfaces
show vrrp
ping 10.50.99.2  # From vyos-01 — HA link test
ping 10.50.99.1  # From vyos-02 — HA link test

# vyos1 base interfaces
set interfaces ethernet eth0 description 'WAN-A-ISPA'
set interfaces ethernet eth0 vrf 'WAN-A'
set interfaces ethernet eth0 address 'dhcp'

set interfaces ethernet eth1 description 'WAN-B-ISPB'
set interfaces ethernet eth1 vrf 'WAN-B'
set interfaces ethernet eth1 address 'dhcp'

set interfaces ethernet eth2 description 'INTERNAL-TRUNK'
set interfaces ethernet eth2 address '10.50.10.11/24'

set interfaces ethernet eth3 description 'HA-SYNC'
set interfaces ethernet eth3 address '10.50.99.1/30'

# Define VRFs
set vrf name WAN-A table '100'
set vrf name WAN-B table '200'

# Assign WAN interfaces to VRFs
set interfaces ethernet eth0 vrf 'WAN-A'
set interfaces ethernet eth1 vrf 'WAN-B'

# Default routes within each VRF (populated by DHCP automatically
# when interface is in the VRF — verify with 'show vrf WAN-A route')

# Static fallback: vyos1 falls back to WAN-B if WAN-A fails
# (lower metric = preferred, higher = fallback)
set protocols static vrf WAN-A route '::/0' interface 'eth0' metric '10'
set protocols static vrf WAN-B route '::/0' interface 'eth1' metric '20'

# vyos2: WAN-B is primary (metric 10), WAN-A is fallback (metric 20)

# Generate keypairs first (run on each node):
# run generate pki wireguard key-pair
# Submit the public-key to iFog and ROUTE64 portals

# vyos1 — iFog tunnel (wg0), sourced via VRF WAN-A
set interfaces wireguard wg0 description 'iFog-Fremont-vyos1'
set interfaces wireguard wg0 address '<ifog-v1-link-addr>/127'
set interfaces wireguard wg0 private-key '<vyos1-wg0-privkey>'
set interfaces wireguard wg0 vrf 'WAN-A'
set interfaces wireguard wg0 peer IFOG public-key '<ifog-pubkey>'
set interfaces wireguard wg0 peer IFOG endpoint '<ifog-v1-endpoint>'
set interfaces wireguard wg0 peer IFOG allowed-ips '::/0'
set interfaces wireguard wg0 peer IFOG persistent-keepalive '25'

# vyos1 — ROUTE64 tunnel (wg1), sourced via VRF WAN-B
set interfaces wireguard wg1 description 'ROUTE64-vyos1'
set interfaces wireguard wg1 address '<r64-v1-link-addr>/127'
set interfaces wireguard wg1 private-key '<vyos1-wg1-privkey>'
set interfaces wireguard wg1 vrf 'WAN-B'
set interfaces wireguard wg1 peer R64 public-key '<r64-pubkey>'
set interfaces wireguard wg1 peer R64 endpoint '<r64-v1-endpoint>'
set interfaces wireguard wg1 peer R64 allowed-ips '::/0'
set interfaces wireguard wg1 peer R64 persistent-keepalive '25'

# vyos2: mirror with vyos2 keypairs/link-addrs
# wg0 (iFog) → VRF WAN-B (vyos2 primary WAN is ISP-B)
# wg1 (ROUTE64) → VRF WAN-A

# vyos1
set protocols bgp system-as '{domus-asn}'
set protocols bgp parameters router-id '10.50.1.11'
set protocols bgp parameters bestpath as-path multipath-relax

# iFog peer (wg0 link address)
set protocols bgp neighbor '<ifog-v1-peer-addr>' remote-as '{ifog-asn}'
set protocols bgp neighbor '<ifog-v1-peer-addr>' description 'iFog-BGPTunnel'
set protocols bgp neighbor '<ifog-v1-peer-addr>' timers holdtime '30'
set protocols bgp neighbor '<ifog-v1-peer-addr>' timers keepalive '10'
set protocols bgp neighbor '<ifog-v1-peer-addr>' bfd
set protocols bgp neighbor '<ifog-v1-peer-addr>' address-family ipv6-unicast
set protocols bgp neighbor '<ifog-v1-peer-addr>' address-family ipv6-unicast \
  route-map export 'RM-OUT-IFOG'
set protocols bgp neighbor '<ifog-v1-peer-addr>' address-family ipv6-unicast \
  route-map import 'RM-IN-IFOG'

# ROUTE64 peer (wg1 link address)
set protocols bgp neighbor '<r64-v1-peer-addr>' remote-as '{route64-asn}'
set protocols bgp neighbor '<r64-v1-peer-addr>' description 'ROUTE64'
set protocols bgp neighbor '<r64-v1-peer-addr>' timers holdtime '30'
set protocols bgp neighbor '<r64-v1-peer-addr>' timers keepalive '10'
set protocols bgp neighbor '<r64-v1-peer-addr>' bfd
set protocols bgp neighbor '<r64-v1-peer-addr>' address-family ipv6-unicast
set protocols bgp neighbor '<r64-v1-peer-addr>' address-family ipv6-unicast \
  route-map export 'RM-OUT-R64'
set protocols bgp neighbor '<r64-v1-peer-addr>' address-family ipv6-unicast \
  route-map import 'RM-IN-R64'

# Advertise PI prefix
set protocols bgp address-family ipv6-unicast network '{domus-prefix}'

# RPKI validation
set protocols rpki cache RIPE address 'rpki.ripe.net'
set protocols rpki cache RIPE port '3323'
set protocols rpki cache RIPE preference '1'

# vyos2: router-id 10.50.1.12
# Route-map local-pref values are flipped (see 2.5)

# Prefix list — PI block exact match only
set policy prefix-list6 PL-MY-PREFIX rule 10 action 'permit'
set policy prefix-list6 PL-MY-PREFIX rule 10 prefix '{domus-prefix}'
set policy prefix-list6 PL-MY-PREFIX rule 10 le '48'

# --- vyos1 route-maps ---

# Outbound to iFog — clean advertisement (vyos1 preferred inbound via iFog)
set policy route-map RM-OUT-IFOG rule 10 action 'permit'
set policy route-map RM-OUT-IFOG rule 10 match ipv6 address \
  prefix-list 'PL-MY-PREFIX'
set policy route-map RM-OUT-IFOG rule 20 action 'deny'

# Outbound to ROUTE64 — prepend twice (signals DFZ to prefer vyos2 via ROUTE64)
set policy route-map RM-OUT-R64 rule 10 action 'permit'
set policy route-map RM-OUT-R64 rule 10 match ipv6 address \
  prefix-list 'PL-MY-PREFIX'
set policy route-map RM-OUT-R64 rule 10 set as-path-prepend \
  '{domus-asn} {domus-asn}'
set policy route-map RM-OUT-R64 rule 20 action 'deny'

# Inbound from iFog — preferred (local-pref 150)
set policy route-map RM-IN-IFOG rule 10 action 'permit'
set policy route-map RM-IN-IFOG rule 10 set local-preference '150'

# Inbound from ROUTE64 — fallback (local-pref 100)
set policy route-map RM-IN-R64 rule 10 action 'permit'
set policy route-map RM-IN-R64 rule 10 set local-preference '100'

# --- vyos2 differences ---
# RM-OUT-IFOG: add prepend (vyos2 not preferred inbound via iFog)
# RM-OUT-R64: no prepend (vyos2 preferred inbound via ROUTE64)
# RM-IN-IFOG: local-pref 100
# RM-IN-R64: local-pref 150

# vyos1 — iFog peer (tunnel over ISP-A wired — tight timers)
set protocols bfd peer '<ifog-v1-peer-addr>' interval multiplier '3'
set protocols bfd peer '<ifog-v1-peer-addr>' interval receive '300'
set protocols bfd peer '<ifog-v1-peer-addr>' interval transmit '300'
# Detection window: 300ms x 3 = 900ms

# vyos1 — ROUTE64 peer (tunnel over ISP-B cellular — relaxed timers)
set protocols bfd peer '<r64-v1-peer-addr>' interval multiplier '4'
set protocols bfd peer '<r64-v1-peer-addr>' interval receive '500'
set protocols bfd peer '<r64-v1-peer-addr>' interval transmit '500'
# Detection window: 500ms x 4 = 2000ms — tolerates cellular jitter

# Link BFD to BGP neighbors
set protocols bgp neighbor '<ifog-v1-peer-addr>' bfd
set protocols bgp neighbor '<r64-v1-peer-addr>' bfd

# vyos2: mirror, but swap timer profiles
# iFog tunnel on vyos2 → ISP-B (cellular) → use 500/4
# ROUTE64 tunnel on vyos2 → ISP-A (wired) → use 300/3

# Before any maintenance — send NOTIFICATION with Administrative Shutdown
# Peers drain traffic gracefully before session drops
set protocols bgp neighbor '<ifog-v1-peer-addr>' shutdown

# After maintenance is complete — restore
delete protocols bgp neighbor '<ifog-v1-peer-addr>' shutdown

# vyos1: MASTER DATA (VRID 10), BACKUP VOICE (VRID 20)

set high-availability vrrp group DATA-GW interface 'eth2'
set high-availability vrrp group DATA-GW vrid '10'
set high-availability vrrp group DATA-GW address '10.50.10.1/24'
set high-availability vrrp group DATA-GW priority '150'
set high-availability vrrp group DATA-GW hello-source-address '10.50.10.11'
set high-availability vrrp group DATA-GW peer-address '10.50.10.12'
set high-availability vrrp group DATA-GW no-preempt

set high-availability vrrp group VOICE-GW interface 'eth2'
set high-availability vrrp group VOICE-GW vrid '20'
set high-availability vrrp group VOICE-GW address '10.50.20.1/24'
set high-availability vrrp group VOICE-GW priority '100'
set high-availability vrrp group VOICE-GW hello-source-address '10.50.10.11'
set high-availability vrrp group VOICE-GW peer-address '10.50.10.12'
set high-availability vrrp group VOICE-GW no-preempt

set high-availability vrrp sync-group SYNC member 'DATA-GW'
set high-availability vrrp sync-group SYNC member 'VOICE-GW'

# vyos2: DATA-GW priority 100, VOICE-GW priority 150

# Both nodes — peer address differs only

set service conntrack-sync accept-protocol 'tcp'
set service conntrack-sync accept-protocol 'udp'
set service conntrack-sync accept-protocol 'icmp'
set service conntrack-sync failover-mechanism vrrp sync-group 'SYNC'
set service conntrack-sync interface 'eth3' peer '10.50.99.2'
set service conntrack-sync event-listen-queue-size '8'
set service conntrack-sync sync-queue-size '8'

# vyos2: peer '10.50.99.1'

# NAT44 — masquerade IPv4 outbound via ISP-A WAN
set nat source rule 10 description 'IPv4 outbound via ISP-A'
set nat source rule 10 outbound-interface name 'eth0'
set nat source rule 10 source address '10.50.0.0/16'
set nat source rule 10 translation address 'masquerade'

# NAT44 — masquerade IPv4 outbound via ISP-B WAN
set nat source rule 20 description 'IPv4 outbound via ISP-B'
set nat source rule 20 outbound-interface name 'eth1'
set nat source rule 20 source address '10.50.0.0/16'
set nat source rule 20 translation address 'masquerade'

# IPv6 downstream — advertise PI /48 subnets via Router Advertisements
# Allocate a /64 per VLAN from your /48
# Example: DATA VLAN gets 2a0e:xxxx:0:10::/64
set service router-advert interface 'eth2' prefix '{domus-prefix}'

show interfaces wireguard wg0
show interfaces wireguard wg1
# latest-handshake should be < 30 seconds

ping6 <ifog-v1-peer-addr> interface wg0 count 5
ping6 <r64-v1-peer-addr> interface wg1 count 5

show vrf WAN-A route
show vrf WAN-B route
# Each should have a default route via its WAN interface

ip -6 route show vrf WAN-A
ip -6 route show vrf WAN-B

show bgp summary
# Expect: 2 neighbors, both Established, non-zero prefixes received

show bgp ipv6 unicast {domus-prefix}
# Expect: 2 paths — one via iFog, one via ROUTE64

show bgp ipv6 unicast neighbors <ifog-peer> advertised-routes
# Expect: {domus-prefix} advertised outbound

show rpki prefix-table
# Your prefix should appear as valid

# External check
curl -s "https://rpki.cloudflare.com/api/v1/validity/\
{domus-asn}/{domus-prefix}" | python3 -m json.tool

show vrrp
# vyos1: DATA-GW MASTER, VOICE-GW BACKUP
# vyos2: DATA-GW BACKUP, VOICE-GW MASTER

show conntrack-sync statistics
# internal-cache non-zero on active node
# external-cache on standby matches internal-cache count

show bfd peers
# Both peers: state Up

# Check DFZ propagation
# Visit bgp.tools → search your ASN
# Prefix should be visible from multiple vantage points

# Verify AS-path prepend is working
# Visit lg.he.net → traceroute/BGP lookup for your prefix
# Via iFog: AS34927 {domus-asn}
# Via ROUTE64 from vyos1: AS{r64} {domus-asn} {domus-asn}
# Via ROUTE64 from vyos2: AS{r64} {domus-asn}

# Establish long-lived TCP session through the network first (e.g. iperf6)
# Bring down iFog WireGuard tunnel on vyos1:
sudo ip link set wg0 down

# Expected sequence:
# t+0.9s  BFD declares iFog peer down (wired timer: 300ms x 3)
# t+1s    BGP withdraws iFog paths on vyos1
# t+3-5s  DFZ reconverges — all inbound via ROUTE64
# TCP session survives — conntrack-sync preserved state on vyos2

sudo ip link set wg0 up
# BGP re-establishes, iFog paths return

# Disable eth0 on vyos1 (simulates cable cut)
set interfaces ethernet eth0 disable
commit

# Expected:
# VRF WAN-A loses default route on vyos1
# iFog wg0 tunnel keepalives stop → BFD triggers
# VRF WAN-B becomes active path for vyos1 tunnels
# vyos2 unaffected — already on ISP-B primary

delete interfaces ethernet eth0 disable
commit

# Power off or pause vyos1 KVM VM
virsh suspend vyos1

# Verify on vyos2:
show vrrp         # Both VRIDs should be MASTER on vyos2
show bgp summary  # Both sessions still Established (vyos2 independent)
show conntrack-sync statistics  # Sessions flowing from external-cache

virsh resume vyos1
# vyos1 re-establishes BGP, VRRP transitions back to split-master

# Reboot T-Mobile gateway (power cycle)
# WireGuard tunnel over ISP-B will drop for ~30-60s during reboot

# Expected on vyos2:
# BFD on ROUTE64 peer (ISP-B path): 500ms x 4 = 2s detection
# BGP withdraws ROUTE64 paths on vyos2
# iFog (via ISP-A fallback VRF) becomes sole path
# After T-Mobile recovers: WireGuard re-handshakes,
# BGP re-establishes, ROUTE64 paths return

# BGP
show bgp summary
show bgp ipv6 unicast
show bgp ipv6 unicast neighbors <addr> advertised-routes
show bgp ipv6 unicast neighbors <addr> received-routes

# Graceful shutdown (before maintenance)
set protocols bgp neighbor <peer> shutdown
delete protocols bgp neighbor <peer> shutdown

# VRRP
show vrrp
show vrrp detail

# conntrack-sync
show conntrack-sync statistics
show conntrack-sync status

# BFD
show bfd peers
show bfd peers detail

# VRF
show vrf
show vrf WAN-A route
show vrf WAN-B route

# WireGuard
show interfaces wireguard wg0
show interfaces wireguard wg1

# IPv6 routing
show ipv6 route
show ipv6 route bgp

# nftables
sudo nft list ruleset | grep -A5 ip6

# RPKI
show rpki prefix-table
show rpki cache-connection

# Port status with bridge membership and link state
ip -br link show | awk '/^(eno|br-)/ {printf "%-12s %-10s %s\n", $1, $2, $3}'

# Which ports are in which bridges
bridge link show | awk '{print $2, "→", $NF}'

# Full picture: port, state, speed, bridge membership
for port in eno{1..8}; do
  state=$(cat /sys/class/net/$port/operstate 2>/dev/null || echo "missing")
  speed=$(cat /sys/class/net/$port/speed 2>/dev/null || echo "?")
  master=$(ip link show $port 2>/dev/null | awk -F'master ' '{print $2}' | awk '{print $1}')
  printf "%-6s  %-6s  %5sMbps  %s\n" "$port" "$state" "$speed" "${master:-[none]}"
done