The network

[http://www.personaltelco.net/~brj/Mississippi.png]

mississippi is the default route for all clients. It decides whether to use its own internet conn, freshpot's, or the coop bookstore's. There appear to be two ways to do this:

Equal Cost Multipath

Using the routing system, we can specify a default route with multiple nexthops. When routing a packet, it picks one semi-randomly. It <i>appears</i> to handle flows properly, routing them all through the same router. This may be a side effect of caching, or it may be intentional. Either way, it works. Also, locally generated packets are routed this way.

When using this method, it seems to be necessary to ping the other routers regularly so that the main router knows they're alive.

Netfilter Marking

We can also use netfilter to tell the routing system which gateway to use. We use the random match (must use patch-o-matic on kernel) to mark each connection, and then copy the connmark to the packet mark. Then we use policy routing to select a default route based on mark. Also have to provide a default route not based on mark in order to route locally generated packets.

Comparison

TODO

net-clear.sh

iptables -t nat -F
iptables -t nat -X
iptables -t mangle -F
iptables -t mangle -X
iptables -t filter -F
iptables -t filter -X

ip route flush table main proto static
ip route flush table 201 proto static
ip route flush table 202 proto static
ip route flush table 203 proto static
ip route flush table 222 proto static

for i in 50 64 128 201 202 203 222 ; do
  ip rule del prio $i
done

net-setup-routing.sh

int_if=eth0
int_ip=10.0.0.1
int_net=10.0.0.0/24
int_mask=255.255.255.0

ext_if=eth1
ext_ip=10.1.1.2
ext_net=10.1.1.0/24
ext_mask=255.255.255.0

gw1_ip=10.1.1.1
gw1_dev=$ext_if
gw1_weight=2

gw2_ip=10.0.0.2
gw2_dev=$int_if
gw2_weight=2

gw3_ip=10.0.0.3
gw3_dev=$int_if
gw3_weight=1

sysctl net.ipv4.ip_forward=1
sysctl net.ipv4.conf.all.send_redirects=0
sysctl net.ipv4.conf.$int_if.send_redirects=0
sysctl net.ipv4.conf.all.rp_filter=0
sysctl net.ipv4.conf.$ext_if.rp_filter=0
sysctl net.ipv4.conf.$int_if.rp_filter=0

ip route add default table 222 proto static \
        nexthop via $gw1_ip dev $gw1_dev weight $gw1_weight \
        nexthop via $gw2_ip dev $gw2_dev weight $gw2_weight \
        nexthop via $gw3_ip dev $gw3_dev weight $gw3_weight

ip rule add prio 50 table main
ip rule add prio 222 table 222

iptables -t nat -A POSTROUTING -o $ext_if \
         -j SNAT --to-source $ext_ip

net-setup-netfilter-marking.sh

int_if=eth0
int_ip=10.0.0.1
int_net=10.0.0.0/24

ext_if=eth1
ext_ip=10.1.1.2

gw1_ip=10.1.1.1
gw1_dev=$ext_if
gw1_pct=40

gw2_ip=10.0.0.2
gw2_dev=$int_if
gw2_pct=66

gw3_ip=10.0.0.3
gw3_dev=$int_if

# percents have to be specified a bit carefully:
# each gateway takes that percent of the _remaining_
# unclaimed traffic.
# So if you want 2:2:1 it's not 40:40:20, but 40:66:100

sysctl net.ipv4.ip_forward=1
sysctl net.ipv4.conf.all.send_redirects=0
sysctl net.ipv4.conf.$int_if.send_redirects=0
sysctl net.ipv4.conf.all.rp_filter=0
sysctl net.ipv4.conf.$ext_if.rp_filter=0
sysctl net.ipv4.conf.$int_if.rp_filter=0

ip route add default table 201 proto static metric 1 via $gw1_ip dev $gw1_dev
ip route add default table 201 proto static metric 2 via $gw2_ip dev $gw2_dev
ip route add default table 201 proto static metric 3 via $gw3_ip dev $gw3_dev

ip route add default table 202 proto static metric 1 via $gw2_ip dev $gw2_dev
ip route add default table 202 proto static metric 2 via $gw1_ip dev $gw1_dev
ip route add default table 202 proto static metric 3 via $gw3_ip dev $gw3_dev

ip route add default table 203 proto static metric 1 via $gw3_ip dev $gw3_dev
ip route add default table 203 proto static metric 2 via $gw1_ip dev $gw1_dev
ip route add default table 203 proto static metric 3 via $gw2_ip dev $gw2_dev

# fallback for local conns.
ip route add default table 222 proto static metric 1 via $gw1_ip dev $gw1_dev
ip route add default table 222 proto static metric 2 via $gw2_ip dev $gw2_dev
ip route add default table 222 proto static metric 3 via $gw3_ip dev $gw3_dev

ip rule add prio 50 table main
ip rule add prio 201 fwmark 1 table 201
ip rule add prio 202 fwmark 2 table 202
ip rule add prio 203 fwmark 3 table 203
ip rule add prio 222 table 222

iptables -t mangle -N BALANCE
iptables -t mangle -A BALANCE \
         -m connmark --mark 0 \
         -m random --average $gw1_pct \
         -j CONNMARK --set-mark 1
iptables -t mangle -A BALANCE \
         -m connmark --mark 1 \
         -j LOG --log-prefix "marked 1: "
iptables -t mangle -A BALANCE \
         -m connmark --mark 0 \
         -m random --average $gw2_pct \
         -j CONNMARK --set-mark 2
iptables -t mangle -A BALANCE \
         -m connmark --mark 2 \
         -j LOG --log-prefix "marked 2: "
iptables -t mangle -A BALANCE \
         -m connmark --mark 0 \
         -j CONNMARK --set-mark 3
iptables -t mangle -A BALANCE \
         -j CONNMARK --restore-mark
iptables -t mangle -A PREROUTING -j BALANCE
#iptables -t mangle -A OUTPUT -j BALANCE

iptables -t nat -A POSTROUTING -o $ext_if \
         -j SNAT --to-source $ext_ip