Enable GLB Healthcheck to announce VIP/Bind status into BGP

Makefile

@@ -4,7 +4,8 @@ mkdeb:
 	cd src/glb-director && script/cibuild
 
 clean:
-	make -C src/glb-redirect clean
+	make -C src/glb-redirect clean && rm -rf ${BUILDDIR}/glb-director_*.deb
 	make -C src/glb-healthcheck clean
 	make -C src/glb-director clean
 	make -C src/glb-director/cli clean
+	make -C src/glb-director/ftctl clean

Vagrantfile

@@ -6,9 +6,10 @@ Vagrant.configure("2") do |config|
 
   config.vm.synced_folder "src/glb-wireshark-dissector/", "/home/vagrant/.config/wireshark/plugins/glb-wireshark-dissector", type: 'rsync'
 
-  config.vm.provision "shell", inline: <<-SHELL
+  config.vm.provision "shell", name: "Base Host Tool Installation", inline: <<-SHELL
     apt-get update
-    DEBIAN_FRONTEND=noninteractive apt-get install -y tcpdump net-tools tshark build-essential libxtables-dev linux-headers-$(uname -r) python-pip jq bird curl
+    DEBIAN_FRONTEND=noninteractive apt-get install -y tcpdump net-tools tshark build-essential libxtables-dev python-pip jq bird curl
+    DEBIAN_FRONTEND=noninteractive apt-get install -y linux-headers-$(uname -r)
     groupadd wireshark || true
     usermod -a -G wireshark vagrant || true
     chgrp wireshark /usr/bin/dumpcap
@@ -21,7 +22,7 @@ Vagrant.configure("2") do |config|
     v.vm.network "private_network", ip: "192.168.50.2", virtualbox__intnet: "glb_datacenter_network", :mac=> "001122334455"
     v.vm.hostname = "router"
 
-    v.vm.provision "shell", inline: <<-SHELL
+    v.vm.provision "shell", name: "Enable IPv4 forwarding", inline: <<-SHELL
       if ! grep -q '^net.ipv4.ip_forward' /etc/sysctl.conf; then
         echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
         sysctl -w net.ipv4.ip_forward=1
@@ -35,7 +36,7 @@ Vagrant.configure("2") do |config|
     v.vm.network "private_network", ip: "192.168.40.2", virtualbox__intnet: "glb_user_network"
     v.vm.hostname = "user"
 
-    v.vm.provision "shell", inline: <<-SHELL
+    v.vm.provision "shell", name: "Configure routes", inline: <<-SHELL
       /vagrant/script/helpers/configure-vagrant-user.sh
 
       ip addr add 192.168.40.50/24 dev eth1 || true
@@ -66,7 +67,7 @@ Vagrant.configure("2") do |config|
         vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
       end
 
-      v.vm.provision "shell", inline: <<-SHELL
+      v.vm.provision "shell", name: "Huge pages configuration", inline: <<-SHELL
         mkdir -p /mnt/huge
         if ! grep -q 'hugetlbfs' /etc/fstab; then
           echo 'hugetlbfs /mnt/huge hugetlbfs mode=1770 0 0' >>/etc/fstab
@@ -78,40 +79,57 @@ Vagrant.configure("2") do |config|
         fi
       SHELL
 
+      v.vm.provision "shell", name: "Update Kernel", inline: <<-SHELL
+        apt-get update
+        DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-amd64
+        DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+      SHELL
+
+      # reload VM to pick up the new kernel so the correct header files can be installed
+      v.vm.provision :reload
+      v.vm.synced_folder ".", "/vagrant", type: "rsync"
+
       # install DPDK et al.
-      v.vm.provision "shell", inline: <<-SHELL
-        apt-get install -y apt-transport-https curl
+      v.vm.provision "shell", run: "always", name: "Install DPDK and enable DPDK modules", inline: <<-SHELL
         echo 'deb http://ftp.debian.org/debian stretch-backports main' >/etc/apt/sources.list.d/backports.list
         apt-get update
         curl -s https://packagecloud.io/install/repositories/github/unofficial-dpdk-stable/script.deb.sh | sudo bash
-        apt-get install -y linux-headers-`(uname -r)` # dpdk requires this for the current kernel, but won't block if not installed
-        apt-get install -y python-pip dpdk-dev=17.11.1-6 dpdk=17.11.1-6 dpdk-rte-kni-dkms dpdk-igb-uio-dkms libjansson-dev
-        apt-get install -y valgrind vim tcpdump clang golang
-
+        DEBIAN_FRONTEND=noninteractive apt-get install -y linux-headers-`(uname -r)` # dpdk requires this for the current kernel, but won't block if not installed
+        DEBIAN_FRONTEND=noninteractive apt-get install -y python-pip dpdk-dev=17.11.1-6 dpdk=17.11.1-6 dpdk-rte-kni-dkms dpdk-igb-uio-dkms libjansson-dev
+        DEBIAN_FRONTEND=noninteractive apt-get install -y valgrind vim tcpdump clang
+
+        #Install an updated version of Golang that matches Debian Latest
+        curl -s https://dl.google.com/go/go1.11.6.linux-amd64.tar.gz -o /tmp/go1.11.6.linux-amd64.tar.gz
+        tar -C /usr/local -xzf /tmp/go1.11.6.linux-amd64.tar.gz
+        ln -s /usr/local/go/bin/go /usr/bin/
+        ln -s /usr/local/go/bin/godoc /usr/bin/
+        ln -s /usr/local/go/bin/gofmt /usr/bin/
+
+        echo "Setting modules to load"
         echo 'rte_kni' >/etc/modules-load.d/dpdk
         echo 'igb_uio' >>/etc/modules-load.d/dpdk
       SHELL
 
-      v.vm.provision "shell", run: "always", inline: <<-SHELL
+      v.vm.provision "shell", run: "always", name: "Modprobe DPDK modules", inline: <<-SHELL
         modprobe rte_kni
         modprobe igb_uio
       SHELL
 
       if install_example_setup
         # example setup
-        v.vm.provision "shell", run: "always", inline: <<-SHELL
+        v.vm.provision "shell", run: "always", name: "Install and configure GLB director", inline: <<-SHELL
           ifdown eth1
           dpdk-devbind --bind=igb_uio eth1
           dpdk-devbind --status
 
-          apt install /vagrant/tmp/build/glb-director_*.deb
-          apt install /vagrant/tmp/build/glb-healthcheck_*.deb
+          DEBIAN_FRONTEND=noninteractive apt install /vagrant/tmp/build/glb-director_*.deb
+          DEBIAN_FRONTEND=noninteractive apt install /vagrant/tmp/build/glb-healthcheck_*.deb
 
           /vagrant/script/helpers/configure-vagrant-director.sh "#{ipv4_addr}"
         SHELL
       else
         # test setup
-        v.vm.provision "shell", run: "always", inline: <<-SHELL
+        v.vm.provision "shell", run: "always", name: "Configure IPv6", inline: <<-SHELL
           ip addr add #{ipv6_addr} dev eth1 || true
         SHELL
       end
@@ -124,12 +142,22 @@ Vagrant.configure("2") do |config|
 
       v.vm.network "private_network", ip: ipv4_addr, virtualbox__intnet: "glb_datacenter_network"
 
-      v.vm.provision "shell", inline: <<-SHELL
+      v.vm.provision "shell", name: "Update Kernel", inline: <<-SHELL
+        apt-get update
+        DEBIAN_FRONTEND=noninteractive apt-get install -y linux-image-amd64
+        DEBIAN_FRONTEND=noninteractive apt-get install -y apt-transport-https curl
+      SHELL
+
+      # reload VM to pick up the new kernel so the correct header files can be installed
+      v.vm.provision :reload
+      v.vm.synced_folder ".", "/vagrant", type: "rsync"
+
+      v.vm.provision "shell", name: "Install hello world HTML", inline: <<-SHELL
         DEBIAN_FRONTEND=noninteractive apt-get install -y nginx
         echo "hello world from #{name} via GLB" >/var/www/html/index.html
       SHELL
 
-      v.vm.provision "shell", run: "always", inline: <<-SHELL
+      v.vm.provision "shell", run: "always", name: "Configure Tunnel Interfaces", inline: <<-SHELL
         modprobe fou
         modprobe sit
         ip link set up dev tunl0 || true

docs/README.md

@@ -8,6 +8,7 @@
  * [glb-director component configuration options](./setup/glb-director-configuration.md) - configuration options for the GLB Director and discussion of NIC/Port and CPU Core configuration.
  * [Forwarding table configuration & healthchecks](./setup/forwarding-table-config.md) - Configuring the GLB forwarding table, and the director->proxy healthcheck process.
  * [Backend proxy server setup](./setup/backend-proxy-setup.md) - Configuring the backend TCP/proxy tier servers.
+ * [Integrating with GoBGP](./setup/gobgp-integration.md) - Optional feature for announcing VIPs/Binds via GoBGP
 
 Some notable known limitations / design decisions of the current implementation:
  * The datacenter internal MTU is expected to be large enough to encapsulate any user packet inside a GUE header. We use jumbo frames (9000+ MTU) within the datacenter with a transit/internet MTU of 1500. GLB Director will not fragment packets if they are too large.

docs/setup/forwarding-table-config.md

@@ -122,8 +122,43 @@ If `glb-healthcheck` is used, a backend will be considered down if either the `g
     "src": "/etc/glb/forwarding_table.src.json",
     "dst": "/etc/glb/forwarding_table.checked.json"
   },
-  "reload_command": "glb-director-cli build-config /etc/glb/forwarding_table.checked.json /etc/glb/forwarding_table.checked.bin && systemctl reload glb-director"
+  "reload_command": "glb-director-cli build-config /etc/glb/forwarding_table.checked.json /etc/glb/forwarding_table.checked.bin && systemctl reload glb-director",
+  "gobgp_config": {
+    "address": "127.0.0.1:50051",
+    "nexthop": "192.168.50.6",
+    "nexthop_ipv6": "fdb4:98ce:52d4::1",
+    "communities": [ "65000:1234", "65000:5678" ], 
+  },
 }
 ```
+### `forwarding_table` 
+
+_required_
+
+specifies which forwarding tables are used for glb-healtcheck
+
+`src` the source forwarding table that is created by the user
+`dst` the destination forwarding table that is generated by glb-healthcheck
+
+### `reload_command` 
+
+_required_
 
 This instructs the healthchecker to load `/etc/glb/forwarding_table.src.json`, perform any checks defined inside it, and keep a `/etc/glb/forwarding_table.checked.json` up to date with valid/live health state. Any time it changes the `dst` file, it also runs the reload command (which in this case compiles the table and reloads the director to pick up those changes).
+
+### `gobgp_config`
+
+_optional_
+
+This instructs the health checker to integrate with a [GoBGP](https://github.com/osrg/gobgp) daemon. It communicates over
+gRPC to announce the status of the GLB binds or VIPs. When a bind/VIP is healthy the path will be announced with the configured
+next hop address. The setting `nexthop` is used for IPv4 bind/VIPs and `nexthop_ipv6` is used for IPv6 bind/VIPs. If the protocol
+specific next hop is not configured, then it will not be able to announce the route. If the `address` is not configured (default) this feature will not be enabled.
+
+`address` _required_ The gRPC address for the GoBGP daemon. This typically is `172.0.0.1:50051` assuming you have the BGP daemon installed locally.
+
+`nexthop` _required_ The IPv4 next hop address to use for route announcements. This is required if you have IPv4 bind/VIPs.
+
+`nexthop_ipv6` _optional_ The IPv6 next hop address to use for route announcements. This is only required if you are using IPv6 binds/VIPs.
+
+`communities` _optional_ A list of BGP communities you wish to use for tagging routes

docs/setup/gobgp-integration.md

@@ -0,0 +1,146 @@
+# GoBGP integration
+
+When using Github Load Balancer (GLB), you need to inform the upstream network
+devices about which bind or VIPs that your services are listening on. In the provided
+example Vagrant topology the BIRD BGP daemon is used for announcing routes for the available
+bind/VIPs configured on GLB. This is an excellent solution for integrating with upstream routers
+and announcing available VIPs. The drawback is that this is a static announcement that provides
+no validation of the actual VIP is healthy.
+
+Within the glb-healthcheck service it provides all the available health checking to understand the
+state for all the backends associated with a VIP. The glb-healthcheck service can to talk to a local
+instance of [GoBGP](https://github.com/osrg/gobgp) and use that as the method for announcing the VIPs
+to the upstream routers via BGP. This allows the state of the binds to determine if a route should be
+announced or not. This prevents the GLB node from potentially black holing or dropping traffic to VIPs
+that are not ready to serve traffic.
+
+## Enabling GoBGP integration
+
+To enable the integration with GoBGP you must configure the glb-healthcheck service to use the local GoBGP
+daemon. The required configurtion must be set in the `healthcheck.conf` file, more details for the configuration
+options can be found [here](./setup/forwarding-table-config.md).
+
+## Running GoBGP
+
+You must install the GoBGP daemon locally so glb-healthcheck is able to communicate with it. You can download the
+[latest release](https://github.com/osrg/gobgp/releases) bundle. It includes gobgpd (daemon) and gobgp (tool to interact with the
+daemon) in its release file. Place the binaries on your system and set it up as a systemd service.
+
+### Systemd service definition
+
+To enable GoBGP as a systemd service this service definition can be placed in `/lib/systemd/system/gobgp.service`. Once 
+added you will need to run `systemctl daemon-reload` to pickup the new service. The service definition below requires there
+to be a local `gobgpd` user for the service to run as. This prevents it from needing to run as `root`.
+
+```
+[Unit]
+Description=GoBGP Daemon
+After=network.target network-online.target
+Wants=network.target
+
+[Service]
+Type=simple
+Restart=always
+Environment="PATH=/usr/local/bin:/usr/bin:/bin"
+ExecStart=/usr/local/sbin/gobgpd --api-hosts=127.0.0.1:50051 -f /etc/gobgp/config.toml
+ExecReload=/bin/kill -HUP $MAINPID
+KillSignal=TERM
+User=gobgpd
+WorkingDirectory=/etc/gobgp
+TimeoutStopSec=3
+
+[Install]
+WantedBy=multi-user.target
+```
+
+### Listening on low port without running as root
+
+To enable the service to listen on TCP port 179 without running as root we can enable the capacity directly on the binary.
+
+```
+# setcap cap_net_bind_service=+ep /usr/local/sbin/gobgpd
+# chown gobgpd:gobgpd /usr/local/sbin/gobgpd
+```
+
+### Base GoBGP Configuration
+
+Below is a simple base configuration that will announce any routes that are added to GoBGP to its upstream
+neighbor. There are many [configuration options](https://github.com/osrg/gobgp/blob/master/docs/sources/configuration.md)
+available for GoBGP. Currently within glb-healthcheck GoBGP is only used for IPv4 or IPv6 route announcements. In the future
+other features of GoBGP could be used such as flowspec integration.
+
+```
+[global.config]
+    as = 65000
+    router-id = "1.1.1.1"
+
+# configure each remote neighbor
+[[neighbors]]
+    [neighbors.config]
+        neighbor-address = "172.31.2.168"
+        peer-as = 65001
+    [neighbors.apply-policy.config] # only export routes, do not recieve them
+        export-policy-list = ["export-all-ipv4", "export-all-ipv6"]
+        default-export-policy = "accept-route"
+    [[neighbors.afi-safis]] # enable upstream IPv4
+        [neighbors.afi-safis.config]
+        afi-safi-name = "ipv4-unicast"
+    [[neighbors.afi-safis]] # enable upstream IPv6
+        [neighbors.afi-safis.config]
+        afi-safi-name = "ipv6-unicast"
+
+# Prefix list to match any IPv4 address
+[[defined-sets.prefix-sets]]
+    prefix-set-name = "any-ipv4"
+    [[defined-sets.prefix-sets.prefix-list]]
+        ip-prefix = "0.0.0.0/0"
+
+# Prefix list to match any IPv6 address
+[[defined-sets.prefix-sets]]
+    prefix-set-name = "any-ipv6"
+    [[defined-sets.prefix-sets.prefix-list]]
+        ip-prefix = "::/0"
+
+# Export policy to export any IPv4 addresses
+[[policy-definitions]]
+    name = "export-all-ipv4"
+    [[policy-definitions.statements]]
+        name = "export-ipv4-all-match"
+        [policy-definitions.statements.conditions.match-prefix-set]
+        prefix-set = "any-ipv4"
+        [policy-definitions.statements.actions]
+        route-disposition = "accept-route"
+
+# Export policy to export any IPv6 addresses
+[[policy-definitions]]
+    name = "export-all-ipv6"
+    [[policy-definitions.statements]]
+        name = "export-ipv6-all-match"
+        [policy-definitions.statements.conditions.match-prefix-set]
+        prefix-set = "any-ipv6"
+        [policy-definitions.statements.actions]
+        route-disposition = "accept-route"
+```
+
+## How it works
+
+After every run of the glb-healthcheck the GoBGP integration will take the results and determine which binds to announce. It
+will only announce routes for binds with healthy backends. The route will be refreshed for every successful health check cycle.
+Once all of the backends for a specific bind have failed health checks, the route will be withdrawn automatically. 
+
+If a bind is removed from one of the forwarding tables it will automatically be removed at the end of the next run. The state
+of the routes is stored within glb-healthcheck. In the event that you want to gracefully retire a VIP you can drain each backend. Once 
+the backends are all set to `inactive` state then the routes will be automatically withdrawn. This prevents accidently dropping traffic
+by sending traffic to a bind in which all backends are set to `inactive`.
+
+## Cleaning up routes
+
+If any announced routes get out of sync you can manually remove them from the GoBGP daemon. You can restart GoBGP and this will
+remove any routes loaded. This will be disruptive as it will force your instance of GoBGP to reset all of its neighbor connections.
+Alternatively you can delete routes using the control plane CLI tool `gobgp`.
+
+```
+# gobgp global rib del <prefix> [-a <address family>]
+$ gobgp global rib del 1.1.1.1/32
+$ gobgp global rib del 2001:dead:beef::1/128 -a ipv6
+```

script/Dockerfile.latest

@@ -0,0 +1,10 @@
+FROM debian:latest
+
+RUN apt-get update && apt-get install -y build-essential debhelper wget pkg-config
+
+# golang
+RUN apt-get update && apt-get install -y golang golang-glide
+
+# fpm for packaging
+RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential
+RUN gem install --no-ri --no-rdoc rake fpm

script/Dockerfile.stretch

@@ -3,20 +3,15 @@ FROM debian:stretch
 RUN echo 'deb http://ftp.debian.org/debian stretch-backports main' >>/etc/apt/sources.list
 RUN apt-get update && apt-get -y install curl
 
-# DPDK 
+# DPDK
 RUN curl -s https://packagecloud.io/install/repositories/github/unofficial-dpdk-stable/script.deb.sh | bash
-RUN apt-get update && apt-get install -y build-essential dpdk dpdk-dev wget pkg-config libjansson-dev
+RUN apt-get update && apt-get install -y build-essential dpdk dpdk-dev wget pkg-config libjansson-dev ruby ruby-dev
 
 # iptables / DKMS
 RUN apt-get update && apt-get install -y iptables-dev dkms debhelper libxtables-dev
 
-# golang
-RUN apt-get update && apt-get install -y golang golang-glide
-
-# fpm for packaging
-RUN apt-get update && apt-get install -y ruby ruby-dev rubygems build-essential
-RUN gem install --no-ri --no-rdoc rake fpm
-
 # patch DKMS for source package generation https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=832558
 ADD helpers/dkms.diff /root/dkms.diff
-RUN patch -d /usr/sbin </root/dkms.diff
+
+RUN patch -d /usr/sbin </root/dkms.diff;
+RUN gem install --no-ri --no-rdoc rake fpm