Merge pull request #385 from ebmdatalab/dokku

Move deployment to dokku

.github/workflows/main.yml

@@ -1,19 +1,26 @@
 ---
 name: CI
 
+env:
+  IMAGE_NAME: bennettbot
+  PUBLIC_IMAGE_NAME: ghcr.io/ebmdatalab/bennettbot
+  REGISTRY: ghcr.io
+  SSH_AUTH_SOCK: /tmp/agent.sock
+
 on:
   push:
+  workflow_dispatch:
 
 jobs:
   check:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: opensafely-core/setup-action@v1
         with:
           install-just: true
-          python-version: "3.8"
+          python-version: "3.9"
           cache-dependency-path: requirements.*.txt
       - name: Check formatting, linting and import sorting
         run: just check
@@ -22,12 +29,72 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: opensafely-core/setup-action@v1
         with:
           install-just: true
-          python-version: "3.8"
+          python-version: "3.9"
           cache-dependency-path: requirements.*.txt
       - name: Run tests
         run: |
           just test
+
+
+  docker-check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: opensafely-core/setup-action@v1
+        with:
+          install-just: true
+      - name: Check formatting, linting and import sorting on docker
+        run: just docker/check
+
+
+  docker-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: opensafely-core/setup-action@v1
+        with:
+          install-just: true
+      - name: Run tests in docker
+        run: just docker/test
+
+  deploy:
+    needs: [check, test, docker-check, docker-test]
+
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+
+    if: github.ref == 'refs/heads/main'
+
+    concurrency: deploy-production
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: opensafely-core/setup-action@v1
+        with:
+          install-just: true
+
+      - name: Build docker image
+        run: |
+            just docker/build prod
+
+      - name: Publish docker image
+        run: |
+            echo ${{ secrets.GITHUB_TOKEN }} | docker login $REGISTRY -u ${{ github.actor }} --password-stdin
+            docker tag $IMAGE_NAME $PUBLIC_IMAGE_NAME:latest
+            docker push $PUBLIC_IMAGE_NAME:latest
+
+      - name: Deploy image
+        run: |
+            ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+            ssh-add - <<< "${{ secrets.DOKKU3_DEPLOY_SSH_KEY }}"
+            SHA=$(docker inspect --format='{{index .RepoDigests 0}}' $PUBLIC_IMAGE_NAME:latest)
+            ssh -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" [email protected] git:from-image bennettbot $SHA

.gitignore

@@ -1,6 +1,8 @@
 environment
 .env
 workspace/techsupport/ooo.json
+gcp-credentials.json
+.bot_startup_check
 
 # System files
 *.py[co]

DEPLOY.md

@@ -1,9 +1,13 @@
-# Deployment
+## Deployment
 
-ebmbot is deployed on smallweb1 and is managed by systemd.
+Deployment uses `dokku` and requires the environment variables defined
+[below](#configure-app-environment-variables).
 
-* To deploy: `fab deploy`
-* To view logs: `sudo journalctl -u app.ebmbot.* -b -f`
+It is deployed to our `dokku3` instance [automatically via GitHub actions](#deployment-via-github-actions).
+
+
+It runs as a single dokku app named `bennettbot`, with multiple processes for each
+service (bot, dispatcher and webserver) as defined in the `Procfile`.
 
 ## Creating the Slack app
 (This should only need to be done once).
@@ -71,10 +75,58 @@ events.  We need to add some more:
 If you update any scopes after installing the app, you'll need to reinstall it (slack will
 usually prompt for this).
 
-### Set environment variables
+## Configuring the dokku app
+(This should only need to be done once).
+
+### Create app
+
+```sh
+$ dokku apps:create bennettbot
+```
+
+### Create storage for sqlite db, logs and fabric job workspaces
+```sh
+$ mkdir -p /var/lib/dokku/data/storage/bennettbot/logs
+$ mkdir -p /var/lib/dokku/data/storage/bennettbot/workspace
+$ dokku storage:mount bennettbot /var/lib/dokku/data/storage/bennettbot/:/storage
+```
+
+### Set up the user
+
+This is done using the ansible playbook in `https://github.com/ebmdatalab/sysadmin/blob/main/infra`.
+
+See <https://github.com/ebmdatalab/sysadmin/blob/main/infra/README.md> for more details.
+
+To run just the bennettbot tasks:
+
+```sh
+just test dokku3 --tags bennettbot
+```
+And if all looks OK:
+
+```sh
+just apply dokku3 --tags bennettbot
+```
+
+This will create the ebmbot user on dokku3 and chown any mounted volumes.
+
+
+### Create ssh key and mount ebmbot user's home directory
+
+Create an ssh key for the ebmbot user, in the usual $HOME/.ssh/ location.
+
+Mount the user's home directory into the app.
+
+```sh
+$ dokku storage:mount bennettbot /home/ebmbot/:/home/ebmbot
+```
+
+Add the ebmbot user's key to any servers that it requires access to
+(i.e. any jobs that run `fab` commands).
+
+### Configure app environment variables
 
-Copy `dotenv-sample` to `.env` and update with relevant environment variables. See also
-comments in `ebmbot/settings.py`.
+See also comments in `ebmbot/settings.py`.
 
 The following slack environment variables need to be set:
 - `SLACK_LOGS_CHANNEL`: channel where scheduled job notifications will be posted
@@ -92,9 +144,75 @@ OpenPrescribing, and are configured at https://github.com/ebmdatalab/openprescri
 
 The following environment variable allows the bot to authenticate with Github to retrieve
 project information.
-- `DATA_TEAM_GITHUB_API_TOKEN`: Note that this must be a classic PAT (not fine-grained) and
-  needs the `repo` and `read:project` scope
+- `DATA_TEAM_GITHUB_API_TOKEN`: Note that this must be a classic PAT (not fine-grained)
+  and needs the `repo` and `read:project` scope
 
 This is the path to credentials for the [email protected]
 service account:
 - `GCP_CREDENTIALS_PATH`
+
+The path for logs; set this to a directory in the dokku mounted storage so the logs
+persist outside of the containers.
+- `LOGS_DIR`
+Also set the alias for the logs dir to the location of the mounted volume on the host,
+for error reporting
+- `HOST_LOGS_DIR`
+
+The path for the sqlite db file; set this to a file in the dokku mounted storage
+- `DB_PATH`
+
+The path for workspaces that are created by the job (i.e. fabric jobs that fetch
+the fabfile for running the commands). Set this to a directory in the dokku mounted
+storage that the docker user will have write access to.
+- `FAB_WORKSPACE_DIR`
+
+Path for file created after bot startup (used in the bot healthcheck in `app.json`).
+- `BOT_CHECK_FILE`
+
+Set each env varible with:
+```sh
+$ dokku config:set bennettbot ENVVAR_NAME=value
+```
+
+e.g.
+```sh
+$ dokku config:set bennettbot LOGS_DIR=/storage/logs
+$ dokku config:set bennettbot HOST_LOGS_DIR=/var/lib/dokku/data/storage/bennettbot/logs
+$ dokku config:set bennettbot DB_PATH=/storage/bennettbot.db
+$ dokku config:set bennettbot FAB_WORKSPACE_DIR=/storage/workspace
+$ dokku config:set bennettbot BOT_CHECK_FILE=/storage/.bot_startup_check
+```
+
+### Map port 9999 for incoming github hooks
+https://dokku.com/docs/networking/port-management/
+
+dokku ports:add bennettbot http:9999:9999
+
+
+## Deployment via GitHub Actions
+
+Merges to the `main` branch will trigger an auto-deploy via GitHub actions.
+
+Note this deploys by building the prod docker image (see `docker/docker-compose.yaml`) and using the dokku [git:from-image](https://dokku.com/docs/deployment/methods/git/#initializing-an-app-repository-from-a-docker-image) command.
+
+
+### Manually deploying
+
+To deploy manually:
+
+```
+# build prod image locally
+just docker/build prod
+
+# tag image and push
+docker tag bennettbot ghcr.io/ebmdatalab/bennettbot:latest
+docker push ghcr.io/ebmdatalab/bennettbot:latest
+
+# get the SHA for the latest image
+SHA=$(docker inspect --format='{{index .RepoDigests 0}}' ghcr.io/ebmdatalab/bennettbot:latest)
+```
+
+On dokku3, as the `dokku` user:
+```
+$ dokku git:from-image bennettbot <SHA>
+```

DEVELOPERS.md

@@ -24,25 +24,76 @@ just #  shortcut for just --list
 
 Set up a local development environment with:
 ```
-just dev_setup
+just devenv
 ```
 
+This will create a virtual environment, install requurements and create a
+`.env` file by copying `dotenv-sample`; update it as necessary with valid dev
+values for environment variables.
+
+
 ### Set up a test slack workspace and app
 
 Create a [new slack workspace](https://slack.com/intl/en-gb/get-started#/createnew) to use for testing.
 
 Follow the steps to [create a slack app with the required scopes](DEPLOY.md#creating-the-slack-app)
 and install it into your test workspace.
 
+
 ### Set up environment
-* Copy environment: `cp dotenv-sample .env`
 
 Edit `.env` with the slack app tokens etc for the test slack app.
 
 
+## Run in docker
+
+### Build docker image
+
+This builds the dev image by default:
+
+```
+just docker/build
+```
+
+### Run checks
+
+```
+just docker/check`
+```
+
+### Run tests
+```
+just docker/test
+```
+
+### Run all services
+
+Run all 3 services (bot, dispatcher and webserver) in separate docker
+containers.
+
+```
+just docker/run-all
+```
+
+### Stop/remove containers
+
+Stop running service container:
+
+```
+just docker/stop-all
+```
+
+Stop running services and remove containers:
+
+```
+just docker/rm-all
+```
+
+## Run locally
+
 ### Run checks
 
-Run linter and formatted:
+Run linter and formatter:
 ```
 just check
 ```

Procfile

@@ -0,0 +1,4 @@
+bot: python -m ebmbot.bot
+dispatcher: python -m ebmbot.dispatcher
+web: python -m ebmbot.webserver
+release: rm -f /storage/.bot_startup_check

app.json

@@ -0,0 +1,26 @@
+{
+    "healthchecks": {
+        "web": [
+            {
+                "type":        "startup",
+                "name":        "web check",
+                "description": "Checking if the app responds to the /check endpoint",
+                "path":        "/check/",
+                "attempts": 10,
+                "wait": 30,
+                "timeout": 60
+            }
+        ],
+        "bot": [
+            {
+                "type":        "startup",
+                "name":        "bot check",
+                "description": "Checking if the bot start up file exists",
+                "command": ["/app/bot_check.sh"],
+                "attempts": 10,
+                "wait": 30,
+                "timeout": 60
+            }
+        ]
+      }
+  }