From 68a389eaf485dfac6333b18f96882a76752b8cc3 Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Fri, 9 Feb 2024 22:39:14 +0900 Subject: [PATCH 1/9] make publish to dockerhub also publish tagged docker images if the commit to build from has a tag --- .github/workflows/publish-to-dockerhub.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-dockerhub.yml b/.github/workflows/publish-to-dockerhub.yml index 39e566bae..6fe24ffc1 100644 --- a/.github/workflows/publish-to-dockerhub.yml +++ b/.github/workflows/publish-to-dockerhub.yml @@ -32,6 +32,7 @@ jobs: DOCKERHUB_ACCOUNT: ${{ secrets.DOCKERHUB_ACCOUNT }} DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} run: | - export COUNTRY_CONFIG_VERSION=`git rev-parse --short=7 HEAD` + # Check if the current commit has a tag and use it; otherwise, use the short SHA of the HEAD commit + COUNTRY_CONFIG_VERSION=$(git describe --tags --exact-match 2>/dev/null || git rev-parse --short=7 HEAD) echo "Publishing a Docker image with a tag $COUNTRY_CONFIG_VERSION" bash build-and-push.sh && unset COUNTRY_CONFIG_VERSION From b7d89d262d9992bfcc7223d63f3eae4fa45e8027 Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Fri, 9 Feb 2024 22:45:34 +0900 Subject: [PATCH 2/9] set fetch-depth so it also gets tags --- .github/workflows/publish-to-dockerhub.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-to-dockerhub.yml b/.github/workflows/publish-to-dockerhub.yml index 6fe24ffc1..650fa95e1 100644 --- a/.github/workflows/publish-to-dockerhub.yml +++ b/.github/workflows/publish-to-dockerhub.yml @@ -19,9 +19,12 @@ jobs: - uses: actions/checkout@v2 if: github.event_name == 'workflow_dispatch' with: + fetch-depth: 2 ref: '${{ github.event.inputs.branch_name }}' + - uses: actions/checkout@v2 if: github.event_name == 'push' + - name: Login to DockerHub uses: docker/login-action@v1 with: From f3be2b34737aa7d93e858889a66d236fc649119e Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Fri, 9 Feb 2024 22:50:56 +0900 Subject: [PATCH 3/9] add block to fetch tags --- .github/workflows/publish-to-dockerhub.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-to-dockerhub.yml b/.github/workflows/publish-to-dockerhub.yml index 650fa95e1..425ea8bfa 100644 --- a/.github/workflows/publish-to-dockerhub.yml +++ b/.github/workflows/publish-to-dockerhub.yml @@ -25,6 +25,9 @@ jobs: - uses: actions/checkout@v2 if: github.event_name == 'push' + - name: Get tags + run: git fetch --tags origin + - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -36,6 +39,6 @@ jobs: DOCKERHUB_REPO: ${{ secrets.DOCKERHUB_REPO }} run: | # Check if the current commit has a tag and use it; otherwise, use the short SHA of the HEAD commit - COUNTRY_CONFIG_VERSION=$(git describe --tags --exact-match 2>/dev/null || git rev-parse --short=7 HEAD) + export COUNTRY_CONFIG_VERSION=$(git describe --tags --exact-match 2>/dev/null || git rev-parse --short=7 HEAD) echo "Publishing a Docker image with a tag $COUNTRY_CONFIG_VERSION" bash build-and-push.sh && unset COUNTRY_CONFIG_VERSION From 4e457fd8e46aa1f659c868ce9ec82d761dd9298a Mon Sep 17 00:00:00 2001 From: euanmillar Date: Thu, 15 Feb 2024 17:57:39 +0000 Subject: [PATCH 4/9] Micellaneous fixes to the release --- infrastructure/deployment/deploy.sh | 7 +++++++ infrastructure/docker-compose.deploy.yml | 2 ++ .../monitoring/elastalert/elastalert.yaml | 2 -- infrastructure/monitoring/kibana/setup-config.sh | 12 ++++++------ infrastructure/server-setup/playbook.yml | 4 +++- .../server-setup/tasks/backups/crontab.yml | 2 +- infrastructure/setup-deploy-config.sh | 13 +++++++++++++ 7 files changed, 32 insertions(+), 10 deletions(-) diff --git a/infrastructure/deployment/deploy.sh b/infrastructure/deployment/deploy.sh index 426cb14d3..b21c137c9 100755 --- a/infrastructure/deployment/deploy.sh +++ b/infrastructure/deployment/deploy.sh @@ -352,6 +352,12 @@ EOF # Setup configuration files and compose file for the deployment domain configured_ssh " + HOST=$HOST + SMTP_HOST=$SMTP_HOST + SMTP_PORT=$SMTP_PORT + ALERT_EMAIL=$ALERT_EMAIL + SENDER_EMAIL_ADDRESS=$SENDER_EMAIL_ADDRESS + DOMAIN=$DOMAIN MINIO_ROOT_USER=$MINIO_ROOT_USER MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD /opt/opencrvs/infrastructure/setup-deploy-config.sh $HOST | tee -a $LOG_LOCATION/setup-deploy-config.log" @@ -366,6 +372,7 @@ echo echo "Waiting 2 mins for mongo to deploy before working with data. Please note it can take up to 10 minutes for the entire stack to deploy in some scenarios." echo +sleep 120 # Required as Kibana cannot be immediately contacted echo "Setting up Kibana config & alerts" while true; do diff --git a/infrastructure/docker-compose.deploy.yml b/infrastructure/docker-compose.deploy.yml index c53c0fd78..7e7b0e834 100644 --- a/infrastructure/docker-compose.deploy.yml +++ b/infrastructure/docker-compose.deploy.yml @@ -405,6 +405,8 @@ services: - '/opt/opencrvs/infrastructure/monitoring/elastalert/rules:/opt/elastalert/rules' networks: - overlay_net + depends_on: + - elasticsearch deploy: labels: - 'traefik.enable=false' diff --git a/infrastructure/monitoring/elastalert/elastalert.yaml b/infrastructure/monitoring/elastalert/elastalert.yaml index 2c81b51bc..b74db9981 100644 --- a/infrastructure/monitoring/elastalert/elastalert.yaml +++ b/infrastructure/monitoring/elastalert/elastalert.yaml @@ -16,8 +16,6 @@ buffer_time: es_host: elasticsearch es_port: 9200 -es_username: '{{ES_USERNAME}}' -es_password: '{{ES_PASSWORD}}' writeback_index: elastalert_status diff --git a/infrastructure/monitoring/kibana/setup-config.sh b/infrastructure/monitoring/kibana/setup-config.sh index 3fe3cdfff..92e456d1d 100755 --- a/infrastructure/monitoring/kibana/setup-config.sh +++ b/infrastructure/monitoring/kibana/setup-config.sh @@ -14,14 +14,14 @@ set -e docker_command="docker run --rm -v /opt/opencrvs/infrastructure/monitoring/kibana/config.ndjson:/config.ndjson --network=opencrvs_overlay_net curlimages/curl" # First delete all alerts. This is because the import doesn't remove alerts that are no longer in the config -$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do - $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id" +$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do + $docker_command --connect-timeout 60 -X DELETE -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id" done -$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null +$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST http://kibana:5601/api/saved_objects/_import?overwrite=true -H 'kbn-xsrf: true' --form file=@/config.ndjson > /dev/null # Re-enable all alerts. This is because after importing a config, all alerts are disabled by default -$docker_command --connect-timeout 60 -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do - $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_disable" - $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:${ELASTICSEARCH_SUPERUSER_PASSWORD} "http://kibana:5601/api/alerting/rule/$id/_enable" +$docker_command --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD http://kibana:5601/api/alerting/rules/_find\?page\=1\&per_page\=100\&default_search_operator\=AND\&sort_field\=name\&sort_order\=asc | docker run --rm -i --network=opencrvs_overlay_net stedolan/jq -r '.data[].id' | while read -r id; do + $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_disable" + $docker_command --connect-timeout 60 -X POST -H 'kbn-xsrf: true' -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "http://kibana:5601/api/alerting/rule/$id/_enable" done \ No newline at end of file diff --git a/infrastructure/server-setup/playbook.yml b/infrastructure/server-setup/playbook.yml index 74376cc44..b12b2c4c3 100644 --- a/infrastructure/server-setup/playbook.yml +++ b/infrastructure/server-setup/playbook.yml @@ -34,7 +34,9 @@ - hosts: docker-manager-first, docker-workers become: yes - become_method: sudo + become_method: sudo + vars: + crontab_user: root tasks: - include_tasks: file: tasks/application.yml diff --git a/infrastructure/server-setup/tasks/backups/crontab.yml b/infrastructure/server-setup/tasks/backups/crontab.yml index e1424628e..f20fe5e51 100644 --- a/infrastructure/server-setup/tasks/backups/crontab.yml +++ b/infrastructure/server-setup/tasks/backups/crontab.yml @@ -18,7 +18,7 @@ - name: Throw an error if periodic_restore_from_backup is true but restore_backup_encryption_passphrase is not defined fail: - msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you haven't set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets' + msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you have not set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets' when: periodic_restore_from_backup and restore_backup_encryption_passphrase is not defined - name: 'Setup crontab to download a backup periodically the opencrvs data' diff --git a/infrastructure/setup-deploy-config.sh b/infrastructure/setup-deploy-config.sh index f9d8bacfd..6ce09263d 100755 --- a/infrastructure/setup-deploy-config.sh +++ b/infrastructure/setup-deploy-config.sh @@ -26,6 +26,19 @@ done KIBANA_ENCRYPTION_KEY=`uuidgen` sed -i "s/{{KIBANA_ENCRYPTION_KEY}}/$KIBANA_ENCRYPTION_KEY/g" /opt/opencrvs/infrastructure/monitoring/kibana/kibana.yml +# Move metabase file +mv /opt/opencrvs/infrastructure/metabase.init.db.sql /data/metabase/metabase.init.db.sql + +# Replace environment variables from all alert definition files +for file in /opt/opencrvs/infrastructure/monitoring/elastalert/rules/*.yaml; do + sed -i -e "s%{{HOST}}%$1%" $file + sed -i -e "s%{{SMTP_HOST}}%$SMTP_HOST%" $file + sed -i -e "s%{{SMTP_PORT}}%$SMTP_PORT%" $file + sed -i -e "s%{{ALERT_EMAIL}}%$ALERT_EMAIL%" $file + sed -i -e "s%{{SENDER_EMAIL_ADDRESS}}%$SENDER_EMAIL_ADDRESS%" $file + sed -i -e "s%{{DOMAIN}}%$DOMAIN%" $file +done + sed -i -e "s%{{MINIO_ROOT_USER}}%$MINIO_ROOT_USER%" /opt/opencrvs/infrastructure/mc-config/config.json sed -i -e "s%{{MINIO_ROOT_PASSWORD}}%$MINIO_ROOT_PASSWORD%" /opt/opencrvs/infrastructure/mc-config/config.json From da6451af8c25e01b6c3346993d5cf3d8122f3c9b Mon Sep 17 00:00:00 2001 From: euanmillar Date: Fri, 16 Feb 2024 12:52:57 +0000 Subject: [PATCH 5/9] QA_ENV should be true in countryconfig in development --- infrastructure/docker-compose.development-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/infrastructure/docker-compose.development-deploy.yml b/infrastructure/docker-compose.development-deploy.yml index 53e03684a..276f4f8c5 100644 --- a/infrastructure/docker-compose.development-deploy.yml +++ b/infrastructure/docker-compose.development-deploy.yml @@ -15,6 +15,7 @@ services: - jwt-public-key.{{ts}} environment: - NODE_ENV=production + - QA_ENV=true - FHIR_URL=http://hearth:3447/fhir - AUTH_URL=http://auth:4040 - APPLICATION_CONFIG_URL=http://config:2021 From 8959de77d1615dbbf5fcdeb2755f7d2dc69d5341 Mon Sep 17 00:00:00 2001 From: euanmillar Date: Sun, 18 Feb 2024 21:49:40 +0000 Subject: [PATCH 6/9] Dont create backup user. Instead instruct via docs --- infrastructure/server-setup/backups.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/infrastructure/server-setup/backups.yml b/infrastructure/server-setup/backups.yml index fa8c4ff0a..ca6b9d31f 100644 --- a/infrastructure/server-setup/backups.yml +++ b/infrastructure/server-setup/backups.yml @@ -87,16 +87,6 @@ vars: manager_hostname: "{{ groups['docker-manager-first'][0] }}" tasks: - - name: Ensure backup user is present - user: - name: '{{ external_backup_server_user }}' - state: present - create_home: true - home: '/home/{{ external_backup_server_user }}' - shell: /bin/bash - tags: - - backups - - set_fact: external_backup_server_user_home: '/home/{{ external_backup_server_user }}' tags: From 29d5a3bf602b62a94d001d092bc984e97e251c7e Mon Sep 17 00:00:00 2001 From: euanmillar Date: Tue, 20 Feb 2024 11:44:00 +0000 Subject: [PATCH 7/9] remove restore-backup-encryption-passphrase --- .github/workflows/provision.yml | 1 - infrastructure/server-setup/staging.yml | 1 - .../server-setup/tasks/backups/crontab.yml | 12 ++++++------ 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/provision.yml b/.github/workflows/provision.yml index 888dd6c9c..e9da7ad77 100644 --- a/.github/workflows/provision.yml +++ b/.github/workflows/provision.yml @@ -74,7 +74,6 @@ jobs: mongodb_admin_username: ${{ secrets.MONGODB_ADMIN_USER }} mongodb_admin_password: ${{ secrets.MONGODB_ADMIN_PASSWORD }} backup_encryption_passphrase: ${{ secrets.BACKUP_ENCRYPTION_PASSPHRASE }} - restore_backup_encryption_passphrase: ${{ secrets.RESTORE_BACKUP_ENCRYPTION_PASSPHRASE }} elasticsearch_superuser_password: ${{ secrets.ELASTICSEARCH_SUPERUSER_PASSWORD }} external_backup_server_ssh_port: 22 external_backup_server_ip: ${{ secrets.BACKUP_HOST }} diff --git a/infrastructure/server-setup/staging.yml b/infrastructure/server-setup/staging.yml index ab7590dd5..1d34f18ec 100644 --- a/infrastructure/server-setup/staging.yml +++ b/infrastructure/server-setup/staging.yml @@ -15,7 +15,6 @@ all: - 165.22.110.53 enable_backups: false periodic_restore_from_backup: true - # restore_backup_encryption_passphrase: Defined in --extra-vars by the provisioning pipeline # external_backup_server_ssh_port: Defined in --extra-vars by the provisioning pipeline # external_backup_server_ip: Defined in --extra-vars by the provisioning pipeline users: diff --git a/infrastructure/server-setup/tasks/backups/crontab.yml b/infrastructure/server-setup/tasks/backups/crontab.yml index f20fe5e51..7527efea2 100644 --- a/infrastructure/server-setup/tasks/backups/crontab.yml +++ b/infrastructure/server-setup/tasks/backups/crontab.yml @@ -16,10 +16,10 @@ periodic_restore_from_backup: false when: periodic_restore_from_backup is not defined -- name: Throw an error if periodic_restore_from_backup is true but restore_backup_encryption_passphrase is not defined +- name: Throw an error if periodic_restore_from_backup is true but backup_encryption_passphrase is not defined fail: - msg: 'Error: restore_backup_encryption_passphrase is not defined. It usually means you have not set RESTORE_BACKUP_ENCRYPTION_PASSPHRASE in your staging environments secrets' - when: periodic_restore_from_backup and restore_backup_encryption_passphrase is not defined + msg: 'Error: backup_encryption_passphrase is not defined. It usually means you have not set backup_encryption_passphrase in your staging environments secrets' + when: periodic_restore_from_backup and backup_encryption_passphrase is not defined - name: 'Setup crontab to download a backup periodically the opencrvs data' cron: @@ -27,8 +27,8 @@ name: 'download opencrvs backup' minute: '30' hour: '0' - job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ restore_backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1' - state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" + job: 'cd / && bash /opt/opencrvs/infrastructure/backups/download.sh --passphrase={{ backup_encryption_passphrase }} --ssh_user={{ external_backup_server_user }} --ssh_host={{ external_backup_server_ip }} --ssh_port={{ external_backup_server_ssh_port }} --remote_dir={{ external_backup_server_remote_directory }} >> /var/log/opencrvs-restore.log 2>&1' + state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" - name: 'Setup crontab to restore the opencrvs data' cron: @@ -37,4 +37,4 @@ minute: '0' hour: '1' job: 'cd / && bash /opt/opencrvs/infrastructure/backups/restore.sh --replicas=1 >> /var/log/opencrvs-restore.log 2>&1' - state: "{{ 'present' if (external_backup_server_ip is defined and restore_backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" + state: "{{ 'present' if (external_backup_server_ip is defined and backup_encryption_passphrase and periodic_restore_from_backup) else 'absent' }}" From 98d4b1ca23290e52b19e035f2fbc926c8223c33e Mon Sep 17 00:00:00 2001 From: euanmillar Date: Tue, 20 Feb 2024 11:57:27 +0000 Subject: [PATCH 8/9] Add remove unused kernels options --- infrastructure/server-setup/tasks/updates.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infrastructure/server-setup/tasks/updates.yml b/infrastructure/server-setup/tasks/updates.yml index 098149e63..2317bbde1 100644 --- a/infrastructure/server-setup/tasks/updates.yml +++ b/infrastructure/server-setup/tasks/updates.yml @@ -25,6 +25,9 @@ content: | Unattended-Upgrade::Package-Blacklist {}; Unattended-Upgrade::DevRelease "auto"; + Unattended-Upgrade::Remove-Unused-Kernel-Packages "true"; + Unattended-Upgrade::Remove-New-Unused-Dependencies "true"; + Unattended-Upgrade::Remove-Unused-Dependencies "true"; Unattended-Upgrade::Allowed-Origins { "${distro_id}:${distro_codename}-security"; }; From e1ac7c1e46191864659ca20115a3f91641c40851 Mon Sep 17 00:00:00 2001 From: euanmillar Date: Tue, 20 Feb 2024 20:33:27 +0000 Subject: [PATCH 9/9] Amends to countryconfig files for configurability --- .github/workflows/deploy-prod.yml | 4 +- .github/workflows/generate-demo-data.yml | 48 ------------------- .../docker-compose.production-deploy.yml | 3 -- .../docker-compose.staging-deploy.yml | 3 -- infrastructure/server-setup/backups.yml | 16 ------- infrastructure/server-setup/development.yml | 4 +- infrastructure/server-setup/production.yml | 2 +- infrastructure/server-setup/qa.yml | 9 ---- infrastructure/server-setup/staging.yml | 2 +- src/data-seeding/employees/handler.ts | 5 +- .../employees/source/prod-employees.csv | 1 + 11 files changed, 11 insertions(+), 86 deletions(-) delete mode 100644 .github/workflows/generate-demo-data.yml create mode 100644 src/data-seeding/employees/source/prod-employees.csv diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index b0dd118a0..01e8f9110 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -30,8 +30,8 @@ jobs: secret: ${{ github.TOKEN }} approvers: euanmillar,rikukissa minimum-approvals: 1 - issue-title: 'Deploy (Prod): core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }}' - issue-body: 'Please approve or deny the deployment of core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }} to production' + issue-title: 'Deploy (${{ github.event.inputs.environment }}): core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }}' + issue-body: 'Please approve or deny the deployment of core: ${{ github.event.inputs.core-image-tag }} country config: ${{ github.event.inputs.countryconfig-image-tag }} to ${{ github.event.inputs.environment }}' exclude-workflow-initiator-as-approver: false - name: Clone core uses: actions/checkout@v3 diff --git a/.github/workflows/generate-demo-data.yml b/.github/workflows/generate-demo-data.yml deleted file mode 100644 index 9ab7fb25b..000000000 --- a/.github/workflows/generate-demo-data.yml +++ /dev/null @@ -1,48 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. -# -# OpenCRVS is also distributed under the terms of the Civil Registration -# & Healthcare Disclaimer located at http://opencrvs.org/license. -# -# Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. -name: Generate demo data - -on: - workflow_dispatch: - inputs: - branch: - description: 'Branch to run from' - required: false - default: 'develop' - environment: - type: choice - description: Environment to deploy to - required: true - options: - - production - - qa - - staging - -jobs: - generate-demo-data: - environment: ${{ github.event.inputs.environment }} - runs-on: ubuntu-20.04 - strategy: - matrix: - node-version: [14.17.0] - steps: - - name: Checking out git repo - uses: actions/checkout@v2 - with: - ref: ${{ github.event.inputs.branch }} - - name: Install dependencies - run: yarn install - - name: Start generating test data - run: yarn data-generator - env: - COUNTRY_CONFIG_HOST: ${{ vars.COUNTRY_CONFIG_HOST }} - AUTH_HOST: ${{ vars.AUTH_HOST }} - GATEWAY_HOST: ${{ vars.GATEWAY_HOST }} - CONFIG_HOST: ${{ vars.CONFIG_HOST }} - CONCURRENCY: ${{ vars.CONCURRENCY }} diff --git a/infrastructure/docker-compose.production-deploy.yml b/infrastructure/docker-compose.production-deploy.yml index bcbb7e900..55849ed59 100644 --- a/infrastructure/docker-compose.production-deploy.yml +++ b/infrastructure/docker-compose.production-deploy.yml @@ -39,7 +39,6 @@ services: metrics: environment: - - QA_ENV=true - NODE_ENV=production - SENTRY_DSN=${SENTRY_DSN} - MONGO_URL=mongodb://metrics:${METRICS_MONGODB_PASSWORD}@mongo1,mongo2/metrics?replicaSet=rs0 @@ -48,7 +47,6 @@ services: auth: environment: - - QA_ENV=true - NODE_ENV=production - SENTRY_DSN=${SENTRY_DSN} deploy: @@ -56,7 +54,6 @@ services: user-mgnt: environment: - - QA_ENV=true - NODE_ENV=production - SENTRY_DSN=${SENTRY_DSN} - MONGO_URL=mongodb://user-mgnt:${USER_MGNT_MONGODB_PASSWORD}@mongo1,mongo2/user-mgnt?replicaSet=rs0 diff --git a/infrastructure/docker-compose.staging-deploy.yml b/infrastructure/docker-compose.staging-deploy.yml index c658af0b3..4adea8ac3 100644 --- a/infrastructure/docker-compose.staging-deploy.yml +++ b/infrastructure/docker-compose.staging-deploy.yml @@ -39,7 +39,6 @@ services: metrics: environment: - - QA_ENV=true - NODE_ENV=production - SENTRY_DSN=${SENTRY_DSN} - MONGO_URL=mongodb://metrics:${METRICS_MONGODB_PASSWORD}@mongo1/metrics?replicaSet=rs0 @@ -48,7 +47,6 @@ services: auth: environment: - - QA_ENV=true - NODE_ENV=production - SENTRY_DSN=${SENTRY_DSN} deploy: @@ -56,7 +54,6 @@ services: user-mgnt: environment: - - QA_ENV=true - NODE_ENV=production - SENTRY_DSN=${SENTRY_DSN} - MONGO_URL=mongodb://user-mgnt:${USER_MGNT_MONGODB_PASSWORD}@mongo1/user-mgnt?replicaSet=rs0 diff --git a/infrastructure/server-setup/backups.yml b/infrastructure/server-setup/backups.yml index ca6b9d31f..e5201a12c 100644 --- a/infrastructure/server-setup/backups.yml +++ b/infrastructure/server-setup/backups.yml @@ -92,22 +92,6 @@ tags: - backups - - name: Check if authorized_keys already exists - stat: - path: '{{ external_backup_server_user_home }}/.ssh/authorized_keys' - register: file_check - tags: - - backups - - - name: Ensure .ssh directory exists for external backup server user - file: - path: '{{ external_backup_server_user_home }}/.ssh' - state: directory - owner: '{{ external_backup_server_user }}' - mode: '0700' - tags: - - backups - - name: Ensure backup application servers can login to backup server blockinfile: path: '{{ external_backup_server_user_home }}/.ssh/authorized_keys' diff --git a/infrastructure/server-setup/development.yml b/infrastructure/server-setup/development.yml index bb41b723e..3779a78d0 100644 --- a/infrastructure/server-setup/development.yml +++ b/infrastructure/server-setup/development.yml @@ -18,8 +18,8 @@ all: sudoer: true docker-manager-first: - dev: - qa: # @todo set this to be the hostname of your target server + hosts: + dev: # @todo set this to be the hostname of your target server ansible_host: '44.44.44.44' # @todo set this to be the IP address of your server data_label: data1 # for manager machines, this should always be "data1" diff --git a/infrastructure/server-setup/production.yml b/infrastructure/server-setup/production.yml index 57f052224..1290fd729 100644 --- a/infrastructure/server-setup/production.yml +++ b/infrastructure/server-setup/production.yml @@ -50,5 +50,5 @@ docker-workers: backups: hosts: - qa: + backup: # @todo set this to be the hostname of your backup server ansible_host: '66.66.66.66' diff --git a/infrastructure/server-setup/qa.yml b/infrastructure/server-setup/qa.yml index 3069e11b0..d584e5987 100644 --- a/infrastructure/server-setup/qa.yml +++ b/infrastructure/server-setup/qa.yml @@ -28,15 +28,6 @@ all: - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-243 # example provision user key - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-244 # example provision user key - # If the machine is repurposed to also be the backup host, we need to add more keys to the authorized_keys file so that - # when the application servers get provisioned, the provision user of this machine can be used. - # - # @todo remove this key if the machine is not used as a backup host. - # Otherwise, add the public key of the private key that's used for the "provision" user on the server of which backups this machine hosts. - additional_keys_for_provisioning_user: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-243 # example provision user key - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABg...Z/rhU= provision@github-runner-244 # example provision user key - docker-manager-first: hosts: qa: # @todo set this to be the hostname of your target server diff --git a/infrastructure/server-setup/staging.yml b/infrastructure/server-setup/staging.yml index b4778341d..74d4d8ee7 100644 --- a/infrastructure/server-setup/staging.yml +++ b/infrastructure/server-setup/staging.yml @@ -46,5 +46,5 @@ docker-workers: {} # @todo if you do not intend to set up automatic sync from the backup server, you can remove this section. backups: hosts: - qa: + backup: # @todo set this to be the hostname of your backup server ansible_host: '66.66.66.66' # set this to be the IP address of your backup server diff --git a/src/data-seeding/employees/handler.ts b/src/data-seeding/employees/handler.ts index 59aee343b..09198a79b 100644 --- a/src/data-seeding/employees/handler.ts +++ b/src/data-seeding/employees/handler.ts @@ -8,12 +8,15 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ +import { QA_ENV } from '@countryconfig/constants' import { readCSVToJSON } from '@countryconfig/utils' import { Request, ResponseToolkit } from '@hapi/hapi' export async function usersHandler(_: Request, h: ResponseToolkit) { const users: unknown[] = await readCSVToJSON( - './src/data-seeding/employees/source/default-employees.csv' + process.env.NODE_ENV === 'production' && !QA_ENV + ? './src/data-seeding/employees/source/prod-employees.csv' + : './src/data-seeding/employees/source/test-employees.csv' ) return h.response(users) } diff --git a/src/data-seeding/employees/source/prod-employees.csv b/src/data-seeding/employees/source/prod-employees.csv new file mode 100644 index 000000000..323e18b82 --- /dev/null +++ b/src/data-seeding/employees/source/prod-employees.csv @@ -0,0 +1 @@ +primaryOfficeId,givenNames,familyName,systemRole,role,mobile,username,email,password \ No newline at end of file