From 2d4c0be554b7999d8d1be5602b5a75accdd2116d Mon Sep 17 00:00:00 2001 From: "Christian R. Garcia" Date: Thu, 16 Nov 2023 08:45:25 -0800 Subject: [PATCH 1/4] Removal of prometheus and grafana deployment code --- Makefile | 9 ++--- .../kube-archive}/grafana.yml | 0 .../kube-archive}/prometheus-config.yml | 0 .../kube-archive}/prometheus.yml | 0 docker-compose.yml | 39 ------------------- kube-template/burndown | 3 -- kube-template/burnup | 3 -- 7 files changed, 3 insertions(+), 51 deletions(-) rename {kube-template => archive/kube-archive}/grafana.yml (100%) rename {kube-template => archive/kube-archive}/prometheus-config.yml (100%) rename {kube-template => archive/kube-archive}/prometheus.yml (100%) diff --git a/Makefile b/Makefile index fad2a54d..4b00d7f2 100644 --- a/Makefile +++ b/Makefile @@ -136,7 +136,7 @@ endif #: Build core image build: vars @echo "Makefile: $(GREEN)build$(NC)" - @echo " 🔨 : Running image build for core-v3, prometheus, and nginx." + @echo " 🔨 : Running image build for core-v3 and nginx." ifeq ($(BACKEND),minikube) $(MAKE) build-minikube @@ -147,26 +147,23 @@ endif build-docker: vars @echo "Makefile: $(GREEN)build$(NC)" - @echo " 🔨 : Running image build for core-v3, prometheus, and nginx." + @echo " 🔨 : Running image build for core-v3 and nginx." @echo " 🌎 : Using backend: $(LCYAN)docker$(NC)" @echo "" docker build -t $(SERVICE_NAME)/core-v3:$$TAG ./ @echo "" - docker build -t $(SERVICE_NAME)/prom:$$TAG images/prometheus/. - @echo "" docker build -t $(SERVICE_NAME)/nginx:$$TAG images/nginx/. @echo "" build-minikube: vars @echo "Makefile: $(GREEN)build$(NC)" - @echo " 🔨 : Running image build for core-v3, prometheus, and nginx." + @echo " 🔨 : Running image build for core-v3 and nginx." @echo " 🌎 : Using backend: $(LCYAN)minikube$(NC)" @echo "" minikube image build -t $(SERVICE_NAME)/core-v3:$$TAG ./ - minikube image build -t $(SERVICE_NAME)/prom:$$TAG images/prometheus/. minikube image build -t $(SERVICE_NAME)/nginx:$$TAG images/nginx/. @echo "" diff --git a/kube-template/grafana.yml b/archive/kube-archive/grafana.yml similarity index 100% rename from kube-template/grafana.yml rename to archive/kube-archive/grafana.yml diff --git a/kube-template/prometheus-config.yml b/archive/kube-archive/prometheus-config.yml similarity index 100% rename from kube-template/prometheus-config.yml rename to archive/kube-archive/prometheus-config.yml diff --git a/kube-template/prometheus.yml b/archive/kube-archive/prometheus.yml similarity index 100% rename from kube-template/prometheus.yml rename to archive/kube-archive/prometheus.yml diff --git a/docker-compose.yml b/docker-compose.yml index 53346573..c3c1e760 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -234,42 +234,3 @@ services: depends_on: - mongo - reg -# - prometheus - - # prometheus: - # # build: ./images/prometheus - # image: abaco/prom:$TAG - # volumes: - # - ./images/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - # - ./images/prometheus/alert.rules.yml:/etc/prometheus/alert.rules.yml - # command: - # - '--config.file=/etc/prometheus/prometheus.yml' - # # - '-storage.local.path=/prometheus' - # ports: - # - 9090:9090 - # networks: - # - abaco - # depends_on: - # - mongo - # - reg - - # grafana: - # image: grafana/grafana - # user: "104" - # depends_on: - # - prometheus - # - mongo - # - reg - # ports: - # - 3000:3000 - # volumes: - # - grafana_data:/var/lib/grafana - # - ./images/prometheus/grafana/provisioning/:/etc/grafana/provisioning/ - # env_file: - # - ./images/prometheus/grafana/config.monitoring - # networks: - # - abaco - # restart: always - -# volumes: -# grafana_data: {} diff --git a/kube-template/burndown b/kube-template/burndown index 4687bf01..b5fc4c5b 100755 --- a/kube-template/burndown +++ b/kube-template/burndown @@ -3,7 +3,6 @@ # config kubectl delete configmap actors-config kubectl delete -f nginx-config.yml -kubectl delete -f prometheus-config.yml kubectl delete -f configmap.yml # apps @@ -11,8 +10,6 @@ kubectl delete -f reg.yml kubectl delete -f mes.yml kubectl delete -f admin.yml kubectl delete -f events.yml -kubectl delete -f prometheus.yml -kubectl delete -f grafana.yml kubectl delete -f metrics.yml kubectl delete -f nginx.yml kubectl delete -f spawner.yml diff --git a/kube-template/burnup b/kube-template/burnup index 7a24370b..661a58f3 100755 --- a/kube-template/burnup +++ b/kube-template/burnup @@ -7,7 +7,6 @@ kubectl apply -f security.yml # config files kubectl create configmap actors-config --from-file=config.json kubectl apply -f nginx-config.yml -kubectl apply -f prometheus-config.yml kubectl apply -f configmap.yml # services files @@ -23,8 +22,6 @@ kubectl apply -f reg.yml kubectl apply -f mes.yml kubectl apply -f admin.yml #kubectl apply -f events.yml -kubectl apply -f prometheus.yml -#kubectl apply -f grafana.yml kubectl apply -f metrics.yml kubectl apply -f spawner.yml kubectl apply -f health.yml From 9062b91a3b3c7bf2f3b467c9c1305d89a1c19996 Mon Sep 17 00:00:00 2001 From: "Christian R. Garcia" Date: Thu, 16 Nov 2023 08:46:27 -0800 Subject: [PATCH 2/4] Faster list actors db call --- actors/controllers.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/actors/controllers.py b/actors/controllers.py index 3820779b..c934772e 100644 --- a/actors/controllers.py +++ b/actors/controllers.py @@ -815,14 +815,27 @@ def get(self): result = Search(args_full, 'actors', g.request_tenant_id, g.request_username).search() return ok(result=result, msg="Actors search completed successfully.") else: - actors = [] - for actor_info in actors_store[site()].items(): - if actor_info['tenant'] == g.request_tenant_id: - actor = Actor.from_db(actor_info) - if check_permissions(g.request_username, actor.db_id, READ): - actors.append(actor.display()) + # Pipeline puts permissions.user = "level" on actor obj. + # Ensure users permissions and proper tenant in one step. + # Previously a for loop, really slow. This should be db speed no matter db size. + pipeline = [ + {'$match': {'tenant': g.request_tenant_id}}, + {'$lookup': + {'from' : 'permissions_store', + 'localField' : '_id', + 'foreignField' : '_id', + 'as' : 'permissions'}}, + {'$unwind': '$permissions'}, + {'$match': {'permissions.' + g.request_username: {'$in': ["EXECUTE", "UPDATE", "READ"]}}} + ] + actors = list(actors_store[site()].aggregate(pipeline)) + # We now need to run actor objects through .display() + display_ready_actors = [] + for actor in actors: + actor = Actor.from_db(actor) + display_ready_actors.append(actor.display()) logger.info("actors retrieved.") - return ok(result=actors, msg="Actors retrieved successfully.") + return ok(result=display_ready_actors, msg="Actors retrieved successfully.") def validate_post(self): logger.debug("top of validate post in /actors") From 162778c044447bf729def7b7581dad42dd72b272 Mon Sep 17 00:00:00 2001 From: "Christian R. Garcia" Date: Thu, 16 Nov 2023 09:35:29 -0800 Subject: [PATCH 3/4] Add deployment action build-push-all-branches.yml --- .github/workflows/build-push-all-branches.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/build-push-all-branches.yml diff --git a/.github/workflows/build-push-all-branches.yml b/.github/workflows/build-push-all-branches.yml new file mode 100644 index 00000000..9e04891e --- /dev/null +++ b/.github/workflows/build-push-all-branches.yml @@ -0,0 +1,49 @@ +### Set Up by: Christian Garcia (TACC) +### Release-* methodology originally authored by Nathan Freeman (TACC) here: +### https://github.com/tapis-project/tapis-workflows/blob/prod/.github/workflows/ci.yml#L59 +# Description. This Github action runs when there are new commits to dev-v3, prod-v3, or v3-release-*. +# The action logs into Docker with the Github action environment secrets `DOCKERHUB_USERNAME` +# and `DOCKERHUB_TOKEN`. +# The action pulls the repo and builds the abaco/core-v3 image and uses branch name as the +# docker tag. In instance of `v3-release-1.2.0`, sed will delete `v3-release-`, leaving `1.2.0`. +# If branch = prod-v3, tag becomes `latest` so latest is also built. +# +# NOVEMBER 16th: +# DOCKERHUB_USERNAME: username for cgarcia +# DOCKERHUB_TOKEN: token for cgarcia + +name: Build & Push - All Branches +on: + push: + branches: [ dev-v3, prod-v3, v3-release-* ] + pull_request: + branches: [ dev-v3, prod-v3, v3-release-* ] + workflow_dispatch: + +jobs: + build-and-push-images: + runs-on: ubuntu-latest + environment: docker + steps: + - name: Get tag name from branch. SED removes `v3-release-`; replaces `prod-v3` with `latest`. + id: get_tag_name + shell: bash + run: echo "tag_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/prod-v3/latest/g' | sed 's/v3-release-//g')" >> $GITHUB_OUTPUT + + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to Docker + uses: docker/login-action@v2 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build & push tagged Docker backend image + uses: docker/build-push-action@v3 + with: + push: true + tags: abaco/core-v3:${{ steps.get_tag_name.outputs.tag_name }} From c1640332e60f121c67a66995b499dfc4a6547751 Mon Sep 17 00:00:00 2001 From: "Christian R. Garcia" Date: Thu, 16 Nov 2023 12:56:27 -0800 Subject: [PATCH 4/4] Update build-push-all-branches.yml --- .github/workflows/build-push-all-branches.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-all-branches.yml b/.github/workflows/build-push-all-branches.yml index 9e04891e..17d5e25a 100644 --- a/.github/workflows/build-push-all-branches.yml +++ b/.github/workflows/build-push-all-branches.yml @@ -6,7 +6,7 @@ # and `DOCKERHUB_TOKEN`. # The action pulls the repo and builds the abaco/core-v3 image and uses branch name as the # docker tag. In instance of `v3-release-1.2.0`, sed will delete `v3-release-`, leaving `1.2.0`. -# If branch = prod-v3, tag becomes `latest` so latest is also built. +# If branch = prod-v3, tag becomes `staging` so staging is also built. # # NOVEMBER 16th: # DOCKERHUB_USERNAME: username for cgarcia @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest environment: docker steps: - - name: Get tag name from branch. SED removes `v3-release-`; replaces `prod-v3` with `latest`. + - name: Get tag name from branch. SED removes `v3-release-`; replaces `prod-v3` with `staging`. id: get_tag_name shell: bash - run: echo "tag_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/prod-v3/latest/g' | sed 's/v3-release-//g')" >> $GITHUB_OUTPUT + run: echo "tag_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/prod-v3/staging/g' | sed 's/v3-release-//g')" >> $GITHUB_OUTPUT - name: Checkout repo uses: actions/checkout@v3