From 95747d7317c66ef5cc35174465f2b6e36d8d77c0 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 23 Sep 2024 13:02:41 +0300 Subject: [PATCH 01/10] create cog creation queue with on-demand ec2 instances --- app/models/pydantic/jobs.py | 5 +-- app/settings/globals.py | 1 + terraform/data.tf | 3 +- terraform/main.tf | 31 +++++++++++++++++++ terraform/modules/batch/main.tf | 9 +++++- terraform/modules/batch/outputs.tf | 6 +++- terraform/modules/batch/variables.tf | 1 + .../templates/container_definition.json.tmpl | 4 +++ 8 files changed, 55 insertions(+), 5 deletions(-) diff --git a/app/models/pydantic/jobs.py b/app/models/pydantic/jobs.py index 295358005..a44030595 100644 --- a/app/models/pydantic/jobs.py +++ b/app/models/pydantic/jobs.py @@ -4,6 +4,7 @@ from ...settings.globals import ( AURORA_JOB_QUEUE, + COG_JOB_QUEUE, DATA_LAKE_JOB_QUEUE, DEFAULT_JOB_DURATION, GDAL_PYTHON_JOB_DEFINITION, @@ -138,9 +139,9 @@ class PixETLJob(Job): class GDALCOGJob(Job): - """Use for creating COG files using GDAL Python docker in PixETL queue.""" + """Use for creating COG files using GDAL Python docker in COG queue.""" - job_queue = PIXETL_JOB_QUEUE + job_queue = COG_JOB_QUEUE job_definition = GDAL_PYTHON_JOB_DEFINITION vcpus = 8 memory = 64000 diff --git a/app/settings/globals.py b/app/settings/globals.py index c72037fe7..df73dbf94 100644 --- a/app/settings/globals.py +++ b/app/settings/globals.py @@ -116,6 +116,7 @@ MAX_MEM = config("MAX_MEM", cast=int, default=760000) PIXETL_JOB_DEFINITION = config("PIXETL_JOB_DEFINITION", cast=str) PIXETL_JOB_QUEUE = config("PIXETL_JOB_QUEUE", cast=str) +COG_JOB_QUEUE = config("COG_JOB_QUEUE", cast=str) PIXETL_CORES = config("PIXETL_CORES", cast=int, default=48) PIXETL_MAX_MEM = config("PIXETL_MAX_MEM", cast=int, default=380000) PIXETL_DEFAULT_RESAMPLING = config( diff --git a/terraform/data.tf b/terraform/data.tf index d3748b037..ba7a49cb4 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -68,6 +68,7 @@ data "template_file" "container_definition" { tile_cache_job_queue = module.batch_job_queues.tile_cache_job_queue_arn pixetl_job_definition = module.batch_job_queues.pixetl_job_definition_arn pixetl_job_queue = module.batch_job_queues.pixetl_job_queue_arn + cog_job_queue = module.batch_job_queues.cog_job_queue_arn raster_analysis_lambda_name = "raster-analysis-tiled_raster_analysis-default" raster_analysis_sfn_arn = data.terraform_remote_state.raster_analysis_lambda.outputs.raster_analysis_state_machine_arn service_url = local.service_url @@ -190,4 +191,4 @@ data "template_file" "step_function_policy" { vars = { raster_analysis_state_machine_arn = data.terraform_remote_state.raster_analysis_lambda.outputs.raster_analysis_state_machine_arn } -} \ No newline at end of file +} diff --git a/terraform/main.tf b/terraform/main.tf index 263d70da0..f9df92ad0 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -183,12 +183,43 @@ module "batch_data_lake_writer" { compute_environment_name = "data_lake_writer" } +module "batch_cog_creator" { + source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/compute_environment?ref=v0.4.2.3" + ecs_role_policy_arns = [ + aws_iam_policy.query_batch_jobs.arn, + aws_iam_policy.s3_read_only.arn, + data.terraform_remote_state.core.outputs.iam_policy_s3_write_data-lake_arn, + data.terraform_remote_state.core.outputs.secrets_postgresql-reader_policy_arn, + data.terraform_remote_state.core.outputs.secrets_postgresql-writer_policy_arn, + data.terraform_remote_state.core.outputs.secrets_read-gfw-gee-export_policy_arn + ] + key_pair = var.key_pair + max_vcpus = var.data_lake_max_vcpus + project = local.project + security_group_ids = [ + data.terraform_remote_state.core.outputs.default_security_group_id, + data.terraform_remote_state.core.outputs.postgresql_security_group_id + ] + subnets = data.terraform_remote_state.core.outputs.private_subnet_ids + suffix = local.name_suffix + tags = local.batch_tags + use_ephemeral_storage = true + launch_type = "EC2" + instance_types = [ + "r6id.large", "r6id.xlarge", "r6id.2xlarge", "r6id.4xlarge", "r6id.8xlarge", "r6id.12xlarge", "r6id.16xlarge", "r6id.24xlarge", + "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", + "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge" + ] + compute_environment_name = "cog_creator" +} + module "batch_job_queues" { source = "./modules/batch" aurora_compute_environment_arn = module.batch_aurora_writer.arn data_lake_compute_environment_arn = module.batch_data_lake_writer.arn pixetl_compute_environment_arn = module.batch_data_lake_writer.arn tile_cache_compute_environment_arn = module.batch_data_lake_writer.arn + cog_compute_environment_arn = module.cog_creator.arn environment = var.environment name_suffix = local.name_suffix project = local.project diff --git a/terraform/modules/batch/main.tf b/terraform/modules/batch/main.tf index 3433ccf4c..e62c38214 100644 --- a/terraform/modules/batch/main.tf +++ b/terraform/modules/batch/main.tf @@ -52,6 +52,13 @@ resource "aws_batch_job_queue" "pixetl" { depends_on = [var.pixetl_compute_environment_arn] } +resource "aws_batch_job_queue" "cog" { + name = substr("${var.project}-cog-job-queue${var.name_suffix}", 0, 64) + state = "ENABLED" + priority = 1 + compute_environments = [var.cog_compute_environment_arn] + depends_on = [var.pixetl_compute_environment_arn] +} resource "aws_batch_job_definition" "tile_cache" { name = substr("${var.project}-tile_cache${var.name_suffix}", 0, 64) @@ -190,4 +197,4 @@ data "template_file" "ecs-task_assume" { vars = { service = "ecs-tasks" } -} \ No newline at end of file +} diff --git a/terraform/modules/batch/outputs.tf b/terraform/modules/batch/outputs.tf index 9d91ec956..863bb90cd 100644 --- a/terraform/modules/batch/outputs.tf +++ b/terraform/modules/batch/outputs.tf @@ -38,6 +38,10 @@ output "pixetl_job_queue_arn" { value = aws_batch_job_queue.pixetl.arn } +output "cog_job_queue_arn" { + value = aws_batch_job_queue.cog.arn +} + output "tile_cache_job_definition_arn" { value = aws_batch_job_definition.tile_cache.arn } @@ -48,4 +52,4 @@ output "tile_cache_job_definition" { output "tile_cache_job_queue_arn" { value = aws_batch_job_queue.tile_cache.arn -} \ No newline at end of file +} diff --git a/terraform/modules/batch/variables.tf b/terraform/modules/batch/variables.tf index 3d6e72aae..7c03c859f 100644 --- a/terraform/modules/batch/variables.tf +++ b/terraform/modules/batch/variables.tf @@ -2,6 +2,7 @@ variable "project" { type = string } variable "name_suffix" { type = string } variable "aurora_compute_environment_arn" { type = string } variable "data_lake_compute_environment_arn" { type = string } +variable "cog_compute_environment_arn" { type = string } variable "tile_cache_compute_environment_arn" { type = string } variable "pixetl_compute_environment_arn" { type = string } variable "gdal_repository_url" { type = string } diff --git a/terraform/templates/container_definition.json.tmpl b/terraform/templates/container_definition.json.tmpl index f031b29ca..d62cb9c47 100644 --- a/terraform/templates/container_definition.json.tmpl +++ b/terraform/templates/container_definition.json.tmpl @@ -73,6 +73,10 @@ "name": "PIXETL_JOB_QUEUE", "value": "${pixetl_job_queue}" }, + { + "name": "COG_JOB_QUEUE", + "value": "${cog_job_queue}" + }, { "name": "API_URL", "value": "${service_url}" From 0f767205270c2da804e6a0e35ecec88f30247966 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 23 Sep 2024 13:37:24 +0300 Subject: [PATCH 02/10] store ec2 instances in a variable --- terraform/main.tf | 24 ++++++++---------------- terraform/variables.tf | 10 ++++++++++ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index f9df92ad0..17c5f92a2 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -174,12 +174,8 @@ module "batch_data_lake_writer" { tags = local.batch_tags use_ephemeral_storage = true # SPOT is actually the default, this is just a placeholder until GTC-1791 is done - launch_type = "SPOT" - instance_types = [ - "r6id.large", "r6id.xlarge", "r6id.2xlarge", "r6id.4xlarge", "r6id.8xlarge", "r6id.12xlarge", "r6id.16xlarge", "r6id.24xlarge", - "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", - "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge" - ] + launch_type = "SPOT" + instance_types = var.data_lake_writer_instance_types compute_environment_name = "data_lake_writer" } @@ -200,16 +196,12 @@ module "batch_cog_creator" { data.terraform_remote_state.core.outputs.default_security_group_id, data.terraform_remote_state.core.outputs.postgresql_security_group_id ] - subnets = data.terraform_remote_state.core.outputs.private_subnet_ids - suffix = local.name_suffix - tags = local.batch_tags - use_ephemeral_storage = true - launch_type = "EC2" - instance_types = [ - "r6id.large", "r6id.xlarge", "r6id.2xlarge", "r6id.4xlarge", "r6id.8xlarge", "r6id.12xlarge", "r6id.16xlarge", "r6id.24xlarge", - "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", - "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge" - ] + subnets = data.terraform_remote_state.core.outputs.private_subnet_ids + suffix = local.name_suffix + tags = local.batch_tags + use_ephemeral_storage = true + launch_type = "EC2" + instance_types = var.data_lake_writer_instance_types compute_environment_name = "cog_creator" } diff --git a/terraform/variables.tf b/terraform/variables.tf index 8b56728b9..142736b0f 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -156,3 +156,13 @@ variable "api_gateway_url" { description = "The invoke url of the API Gateway stage" default = "" } + +variable "data_lake_writer_instance_types" { + type = list(string) + description = "memory optimized EC2 instances with local NVMe SSDs for data lake writer batche queues" + default = [ + "r6id.large", "r6id.xlarge", "r6id.2xlarge", "r6id.4xlarge", "r6id.8xlarge", "r6id.12xlarge", "r6id.16xlarge", "r6id.24xlarge", + "r5ad.large", "r5ad.xlarge", "r5ad.2xlarge", "r5ad.4xlarge", "r5ad.8xlarge", "r5ad.12xlarge", "r5ad.16xlarge", "r5ad.24xlarge", + "r5d.large", "r5d.xlarge", "r5d.2xlarge", "r5d.4xlarge", "r5d.8xlarge", "r5d.12xlarge", "r5d.16xlarge", "r5d.24xlarge" + ] +} From 79a7d2e04225a2c3a5415111f0e6bb9107cb1ace Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 23 Sep 2024 16:59:56 +0300 Subject: [PATCH 03/10] cog -> cogify --- app/models/pydantic/jobs.py | 4 ++-- app/settings/globals.py | 2 +- docker-compose.dev.yml | 1 + docker-compose.prod.yml | 1 + docker-compose.test.yml | 1 + terraform/data.tf | 2 +- terraform/modules/batch/outputs.tf | 2 +- terraform/templates/container_definition.json.tmpl | 4 ++-- 8 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/models/pydantic/jobs.py b/app/models/pydantic/jobs.py index a44030595..52bc0e1bb 100644 --- a/app/models/pydantic/jobs.py +++ b/app/models/pydantic/jobs.py @@ -4,7 +4,7 @@ from ...settings.globals import ( AURORA_JOB_QUEUE, - COG_JOB_QUEUE, + COGIFY_JOB_QUEUE, DATA_LAKE_JOB_QUEUE, DEFAULT_JOB_DURATION, GDAL_PYTHON_JOB_DEFINITION, @@ -141,7 +141,7 @@ class PixETLJob(Job): class GDALCOGJob(Job): """Use for creating COG files using GDAL Python docker in COG queue.""" - job_queue = COG_JOB_QUEUE + job_queue = COGIFY_JOB_QUEUE job_definition = GDAL_PYTHON_JOB_DEFINITION vcpus = 8 memory = 64000 diff --git a/app/settings/globals.py b/app/settings/globals.py index df73dbf94..9d3fa4156 100644 --- a/app/settings/globals.py +++ b/app/settings/globals.py @@ -116,7 +116,7 @@ MAX_MEM = config("MAX_MEM", cast=int, default=760000) PIXETL_JOB_DEFINITION = config("PIXETL_JOB_DEFINITION", cast=str) PIXETL_JOB_QUEUE = config("PIXETL_JOB_QUEUE", cast=str) -COG_JOB_QUEUE = config("COG_JOB_QUEUE", cast=str) +COGIFY_JOB_QUEUE = config("COGIFY_JOB_QUEUE", cast=str) PIXETL_CORES = config("PIXETL_CORES", cast=int, default=48) PIXETL_MAX_MEM = config("PIXETL_MAX_MEM", cast=int, default=380000) PIXETL_DEFAULT_RESAMPLING = config( diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 8a60157cf..b755f2eb6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -39,6 +39,7 @@ services: - TILE_CACHE_CLUSTER=tile_cache_cluster - TILE_CACHE_SERVICE=tile_cache_service - PIXETL_JOB_QUEUE=pixetl_jq + - COGIFY_JOB_QUEUE=cogify_jq - API_URL=http://app_dev:80 - RASTER_ANALYSIS_LAMBDA_NAME=raster-analysis-tiled_raster_analysis-default - RW_API_URL=https://staging-api.resourcewatch.org diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 0f764358a..7f13c9f1e 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -34,6 +34,7 @@ services: - DATA_LAKE_JOB_QUEUE=data_lake_jq - TILE_CACHE_JOB_QUEUE=tile_cache_jq - PIXETL_JOB_QUEUE=pixetl_jq + - COGIFY_JOB_QUEUE=cogify_jq - RASTER_ANALYSIS_LAMBDA_NAME=raster_analysis - API_URL="http://app_dev:80" - RW_API_URL=https://api.resourcewatch.org diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 1c9c155ba..ba287eff3 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -49,6 +49,7 @@ services: - TILE_CACHE_CLUSTER=tile_cache_cluster - TILE_CACHE_SERVICE=tile_cache_service - PIXETL_JOB_QUEUE=pixetl_jq + - COGIFY_JOB_QUEUE=cogify_jq - PIXETL_CORES=1 - MAX_CORES=1 - NUM_PROCESSES=1 diff --git a/terraform/data.tf b/terraform/data.tf index ba7a49cb4..4064a2c88 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -68,7 +68,7 @@ data "template_file" "container_definition" { tile_cache_job_queue = module.batch_job_queues.tile_cache_job_queue_arn pixetl_job_definition = module.batch_job_queues.pixetl_job_definition_arn pixetl_job_queue = module.batch_job_queues.pixetl_job_queue_arn - cog_job_queue = module.batch_job_queues.cog_job_queue_arn + cogify_job_queue = module.batch_job_queues.cogify_job_queue_arn raster_analysis_lambda_name = "raster-analysis-tiled_raster_analysis-default" raster_analysis_sfn_arn = data.terraform_remote_state.raster_analysis_lambda.outputs.raster_analysis_state_machine_arn service_url = local.service_url diff --git a/terraform/modules/batch/outputs.tf b/terraform/modules/batch/outputs.tf index 863bb90cd..b4d458b71 100644 --- a/terraform/modules/batch/outputs.tf +++ b/terraform/modules/batch/outputs.tf @@ -38,7 +38,7 @@ output "pixetl_job_queue_arn" { value = aws_batch_job_queue.pixetl.arn } -output "cog_job_queue_arn" { +output "cogify_job_queue_arn" { value = aws_batch_job_queue.cog.arn } diff --git a/terraform/templates/container_definition.json.tmpl b/terraform/templates/container_definition.json.tmpl index d62cb9c47..b24963a12 100644 --- a/terraform/templates/container_definition.json.tmpl +++ b/terraform/templates/container_definition.json.tmpl @@ -74,8 +74,8 @@ "value": "${pixetl_job_queue}" }, { - "name": "COG_JOB_QUEUE", - "value": "${cog_job_queue}" + "name": "COGIFY_JOB_QUEUE", + "value": "${cogify_job_queue}" }, { "name": "API_URL", From 9e2e67c8582d77c2bbc535d75af79adf77d90465 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 23 Sep 2024 17:06:44 +0300 Subject: [PATCH 04/10] fix module name --- terraform/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 17c5f92a2..8feed191e 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -202,7 +202,7 @@ module "batch_cog_creator" { use_ephemeral_storage = true launch_type = "EC2" instance_types = var.data_lake_writer_instance_types - compute_environment_name = "cog_creator" + compute_environment_name = "batch_cog_creator" } module "batch_job_queues" { @@ -211,7 +211,7 @@ module "batch_job_queues" { data_lake_compute_environment_arn = module.batch_data_lake_writer.arn pixetl_compute_environment_arn = module.batch_data_lake_writer.arn tile_cache_compute_environment_arn = module.batch_data_lake_writer.arn - cog_compute_environment_arn = module.cog_creator.arn + cog_compute_environment_arn = module.batch_cog_creator.arn environment = var.environment name_suffix = local.name_suffix project = local.project From 451eda186d1192c061407741dfca4c38ac0b9c03 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Mon, 23 Sep 2024 18:45:31 +0300 Subject: [PATCH 05/10] add cogify batch queue to moto --- tests/conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 50af56e3a..b2d8204e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,7 @@ GDAL_PYTHON_JOB_DEFINITION, PIXETL_JOB_DEFINITION, PIXETL_JOB_QUEUE, + COGIFY_JOB_QUEUE, POSTGRESQL_CLIENT_JOB_DEFINITION, TILE_CACHE_BUCKET, TILE_CACHE_JOB_DEFINITION, @@ -167,6 +168,7 @@ def patch_run(self, *k, **kwargs): "s3_writer", subnet_id, sg_id, iam_arn ) pixetl_env = aws_mock.add_compute_environment("pixetl", subnet_id, sg_id, iam_arn) + cogify_env = aws_mock.add_compute_environment("cogify", subnet_id, sg_id, iam_arn) aws_mock.add_job_queue(AURORA_JOB_QUEUE, aurora_writer_env["computeEnvironmentArn"]) aws_mock.add_job_queue( @@ -175,6 +177,7 @@ def patch_run(self, *k, **kwargs): aws_mock.add_job_queue(DATA_LAKE_JOB_QUEUE, s3_writer_env["computeEnvironmentArn"]) aws_mock.add_job_queue(TILE_CACHE_JOB_QUEUE, s3_writer_env["computeEnvironmentArn"]) aws_mock.add_job_queue(PIXETL_JOB_QUEUE, pixetl_env["computeEnvironmentArn"]) + aws_mock.add_job_queue(COGIFY_JOB_QUEUE, cogify_env["computeEnvironmentArn"]) aws_mock.add_job_definition(GDAL_PYTHON_JOB_DEFINITION, "batch_gdal-python_test") aws_mock.add_job_definition( From 10d357e7248a602f218c70cc4eb01a5e4f8f8cf3 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Tue, 24 Sep 2024 12:16:07 +0300 Subject: [PATCH 06/10] generalize on-demand job queue name --- app/models/pydantic/jobs.py | 6 +++--- app/settings/globals.py | 2 +- docker-compose.dev.yml | 2 +- docker-compose.prod.yml | 2 +- docker-compose.test.yml | 2 +- terraform/data.tf | 2 +- terraform/main.tf | 6 +++--- terraform/modules/batch/main.tf | 6 +++--- terraform/modules/batch/outputs.tf | 4 ++-- terraform/modules/batch/variables.tf | 2 +- terraform/templates/container_definition.json.tmpl | 4 ++-- tests/conftest.py | 6 ++++-- 12 files changed, 23 insertions(+), 21 deletions(-) diff --git a/app/models/pydantic/jobs.py b/app/models/pydantic/jobs.py index 52bc0e1bb..643292f2e 100644 --- a/app/models/pydantic/jobs.py +++ b/app/models/pydantic/jobs.py @@ -4,7 +4,7 @@ from ...settings.globals import ( AURORA_JOB_QUEUE, - COGIFY_JOB_QUEUE, + ON_DEMAND_COMPUTE_JOB_QUEUE, DATA_LAKE_JOB_QUEUE, DEFAULT_JOB_DURATION, GDAL_PYTHON_JOB_DEFINITION, @@ -139,9 +139,9 @@ class PixETLJob(Job): class GDALCOGJob(Job): - """Use for creating COG files using GDAL Python docker in COG queue.""" + """Use for creating COG files using GDAL Python docker in on-demand compute queue.""" - job_queue = COGIFY_JOB_QUEUE + job_queue = ON_DEMAND_COMPUTE_JOB_QUEUE job_definition = GDAL_PYTHON_JOB_DEFINITION vcpus = 8 memory = 64000 diff --git a/app/settings/globals.py b/app/settings/globals.py index 9d3fa4156..47d9ac728 100644 --- a/app/settings/globals.py +++ b/app/settings/globals.py @@ -116,7 +116,7 @@ MAX_MEM = config("MAX_MEM", cast=int, default=760000) PIXETL_JOB_DEFINITION = config("PIXETL_JOB_DEFINITION", cast=str) PIXETL_JOB_QUEUE = config("PIXETL_JOB_QUEUE", cast=str) -COGIFY_JOB_QUEUE = config("COGIFY_JOB_QUEUE", cast=str) +ON_DEMAND_COMPUTE_JOB_QUEUE = config("ON_DEMAND_COMPUTE_JOB_QUEUE", cast=str) PIXETL_CORES = config("PIXETL_CORES", cast=int, default=48) PIXETL_MAX_MEM = config("PIXETL_MAX_MEM", cast=int, default=380000) PIXETL_DEFAULT_RESAMPLING = config( diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index b755f2eb6..98a0e5f83 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -39,7 +39,7 @@ services: - TILE_CACHE_CLUSTER=tile_cache_cluster - TILE_CACHE_SERVICE=tile_cache_service - PIXETL_JOB_QUEUE=pixetl_jq - - COGIFY_JOB_QUEUE=cogify_jq + - ON_DEMAND_COMPUTE_JOB_QUEUE=cogify_jq - API_URL=http://app_dev:80 - RASTER_ANALYSIS_LAMBDA_NAME=raster-analysis-tiled_raster_analysis-default - RW_API_URL=https://staging-api.resourcewatch.org diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 7f13c9f1e..461ed1749 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -34,7 +34,7 @@ services: - DATA_LAKE_JOB_QUEUE=data_lake_jq - TILE_CACHE_JOB_QUEUE=tile_cache_jq - PIXETL_JOB_QUEUE=pixetl_jq - - COGIFY_JOB_QUEUE=cogify_jq + - ON_DEMAND_COMPUTE_JOB_QUEUE=cogify_jq - RASTER_ANALYSIS_LAMBDA_NAME=raster_analysis - API_URL="http://app_dev:80" - RW_API_URL=https://api.resourcewatch.org diff --git a/docker-compose.test.yml b/docker-compose.test.yml index ba287eff3..0288e1675 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -49,7 +49,7 @@ services: - TILE_CACHE_CLUSTER=tile_cache_cluster - TILE_CACHE_SERVICE=tile_cache_service - PIXETL_JOB_QUEUE=pixetl_jq - - COGIFY_JOB_QUEUE=cogify_jq + - ON_DEMAND_COMPUTE_JOB_QUEUE=cogify_jq - PIXETL_CORES=1 - MAX_CORES=1 - NUM_PROCESSES=1 diff --git a/terraform/data.tf b/terraform/data.tf index 4064a2c88..d9e2861aa 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -68,7 +68,7 @@ data "template_file" "container_definition" { tile_cache_job_queue = module.batch_job_queues.tile_cache_job_queue_arn pixetl_job_definition = module.batch_job_queues.pixetl_job_definition_arn pixetl_job_queue = module.batch_job_queues.pixetl_job_queue_arn - cogify_job_queue = module.batch_job_queues.cogify_job_queue_arn + on_demand_compute_job_queue = module.batch_job_queues.on_demand_compute_job_queue_arn raster_analysis_lambda_name = "raster-analysis-tiled_raster_analysis-default" raster_analysis_sfn_arn = data.terraform_remote_state.raster_analysis_lambda.outputs.raster_analysis_state_machine_arn service_url = local.service_url diff --git a/terraform/main.tf b/terraform/main.tf index 8feed191e..80d5249ca 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -179,7 +179,7 @@ module "batch_data_lake_writer" { compute_environment_name = "data_lake_writer" } -module "batch_cog_creator" { +module "batch_cogify" { source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/compute_environment?ref=v0.4.2.3" ecs_role_policy_arns = [ aws_iam_policy.query_batch_jobs.arn, @@ -202,7 +202,7 @@ module "batch_cog_creator" { use_ephemeral_storage = true launch_type = "EC2" instance_types = var.data_lake_writer_instance_types - compute_environment_name = "batch_cog_creator" + compute_environment_name = "batch_cogify" } module "batch_job_queues" { @@ -211,7 +211,7 @@ module "batch_job_queues" { data_lake_compute_environment_arn = module.batch_data_lake_writer.arn pixetl_compute_environment_arn = module.batch_data_lake_writer.arn tile_cache_compute_environment_arn = module.batch_data_lake_writer.arn - cog_compute_environment_arn = module.batch_cog_creator.arn + cogify_compute_environment_arn = module.batch_cogify.arn environment = var.environment name_suffix = local.name_suffix project = local.project diff --git a/terraform/modules/batch/main.tf b/terraform/modules/batch/main.tf index e62c38214..de99d49ba 100644 --- a/terraform/modules/batch/main.tf +++ b/terraform/modules/batch/main.tf @@ -52,12 +52,12 @@ resource "aws_batch_job_queue" "pixetl" { depends_on = [var.pixetl_compute_environment_arn] } -resource "aws_batch_job_queue" "cog" { +resource "aws_batch_job_queue" "on_demand" { name = substr("${var.project}-cog-job-queue${var.name_suffix}", 0, 64) state = "ENABLED" priority = 1 - compute_environments = [var.cog_compute_environment_arn] - depends_on = [var.pixetl_compute_environment_arn] + compute_environments = [var.cogify_compute_environment_arn] + depends_on = [var.cogify_compute_environment_arn] } resource "aws_batch_job_definition" "tile_cache" { diff --git a/terraform/modules/batch/outputs.tf b/terraform/modules/batch/outputs.tf index b4d458b71..2962aa93c 100644 --- a/terraform/modules/batch/outputs.tf +++ b/terraform/modules/batch/outputs.tf @@ -38,8 +38,8 @@ output "pixetl_job_queue_arn" { value = aws_batch_job_queue.pixetl.arn } -output "cogify_job_queue_arn" { - value = aws_batch_job_queue.cog.arn +output "on_demand_compute_job_queue_arn" { + value = aws_batch_job_queue.on_demand.arn } output "tile_cache_job_definition_arn" { diff --git a/terraform/modules/batch/variables.tf b/terraform/modules/batch/variables.tf index 7c03c859f..ff2005c07 100644 --- a/terraform/modules/batch/variables.tf +++ b/terraform/modules/batch/variables.tf @@ -2,7 +2,7 @@ variable "project" { type = string } variable "name_suffix" { type = string } variable "aurora_compute_environment_arn" { type = string } variable "data_lake_compute_environment_arn" { type = string } -variable "cog_compute_environment_arn" { type = string } +variable "cogify_compute_environment_arn" { type = string } variable "tile_cache_compute_environment_arn" { type = string } variable "pixetl_compute_environment_arn" { type = string } variable "gdal_repository_url" { type = string } diff --git a/terraform/templates/container_definition.json.tmpl b/terraform/templates/container_definition.json.tmpl index b24963a12..e2de276e7 100644 --- a/terraform/templates/container_definition.json.tmpl +++ b/terraform/templates/container_definition.json.tmpl @@ -74,8 +74,8 @@ "value": "${pixetl_job_queue}" }, { - "name": "COGIFY_JOB_QUEUE", - "value": "${cogify_job_queue}" + "name": "ON_DEMAND_COMPUTE_JOB_QUEUE", + "value": "${on_demand_compute_job_queue}" }, { "name": "API_URL", diff --git a/tests/conftest.py b/tests/conftest.py index b2d8204e7..74e415d2d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,7 +33,7 @@ GDAL_PYTHON_JOB_DEFINITION, PIXETL_JOB_DEFINITION, PIXETL_JOB_QUEUE, - COGIFY_JOB_QUEUE, + ON_DEMAND_COMPUTE_JOB_QUEUE, POSTGRESQL_CLIENT_JOB_DEFINITION, TILE_CACHE_BUCKET, TILE_CACHE_JOB_DEFINITION, @@ -177,7 +177,9 @@ def patch_run(self, *k, **kwargs): aws_mock.add_job_queue(DATA_LAKE_JOB_QUEUE, s3_writer_env["computeEnvironmentArn"]) aws_mock.add_job_queue(TILE_CACHE_JOB_QUEUE, s3_writer_env["computeEnvironmentArn"]) aws_mock.add_job_queue(PIXETL_JOB_QUEUE, pixetl_env["computeEnvironmentArn"]) - aws_mock.add_job_queue(COGIFY_JOB_QUEUE, cogify_env["computeEnvironmentArn"]) + aws_mock.add_job_queue( + ON_DEMAND_COMPUTE_JOB_QUEUE, cogify_env["computeEnvironmentArn"] + ) aws_mock.add_job_definition(GDAL_PYTHON_JOB_DEFINITION, "batch_gdal-python_test") aws_mock.add_job_definition( From d9d2473bb2d564708606bc6a5ff2d1ab6ea6d4f0 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Tue, 24 Sep 2024 12:19:43 +0300 Subject: [PATCH 07/10] update queue name --- terraform/modules/batch/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/modules/batch/main.tf b/terraform/modules/batch/main.tf index de99d49ba..fb5207473 100644 --- a/terraform/modules/batch/main.tf +++ b/terraform/modules/batch/main.tf @@ -53,7 +53,7 @@ resource "aws_batch_job_queue" "pixetl" { } resource "aws_batch_job_queue" "on_demand" { - name = substr("${var.project}-cog-job-queue${var.name_suffix}", 0, 64) + name = substr("${var.project}-on-demand-job-queue${var.name_suffix}", 0, 64) state = "ENABLED" priority = 1 compute_environments = [var.cogify_compute_environment_arn] From 62b7db4adc5a892df2694ffa1ea6fb21d04776fe Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Tue, 8 Oct 2024 11:01:31 +0300 Subject: [PATCH 08/10] give ecs task role permission to run on on-demand compute job queue --- terraform/data.tf | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/terraform/data.tf b/terraform/data.tf index d9e2861aa..dcb8aacd0 100644 --- a/terraform/data.tf +++ b/terraform/data.tf @@ -96,15 +96,16 @@ data "template_file" "container_definition" { data "template_file" "task_batch_policy" { template = file("${path.root}/templates/run_batch_policy.json.tmpl") vars = { - aurora_job_definition_arn = module.batch_job_queues.aurora_job_definition_arn - aurora_job_queue_arn = module.batch_job_queues.aurora_job_queue_arn - aurora_job_queue_fast_arn = module.batch_job_queues.aurora_job_queue_fast_arn - data_lake_job_definition_arn = module.batch_job_queues.data_lake_job_definition_arn - data_lake_job_queue_arn = module.batch_job_queues.data_lake_job_queue_arn - tile_cache_job_definition_arn = module.batch_job_queues.tile_cache_job_definition_arn - tile_cache_job_queue_arn = module.batch_job_queues.tile_cache_job_queue_arn - pixetl_job_definition_arn = module.batch_job_queues.pixetl_job_definition_arn - pixetl_job_queue_arn = module.batch_job_queues.pixetl_job_queue_arn + aurora_job_definition_arn = module.batch_job_queues.aurora_job_definition_arn + aurora_job_queue_arn = module.batch_job_queues.aurora_job_queue_arn + aurora_job_queue_fast_arn = module.batch_job_queues.aurora_job_queue_fast_arn + data_lake_job_definition_arn = module.batch_job_queues.data_lake_job_definition_arn + data_lake_job_queue_arn = module.batch_job_queues.data_lake_job_queue_arn + tile_cache_job_definition_arn = module.batch_job_queues.tile_cache_job_definition_arn + tile_cache_job_queue_arn = module.batch_job_queues.tile_cache_job_queue_arn + pixetl_job_definition_arn = module.batch_job_queues.pixetl_job_definition_arn + pixetl_job_queue_arn = module.batch_job_queues.pixetl_job_queue_arn + on_demand_compute_job_queue_arn = module.batch_job_queues.on_demand_compute_job_queue_arn } depends_on = [ module.batch_job_queues.aurora_job_definition, From b26841483559474ea9802104cc6aef768bf67cf8 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Tue, 8 Oct 2024 11:16:37 +0300 Subject: [PATCH 09/10] add missing variable in template --- terraform/templates/run_batch_policy.json.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/templates/run_batch_policy.json.tmpl b/terraform/templates/run_batch_policy.json.tmpl index 31c2886a8..a26d9f739 100644 --- a/terraform/templates/run_batch_policy.json.tmpl +++ b/terraform/templates/run_batch_policy.json.tmpl @@ -21,7 +21,9 @@ "${tile_cache_job_definition_arn}", "${pixetl_job_queue_arn}", - "${pixetl_job_definition_arn}" + "${pixetl_job_definition_arn}", + + "{on_demand_compute_job_queue_arn}" ] }, { From 50ab46a6bfea052fb1aa2be3ccbec8a311b30c63 Mon Sep 17 00:00:00 2001 From: Solomon Negusse Date: Tue, 8 Oct 2024 11:24:04 +0300 Subject: [PATCH 10/10] syntax fix --- terraform/templates/run_batch_policy.json.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/templates/run_batch_policy.json.tmpl b/terraform/templates/run_batch_policy.json.tmpl index a26d9f739..044608630 100644 --- a/terraform/templates/run_batch_policy.json.tmpl +++ b/terraform/templates/run_batch_policy.json.tmpl @@ -23,7 +23,7 @@ "${pixetl_job_queue_arn}", "${pixetl_job_definition_arn}", - "{on_demand_compute_job_queue_arn}" + "${on_demand_compute_job_queue_arn}" ] }, {