fix: keep metadata service images in sync (#4)

* fix: keep metadata service images in sync

* refactor: DRY up metaflow service container image reference

* fix: rename common module output to match usages

* fix: represent common module output properly and regenerate docs

* fix: switch back to  attribute

* fix: commit pre-commit hook fixes

* fix: update metadata service image name

Co-authored-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>

Co-authored-by: Oleg Avdeev <oleg.v.avdeev@gmail.com>

Author: ctrombley

README.md

@@ -23,6 +23,7 @@ You can find a complete example that uses this module but also includes setting
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_metaflow-common"></a> [metaflow-common](#module\_metaflow-common) | ./modules/common | n/a |
 | <a name="module_metaflow-computation"></a> [metaflow-computation](#module\_metaflow-computation) | ./modules/computation | n/a |
 | <a name="module_metaflow-datastore"></a> [metaflow-datastore](#module\_metaflow-datastore) | ./modules/datastore | n/a |
 | <a name="module_metaflow-metadata-service"></a> [metaflow-metadata-service](#module\_metaflow-metadata-service) | ./modules/metadata-service | n/a |
@@ -45,6 +46,7 @@ You can find a complete example that uses this module but also includes setting
 | <a name="input_extra_ui_backend_env_vars"></a> [extra\_ui\_backend\_env\_vars](#input\_extra\_ui\_backend\_env\_vars) | Additional environment variables for UI backend container | `map(string)` | `{}` | no |
 | <a name="input_extra_ui_static_env_vars"></a> [extra\_ui\_static\_env\_vars](#input\_extra\_ui\_static\_env\_vars) | Additional environment variables for UI static app | `map(string)` | `{}` | no |
 | <a name="input_iam_partition"></a> [iam\_partition](#input\_iam\_partition) | IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is) | `string` | `"aws"` | no |
+| <a name="input_metadata_service_container_image"></a> [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no |
 | <a name="input_resource_prefix"></a> [resource\_prefix](#input\_resource\_prefix) | string prefix for all resources | `string` | `"metaflow"` | no |
 | <a name="input_resource_suffix"></a> [resource\_suffix](#input\_resource\_suffix) | string suffix for all resources | `string` | `""` | no |
 | <a name="input_subnet1_id"></a> [subnet1\_id](#input\_subnet1\_id) | First subnet used for availability zone redundancy | `string` | n/a | yes |

locals.tf

@@ -1,3 +1,7 @@
+module "metaflow-common" {
+  source = "./modules/common"
+}
+
 locals {
   resource_prefix = length(var.resource_prefix) > 0 ? "${var.resource_prefix}-" : ""
   resource_suffix = length(var.resource_suffix) > 0 ? "-${var.resource_suffix}" : ""
@@ -7,4 +11,9 @@ locals {
 
   batch_s3_task_role_name   = "${local.resource_prefix}batch_s3_task_role${local.resource_suffix}"
   metaflow_batch_image_name = "${local.resource_prefix}batch${local.resource_suffix}"
+  metadata_service_container_image  = (
+    var.metadata_service_container_image == "" ?
+    module.metaflow-common.default_metadata_service_container_image :
+    var.metadata_service_container_image
+  )
 }

main.tf

@@ -20,19 +20,20 @@ module "metaflow-metadata-service" {
   resource_prefix = local.resource_prefix
   resource_suffix = local.resource_suffix
 
-  access_list_cidr_blocks         = var.access_list_cidr_blocks
-  api_basic_auth                  = var.api_basic_auth
-  database_password               = module.metaflow-datastore.database_password
-  database_username               = module.metaflow-datastore.database_username
-  datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
-  fargate_execution_role_arn      = module.metaflow-computation.ecs_execution_role_arn
-  iam_partition                   = var.iam_partition
-  metaflow_vpc_id                 = var.vpc_id
-  rds_master_instance_endpoint    = module.metaflow-datastore.rds_master_instance_endpoint
-  s3_bucket_arn                   = module.metaflow-datastore.s3_bucket_arn
-  subnet1_id                      = var.subnet1_id
-  subnet2_id                      = var.subnet2_id
-  vpc_cidr_block                  = var.vpc_cidr_block
+  access_list_cidr_blocks           = var.access_list_cidr_blocks
+  api_basic_auth                    = var.api_basic_auth
+  database_password                 = module.metaflow-datastore.database_password
+  database_username                 = module.metaflow-datastore.database_username
+  datastore_s3_bucket_kms_key_arn   = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
+  fargate_execution_role_arn        = module.metaflow-computation.ecs_execution_role_arn
+  iam_partition                     = var.iam_partition
+  metadata_service_container_image  = local.metadata_service_container_image
+  metaflow_vpc_id                   = var.vpc_id
+  rds_master_instance_endpoint      = module.metaflow-datastore.rds_master_instance_endpoint
+  s3_bucket_arn                     = module.metaflow-datastore.s3_bucket_arn
+  subnet1_id                        = var.subnet1_id
+  subnet2_id                        = var.subnet2_id
+  vpc_cidr_block                    = var.vpc_cidr_block
 
   standard_tags = var.tags
 }
@@ -53,6 +54,7 @@ module "metaflow-ui" {
   s3_bucket_arn                   = module.metaflow-datastore.s3_bucket_arn
   subnet1_id                      = var.subnet1_id
   subnet2_id                      = var.subnet2_id
+  ui_backend_container_image      = local.metadata_service_container_image
 
   METAFLOW_DATASTORE_SYSROOT_S3   = module.metaflow-datastore.METAFLOW_DATASTORE_SYSROOT_S3
   certificate_arn = var.ui_certificate_arn

modules/common/README.md

@@ -0,0 +1,11 @@
+<!-- BEGIN_TF_DOCS -->
+## Inputs
+
+No inputs.
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_default_metadata_service_container_image"></a> [default\_metadata\_service\_container\_image](#output\_default\_metadata\_service\_container\_image) | n/a |
+<!-- END_TF_DOCS -->

modules/common/locals.tf

@@ -0,0 +1,3 @@
+locals {
+  default_metadata_service_container_image = "netflixoss/metaflow_metadata_service:v2.2.2"
+}

modules/common/outputs.tf

@@ -0,0 +1,3 @@
+output "default_metadata_service_container_image" {
+  value = local.default_metadata_service_container_image
+}

modules/metadata-service/README.md

@@ -23,6 +23,7 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
 | <a name="input_fargate_execution_role_arn"></a> [fargate\_execution\_role\_arn](#input\_fargate\_execution\_role\_arn) | The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution\_role for our Fargate instance | `string` | n/a | yes |
 | <a name="input_iam_partition"></a> [iam\_partition](#input\_iam\_partition) | IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is) | `string` | `"aws"` | no |
 | <a name="input_is_gov"></a> [is\_gov](#input\_is\_gov) | Set to true if IAM partition is 'aws-us-gov' | `bool` | `false` | no |
+| <a name="input_metadata_service_container_image"></a> [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no |
 | <a name="input_metaflow_vpc_id"></a> [metaflow\_vpc\_id](#input\_metaflow\_vpc\_id) | ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in | `string` | n/a | yes |
 | <a name="input_rds_master_instance_endpoint"></a> [rds\_master\_instance\_endpoint](#input\_rds\_master\_instance\_endpoint) | The database connection endpoint in address:port format | `string` | n/a | yes |
 | <a name="input_resource_prefix"></a> [resource\_prefix](#input\_resource\_prefix) | Prefix given to all AWS resources to differentiate between applications | `string` | n/a | yes |

modules/metadata-service/ecs.tf

@@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "this" {
 [
   {
     "name": "${var.resource_prefix}service${var.resource_suffix}",
-    "image": "netflixoss/metaflow_metadata_service",
+    "image": "${var.metadata_service_container_image}",
     "essential": true,
     "cpu": 512,
     "memory": 1024,

modules/metadata-service/locals.tf

@@ -1,3 +1,7 @@
+module "metaflow-common" {
+  source = "../common"
+}
+
 locals {
   # Name of ECS cluster.
   # replace() ensures names that are composed of just prefix + suffix do not have duplicate dashes
@@ -7,6 +11,11 @@ locals {
 
   # Name of Fargate security group used by the Metadata Service
   metadata_service_security_group_name = "${var.resource_prefix}metadata-service-security-group${var.resource_suffix}"
+  metadata_service_container_image  = (
+    var.metadata_service_container_image == "" ?
+    module.metaflow-common.default_metadata_service_container_image :
+    var.metadata_service_container_image
+  )
 
   api_gateway_endpoint_configuration_type = local.is_gov ? "REGIONAL" : "EDGE"
   api_gateway_key_name                    = "${var.resource_prefix}key${var.resource_suffix}"

modules/metadata-service/variables.tf

@@ -41,6 +41,12 @@ variable "is_gov" {
   description = "Set to true if IAM partition is 'aws-us-gov'"
 }
 
+variable "metadata_service_container_image" {
+  type = string
+  default = ""
+  description = "Container image for metadata service"
+}
+
 variable "metaflow_vpc_id" {
   type        = string
   description = "ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in"

modules/ui/README.md

@@ -29,7 +29,7 @@ The services are deployed behind an AWS ALB, and the module will output the ALB
 | <a name="input_subnet1_id"></a> [subnet1\_id](#input\_subnet1\_id) | First private subnet used for availability zone redundancy | `string` | n/a | yes |
 | <a name="input_subnet2_id"></a> [subnet2\_id](#input\_subnet2\_id) | Second private subnet used for availability zone redundancy | `string` | n/a | yes |
 | <a name="input_ui_allow_list"></a> [ui\_allow\_list](#input\_ui\_allow\_list) | A list of CIDRs the UI will be available to | `list(string)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
-| <a name="input_ui_backend_container_image"></a> [ui\_backend\_container\_image](#input\_ui\_backend\_container\_image) | Container image for UI backend | `string` | `"netflixoss/metaflow_metadata_service:2.1.0"` | no |
+| <a name="input_ui_backend_container_image"></a> [ui\_backend\_container\_image](#input\_ui\_backend\_container\_image) | Container image for UI backend | `string` | `""` | no |
 | <a name="input_ui_static_container_image"></a> [ui\_static\_container\_image](#input\_ui\_static\_container\_image) | Container image for the UI frontend app | `string` | `"public.ecr.aws/outerbounds/metaflow_ui:v1.0.1"` | no |
 
 ## Outputs

modules/ui/locals.tf

@@ -1,3 +1,7 @@
+module "metaflow-common" {
+  source = "../common"
+}
+
 locals {
   # Name of ECS cluster.
   # replace() ensures names that are composed of just prefix + suffix do not have duplicate dashes
@@ -27,4 +31,10 @@ locals {
   }
 
   default_ui_static_env_vars = {}
+
+  metadata_service_container_image  = (
+    var.ui_backend_container_image == "" ?
+    module.metaflow-common.default_metadata_service_container_image :
+    var.ui_backend_container_image
+  )
 }

modules/ui/variables.tf

@@ -100,7 +100,7 @@ variable "extra_ui_static_env_vars" {
 
 variable "ui_backend_container_image" {
   type = string
-  default = "netflixoss/metaflow_metadata_service:2.1.0"
+  default = ""
   description = "Container image for UI backend"
 }
 

variables.tf

@@ -67,6 +67,12 @@ variable "iam_partition" {
   description = "IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is)"
 }
 
+variable "metadata_service_container_image" {
+  type = string
+  default = ""
+  description = "Container image for metadata service"
+}
+
 variable "tags" {
   description = "aws tags"
   type        = map(string)