eks example (#10)

Author: oavdeev

.gitignore

@@ -0,0 +1,4 @@
+.terraform
+.terraform.lock.hcl
+terraform.tfstate
+terraform.tfstate.backup

examples/eks/README.md

@@ -0,0 +1,49 @@
+# An example of deploying Metaflow with a EKS cluster
+
+This example will create Metaflow infrastructure from scratch, with a Kubernetes cluster using Amazon EKS. It uses [`datastore`](../../modules/datastore/) and [`metadata-service`](../../modules/metadata-service/) submodules to provision S3 bucket, RDS database and Metaflow Metadata service running on AWS Fargate.
+
+To run Metaflow jobs, it provisions a EKS cluster using [this popular open source terraform module](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest). In that cluster, it also installs [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) and [Argo Workflows](https://argoproj.github.io/argo-workflows/) using Helm.
+
+Specifically, it'll create following resources in your AWS account:
+* General networking infra:
+    * AWS VPC
+    * NAT gateway for private subnets in the VPC
+* For storing data artifacts:
+    * S3 bucket
+* For Metaflow metadata:
+    * RDS Database instance (on-demand, Multi-AZ, db.t2.small)
+    * ECS service for Metaflow Metadata service
+    * Network load balancer
+    * API Gateway
+* For executing Metaflow tasks:
+    * Autoscaling EKS cluster with at least one instance running
+
+Note that all this infrastructure costs a non-trivial amount at rest, up to $400/month and more if being actively used.
+
+## Instructions
+
+0. Run `terraform init`
+1. Run `terraform apply` to create infrastructure. This command will typically take ~20 minutes to execute.
+2. Make note of the EKS cluster name (it is a short string that starts with `mf-`). Use AWS CLI to generate cluster configuration:
+    ```bash
+    aws eks update-kubeconfig --name <CLUSTER NAME>
+    ```
+2. Copy `config.json` to `~/.metaflowconfig/`
+3. You should be ready to run Metaflow flows using `@kubernetes`
+and be able to deploy them to Argo workflows.
+
+Argo Workflows UI is not accessible from outside the cluster, but you can use port forwarding to see it. Run
+```bash
+kubectl port-forward -n argo service/argo-argo-workflows-server 2746:2746
+```
+..and you should be able to access it at `localhost:2746`.
+
+## Destroying the infrastructure
+
+Run `terraform destroy`
+
+# What's missing
+
+⚠️ This is meant as a reference example, with many things omitted for simplicity, such as proper RBAC setup, production-grade autoscaling and UI. For example, all workloads running in the cluster use the same AWS IAM role. We do not recommend using this as a production deployment of Metaflow on Kubernetes.
+
+For learn more about production-grade deployments, you can talk to us on [the Outerbounds slack](http://slack.outerbounds.co). We are happy to help you there!

examples/eks/eks.tf

@@ -0,0 +1,123 @@
+
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "17.23.0"
+
+  cluster_name    = local.cluster_name
+  cluster_version = "1.21"
+  subnets         = module.vpc.private_subnets
+  enable_irsa     = true
+  tags            = local.tags
+
+  vpc_id = module.vpc.vpc_id
+
+  node_groups_defaults = {
+    ami_type  = "AL2_x86_64"
+    disk_size = 50
+  }
+
+
+  node_groups = {
+    main = {
+      desired_capacity = 1
+      max_capacity     = 5
+      min_capacity     = 1
+
+      instance_types = ["r5.large"]
+      update_config = {
+        max_unavailable_percentage = 50
+      }
+    }
+  }
+
+  workers_additional_policies = [
+    aws_iam_policy.default_node.arn,
+    aws_iam_policy.cluster_autoscaler.arn,
+  ]
+}
+
+
+resource "aws_iam_policy" "default_node" {
+  name_prefix = "${local.cluster_name}-default"
+  description = "Default policy for cluster ${module.eks.cluster_id}"
+  policy      = data.aws_iam_policy_document.default_node.json
+}
+
+data "aws_iam_policy_document" "default_node" {
+  statement {
+    sid    = "S3"
+    effect = "Allow"
+
+    actions = [
+      "s3:*",
+      "kms:*",
+    ]
+
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_policy" "cluster_autoscaler" {
+  name_prefix = "cluster-autoscaler"
+  description = "EKS cluster-autoscaler policy for cluster ${module.eks.cluster_id}"
+  policy      = data.aws_iam_policy_document.cluster_autoscaler.json
+}
+
+data "aws_iam_policy_document" "cluster_autoscaler" {
+  statement {
+    sid    = "clusterAutoscalerAll"
+    effect = "Allow"
+
+    actions = [
+      "autoscaling:DescribeAutoScalingGroups",
+      "autoscaling:DescribeAutoScalingInstances",
+      "autoscaling:DescribeLaunchConfigurations",
+      "autoscaling:DescribeTags",
+      "ec2:DescribeLaunchTemplateVersions",
+    ]
+
+    resources = ["*"]
+  }
+
+  statement {
+    sid    = "clusterAutoscalerOwn"
+    effect = "Allow"
+
+    actions = [
+      "autoscaling:SetDesiredCapacity",
+      "autoscaling:TerminateInstanceInAutoScalingGroup",
+      "autoscaling:UpdateAutoScalingGroup",
+    ]
+
+    resources = ["*"]
+
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/kubernetes.io/cluster/${module.eks.cluster_id}"
+      values   = ["owned"]
+    }
+
+    condition {
+      test     = "StringEquals"
+      variable = "autoscaling:ResourceTag/k8s.io/cluster-autoscaler/enabled"
+      values   = ["true"]
+    }
+  }
+}
+
+
+data "aws_eks_cluster" "cluster" {
+  name = module.eks.cluster_id
+}
+
+data "aws_eks_cluster_auth" "cluster" {
+  name = module.eks.cluster_id
+}
+
+data "aws_caller_identity" "current" {}
+
+provider "kubernetes" {
+  host                   = data.aws_eks_cluster.cluster.endpoint
+  cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
+  token                  = data.aws_eks_cluster_auth.cluster.token
+}

examples/eks/helm.tf

@@ -0,0 +1,85 @@
+provider "helm" {
+  kubernetes {
+    host                   = data.aws_eks_cluster.cluster.endpoint
+    cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data)
+    token                  = data.aws_eks_cluster_auth.cluster.token
+  }
+}
+
+resource "helm_release" "cluster_autoscaler" {
+  name = "autoscaler"
+
+  depends_on = [module.eks]
+
+  repository = "https://kubernetes.github.io/autoscaler"
+  chart      = "cluster-autoscaler"
+  namespace  = "kube-system"
+
+  set {
+    name  = "autoDiscovery.clusterName"
+    value = local.cluster_name
+  }
+
+  set {
+    name  = "awsRegion"
+    value = data.aws_region.current.name
+  }
+}
+
+
+resource "kubernetes_namespace" "argo" {
+  metadata {
+    name = "argo"
+  }
+}
+
+resource "kubernetes_default_service_account" "default" {
+  metadata {
+    namespace = kubernetes_namespace.argo.metadata[0].name
+  }
+}
+
+data "aws_region" "current" {}
+
+locals {
+  argo_values = {
+    "server" = {
+      "extraArgs" = ["--auth-mode=server"]
+    }
+    "workflow" = {
+      "serviceAccount" = {
+        "create" = true
+      }
+    }
+    "controller" = {
+      "containerRuntimeExecutor" = "emissary"
+    }
+    "useDefaultArtifactRepo" = true
+    "useStaticCredentials" = false
+    "artifactRepository" = {
+      "s3" = {
+        "bucket" = module.metaflow-datastore.s3_bucket_name
+        "keyFormat" = "argo-artifacts/{{workflow.creationTimestamp.Y}}/{{workflow.creationTimestamp.m}}/{{workflow.creationTimestamp.d}}/{{workflow.name}}/{{pod.name}}"
+        "region" = data.aws_region.current.name
+        "endpoint" = "s3.amazonaws.com"
+        "useSDKCreds" = true
+        "insecure" = false
+      }
+    }
+  }
+}
+
+resource "helm_release" "argo" {
+  name = "argo"
+
+  depends_on = [module.eks]
+
+  repository = "https://argoproj.github.io/argo-helm"
+  chart      = "argo-workflows"
+  namespace  = kubernetes_namespace.argo.metadata[0].name
+  force_update = true
+
+  values = [
+    yamlencode(local.argo_values)
+  ]
+}

examples/eks/iam-ecs-execution.tf

@@ -0,0 +1,55 @@
+data "aws_iam_policy_document" "ecs_execution_role_assume_role" {
+  statement {
+    actions = [
+      "sts:AssumeRole"
+    ]
+
+    effect = "Allow"
+
+    principals {
+      identifiers = [
+        "ec2.amazonaws.com",
+        "ecs.amazonaws.com",
+        "ecs-tasks.amazonaws.com",
+        "batch.amazonaws.com"
+      ]
+      type = "Service"
+    }
+  }
+}
+
+resource "aws_iam_role" "ecs_execution_role" {
+  name = "${local.resource_prefix}ecs-execution-role${local.resource_suffix}"
+  # Read more about ECS' `task_role` and `execution_role` here https://stackoverflow.com/a/49947471
+  description        = "This role is passed to our AWS ECS' task definition as the `execution_role`. This allows things like the correct image to be pulled and logs to be stored."
+  assume_role_policy = data.aws_iam_policy_document.ecs_execution_role_assume_role.json
+
+  tags = local.tags
+}
+
+data "aws_iam_policy_document" "ecs_task_execution_policy" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "ecr:GetAuthorizationToken",
+      "ecr:BatchCheckLayerAvailability",
+      "ecr:GetDownloadUrlForLayer",
+      "ecr:BatchGetImage",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+
+    # The `"Resource": "*"` is not a concern and the policy that Amazon suggests using
+    # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_execution_IAM_role.html
+    resources = [
+      "*"
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "grant_ecs_access" {
+  name   = "ecs_access"
+  role   = aws_iam_role.ecs_execution_role.name
+  policy = data.aws_iam_policy_document.ecs_task_execution_policy.json
+}

examples/eks/metaflow.tf

@@ -0,0 +1,64 @@
+resource "random_string" "suffix" {
+  length  = 8
+  special = false
+  upper = false
+}
+
+locals {
+  resource_prefix = "metaflow"
+  resource_suffix = random_string.suffix.result
+  tags = {
+      "managedBy" = "terraform"
+      "application" = "metaflow-eks-example"
+  }
+  cluster_name = "mf-${local.resource_suffix}"
+}
+
+data "aws_availability_zones" "available" {
+}
+
+
+module "metaflow-datastore" {
+  source = "outerbounds/metaflow/aws//modules/datastore"
+  version = "0.3.1"
+
+  resource_prefix = local.resource_prefix
+  resource_suffix = local.resource_suffix
+
+  metadata_service_security_group_id = module.metaflow-metadata-service.metadata_service_security_group_id
+  metaflow_vpc_id                    = module.vpc.vpc_id
+  subnet1_id                         = module.vpc.private_subnets[0]
+  subnet2_id                         = module.vpc.private_subnets[1]
+
+  standard_tags = local.tags
+}
+
+module "metaflow-common" {
+  source = "outerbounds/metaflow/aws//modules/common"
+  version = "0.3.1"
+}
+
+module "metaflow-metadata-service" {
+  # source = "outerbounds/metaflow/aws//modules/metadata-service"
+  # version = "0.3.1"
+  source = "../../modules/metadata-service"
+
+  resource_prefix = local.resource_prefix
+  resource_suffix = local.resource_suffix
+
+  access_list_cidr_blocks           = []
+  api_basic_auth                    = true
+  database_password                 = module.metaflow-datastore.database_password
+  database_username                 = module.metaflow-datastore.database_username
+  datastore_s3_bucket_kms_key_arn   = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
+  fargate_execution_role_arn        = aws_iam_role.ecs_execution_role.arn
+  metaflow_vpc_id                   = module.vpc.vpc_id
+  metadata_service_container_image = module.metaflow-common.default_metadata_service_container_image
+  rds_master_instance_endpoint      = module.metaflow-datastore.rds_master_instance_endpoint
+  s3_bucket_arn                     = module.metaflow-datastore.s3_bucket_arn
+  subnet1_id                        = module.vpc.private_subnets[0]
+  subnet2_id                        = module.vpc.private_subnets[1]
+  vpc_cidr_block                    = module.vpc.vpc_cidr_block
+
+  standard_tags = local.tags
+}