Migrate azure to workload identity

osamamagdy · osamamagdy · commit 4f13722836c4 · 2023-12-17T02:38:48.000+02:00
Signed-off-by: osamamagdy &lt;osamamagdy174@gmail.com&gt;
diff --git a/azure/README.md b/azure/README.md
@@ -114,6 +114,7 @@ The documentation below is auto-generated to give insight on what's created via
 
 | Name | Version |
 |------|---------|
+| <a name="provider_azuread"></a> [azuread](#provider\_azuread) | n/a |
 | <a name="provider_azurerm"></a> [azurerm](#provider\_azurerm) | 3.83.0 |
 | <a name="provider_http"></a> [http](#provider\_http) | 3.4.0 |
 | <a name="provider_random"></a> [random](#provider\_random) | 3.5.1 |
@@ -126,6 +127,10 @@ No modules.
 
 | Name | Type |
 |------|------|
+| [azuread_application.app](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application) | resource |
+| [azuread_application_federated_identity_credential.app](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/application_federated_identity_credential) | resource |
+| [azuread_service_principal.app](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/service_principal) | resource |
+| [azuread_service_principal_password.app](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/resources/service_principal_password) | resource |
 | [azurerm_key_vault.vault](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault) | resource |
 | [azurerm_key_vault_access_policy.extra_identity_access](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource |
 | [azurerm_key_vault_access_policy.identity_access](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource |
@@ -135,11 +140,6 @@ No modules.
 | [azurerm_key_vault_secret.wrongsecret_3](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource |
 | [azurerm_kubernetes_cluster.cluster](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster) | resource |
 | [azurerm_resource_group.default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
-| [azurerm_role_assignment.aks_extra_identity_operator](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
-| [azurerm_role_assignment.aks_identity_operator](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
-| [azurerm_role_assignment.aks_vm_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource |
-| [azurerm_user_assigned_identity.aks_extra_pod_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) | resource |
-| [azurerm_user_assigned_identity.aks_pod_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/user_assigned_identity) | resource |
 | [random_integer.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
 | [random_password.password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
 | [random_string.suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
@@ -158,12 +158,10 @@ No modules.
 
 | Name | Description |
 |------|-------------|
-| <a name="output_aad_extra_pod_identity_client_id"></a> [aad\_extra\_pod\_identity\_client\_id](#output\_aad\_extra\_pod\_identity\_client\_id) | Client ID for the Managed Identity for AAD Pod Identity |
-| <a name="output_aad_extra_pod_identity_resource_id"></a> [aad\_extra\_pod\_identity\_resource\_id](#output\_aad\_extra\_pod\_identity\_resource\_id) | Resource ID for the Managed Identity for AAD Pod Identity |
-| <a name="output_aad_pod_identity_client_id"></a> [aad\_pod\_identity\_client\_id](#output\_aad\_pod\_identity\_client\_id) | Client ID for the Managed Identity for AAD Pod Identity |
-| <a name="output_aad_pod_identity_resource_id"></a> [aad\_pod\_identity\_resource\_id](#output\_aad\_pod\_identity\_resource\_id) | Resource ID for the Managed Identity for AAD Pod Identity |
+| <a name="output_app_client_id"></a> [app\_client\_id](#output\_app\_client\_id) | n/a |
 | <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | AKS Cluster name |
 | <a name="output_key_vault_url"></a> [key\_vault\_url](#output\_key\_vault\_url) | Azure KeyVault URI for the Demo Container |
+| <a name="output_oidc_issuer_url"></a> [oidc\_issuer\_url](#output\_oidc\_issuer\_url) | AKS Cluster OIDC Issuer URL |
 | <a name="output_resource_group"></a> [resource\_group](#output\_resource\_group) | Resource group name |
 | <a name="output_tenant_id"></a> [tenant\_id](#output\_tenant\_id) | Azure tenant ID |
 | <a name="output_vault_name"></a> [vault\_name](#output\_vault\_name) | Vault name |
diff --git a/azure/iam.tf b/azure/iam.tf
diff --git a/azure/identity.tf b/azure/identity.tf
@@ -0,0 +1,35 @@
+############################################################################################################################
+## Here we need to create an Azure AD Application + a Service Principal and federate the application with the OIDC Issuer ##
+## so that Azure AD can exchange a token issued to the pod with a token that can be used to access other Azure resources. ##
+############################################################################################################################
+
+
+locals {
+  namespace_name = "default"
+  ## This should match the name of the service account created by helm chart
+  service_account_name = "wrongsecrets-sa"
+}
+
+## Azure AD application that represents the app
+resource "azuread_application" "app" {
+  display_name = "sp-wrongsecrets"
+}
+
+resource "azuread_service_principal" "app" {
+  client_id                    = azuread_application.app.client_id
+  app_role_assignment_required = false
+}
+
+resource "azuread_service_principal_password" "app" {
+  service_principal_id = azuread_service_principal.app.id
+}
+
+## Azure AD federated identity used to federate kubernetes with Azure AD
+resource "azuread_application_federated_identity_credential" "app" {
+  application_id = azuread_application.app.application_id
+  display_name          = "fed-identity-app-wrongsecrets"
+  description           = "The federated identity used to federate K8s with Azure AD with the app service running in k8s wrongsecrets"
+  audiences             = ["api://AzureADTokenExchange"]
+  issuer                = azurerm_kubernetes_cluster.cluster.oidc_issuer_url
+  subject               = "system:serviceaccount:${local.namespace_name}:${local.service_account_name}"
+}
diff --git a/azure/k8s-vault-azure-start.sh b/azure/k8s-vault-azure-start.sh
@@ -21,11 +21,7 @@ export CLUSTER_NAME="$(terraform output -raw cluster_name)"
 export IDENTITY_RESOURCE_GROUP="$(az aks show -g ${RESOURCE_GROUP} -n ${CLUSTER_NAME} --query nodeResourceGroup -otsv)"
 export IDENTITY_NAME="wrongsecrets-identity"
 
-export AZ_POD_RESOURCE_ID="$(terraform output -raw aad_pod_identity_resource_id)"
-export AZ_POD_CLIENT_ID="$(terraform output -raw aad_pod_identity_client_id)"
-
-export AZ_EXTRA_POD_RESOURCE_ID="$(terraform output -raw aad_extra_pod_identity_resource_id)"
-export AZ_EXTRA_POD_CLIENT_ID="$(terraform output -raw aad_extra_pod_identity_client_id)"
+export AZ_AD_APP_CLIENT_ID="$(terraform output -raw app_client_id)"
 
 export AZ_VAULT_URI="$(terraform output -raw vault_uri)"
 export AZ_KEY_VAULT_TENANT_ID="$(terraform output -raw tenant_id)"
@@ -67,19 +63,19 @@ else
   helm install csi csi-secrets-store-provider-azure/csi-secrets-store-provider-azure --namespace kube-system
 fi
 
-#TO BE REPLACED WITH https://azure.github.io/azure-workload-identity/docs/installation.html
-echo "Add Azure pod identity to repo"
-helm repo add aad-pod-identity https://raw.githubusercontent.com/Azure/aad-pod-identity/master/charts
+echo "Add Azure workload identity to repo"
+helm repo add azure-workload-identity https://azure.github.io/azure-workload-identity/charts
 
-helm list --namespace kube-system | grep 'aad-pod-identity' &>/dev/null
+helm list --namespace kube-system | grep 'workload-identity-webhook' &>/dev/null
 if [ $? == 0 ]; then
-  echo "Azure pod identity chart already installed"
+  echo "Azure workload identity chart already installed"
 else
-  helm upgrade --install aad-pod-identity aad-pod-identity/aad-pod-identity #NO LONGER WORKS BECAUSE OF OUR CONFIUGRATION (RESTRICTED IN DEFAULT)
+helm install workload-identity-webhook azure-workload-identity/workload-identity-webhook \
+   --namespace azure-workload-identity-system \
+   --create-namespace \
+   --set azureTenantID="${AZURE_TENANT_ID}"
 fi
 
-#END TO BE REPLACED WITH https://azure.github.io/azure-workload-identity/docs/installation.html
-
 echo "Generate secret manager challenge secret 2"
 az keyvault secret set --name wrongsecret-2 --vault-name "${AZ_KEY_VAULT_NAME}" --value "$(openssl rand -base64 16)" >/dev/null
 
@@ -92,10 +88,10 @@ envsubst <./k8s/secret-volume.yml.tpl >./k8s/secret-volume.yml
 echo "Apply secretsmanager storage volume"
 kubectl apply -f./k8s/secret-volume.yml
 
-envsubst <./k8s/pod-id.yml.tpl >./k8s/pod-id.yml
+envsubst <./k8s/serviceAccount.yml.tpl >./k8s/serviceAccount.yml
 envsubst <./k8s/secret-challenge-vault-deployment.yml.tpl >./k8s/secret-challenge-vault-deployment.yml
 
-kubectl apply -f./k8s/pod-id.yml
+kubectl apply -f./k8s/serviceAccount.yml
 
 while [[ $(kubectl --namespace=default get pods -l "app.kubernetes.io/component=mic" -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True True" ]]; do echo "waiting for component=mic" && sleep 2; done
 while [[ $(kubectl --namespace=default get pods -l "app.kubernetes.io/component=nmi" -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting for component=nmi" && sleep 2; done
diff --git a/azure/k8s/pod-id.yml.tpl b/azure/k8s/pod-id.yml.tpl
diff --git a/azure/k8s/secret-challenge-vault-deployment.yml.tpl b/azure/k8s/secret-challenge-vault-deployment.yml.tpl
@@ -3,7 +3,6 @@ kind: Deployment
 metadata:
   labels:
     app: secret-challenge
-    aadpodidbinding: wrongsecrets-pod-id
   name: secret-challenge
   namespace: default
 spec:
@@ -20,17 +19,15 @@ spec:
     type: RollingUpdate
   template:
     metadata:
-      creationTimestamp: "2020-10-28T20:21:04Z"
       labels:
         app: secret-challenge
-        aadpodidbinding: wrongsecrets-pod-id
       name: secret-challenge
     spec:
       securityContext:
         runAsUser: 2000
         runAsGroup: 2000
         fsGroup: 2000
-      serviceAccountName: vault
+      serviceAccountName: wrongsecrets-sa
       volumes:
         - name: 'ephemeral'
           emptyDir: {}
@@ -92,8 +89,6 @@ spec:
               value: wrongsecret-3
             - name: SPRING_CLOUD_AZURE_KEYVAULT_SECRET_PROPERTYSOURCES_0_ENDPOINT
               value: ${AZ_VAULT_URI}
-            - name: SPRING_CLOUD_AZURE_KEYVAULT_SECRET_PROPERTYSOURCES_0_CREDENTIAL_CLIENTID
-              value: ${AZ_POD_CLIENT_ID}
             - name: SPRING_CLOUD_AZURE_KEYVAULT_SECRET_PROPERTYSOURCES_0_CREDENTIAL_MANAGEDIDENTITYENABLED
               value: "true"
             - name: SPECIAL_K8S_SECRET
diff --git a/azure/k8s/secret-volume.yml.tpl b/azure/k8s/secret-volume.yml.tpl
@@ -5,7 +5,6 @@ metadata:
 spec:
   provider: azure
   parameters:
-    usePodIdentity: "true"
     tenantId: ${AZ_KEY_VAULT_TENANT_ID}
     keyvaultName: ${AZ_KEY_VAULT_NAME}
     objects: |
diff --git a/azure/k8s/serviceAccount.yaml.tpl b/azure/k8s/serviceAccount.yaml.tpl
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: wrongsecrets-sa
+  labels:
+    azure.workload.identity/use: "true" # Represents the service account is to be used for workload identity, see https://azure.github.io/azure-workload-identity/docs/topics/service-account-labels-and-annotations.html
+  annotations:
+    azure.workload.identity/client-id: ${AZ_AD_APP_CLIENT_ID}
+    azure.workload.identity/tenant-id: ${AZURE_TENANT_ID}
+    azure.workload.identity/service-account-token-expiration: "86400" # Token is valid for 1 day
diff --git a/azure/main.tf b/azure/main.tf
@@ -51,6 +51,10 @@ resource "azurerm_kubernetes_cluster" "cluster" {
 
   kubernetes_version = var.cluster_version
 
+  oidc_issuer_enabled = true
+
+  workload_identity_enabled = true
+
   api_server_access_profile {
     authorized_ip_ranges = ["${data.http.ip.response_body}/32"]
   }
diff --git a/azure/outputs.tf b/azure/outputs.tf
@@ -8,29 +8,19 @@ output "cluster_name" {
   description = "AKS Cluster name"
 }
 
-output "resource_group" {
-  value       = azurerm_kubernetes_cluster.cluster.resource_group_name
-  description = "Resource group name"
+output "oidc_issuer_url" {
+  value       = azurerm_kubernetes_cluster.cluster.oidc_issuer_url
+  description = "AKS Cluster OIDC Issuer URL"
 }
 
-output "aad_pod_identity_resource_id" {
-  value       = azurerm_user_assigned_identity.aks_pod_identity.id
-  description = "Resource ID for the Managed Identity for AAD Pod Identity"
-}
 
-output "aad_pod_identity_client_id" {
-  value       = azurerm_user_assigned_identity.aks_pod_identity.client_id
-  description = "Client ID for the Managed Identity for AAD Pod Identity"
+output "app_client_id" {
+  value = azuread_application.app.application_id
 }
 
-output "aad_extra_pod_identity_resource_id" {
-  value       = azurerm_user_assigned_identity.aks_extra_pod_identity.id
-  description = "Resource ID for the Managed Identity for AAD Pod Identity"
-}
-
-output "aad_extra_pod_identity_client_id" {
-  value       = azurerm_user_assigned_identity.aks_extra_pod_identity.client_id
-  description = "Client ID for the Managed Identity for AAD Pod Identity"
+output "resource_group" {
+  value       = azurerm_kubernetes_cluster.cluster.resource_group_name
+  description = "Resource group name"
 }
 
 output "vault_uri" {
diff --git a/azure/secrets.tf b/azure/secrets.tf
@@ -39,7 +39,7 @@ resource "azurerm_key_vault_access_policy" "user" {
 resource "azurerm_key_vault_access_policy" "identity_access" {
   key_vault_id = azurerm_key_vault.vault.id
   tenant_id    = data.azurerm_client_config.current.tenant_id
-  object_id    = azurerm_user_assigned_identity.aks_pod_identity.principal_id
+  object_id    = azuread_service_principal.app.id
 
   secret_permissions = [
     "Get", "List"
@@ -107,7 +107,7 @@ resource "azurerm_key_vault_secret" "wrongsecret_3" {
 resource "azurerm_key_vault_access_policy" "extra_identity_access" {
   key_vault_id = azurerm_key_vault.vault.id
   tenant_id    = data.azurerm_client_config.current.tenant_id
-  object_id    = azurerm_user_assigned_identity.aks_extra_pod_identity.principal_id
+  object_id    = azuread_service_principal.app.id
 
   secret_permissions = [
     "Get", "List"
diff --git a/azure/versions.tf b/azure/versions.tf
@@ -14,5 +14,9 @@ terraform {
       source  = "hashicorp/http"
       version = "~> 3.4.0"
     }
+    azuread = {
+      source = "hashicorp/azuread"
+      version = "2.47.0"
+    }
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,10 @@ resource "azurerm_kubernetes_cluster" "cluster" {`
`51`	`51`
`52`	`52`	`kubernetes_version = var.cluster_version`
`53`	`53`
	`54`	`+ oidc_issuer_enabled = true`
	`55`	`+`
	`56`	`+ workload_identity_enabled = true`
	`57`	`+`
`54`	`58`	`api_server_access_profile {`
`55`	`59`	`authorized_ip_ranges = ["${data.http.ip.response_body}/32"]`
`56`	`60`	`}`
Original file line number	Diff line number	Diff line change
`@@ -14,5 +14,9 @@ terraform {`
`14`	`14`	`source = "hashicorp/http"`
`15`	`15`	`version = "~> 3.4.0"`
`16`	`16`	`}`
	`17`	`+ azuread = {`
	`18`	`+ source = "hashicorp/azuread"`
	`19`	`+ version = "2.47.0"`
	`20`	`+ }`
`17`	`21`	`}`
`18`	`22`	`}`